drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.c

   1 /*
   2  * Copyright (c) 2014-2018 The Linux Foundation. All rights reserved.
   3  * Copyright (C) 2013 Red Hat
   4  * Author: Rob Clark <robdclark@gmail.com>
   5  *
   6  * This program is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 as published by
   8  * the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13  * more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along with
  16  * this program.  If not, see <http://www.gnu.org/licenses/>.
  17  */
  18
  19 #define pr_fmt(fmt)     "[drm:%s:%d] " fmt, __func__, __LINE__
  20 #include <linux/sort.h>
  21 #include <linux/debugfs.h>
  22 #include <linux/ktime.h>
  23 #include <drm/drm_mode.h>
  24 #include <drm/drm_crtc.h>
  25 #include <drm/drm_crtc_helper.h>
  26 #include <drm/drm_flip_work.h>
  27 #include <drm/drm_rect.h>
  28
  29 #include "dpu_kms.h"
  30 #include "dpu_hw_lm.h"
  31 #include "dpu_hw_ctl.h"
  32 #include "dpu_crtc.h"
  33 #include "dpu_plane.h"
  34 #include "dpu_encoder.h"
  35 #include "dpu_vbif.h"
  36 #include "dpu_power_handle.h"
  37 #include "dpu_core_perf.h"
  38 #include "dpu_trace.h"
  39
  40 #define DPU_DRM_BLEND_OP_NOT_DEFINED    0
  41 #define DPU_DRM_BLEND_OP_OPAQUE         1
  42 #define DPU_DRM_BLEND_OP_PREMULTIPLIED  2
  43 #define DPU_DRM_BLEND_OP_COVERAGE       3
  44 #define DPU_DRM_BLEND_OP_MAX            4
  45
  46 /* layer mixer index on dpu_crtc */
  47 #define LEFT_MIXER 0
  48 #define RIGHT_MIXER 1
  49
  50 static inline struct dpu_kms *_dpu_crtc_get_kms(struct drm_crtc *crtc)
  51 {
  52         struct msm_drm_private *priv;
  53
  54         if (!crtc || !crtc->dev || !crtc->dev->dev_private) {
  55                 DPU_ERROR("invalid crtc\n");
  56                 return NULL;
  57         }
  58         priv = crtc->dev->dev_private;
  59         if (!priv || !priv->kms) {
  60                 DPU_ERROR("invalid kms\n");
  61                 return NULL;
  62         }
  63
  64         return to_dpu_kms(priv->kms);
  65 }
  66
  67 static inline int _dpu_crtc_power_enable(struct dpu_crtc *dpu_crtc, bool enable)
  68 {
  69         struct drm_crtc *crtc;
  70         struct msm_drm_private *priv;
  71         struct dpu_kms *dpu_kms;
  72
  73         if (!dpu_crtc) {
  74                 DPU_ERROR("invalid dpu crtc\n");
  75                 return -EINVAL;
  76         }
  77
  78         crtc = &dpu_crtc->base;
  79         if (!crtc->dev || !crtc->dev->dev_private) {
  80                 DPU_ERROR("invalid drm device\n");
  81                 return -EINVAL;
  82         }
  83
  84         priv = crtc->dev->dev_private;
  85         if (!priv->kms) {
  86                 DPU_ERROR("invalid kms\n");
  87                 return -EINVAL;
  88         }
  89
  90         dpu_kms = to_dpu_kms(priv->kms);
  91
  92         if (enable)
  93                 pm_runtime_get_sync(&dpu_kms->pdev->dev);
  94         else
  95                 pm_runtime_put_sync(&dpu_kms->pdev->dev);
  96
  97         return 0;
  98 }
  99
 100 /**
 101  * _dpu_crtc_rp_to_crtc - get crtc from resource pool object
 102  * @rp: Pointer to resource pool
 103  * return: Pointer to drm crtc if success; null otherwise
 104  */
 105 static struct drm_crtc *_dpu_crtc_rp_to_crtc(struct dpu_crtc_respool *rp)
 106 {
 107         if (!rp)
 108                 return NULL;
 109
 110         return container_of(rp, struct dpu_crtc_state, rp)->base.crtc;
 111 }
 112
 113 /**
 114  * _dpu_crtc_rp_reclaim - reclaim unused, or all if forced, resources in pool
 115  * @rp: Pointer to resource pool
 116  * @force: True to reclaim all resources; otherwise, reclaim only unused ones
 117  * return: None
 118  */
 119 static void _dpu_crtc_rp_reclaim(struct dpu_crtc_respool *rp, bool force)
 120 {
 121         struct dpu_crtc_res *res, *next;
 122         struct drm_crtc *crtc;
 123
 124         crtc = _dpu_crtc_rp_to_crtc(rp);
 125         if (!crtc) {
 126                 DPU_ERROR("invalid crtc\n");
 127                 return;
 128         }
 129
 130         DPU_DEBUG("crtc%d.%u %s\n", crtc->base.id, rp->sequence_id,
 131                         force ? "destroy" : "free_unused");
 132
 133         list_for_each_entry_safe(res, next, &rp->res_list, list) {
 134                 if (!force && !(res->flags & DPU_CRTC_RES_FLAG_FREE))
 135                         continue;
 136                 DPU_DEBUG("crtc%d.%u reclaim res:0x%x/0x%llx/%pK/%d\n",
 137                                 crtc->base.id, rp->sequence_id,
 138                                 res->type, res->tag, res->val,
 139                                 atomic_read(&res->refcount));
 140                 list_del(&res->list);
 141                 if (res->ops.put)
 142                         res->ops.put(res->val);
 143                 kfree(res);
 144         }
 145 }
 146
 147 /**
 148  * _dpu_crtc_rp_free_unused - free unused resource in pool
 149  * @rp: Pointer to resource pool
 150  * return: none
 151  */
 152 static void _dpu_crtc_rp_free_unused(struct dpu_crtc_respool *rp)
 153 {
 154         mutex_lock(rp->rp_lock);
 155         _dpu_crtc_rp_reclaim(rp, false);
 156         mutex_unlock(rp->rp_lock);
 157 }
 158
 159 /**
 160  * _dpu_crtc_rp_destroy - destroy resource pool
 161  * @rp: Pointer to resource pool
 162  * return: None
 163  */
 164 static void _dpu_crtc_rp_destroy(struct dpu_crtc_respool *rp)
 165 {
 166         mutex_lock(rp->rp_lock);
 167         list_del_init(&rp->rp_list);
 168         _dpu_crtc_rp_reclaim(rp, true);
 169         mutex_unlock(rp->rp_lock);
 170 }
 171
 172 /**
 173  * _dpu_crtc_hw_blk_get - get callback for hardware block
 174  * @val: Resource handle
 175  * @type: Resource type
 176  * @tag: Search tag for given resource
 177  * return: Resource handle
 178  */
 179 static void *_dpu_crtc_hw_blk_get(void *val, u32 type, u64 tag)
 180 {
 181         DPU_DEBUG("res:%d/0x%llx/%pK\n", type, tag, val);
 182         return dpu_hw_blk_get(val, type, tag);
 183 }
 184
 185 /**
 186  * _dpu_crtc_hw_blk_put - put callback for hardware block
 187  * @val: Resource handle
 188  * return: None
 189  */
 190 static void _dpu_crtc_hw_blk_put(void *val)
 191 {
 192         DPU_DEBUG("res://%pK\n", val);
 193         dpu_hw_blk_put(val);
 194 }
 195
 196 /**
 197  * _dpu_crtc_rp_duplicate - duplicate resource pool and reset reference count
 198  * @rp: Pointer to original resource pool
 199  * @dup_rp: Pointer to duplicated resource pool
 200  * return: None
 201  */
 202 static void _dpu_crtc_rp_duplicate(struct dpu_crtc_respool *rp,
 203                 struct dpu_crtc_respool *dup_rp)
 204 {
 205         struct dpu_crtc_res *res, *dup_res;
 206         struct drm_crtc *crtc;
 207
 208         if (!rp || !dup_rp || !rp->rp_head) {
 209                 DPU_ERROR("invalid resource pool\n");
 210                 return;
 211         }
 212
 213         crtc = _dpu_crtc_rp_to_crtc(rp);
 214         if (!crtc) {
 215                 DPU_ERROR("invalid crtc\n");
 216                 return;
 217         }
 218
 219         DPU_DEBUG("crtc%d.%u duplicate\n", crtc->base.id, rp->sequence_id);
 220
 221         mutex_lock(rp->rp_lock);
 222         dup_rp->sequence_id = rp->sequence_id + 1;
 223         INIT_LIST_HEAD(&dup_rp->res_list);
 224         dup_rp->ops = rp->ops;
 225         list_for_each_entry(res, &rp->res_list, list) {
 226                 dup_res = kzalloc(sizeof(struct dpu_crtc_res), GFP_KERNEL);
 227                 if (!dup_res) {
 228                         mutex_unlock(rp->rp_lock);
 229                         return;
 230                 }
 231                 INIT_LIST_HEAD(&dup_res->list);
 232                 atomic_set(&dup_res->refcount, 0);
 233                 dup_res->type = res->type;
 234                 dup_res->tag = res->tag;
 235                 dup_res->val = res->val;
 236                 dup_res->ops = res->ops;
 237                 dup_res->flags = DPU_CRTC_RES_FLAG_FREE;
 238                 DPU_DEBUG("crtc%d.%u dup res:0x%x/0x%llx/%pK/%d\n",
 239                                 crtc->base.id, dup_rp->sequence_id,
 240                                 dup_res->type, dup_res->tag, dup_res->val,
 241                                 atomic_read(&dup_res->refcount));
 242                 list_add_tail(&dup_res->list, &dup_rp->res_list);
 243                 if (dup_res->ops.get)
 244                         dup_res->ops.get(dup_res->val, 0, -1);
 245         }
 246
 247         dup_rp->rp_lock = rp->rp_lock;
 248         dup_rp->rp_head = rp->rp_head;
 249         INIT_LIST_HEAD(&dup_rp->rp_list);
 250         list_add_tail(&dup_rp->rp_list, rp->rp_head);
 251         mutex_unlock(rp->rp_lock);
 252 }
 253
 254 /**
 255  * _dpu_crtc_rp_reset - reset resource pool after allocation
 256  * @rp: Pointer to original resource pool
 257  * @rp_lock: Pointer to serialization resource pool lock
 258  * @rp_head: Pointer to crtc resource pool head
 259  * return: None
 260  */
 261 static void _dpu_crtc_rp_reset(struct dpu_crtc_respool *rp,
 262                 struct mutex *rp_lock, struct list_head *rp_head)
 263 {
 264         if (!rp || !rp_lock || !rp_head) {
 265                 DPU_ERROR("invalid resource pool\n");
 266                 return;
 267         }
 268
 269         mutex_lock(rp_lock);
 270         rp->rp_lock = rp_lock;
 271         rp->rp_head = rp_head;
 272         INIT_LIST_HEAD(&rp->rp_list);
 273         rp->sequence_id = 0;
 274         INIT_LIST_HEAD(&rp->res_list);
 275         rp->ops.get = _dpu_crtc_hw_blk_get;
 276         rp->ops.put = _dpu_crtc_hw_blk_put;
 277         list_add_tail(&rp->rp_list, rp->rp_head);
 278         mutex_unlock(rp_lock);
 279 }
 280
 281 static void dpu_crtc_destroy(struct drm_crtc *crtc)
 282 {
 283         struct dpu_crtc *dpu_crtc = to_dpu_crtc(crtc);
 284
 285         DPU_DEBUG("\n");
 286
 287         if (!crtc)
 288                 return;
 289
 290         dpu_crtc->phandle = NULL;
 291
 292         drm_crtc_cleanup(crtc);
 293         mutex_destroy(&dpu_crtc->crtc_lock);
 294         kfree(dpu_crtc);
 295 }
 296
 297 static void _dpu_crtc_setup_blend_cfg(struct dpu_crtc_mixer *mixer,
 298                 struct dpu_plane_state *pstate, struct dpu_format *format)
 299 {
 300         struct dpu_hw_mixer *lm = mixer->hw_lm;
 301         uint32_t blend_op;
 302         struct drm_format_name_buf format_name;
 303
 304         /* default to opaque blending */
 305         blend_op = DPU_BLEND_FG_ALPHA_FG_CONST |
 306                 DPU_BLEND_BG_ALPHA_BG_CONST;
 307
 308         if (format->alpha_enable) {
 309                 /* coverage blending */
 310                 blend_op = DPU_BLEND_FG_ALPHA_FG_PIXEL |
 311                         DPU_BLEND_BG_ALPHA_FG_PIXEL |
 312                         DPU_BLEND_BG_INV_ALPHA;
 313         }
 314
 315         lm->ops.setup_blend_config(lm, pstate->stage,
 316                                 0xFF, 0, blend_op);
 317
 318         DPU_DEBUG("format:%s, alpha_en:%u blend_op:0x%x\n",
 319                 drm_get_format_name(format->base.pixel_format, &format_name),
 320                 format->alpha_enable, blend_op);
 321 }
 322
 323 static void _dpu_crtc_program_lm_output_roi(struct drm_crtc *crtc)
 324 {
 325         struct dpu_crtc *dpu_crtc;
 326         struct dpu_crtc_state *crtc_state;
 327         int lm_idx, lm_horiz_position;
 328
 329         dpu_crtc = to_dpu_crtc(crtc);
 330         crtc_state = to_dpu_crtc_state(crtc->state);
 331
 332         lm_horiz_position = 0;
 333         for (lm_idx = 0; lm_idx < dpu_crtc->num_mixers; lm_idx++) {
 334                 const struct drm_rect *lm_roi = &crtc_state->lm_bounds[lm_idx];
 335                 struct dpu_hw_mixer *hw_lm = dpu_crtc->mixers[lm_idx].hw_lm;
 336                 struct dpu_hw_mixer_cfg cfg;
 337
 338                 if (!lm_roi || !drm_rect_visible(lm_roi))
 339                         continue;
 340
 341                 cfg.out_width = drm_rect_width(lm_roi);
 342                 cfg.out_height = drm_rect_height(lm_roi);
 343                 cfg.right_mixer = lm_horiz_position++;
 344                 cfg.flags = 0;
 345                 hw_lm->ops.setup_mixer_out(hw_lm, &cfg);
 346         }
 347 }
 348
 349 static void _dpu_crtc_blend_setup_mixer(struct drm_crtc *crtc,
 350         struct dpu_crtc *dpu_crtc, struct dpu_crtc_mixer *mixer)
 351 {
 352         struct drm_plane *plane;
 353         struct drm_framebuffer *fb;
 354         struct drm_plane_state *state;
 355         struct dpu_crtc_state *cstate;
 356         struct dpu_plane_state *pstate = NULL;
 357         struct dpu_format *format;
 358         struct dpu_hw_ctl *ctl;
 359         struct dpu_hw_mixer *lm;
 360         struct dpu_hw_stage_cfg *stage_cfg;
 361
 362         u32 flush_mask;
 363         uint32_t stage_idx, lm_idx;
 364         int zpos_cnt[DPU_STAGE_MAX + 1] = { 0 };
 365         bool bg_alpha_enable = false;
 366
 367         if (!dpu_crtc || !mixer) {
 368                 DPU_ERROR("invalid dpu_crtc or mixer\n");
 369                 return;
 370         }
 371
 372         ctl = mixer->hw_ctl;
 373         lm = mixer->hw_lm;
 374         stage_cfg = &dpu_crtc->stage_cfg;
 375         cstate = to_dpu_crtc_state(crtc->state);
 376
 377         drm_atomic_crtc_for_each_plane(plane, crtc) {
 378                 state = plane->state;
 379                 if (!state)
 380                         continue;
 381
 382                 pstate = to_dpu_plane_state(state);
 383                 fb = state->fb;
 384
 385                 dpu_plane_get_ctl_flush(plane, ctl, &flush_mask);
 386
 387                 DPU_DEBUG("crtc %d stage:%d - plane %d sspp %d fb %d\n",
 388                                 crtc->base.id,
 389                                 pstate->stage,
 390                                 plane->base.id,
 391                                 dpu_plane_pipe(plane) - SSPP_VIG0,
 392                                 state->fb ? state->fb->base.id : -1);
 393
 394                 format = to_dpu_format(msm_framebuffer_format(pstate->base.fb));
 395                 if (!format) {
 396                         DPU_ERROR("invalid format\n");
 397                         return;
 398                 }
 399
 400                 if (pstate->stage == DPU_STAGE_BASE && format->alpha_enable)
 401                         bg_alpha_enable = true;
 402
 403                 stage_idx = zpos_cnt[pstate->stage]++;
 404                 stage_cfg->stage[pstate->stage][stage_idx] =
 405                                         dpu_plane_pipe(plane);
 406                 stage_cfg->multirect_index[pstate->stage][stage_idx] =
 407                                         pstate->multirect_index;
 408
 409                 trace_dpu_crtc_setup_mixer(DRMID(crtc), DRMID(plane),
 410                                            state, pstate, stage_idx,
 411                                            dpu_plane_pipe(plane) - SSPP_VIG0,
 412                                            format->base.pixel_format,
 413                                            fb ? fb->modifier : 0);
 414
 415                 /* blend config update */
 416                 for (lm_idx = 0; lm_idx < dpu_crtc->num_mixers; lm_idx++) {
 417                         _dpu_crtc_setup_blend_cfg(mixer + lm_idx,
 418                                                 pstate, format);
 419
 420                         mixer[lm_idx].flush_mask |= flush_mask;
 421
 422                         if (bg_alpha_enable && !format->alpha_enable)
 423                                 mixer[lm_idx].mixer_op_mode = 0;
 424                         else
 425                                 mixer[lm_idx].mixer_op_mode |=
 426                                                 1 << pstate->stage;
 427                 }
 428         }
 429
 430          _dpu_crtc_program_lm_output_roi(crtc);
 431 }
 432
 433 /**
 434  * _dpu_crtc_blend_setup - configure crtc mixers
 435  * @crtc: Pointer to drm crtc structure
 436  */
 437 static void _dpu_crtc_blend_setup(struct drm_crtc *crtc)
 438 {
 439         struct dpu_crtc *dpu_crtc;
 440         struct dpu_crtc_state *dpu_crtc_state;
 441         struct dpu_crtc_mixer *mixer;
 442         struct dpu_hw_ctl *ctl;
 443         struct dpu_hw_mixer *lm;
 444
 445         int i;
 446
 447         if (!crtc)
 448                 return;
 449
 450         dpu_crtc = to_dpu_crtc(crtc);
 451         dpu_crtc_state = to_dpu_crtc_state(crtc->state);
 452         mixer = dpu_crtc->mixers;
 453
 454         DPU_DEBUG("%s\n", dpu_crtc->name);
 455
 456         if (dpu_crtc->num_mixers > CRTC_DUAL_MIXERS) {
 457                 DPU_ERROR("invalid number mixers: %d\n", dpu_crtc->num_mixers);
 458                 return;
 459         }
 460
 461         for (i = 0; i < dpu_crtc->num_mixers; i++) {
 462                 if (!mixer[i].hw_lm || !mixer[i].hw_ctl) {
 463                         DPU_ERROR("invalid lm or ctl assigned to mixer\n");
 464                         return;
 465                 }
 466                 mixer[i].mixer_op_mode = 0;
 467                 mixer[i].flush_mask = 0;
 468                 if (mixer[i].hw_ctl->ops.clear_all_blendstages)
 469                         mixer[i].hw_ctl->ops.clear_all_blendstages(
 470                                         mixer[i].hw_ctl);
 471         }
 472
 473         /* initialize stage cfg */
 474         memset(&dpu_crtc->stage_cfg, 0, sizeof(struct dpu_hw_stage_cfg));
 475
 476         _dpu_crtc_blend_setup_mixer(crtc, dpu_crtc, mixer);
 477
 478         for (i = 0; i < dpu_crtc->num_mixers; i++) {
 479                 ctl = mixer[i].hw_ctl;
 480                 lm = mixer[i].hw_lm;
 481
 482                 lm->ops.setup_alpha_out(lm, mixer[i].mixer_op_mode);
 483
 484                 mixer[i].flush_mask |= ctl->ops.get_bitmask_mixer(ctl,
 485                         mixer[i].hw_lm->idx);
 486
 487                 /* stage config flush mask */
 488                 ctl->ops.update_pending_flush(ctl, mixer[i].flush_mask);
 489
 490                 DPU_DEBUG("lm %d, op_mode 0x%X, ctl %d, flush mask 0x%x\n",
 491                         mixer[i].hw_lm->idx - LM_0,
 492                         mixer[i].mixer_op_mode,
 493                         ctl->idx - CTL_0,
 494                         mixer[i].flush_mask);
 495
 496                 ctl->ops.setup_blendstage(ctl, mixer[i].hw_lm->idx,
 497                         &dpu_crtc->stage_cfg);
 498         }
 499 }
 500
 501 /**
 502  *  _dpu_crtc_complete_flip - signal pending page_flip events
 503  * Any pending vblank events are added to the vblank_event_list
 504  * so that the next vblank interrupt shall signal them.
 505  * However PAGE_FLIP events are not handled through the vblank_event_list.
 506  * This API signals any pending PAGE_FLIP events requested through
 507  * DRM_IOCTL_MODE_PAGE_FLIP and are cached in the dpu_crtc->event.
 508  * @crtc: Pointer to drm crtc structure
 509  */
 510 static void _dpu_crtc_complete_flip(struct drm_crtc *crtc)
 511 {
 512         struct dpu_crtc *dpu_crtc = to_dpu_crtc(crtc);
 513         struct drm_device *dev = crtc->dev;
 514         unsigned long flags;
 515
 516         spin_lock_irqsave(&dev->event_lock, flags);
 517         if (dpu_crtc->event) {
 518                 DRM_DEBUG_VBL("%s: send event: %pK\n", dpu_crtc->name,
 519                               dpu_crtc->event);
 520                 trace_dpu_crtc_complete_flip(DRMID(crtc));
 521                 drm_crtc_send_vblank_event(crtc, dpu_crtc->event);
 522                 dpu_crtc->event = NULL;
 523         }
 524         spin_unlock_irqrestore(&dev->event_lock, flags);
 525 }
 526
 527 enum dpu_intf_mode dpu_crtc_get_intf_mode(struct drm_crtc *crtc)
 528 {
 529         struct drm_encoder *encoder;
 530
 531         if (!crtc || !crtc->dev) {
 532                 DPU_ERROR("invalid crtc\n");
 533                 return INTF_MODE_NONE;
 534         }
 535
 536         drm_for_each_encoder(encoder, crtc->dev)
 537                 if (encoder->crtc == crtc)
 538                         return dpu_encoder_get_intf_mode(encoder);
 539
 540         return INTF_MODE_NONE;
 541 }
 542
 543 static void dpu_crtc_vblank_cb(void *data)
 544 {
 545         struct drm_crtc *crtc = (struct drm_crtc *)data;
 546         struct dpu_crtc *dpu_crtc = to_dpu_crtc(crtc);
 547
 548         /* keep statistics on vblank callback - with auto reset via debugfs */
 549         if (ktime_compare(dpu_crtc->vblank_cb_time, ktime_set(0, 0)) == 0)
 550                 dpu_crtc->vblank_cb_time = ktime_get();
 551         else
 552                 dpu_crtc->vblank_cb_count++;
 553         _dpu_crtc_complete_flip(crtc);
 554         drm_crtc_handle_vblank(crtc);
 555         trace_dpu_crtc_vblank_cb(DRMID(crtc));
 556 }
 557
 558 static void dpu_crtc_frame_event_work(struct kthread_work *work)
 559 {
 560         struct msm_drm_private *priv;
 561         struct dpu_crtc_frame_event *fevent;
 562         struct drm_crtc *crtc;
 563         struct dpu_crtc *dpu_crtc;
 564         struct dpu_kms *dpu_kms;
 565         unsigned long flags;
 566         bool frame_done = false;
 567
 568         if (!work) {
 569                 DPU_ERROR("invalid work handle\n");
 570                 return;
 571         }
 572
 573         fevent = container_of(work, struct dpu_crtc_frame_event, work);
 574         if (!fevent->crtc || !fevent->crtc->state) {
 575                 DPU_ERROR("invalid crtc\n");
 576                 return;
 577         }
 578
 579         crtc = fevent->crtc;
 580         dpu_crtc = to_dpu_crtc(crtc);
 581
 582         dpu_kms = _dpu_crtc_get_kms(crtc);
 583         if (!dpu_kms) {
 584                 DPU_ERROR("invalid kms handle\n");
 585                 return;
 586         }
 587         priv = dpu_kms->dev->dev_private;
 588         DPU_ATRACE_BEGIN("crtc_frame_event");
 589
 590         DRM_DEBUG_KMS("crtc%d event:%u ts:%lld\n", crtc->base.id, fevent->event,
 591                         ktime_to_ns(fevent->ts));
 592
 593         if (fevent->event & (DPU_ENCODER_FRAME_EVENT_DONE
 594                                 | DPU_ENCODER_FRAME_EVENT_ERROR
 595                                 | DPU_ENCODER_FRAME_EVENT_PANEL_DEAD)) {
 596
 597                 if (atomic_read(&dpu_crtc->frame_pending) < 1) {
 598                         /* this should not happen */
 599                         DRM_ERROR("crtc%d ev:%u ts:%lld frame_pending:%d\n",
 600                                         crtc->base.id,
 601                                         fevent->event,
 602                                         ktime_to_ns(fevent->ts),
 603                                         atomic_read(&dpu_crtc->frame_pending));
 604                 } else if (atomic_dec_return(&dpu_crtc->frame_pending) == 0) {
 605                         /* release bandwidth and other resources */
 606                         trace_dpu_crtc_frame_event_done(DRMID(crtc),
 607                                                         fevent->event);
 608                         dpu_core_perf_crtc_release_bw(crtc);
 609                 } else {
 610                         trace_dpu_crtc_frame_event_more_pending(DRMID(crtc),
 611                                                                 fevent->event);
 612                 }
 613
 614                 if (fevent->event & DPU_ENCODER_FRAME_EVENT_DONE)
 615                         dpu_core_perf_crtc_update(crtc, 0, false);
 616
 617                 if (fevent->event & (DPU_ENCODER_FRAME_EVENT_DONE
 618                                         | DPU_ENCODER_FRAME_EVENT_ERROR))
 619                         frame_done = true;
 620         }
 621
 622         if (fevent->event & DPU_ENCODER_FRAME_EVENT_PANEL_DEAD)
 623                 DPU_ERROR("crtc%d ts:%lld received panel dead event\n",
 624                                 crtc->base.id, ktime_to_ns(fevent->ts));
 625
 626         if (frame_done)
 627                 complete_all(&dpu_crtc->frame_done_comp);
 628
 629         spin_lock_irqsave(&dpu_crtc->spin_lock, flags);
 630         list_add_tail(&fevent->list, &dpu_crtc->frame_event_list);
 631         spin_unlock_irqrestore(&dpu_crtc->spin_lock, flags);
 632         DPU_ATRACE_END("crtc_frame_event");
 633 }
 634
 635 /*
 636  * dpu_crtc_frame_event_cb - crtc frame event callback API. CRTC module
 637  * registers this API to encoder for all frame event callbacks like
 638  * frame_error, frame_done, idle_timeout, etc. Encoder may call different events
 639  * from different context - IRQ, user thread, commit_thread, etc. Each event
 640  * should be carefully reviewed and should be processed in proper task context
 641  * to avoid schedulin delay or properly manage the irq context's bottom half
 642  * processing.
 643  */
 644 static void dpu_crtc_frame_event_cb(void *data, u32 event)
 645 {
 646         struct drm_crtc *crtc = (struct drm_crtc *)data;
 647         struct dpu_crtc *dpu_crtc;
 648         struct msm_drm_private *priv;
 649         struct dpu_crtc_frame_event *fevent;
 650         unsigned long flags;
 651         u32 crtc_id;
 652
 653         if (!crtc || !crtc->dev || !crtc->dev->dev_private) {
 654                 DPU_ERROR("invalid parameters\n");
 655                 return;
 656         }
 657
 658         /* Nothing to do on idle event */
 659         if (event & DPU_ENCODER_FRAME_EVENT_IDLE)
 660                 return;
 661
 662         dpu_crtc = to_dpu_crtc(crtc);
 663         priv = crtc->dev->dev_private;
 664         crtc_id = drm_crtc_index(crtc);
 665
 666         trace_dpu_crtc_frame_event_cb(DRMID(crtc), event);
 667
 668         spin_lock_irqsave(&dpu_crtc->spin_lock, flags);
 669         fevent = list_first_entry_or_null(&dpu_crtc->frame_event_list,
 670                         struct dpu_crtc_frame_event, list);
 671         if (fevent)
 672                 list_del_init(&fevent->list);
 673         spin_unlock_irqrestore(&dpu_crtc->spin_lock, flags);
 674
 675         if (!fevent) {
 676                 DRM_ERROR("crtc%d event %d overflow\n", crtc->base.id, event);
 677                 return;
 678         }
 679
 680         fevent->event = event;
 681         fevent->crtc = crtc;
 682         fevent->ts = ktime_get();
 683         kthread_queue_work(&priv->event_thread[crtc_id].worker, &fevent->work);
 684 }
 685
 686 void dpu_crtc_complete_commit(struct drm_crtc *crtc,
 687                 struct drm_crtc_state *old_state)
 688 {
 689         if (!crtc || !crtc->state) {
 690                 DPU_ERROR("invalid crtc\n");
 691                 return;
 692         }
 693         trace_dpu_crtc_complete_commit(DRMID(crtc));
 694 }
 695
 696 static void _dpu_crtc_setup_mixer_for_encoder(
 697                 struct drm_crtc *crtc,
 698                 struct drm_encoder *enc)
 699 {
 700         struct dpu_crtc *dpu_crtc = to_dpu_crtc(crtc);
 701         struct dpu_kms *dpu_kms = _dpu_crtc_get_kms(crtc);
 702         struct dpu_rm *rm = &dpu_kms->rm;
 703         struct dpu_crtc_mixer *mixer;
 704         struct dpu_hw_ctl *last_valid_ctl = NULL;
 705         int i;
 706         struct dpu_rm_hw_iter lm_iter, ctl_iter;
 707
 708         dpu_rm_init_hw_iter(&lm_iter, enc->base.id, DPU_HW_BLK_LM);
 709         dpu_rm_init_hw_iter(&ctl_iter, enc->base.id, DPU_HW_BLK_CTL);
 710
 711         /* Set up all the mixers and ctls reserved by this encoder */
 712         for (i = dpu_crtc->num_mixers; i < ARRAY_SIZE(dpu_crtc->mixers); i++) {
 713                 mixer = &dpu_crtc->mixers[i];
 714
 715                 if (!dpu_rm_get_hw(rm, &lm_iter))
 716                         break;
 717                 mixer->hw_lm = (struct dpu_hw_mixer *)lm_iter.hw;
 718
 719                 /* CTL may be <= LMs, if <, multiple LMs controlled by 1 CTL */
 720                 if (!dpu_rm_get_hw(rm, &ctl_iter)) {
 721                         DPU_DEBUG("no ctl assigned to lm %d, using previous\n",
 722                                         mixer->hw_lm->idx - LM_0);
 723                         mixer->hw_ctl = last_valid_ctl;
 724                 } else {
 725                         mixer->hw_ctl = (struct dpu_hw_ctl *)ctl_iter.hw;
 726                         last_valid_ctl = mixer->hw_ctl;
 727                 }
 728
 729                 /* Shouldn't happen, mixers are always >= ctls */
 730                 if (!mixer->hw_ctl) {
 731                         DPU_ERROR("no valid ctls found for lm %d\n",
 732                                         mixer->hw_lm->idx - LM_0);
 733                         return;
 734                 }
 735
 736                 mixer->encoder = enc;
 737
 738                 dpu_crtc->num_mixers++;
 739                 DPU_DEBUG("setup mixer %d: lm %d\n",
 740                                 i, mixer->hw_lm->idx - LM_0);
 741                 DPU_DEBUG("setup mixer %d: ctl %d\n",
 742                                 i, mixer->hw_ctl->idx - CTL_0);
 743         }
 744 }
 745
 746 static void _dpu_crtc_setup_mixers(struct drm_crtc *crtc)
 747 {
 748         struct dpu_crtc *dpu_crtc = to_dpu_crtc(crtc);
 749         struct drm_encoder *enc;
 750
 751         dpu_crtc->num_mixers = 0;
 752         dpu_crtc->mixers_swapped = false;
 753         memset(dpu_crtc->mixers, 0, sizeof(dpu_crtc->mixers));
 754
 755         mutex_lock(&dpu_crtc->crtc_lock);
 756         /* Check for mixers on all encoders attached to this crtc */
 757         list_for_each_entry(enc, &crtc->dev->mode_config.encoder_list, head) {
 758                 if (enc->crtc != crtc)
 759                         continue;
 760
 761                 _dpu_crtc_setup_mixer_for_encoder(crtc, enc);
 762         }
 763
 764         mutex_unlock(&dpu_crtc->crtc_lock);
 765 }
 766
 767 static void _dpu_crtc_setup_lm_bounds(struct drm_crtc *crtc,
 768                 struct drm_crtc_state *state)
 769 {
 770         struct dpu_crtc *dpu_crtc;
 771         struct dpu_crtc_state *cstate;
 772         struct drm_display_mode *adj_mode;
 773         u32 crtc_split_width;
 774         int i;
 775
 776         if (!crtc || !state) {
 777                 DPU_ERROR("invalid args\n");
 778                 return;
 779         }
 780
 781         dpu_crtc = to_dpu_crtc(crtc);
 782         cstate = to_dpu_crtc_state(state);
 783
 784         adj_mode = &state->adjusted_mode;
 785         crtc_split_width = dpu_crtc_get_mixer_width(dpu_crtc, cstate, adj_mode);
 786
 787         for (i = 0; i < dpu_crtc->num_mixers; i++) {
 788                 struct drm_rect *r = &cstate->lm_bounds[i];
 789                 r->x1 = crtc_split_width * i;
 790                 r->y1 = 0;
 791                 r->x2 = r->x1 + crtc_split_width;
 792                 r->y2 = dpu_crtc_get_mixer_height(dpu_crtc, cstate, adj_mode);
 793
 794                 trace_dpu_crtc_setup_lm_bounds(DRMID(crtc), i, r);
 795         }
 796
 797         drm_mode_debug_printmodeline(adj_mode);
 798 }
 799
 800 static void dpu_crtc_atomic_begin(struct drm_crtc *crtc,
 801                 struct drm_crtc_state *old_state)
 802 {
 803         struct dpu_crtc *dpu_crtc;
 804         struct drm_encoder *encoder;
 805         struct drm_device *dev;
 806         unsigned long flags;
 807         struct dpu_crtc_smmu_state_data *smmu_state;
 808
 809         if (!crtc) {
 810                 DPU_ERROR("invalid crtc\n");
 811                 return;
 812         }
 813
 814         if (!crtc->state->enable) {
 815                 DPU_DEBUG("crtc%d -> enable %d, skip atomic_begin\n",
 816                                 crtc->base.id, crtc->state->enable);
 817                 return;
 818         }
 819
 820         DPU_DEBUG("crtc%d\n", crtc->base.id);
 821
 822         dpu_crtc = to_dpu_crtc(crtc);
 823         dev = crtc->dev;
 824         smmu_state = &dpu_crtc->smmu_state;
 825
 826         if (!dpu_crtc->num_mixers) {
 827                 _dpu_crtc_setup_mixers(crtc);
 828                 _dpu_crtc_setup_lm_bounds(crtc, crtc->state);
 829         }
 830
 831         if (dpu_crtc->event) {
 832                 WARN_ON(dpu_crtc->event);
 833         } else {
 834                 spin_lock_irqsave(&dev->event_lock, flags);
 835                 dpu_crtc->event = crtc->state->event;
 836                 crtc->state->event = NULL;
 837                 spin_unlock_irqrestore(&dev->event_lock, flags);
 838         }
 839
 840         list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) {
 841                 if (encoder->crtc != crtc)
 842                         continue;
 843
 844                 /* encoder will trigger pending mask now */
 845                 dpu_encoder_trigger_kickoff_pending(encoder);
 846         }
 847
 848         /*
 849          * If no mixers have been allocated in dpu_crtc_atomic_check(),
 850          * it means we are trying to flush a CRTC whose state is disabled:
 851          * nothing else needs to be done.
 852          */
 853         if (unlikely(!dpu_crtc->num_mixers))
 854                 return;
 855
 856         _dpu_crtc_blend_setup(crtc);
 857
 858         /*
 859          * PP_DONE irq is only used by command mode for now.
 860          * It is better to request pending before FLUSH and START trigger
 861          * to make sure no pp_done irq missed.
 862          * This is safe because no pp_done will happen before SW trigger
 863          * in command mode.
 864          */
 865 }
 866
 867 static void dpu_crtc_atomic_flush(struct drm_crtc *crtc,
 868                 struct drm_crtc_state *old_crtc_state)
 869 {
 870         struct dpu_crtc *dpu_crtc;
 871         struct drm_device *dev;
 872         struct drm_plane *plane;
 873         struct msm_drm_private *priv;
 874         struct msm_drm_thread *event_thread;
 875         unsigned long flags;
 876         struct dpu_crtc_state *cstate;
 877
 878         if (!crtc || !crtc->dev || !crtc->dev->dev_private) {
 879                 DPU_ERROR("invalid crtc\n");
 880                 return;
 881         }
 882
 883         if (!crtc->state->enable) {
 884                 DPU_DEBUG("crtc%d -> enable %d, skip atomic_flush\n",
 885                                 crtc->base.id, crtc->state->enable);
 886                 return;
 887         }
 888
 889         DPU_DEBUG("crtc%d\n", crtc->base.id);
 890
 891         dpu_crtc = to_dpu_crtc(crtc);
 892         cstate = to_dpu_crtc_state(crtc->state);
 893         dev = crtc->dev;
 894         priv = dev->dev_private;
 895
 896         if (crtc->index >= ARRAY_SIZE(priv->event_thread)) {
 897                 DPU_ERROR("invalid crtc index[%d]\n", crtc->index);
 898                 return;
 899         }
 900
 901         event_thread = &priv->event_thread[crtc->index];
 902
 903         if (dpu_crtc->event) {
 904                 DPU_DEBUG("already received dpu_crtc->event\n");
 905         } else {
 906                 spin_lock_irqsave(&dev->event_lock, flags);
 907                 dpu_crtc->event = crtc->state->event;
 908                 crtc->state->event = NULL;
 909                 spin_unlock_irqrestore(&dev->event_lock, flags);
 910         }
 911
 912         /*
 913          * If no mixers has been allocated in dpu_crtc_atomic_check(),
 914          * it means we are trying to flush a CRTC whose state is disabled:
 915          * nothing else needs to be done.
 916          */
 917         if (unlikely(!dpu_crtc->num_mixers))
 918                 return;
 919
 920         /*
 921          * For planes without commit update, drm framework will not add
 922          * those planes to current state since hardware update is not
 923          * required. However, if those planes were power collapsed since
 924          * last commit cycle, driver has to restore the hardware state
 925          * of those planes explicitly here prior to plane flush.
 926          */
 927         drm_atomic_crtc_for_each_plane(plane, crtc)
 928                 dpu_plane_restore(plane);
 929
 930         /* update performance setting before crtc kickoff */
 931         dpu_core_perf_crtc_update(crtc, 1, false);
 932
 933         /*
 934          * Final plane updates: Give each plane a chance to complete all
 935          *                      required writes/flushing before crtc's "flush
 936          *                      everything" call below.
 937          */
 938         drm_atomic_crtc_for_each_plane(plane, crtc) {
 939                 if (dpu_crtc->smmu_state.transition_error)
 940                         dpu_plane_set_error(plane, true);
 941                 dpu_plane_flush(plane);
 942         }
 943
 944         /* Kickoff will be scheduled by outer layer */
 945 }
 946
 947 /**
 948  * dpu_crtc_destroy_state - state destroy hook
 949  * @crtc: drm CRTC
 950  * @state: CRTC state object to release
 951  */
 952 static void dpu_crtc_destroy_state(struct drm_crtc *crtc,
 953                 struct drm_crtc_state *state)
 954 {
 955         struct dpu_crtc *dpu_crtc;
 956         struct dpu_crtc_state *cstate;
 957
 958         if (!crtc || !state) {
 959                 DPU_ERROR("invalid argument(s)\n");
 960                 return;
 961         }
 962
 963         dpu_crtc = to_dpu_crtc(crtc);
 964         cstate = to_dpu_crtc_state(state);
 965
 966         DPU_DEBUG("crtc%d\n", crtc->base.id);
 967
 968         _dpu_crtc_rp_destroy(&cstate->rp);
 969
 970         __drm_atomic_helper_crtc_destroy_state(state);
 971
 972         kfree(cstate);
 973 }
 974
 975 static int _dpu_crtc_wait_for_frame_done(struct drm_crtc *crtc)
 976 {
 977         struct dpu_crtc *dpu_crtc;
 978         int ret, rc = 0;
 979
 980         if (!crtc) {
 981                 DPU_ERROR("invalid argument\n");
 982                 return -EINVAL;
 983         }
 984         dpu_crtc = to_dpu_crtc(crtc);
 985
 986         if (!atomic_read(&dpu_crtc->frame_pending)) {
 987                 DPU_DEBUG("no frames pending\n");
 988                 return 0;
 989         }
 990
 991         DPU_ATRACE_BEGIN("frame done completion wait");
 992         ret = wait_for_completion_timeout(&dpu_crtc->frame_done_comp,
 993                         msecs_to_jiffies(DPU_FRAME_DONE_TIMEOUT));
 994         if (!ret) {
 995                 DRM_ERROR("frame done wait timed out, ret:%d\n", ret);
 996                 rc = -ETIMEDOUT;
 997         }
 998         DPU_ATRACE_END("frame done completion wait");
 999
1000         return rc;
1001 }
1002
1003 void dpu_crtc_commit_kickoff(struct drm_crtc *crtc)
1004 {
1005         struct drm_encoder *encoder;
1006         struct drm_device *dev;
1007         struct dpu_crtc *dpu_crtc;
1008         struct msm_drm_private *priv;
1009         struct dpu_kms *dpu_kms;
1010         struct dpu_crtc_state *cstate;
1011         int ret;
1012
1013         if (!crtc) {
1014                 DPU_ERROR("invalid argument\n");
1015                 return;
1016         }
1017         dev = crtc->dev;
1018         dpu_crtc = to_dpu_crtc(crtc);
1019         dpu_kms = _dpu_crtc_get_kms(crtc);
1020
1021         if (!dpu_kms || !dpu_kms->dev || !dpu_kms->dev->dev_private) {
1022                 DPU_ERROR("invalid argument\n");
1023                 return;
1024         }
1025
1026         priv = dpu_kms->dev->dev_private;
1027         cstate = to_dpu_crtc_state(crtc->state);
1028
1029         /*
1030          * If no mixers has been allocated in dpu_crtc_atomic_check(),
1031          * it means we are trying to start a CRTC whose state is disabled:
1032          * nothing else needs to be done.
1033          */
1034         if (unlikely(!dpu_crtc->num_mixers))
1035                 return;
1036
1037         DPU_ATRACE_BEGIN("crtc_commit");
1038
1039         list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) {
1040                 struct dpu_encoder_kickoff_params params = { 0 };
1041
1042                 if (encoder->crtc != crtc)
1043                         continue;
1044
1045                 /*
1046                  * Encoder will flush/start now, unless it has a tx pending.
1047                  * If so, it may delay and flush at an irq event (e.g. ppdone)
1048                  */
1049                 dpu_encoder_prepare_for_kickoff(encoder, &params);
1050         }
1051
1052         /* wait for frame_event_done completion */
1053         DPU_ATRACE_BEGIN("wait_for_frame_done_event");
1054         ret = _dpu_crtc_wait_for_frame_done(crtc);
1055         DPU_ATRACE_END("wait_for_frame_done_event");
1056         if (ret) {
1057                 DPU_ERROR("crtc%d wait for frame done failed;frame_pending%d\n",
1058                                 crtc->base.id,
1059                                 atomic_read(&dpu_crtc->frame_pending));
1060                 goto end;
1061         }
1062
1063         if (atomic_inc_return(&dpu_crtc->frame_pending) == 1) {
1064                 /* acquire bandwidth and other resources */
1065                 DPU_DEBUG("crtc%d first commit\n", crtc->base.id);
1066         } else
1067                 DPU_DEBUG("crtc%d commit\n", crtc->base.id);
1068
1069         dpu_crtc->play_count++;
1070
1071         dpu_vbif_clear_errors(dpu_kms);
1072
1073         list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) {
1074                 if (encoder->crtc != crtc)
1075                         continue;
1076
1077                 dpu_encoder_kickoff(encoder);
1078         }
1079
1080 end:
1081         reinit_completion(&dpu_crtc->frame_done_comp);
1082         DPU_ATRACE_END("crtc_commit");
1083 }
1084
1085 /**
1086  * _dpu_crtc_vblank_enable_no_lock - update power resource and vblank request
1087  * @dpu_crtc: Pointer to dpu crtc structure
1088  * @enable: Whether to enable/disable vblanks
1089  *
1090  * @Return: error code
1091  */
1092 static int _dpu_crtc_vblank_enable_no_lock(
1093                 struct dpu_crtc *dpu_crtc, bool enable)
1094 {
1095         struct drm_device *dev;
1096         struct drm_crtc *crtc;
1097         struct drm_encoder *enc;
1098
1099         if (!dpu_crtc) {
1100                 DPU_ERROR("invalid crtc\n");
1101                 return -EINVAL;
1102         }
1103
1104         crtc = &dpu_crtc->base;
1105         dev = crtc->dev;
1106
1107         if (enable) {
1108                 int ret;
1109
1110                 /* drop lock since power crtc cb may try to re-acquire lock */
1111                 mutex_unlock(&dpu_crtc->crtc_lock);
1112                 ret = _dpu_crtc_power_enable(dpu_crtc, true);
1113                 mutex_lock(&dpu_crtc->crtc_lock);
1114                 if (ret)
1115                         return ret;
1116
1117                 list_for_each_entry(enc, &dev->mode_config.encoder_list, head) {
1118                         if (enc->crtc != crtc)
1119                                 continue;
1120
1121                         trace_dpu_crtc_vblank_enable(DRMID(&dpu_crtc->base),
1122                                                      DRMID(enc), enable,
1123                                                      dpu_crtc);
1124
1125                         dpu_encoder_register_vblank_callback(enc,
1126                                         dpu_crtc_vblank_cb, (void *)crtc);
1127                 }
1128         } else {
1129                 list_for_each_entry(enc, &dev->mode_config.encoder_list, head) {
1130                         if (enc->crtc != crtc)
1131                                 continue;
1132
1133                         trace_dpu_crtc_vblank_enable(DRMID(&dpu_crtc->base),
1134                                                      DRMID(enc), enable,
1135                                                      dpu_crtc);
1136
1137                         dpu_encoder_register_vblank_callback(enc, NULL, NULL);
1138                 }
1139
1140                 /* drop lock since power crtc cb may try to re-acquire lock */
1141                 mutex_unlock(&dpu_crtc->crtc_lock);
1142                 _dpu_crtc_power_enable(dpu_crtc, false);
1143                 mutex_lock(&dpu_crtc->crtc_lock);
1144         }
1145
1146         return 0;
1147 }
1148
1149 /**
1150  * _dpu_crtc_set_suspend - notify crtc of suspend enable/disable
1151  * @crtc: Pointer to drm crtc object
1152  * @enable: true to enable suspend, false to indicate resume
1153  */
1154 static void _dpu_crtc_set_suspend(struct drm_crtc *crtc, bool enable)
1155 {
1156         struct dpu_crtc *dpu_crtc;
1157         struct msm_drm_private *priv;
1158         struct dpu_kms *dpu_kms;
1159         int ret = 0;
1160
1161         if (!crtc || !crtc->dev || !crtc->dev->dev_private) {
1162                 DPU_ERROR("invalid crtc\n");
1163                 return;
1164         }
1165         dpu_crtc = to_dpu_crtc(crtc);
1166         priv = crtc->dev->dev_private;
1167
1168         if (!priv->kms) {
1169                 DPU_ERROR("invalid crtc kms\n");
1170                 return;
1171         }
1172         dpu_kms = to_dpu_kms(priv->kms);
1173
1174         DRM_DEBUG_KMS("crtc%d suspend = %d\n", crtc->base.id, enable);
1175
1176         mutex_lock(&dpu_crtc->crtc_lock);
1177
1178         /*
1179          * If the vblank is enabled, release a power reference on suspend
1180          * and take it back during resume (if it is still enabled).
1181          */
1182         trace_dpu_crtc_set_suspend(DRMID(&dpu_crtc->base), enable, dpu_crtc);
1183         if (dpu_crtc->suspend == enable)
1184                 DPU_DEBUG("crtc%d suspend already set to %d, ignoring update\n",
1185                                 crtc->base.id, enable);
1186         else if (dpu_crtc->enabled && dpu_crtc->vblank_requested) {
1187                 ret = _dpu_crtc_vblank_enable_no_lock(dpu_crtc, !enable);
1188                 if (ret)
1189                         DPU_ERROR("%s vblank enable failed: %d\n",
1190                                         dpu_crtc->name, ret);
1191         }
1192
1193         dpu_crtc->suspend = enable;
1194         mutex_unlock(&dpu_crtc->crtc_lock);
1195 }
1196
1197 /**
1198  * dpu_crtc_duplicate_state - state duplicate hook
1199  * @crtc: Pointer to drm crtc structure
1200  * @Returns: Pointer to new drm_crtc_state structure
1201  */
1202 static struct drm_crtc_state *dpu_crtc_duplicate_state(struct drm_crtc *crtc)
1203 {
1204         struct dpu_crtc *dpu_crtc;
1205         struct dpu_crtc_state *cstate, *old_cstate;
1206
1207         if (!crtc || !crtc->state) {
1208                 DPU_ERROR("invalid argument(s)\n");
1209                 return NULL;
1210         }
1211
1212         dpu_crtc = to_dpu_crtc(crtc);
1213         old_cstate = to_dpu_crtc_state(crtc->state);
1214         cstate = kmemdup(old_cstate, sizeof(*old_cstate), GFP_KERNEL);
1215         if (!cstate) {
1216                 DPU_ERROR("failed to allocate state\n");
1217                 return NULL;
1218         }
1219
1220         /* duplicate base helper */
1221         __drm_atomic_helper_crtc_duplicate_state(crtc, &cstate->base);
1222
1223         _dpu_crtc_rp_duplicate(&old_cstate->rp, &cstate->rp);
1224
1225         return &cstate->base;
1226 }
1227
1228 /**
1229  * dpu_crtc_reset - reset hook for CRTCs
1230  * Resets the atomic state for @crtc by freeing the state pointer (which might
1231  * be NULL, e.g. at driver load time) and allocating a new empty state object.
1232  * @crtc: Pointer to drm crtc structure
1233  */
1234 static void dpu_crtc_reset(struct drm_crtc *crtc)
1235 {
1236         struct dpu_crtc *dpu_crtc;
1237         struct dpu_crtc_state *cstate;
1238
1239         if (!crtc) {
1240                 DPU_ERROR("invalid crtc\n");
1241                 return;
1242         }
1243
1244         /* revert suspend actions, if necessary */
1245         if (dpu_kms_is_suspend_state(crtc->dev))
1246                 _dpu_crtc_set_suspend(crtc, false);
1247
1248         /* remove previous state, if present */
1249         if (crtc->state) {
1250                 dpu_crtc_destroy_state(crtc, crtc->state);
1251                 crtc->state = 0;
1252         }
1253
1254         dpu_crtc = to_dpu_crtc(crtc);
1255         cstate = kzalloc(sizeof(*cstate), GFP_KERNEL);
1256         if (!cstate) {
1257                 DPU_ERROR("failed to allocate state\n");
1258                 return;
1259         }
1260
1261         _dpu_crtc_rp_reset(&cstate->rp, &dpu_crtc->rp_lock,
1262                         &dpu_crtc->rp_head);
1263
1264         cstate->base.crtc = crtc;
1265         crtc->state = &cstate->base;
1266 }
1267
1268 static void dpu_crtc_handle_power_event(u32 event_type, void *arg)
1269 {
1270         struct drm_crtc *crtc = arg;
1271         struct dpu_crtc *dpu_crtc;
1272         struct drm_encoder *encoder;
1273
1274         if (!crtc) {
1275                 DPU_ERROR("invalid crtc\n");
1276                 return;
1277         }
1278         dpu_crtc = to_dpu_crtc(crtc);
1279
1280         mutex_lock(&dpu_crtc->crtc_lock);
1281
1282         trace_dpu_crtc_handle_power_event(DRMID(crtc), event_type);
1283
1284         /* restore encoder; crtc will be programmed during commit */
1285         drm_for_each_encoder(encoder, crtc->dev) {
1286                 if (encoder->crtc != crtc)
1287                         continue;
1288
1289                 dpu_encoder_virt_restore(encoder);
1290         }
1291
1292         mutex_unlock(&dpu_crtc->crtc_lock);
1293 }
1294
1295 static void dpu_crtc_disable(struct drm_crtc *crtc)
1296 {
1297         struct dpu_crtc *dpu_crtc;
1298         struct dpu_crtc_state *cstate;
1299         struct drm_display_mode *mode;
1300         struct drm_encoder *encoder;
1301         struct msm_drm_private *priv;
1302         int ret;
1303         unsigned long flags;
1304
1305         if (!crtc || !crtc->dev || !crtc->dev->dev_private || !crtc->state) {
1306                 DPU_ERROR("invalid crtc\n");
1307                 return;
1308         }
1309         dpu_crtc = to_dpu_crtc(crtc);
1310         cstate = to_dpu_crtc_state(crtc->state);
1311         mode = &cstate->base.adjusted_mode;
1312         priv = crtc->dev->dev_private;
1313
1314         DRM_DEBUG_KMS("crtc%d\n", crtc->base.id);
1315
1316         if (dpu_kms_is_suspend_state(crtc->dev))
1317                 _dpu_crtc_set_suspend(crtc, true);
1318
1319         /* Disable/save vblank irq handling */
1320         drm_crtc_vblank_off(crtc);
1321
1322         mutex_lock(&dpu_crtc->crtc_lock);
1323
1324         /* wait for frame_event_done completion */
1325         if (_dpu_crtc_wait_for_frame_done(crtc))
1326                 DPU_ERROR("crtc%d wait for frame done failed;frame_pending%d\n",
1327                                 crtc->base.id,
1328                                 atomic_read(&dpu_crtc->frame_pending));
1329
1330         trace_dpu_crtc_disable(DRMID(crtc), false, dpu_crtc);
1331         if (dpu_crtc->enabled && !dpu_crtc->suspend &&
1332                         dpu_crtc->vblank_requested) {
1333                 ret = _dpu_crtc_vblank_enable_no_lock(dpu_crtc, false);
1334                 if (ret)
1335                         DPU_ERROR("%s vblank enable failed: %d\n",
1336                                         dpu_crtc->name, ret);
1337         }
1338         dpu_crtc->enabled = false;
1339
1340         if (atomic_read(&dpu_crtc->frame_pending)) {
1341                 trace_dpu_crtc_disable_frame_pending(DRMID(crtc),
1342                                      atomic_read(&dpu_crtc->frame_pending));
1343                 dpu_core_perf_crtc_release_bw(crtc);
1344                 atomic_set(&dpu_crtc->frame_pending, 0);
1345         }
1346
1347         dpu_core_perf_crtc_update(crtc, 0, true);
1348
1349         drm_for_each_encoder(encoder, crtc->dev) {
1350                 if (encoder->crtc != crtc)
1351                         continue;
1352                 dpu_encoder_register_frame_event_callback(encoder, NULL, NULL);
1353         }
1354
1355         if (dpu_crtc->power_event)
1356                 dpu_power_handle_unregister_event(dpu_crtc->phandle,
1357                                 dpu_crtc->power_event);
1358
1359         memset(dpu_crtc->mixers, 0, sizeof(dpu_crtc->mixers));
1360         dpu_crtc->num_mixers = 0;
1361         dpu_crtc->mixers_swapped = false;
1362
1363         /* disable clk & bw control until clk & bw properties are set */
1364         cstate->bw_control = false;
1365         cstate->bw_split_vote = false;
1366
1367         mutex_unlock(&dpu_crtc->crtc_lock);
1368
1369         if (crtc->state->event && !crtc->state->active) {
1370                 spin_lock_irqsave(&crtc->dev->event_lock, flags);
1371                 drm_crtc_send_vblank_event(crtc, crtc->state->event);
1372                 crtc->state->event = NULL;
1373                 spin_unlock_irqrestore(&crtc->dev->event_lock, flags);
1374         }
1375 }
1376
1377 static void dpu_crtc_enable(struct drm_crtc *crtc,
1378                 struct drm_crtc_state *old_crtc_state)
1379 {
1380         struct dpu_crtc *dpu_crtc;
1381         struct drm_encoder *encoder;
1382         struct msm_drm_private *priv;
1383         int ret;
1384
1385         if (!crtc || !crtc->dev || !crtc->dev->dev_private) {
1386                 DPU_ERROR("invalid crtc\n");
1387                 return;
1388         }
1389         priv = crtc->dev->dev_private;
1390
1391         DRM_DEBUG_KMS("crtc%d\n", crtc->base.id);
1392         dpu_crtc = to_dpu_crtc(crtc);
1393
1394         drm_for_each_encoder(encoder, crtc->dev) {
1395                 if (encoder->crtc != crtc)
1396                         continue;
1397                 dpu_encoder_register_frame_event_callback(encoder,
1398                                 dpu_crtc_frame_event_cb, (void *)crtc);
1399         }
1400
1401         mutex_lock(&dpu_crtc->crtc_lock);
1402         trace_dpu_crtc_enable(DRMID(crtc), true, dpu_crtc);
1403         if (!dpu_crtc->enabled && !dpu_crtc->suspend &&
1404                         dpu_crtc->vblank_requested) {
1405                 ret = _dpu_crtc_vblank_enable_no_lock(dpu_crtc, true);
1406                 if (ret)
1407                         DPU_ERROR("%s vblank enable failed: %d\n",
1408                                         dpu_crtc->name, ret);
1409         }
1410         dpu_crtc->enabled = true;
1411
1412         mutex_unlock(&dpu_crtc->crtc_lock);
1413
1414         /* Enable/restore vblank irq handling */
1415         drm_crtc_vblank_on(crtc);
1416
1417         dpu_crtc->power_event = dpu_power_handle_register_event(
1418                 dpu_crtc->phandle, DPU_POWER_EVENT_ENABLE,
1419                 dpu_crtc_handle_power_event, crtc, dpu_crtc->name);
1420
1421 }
1422
1423 struct plane_state {
1424         struct dpu_plane_state *dpu_pstate;
1425         const struct drm_plane_state *drm_pstate;
1426         int stage;
1427         u32 pipe_id;
1428 };
1429
1430 static int dpu_crtc_atomic_check(struct drm_crtc *crtc,
1431                 struct drm_crtc_state *state)
1432 {
1433         struct dpu_crtc *dpu_crtc;
1434         struct plane_state *pstates;
1435         struct dpu_crtc_state *cstate;
1436
1437         const struct drm_plane_state *pstate;
1438         struct drm_plane *plane;
1439         struct drm_display_mode *mode;
1440
1441         int cnt = 0, rc = 0, mixer_width, i, z_pos;
1442
1443         struct dpu_multirect_plane_states multirect_plane[DPU_STAGE_MAX * 2];
1444         int multirect_count = 0;
1445         const struct drm_plane_state *pipe_staged[SSPP_MAX];
1446         int left_zpos_cnt = 0, right_zpos_cnt = 0;
1447         struct drm_rect crtc_rect = { 0 };
1448
1449         if (!crtc) {
1450                 DPU_ERROR("invalid crtc\n");
1451                 return -EINVAL;
1452         }
1453
1454         pstates = kzalloc(sizeof(*pstates) * DPU_STAGE_MAX * 4, GFP_KERNEL);
1455
1456         dpu_crtc = to_dpu_crtc(crtc);
1457         cstate = to_dpu_crtc_state(state);
1458
1459         if (!state->enable || !state->active) {
1460                 DPU_DEBUG("crtc%d -> enable %d, active %d, skip atomic_check\n",
1461                                 crtc->base.id, state->enable, state->active);
1462                 goto end;
1463         }
1464
1465         mode = &state->adjusted_mode;
1466         DPU_DEBUG("%s: check", dpu_crtc->name);
1467
1468         /* force a full mode set if active state changed */
1469         if (state->active_changed)
1470                 state->mode_changed = true;
1471
1472         memset(pipe_staged, 0, sizeof(pipe_staged));
1473
1474         mixer_width = dpu_crtc_get_mixer_width(dpu_crtc, cstate, mode);
1475
1476         _dpu_crtc_setup_lm_bounds(crtc, state);
1477
1478         crtc_rect.x2 = mode->hdisplay;
1479         crtc_rect.y2 = mode->vdisplay;
1480
1481          /* get plane state for all drm planes associated with crtc state */
1482         drm_atomic_crtc_state_for_each_plane_state(plane, pstate, state) {
1483                 struct drm_rect dst, clip = crtc_rect;
1484
1485                 if (IS_ERR_OR_NULL(pstate)) {
1486                         rc = PTR_ERR(pstate);
1487                         DPU_ERROR("%s: failed to get plane%d state, %d\n",
1488                                         dpu_crtc->name, plane->base.id, rc);
1489                         goto end;
1490                 }
1491                 if (cnt >= DPU_STAGE_MAX * 4)
1492                         continue;
1493
1494                 pstates[cnt].dpu_pstate = to_dpu_plane_state(pstate);
1495                 pstates[cnt].drm_pstate = pstate;
1496                 pstates[cnt].stage = pstate->normalized_zpos;
1497                 pstates[cnt].pipe_id = dpu_plane_pipe(plane);
1498
1499                 if (pipe_staged[pstates[cnt].pipe_id]) {
1500                         multirect_plane[multirect_count].r0 =
1501                                 pipe_staged[pstates[cnt].pipe_id];
1502                         multirect_plane[multirect_count].r1 = pstate;
1503                         multirect_count++;
1504
1505                         pipe_staged[pstates[cnt].pipe_id] = NULL;
1506                 } else {
1507                         pipe_staged[pstates[cnt].pipe_id] = pstate;
1508                 }
1509
1510                 cnt++;
1511
1512                 dst = drm_plane_state_dest(pstate);
1513                 if (!drm_rect_intersect(&clip, &dst)) {
1514                         DPU_ERROR("invalid vertical/horizontal destination\n");
1515                         DPU_ERROR("display: " DRM_RECT_FMT " plane: "
1516                                   DRM_RECT_FMT "\n", DRM_RECT_ARG(&crtc_rect),
1517                                   DRM_RECT_ARG(&dst));
1518                         rc = -E2BIG;
1519                         goto end;
1520                 }
1521         }
1522
1523         for (i = 1; i < SSPP_MAX; i++) {
1524                 if (pipe_staged[i]) {
1525                         dpu_plane_clear_multirect(pipe_staged[i]);
1526
1527                         if (is_dpu_plane_virtual(pipe_staged[i]->plane)) {
1528                                 DPU_ERROR(
1529                                         "r1 only virt plane:%d not supported\n",
1530                                         pipe_staged[i]->plane->base.id);
1531                                 rc  = -EINVAL;
1532                                 goto end;
1533                         }
1534                 }
1535         }
1536
1537         z_pos = -1;
1538         for (i = 0; i < cnt; i++) {
1539                 /* reset counts at every new blend stage */
1540                 if (pstates[i].stage != z_pos) {
1541                         left_zpos_cnt = 0;
1542                         right_zpos_cnt = 0;
1543                         z_pos = pstates[i].stage;
1544                 }
1545
1546                 /* verify z_pos setting before using it */
1547                 if (z_pos >= DPU_STAGE_MAX - DPU_STAGE_0) {
1548                         DPU_ERROR("> %d plane stages assigned\n",
1549                                         DPU_STAGE_MAX - DPU_STAGE_0);
1550                         rc = -EINVAL;
1551                         goto end;
1552                 } else if (pstates[i].drm_pstate->crtc_x < mixer_width) {
1553                         if (left_zpos_cnt == 2) {
1554                                 DPU_ERROR("> 2 planes @ stage %d on left\n",
1555                                         z_pos);
1556                                 rc = -EINVAL;
1557                                 goto end;
1558                         }
1559                         left_zpos_cnt++;
1560
1561                 } else {
1562                         if (right_zpos_cnt == 2) {
1563                                 DPU_ERROR("> 2 planes @ stage %d on right\n",
1564                                         z_pos);
1565                                 rc = -EINVAL;
1566                                 goto end;
1567                         }
1568                         right_zpos_cnt++;
1569                 }
1570
1571                 pstates[i].dpu_pstate->stage = z_pos + DPU_STAGE_0;
1572                 DPU_DEBUG("%s: zpos %d", dpu_crtc->name, z_pos);
1573         }
1574
1575         for (i = 0; i < multirect_count; i++) {
1576                 if (dpu_plane_validate_multirect_v2(&multirect_plane[i])) {
1577                         DPU_ERROR(
1578                         "multirect validation failed for planes (%d - %d)\n",
1579                                         multirect_plane[i].r0->plane->base.id,
1580                                         multirect_plane[i].r1->plane->base.id);
1581                         rc = -EINVAL;
1582                         goto end;
1583                 }
1584         }
1585
1586         rc = dpu_core_perf_crtc_check(crtc, state);
1587         if (rc) {
1588                 DPU_ERROR("crtc%d failed performance check %d\n",
1589                                 crtc->base.id, rc);
1590                 goto end;
1591         }
1592
1593         /* validate source split:
1594          * use pstates sorted by stage to check planes on same stage
1595          * we assume that all pipes are in source split so its valid to compare
1596          * without taking into account left/right mixer placement
1597          */
1598         for (i = 1; i < cnt; i++) {
1599                 struct plane_state *prv_pstate, *cur_pstate;
1600                 struct drm_rect left_rect, right_rect;
1601                 int32_t left_pid, right_pid;
1602                 int32_t stage;
1603
1604                 prv_pstate = &pstates[i - 1];
1605                 cur_pstate = &pstates[i];
1606                 if (prv_pstate->stage != cur_pstate->stage)
1607                         continue;
1608
1609                 stage = cur_pstate->stage;
1610
1611                 left_pid = prv_pstate->dpu_pstate->base.plane->base.id;
1612                 left_rect = drm_plane_state_dest(prv_pstate->drm_pstate);
1613
1614                 right_pid = cur_pstate->dpu_pstate->base.plane->base.id;
1615                 right_rect = drm_plane_state_dest(cur_pstate->drm_pstate);
1616
1617                 if (right_rect.x1 < left_rect.x1) {
1618                         swap(left_pid, right_pid);
1619                         swap(left_rect, right_rect);
1620                 }
1621
1622                 /**
1623                  * - planes are enumerated in pipe-priority order such that
1624                  *   planes with lower drm_id must be left-most in a shared
1625                  *   blend-stage when using source split.
1626                  * - planes in source split must be contiguous in width
1627                  * - planes in source split must have same dest yoff and height
1628                  */
1629                 if (right_pid < left_pid) {
1630                         DPU_ERROR(
1631                                 "invalid src split cfg. priority mismatch. stage: %d left: %d right: %d\n",
1632                                 stage, left_pid, right_pid);
1633                         rc = -EINVAL;
1634                         goto end;
1635                 } else if (right_rect.x1 != drm_rect_width(&left_rect)) {
1636                         DPU_ERROR("non-contiguous coordinates for src split. "
1637                                   "stage: %d left: " DRM_RECT_FMT " right: "
1638                                   DRM_RECT_FMT "\n", stage,
1639                                   DRM_RECT_ARG(&left_rect),
1640                                   DRM_RECT_ARG(&right_rect));
1641                         rc = -EINVAL;
1642                         goto end;
1643                 } else if (left_rect.y1 != right_rect.y1 ||
1644                            drm_rect_height(&left_rect) != drm_rect_height(&right_rect)) {
1645                         DPU_ERROR("source split at stage: %d. invalid "
1646                                   "yoff/height: left: " DRM_RECT_FMT " right: "
1647                                   DRM_RECT_FMT "\n", stage,
1648                                   DRM_RECT_ARG(&left_rect),
1649                                   DRM_RECT_ARG(&right_rect));
1650                         rc = -EINVAL;
1651                         goto end;
1652                 }
1653         }
1654
1655 end:
1656         _dpu_crtc_rp_free_unused(&cstate->rp);
1657         kfree(pstates);
1658         return rc;
1659 }
1660
1661 int dpu_crtc_vblank(struct drm_crtc *crtc, bool en)
1662 {
1663         struct dpu_crtc *dpu_crtc;
1664         int ret;
1665
1666         if (!crtc) {
1667                 DPU_ERROR("invalid crtc\n");
1668                 return -EINVAL;
1669         }
1670         dpu_crtc = to_dpu_crtc(crtc);
1671
1672         mutex_lock(&dpu_crtc->crtc_lock);
1673         trace_dpu_crtc_vblank(DRMID(&dpu_crtc->base), en, dpu_crtc);
1674         if (dpu_crtc->enabled && !dpu_crtc->suspend) {
1675                 ret = _dpu_crtc_vblank_enable_no_lock(dpu_crtc, en);
1676                 if (ret)
1677                         DPU_ERROR("%s vblank enable failed: %d\n",
1678                                         dpu_crtc->name, ret);
1679         }
1680         dpu_crtc->vblank_requested = en;
1681         mutex_unlock(&dpu_crtc->crtc_lock);
1682
1683         return 0;
1684 }
1685
1686 #ifdef CONFIG_DEBUG_FS
1687 static int _dpu_debugfs_status_show(struct seq_file *s, void *data)
1688 {
1689         struct dpu_crtc *dpu_crtc;
1690         struct dpu_plane_state *pstate = NULL;
1691         struct dpu_crtc_mixer *m;
1692
1693         struct drm_crtc *crtc;
1694         struct drm_plane *plane;
1695         struct drm_display_mode *mode;
1696         struct drm_framebuffer *fb;
1697         struct drm_plane_state *state;
1698         struct dpu_crtc_state *cstate;
1699
1700         int i, out_width;
1701
1702         if (!s || !s->private)
1703                 return -EINVAL;
1704
1705         dpu_crtc = s->private;
1706         crtc = &dpu_crtc->base;
1707         cstate = to_dpu_crtc_state(crtc->state);
1708
1709         mutex_lock(&dpu_crtc->crtc_lock);
1710         mode = &crtc->state->adjusted_mode;
1711         out_width = dpu_crtc_get_mixer_width(dpu_crtc, cstate, mode);
1712
1713         seq_printf(s, "crtc:%d width:%d height:%d\n", crtc->base.id,
1714                                 mode->hdisplay, mode->vdisplay);
1715
1716         seq_puts(s, "\n");
1717
1718         for (i = 0; i < dpu_crtc->num_mixers; ++i) {
1719                 m = &dpu_crtc->mixers[i];
1720                 if (!m->hw_lm)
1721                         seq_printf(s, "\tmixer[%d] has no lm\n", i);
1722                 else if (!m->hw_ctl)
1723                         seq_printf(s, "\tmixer[%d] has no ctl\n", i);
1724                 else
1725                         seq_printf(s, "\tmixer:%d ctl:%d width:%d height:%d\n",
1726                                 m->hw_lm->idx - LM_0, m->hw_ctl->idx - CTL_0,
1727                                 out_width, mode->vdisplay);
1728         }
1729
1730         seq_puts(s, "\n");
1731
1732         drm_atomic_crtc_for_each_plane(plane, crtc) {
1733                 pstate = to_dpu_plane_state(plane->state);
1734                 state = plane->state;
1735
1736                 if (!pstate || !state)
1737                         continue;
1738
1739                 seq_printf(s, "\tplane:%u stage:%d\n", plane->base.id,
1740                         pstate->stage);
1741
1742                 if (plane->state->fb) {
1743                         fb = plane->state->fb;
1744
1745                         seq_printf(s, "\tfb:%d image format:%4.4s wxh:%ux%u ",
1746                                 fb->base.id, (char *) &fb->format->format,
1747                                 fb->width, fb->height);
1748                         for (i = 0; i < ARRAY_SIZE(fb->format->cpp); ++i)
1749                                 seq_printf(s, "cpp[%d]:%u ",
1750                                                 i, fb->format->cpp[i]);
1751                         seq_puts(s, "\n\t");
1752
1753                         seq_printf(s, "modifier:%8llu ", fb->modifier);
1754                         seq_puts(s, "\n");
1755
1756                         seq_puts(s, "\t");
1757                         for (i = 0; i < ARRAY_SIZE(fb->pitches); i++)
1758                                 seq_printf(s, "pitches[%d]:%8u ", i,
1759                                                         fb->pitches[i]);
1760                         seq_puts(s, "\n");
1761
1762                         seq_puts(s, "\t");
1763                         for (i = 0; i < ARRAY_SIZE(fb->offsets); i++)
1764                                 seq_printf(s, "offsets[%d]:%8u ", i,
1765                                                         fb->offsets[i]);
1766                         seq_puts(s, "\n");
1767                 }
1768
1769                 seq_printf(s, "\tsrc_x:%4d src_y:%4d src_w:%4d src_h:%4d\n",
1770                         state->src_x, state->src_y, state->src_w, state->src_h);
1771
1772                 seq_printf(s, "\tdst x:%4d dst_y:%4d dst_w:%4d dst_h:%4d\n",
1773                         state->crtc_x, state->crtc_y, state->crtc_w,
1774                         state->crtc_h);
1775                 seq_printf(s, "\tmultirect: mode: %d index: %d\n",
1776                         pstate->multirect_mode, pstate->multirect_index);
1777
1778                 seq_puts(s, "\n");
1779         }
1780         if (dpu_crtc->vblank_cb_count) {
1781                 ktime_t diff = ktime_sub(ktime_get(), dpu_crtc->vblank_cb_time);
1782                 s64 diff_ms = ktime_to_ms(diff);
1783                 s64 fps = diff_ms ? div_s64(
1784                                 dpu_crtc->vblank_cb_count * 1000, diff_ms) : 0;
1785
1786                 seq_printf(s,
1787                         "vblank fps:%lld count:%u total:%llums total_framecount:%llu\n",
1788                                 fps, dpu_crtc->vblank_cb_count,
1789                                 ktime_to_ms(diff), dpu_crtc->play_count);
1790
1791                 /* reset time & count for next measurement */
1792                 dpu_crtc->vblank_cb_count = 0;
1793                 dpu_crtc->vblank_cb_time = ktime_set(0, 0);
1794         }
1795
1796         seq_printf(s, "vblank_enable:%d\n", dpu_crtc->vblank_requested);
1797
1798         mutex_unlock(&dpu_crtc->crtc_lock);
1799
1800         return 0;
1801 }
1802
1803 static int _dpu_debugfs_status_open(struct inode *inode, struct file *file)
1804 {
1805         return single_open(file, _dpu_debugfs_status_show, inode->i_private);
1806 }
1807
1808 #define DEFINE_DPU_DEBUGFS_SEQ_FOPS(__prefix)                          \
1809 static int __prefix ## _open(struct inode *inode, struct file *file)    \
1810 {                                                                       \
1811         return single_open(file, __prefix ## _show, inode->i_private);  \
1812 }                                                                       \
1813 static const struct file_operations __prefix ## _fops = {               \
1814         .owner = THIS_MODULE,                                           \
1815         .open = __prefix ## _open,                                      \
1816         .release = single_release,                                      \
1817         .read = seq_read,                                               \
1818         .llseek = seq_lseek,                                            \
1819 }
1820
1821 static int dpu_crtc_debugfs_state_show(struct seq_file *s, void *v)
1822 {
1823         struct drm_crtc *crtc = (struct drm_crtc *) s->private;
1824         struct dpu_crtc *dpu_crtc = to_dpu_crtc(crtc);
1825         struct dpu_crtc_res *res;
1826         struct dpu_crtc_respool *rp;
1827         int i;
1828
1829         seq_printf(s, "client type: %d\n", dpu_crtc_get_client_type(crtc));
1830         seq_printf(s, "intf_mode: %d\n", dpu_crtc_get_intf_mode(crtc));
1831         seq_printf(s, "core_clk_rate: %llu\n",
1832                         dpu_crtc->cur_perf.core_clk_rate);
1833         for (i = DPU_POWER_HANDLE_DBUS_ID_MNOC;
1834                         i < DPU_POWER_HANDLE_DBUS_ID_MAX; i++) {
1835                 seq_printf(s, "bw_ctl[%s]: %llu\n",
1836                                 dpu_power_handle_get_dbus_name(i),
1837                                 dpu_crtc->cur_perf.bw_ctl[i]);
1838                 seq_printf(s, "max_per_pipe_ib[%s]: %llu\n",
1839                                 dpu_power_handle_get_dbus_name(i),
1840                                 dpu_crtc->cur_perf.max_per_pipe_ib[i]);
1841         }
1842
1843         mutex_lock(&dpu_crtc->rp_lock);
1844         list_for_each_entry(rp, &dpu_crtc->rp_head, rp_list) {
1845                 seq_printf(s, "rp.%d: ", rp->sequence_id);
1846                 list_for_each_entry(res, &rp->res_list, list)
1847                         seq_printf(s, "0x%x/0x%llx/%pK/%d ",
1848                                         res->type, res->tag, res->val,
1849                                         atomic_read(&res->refcount));
1850                 seq_puts(s, "\n");
1851         }
1852         mutex_unlock(&dpu_crtc->rp_lock);
1853
1854         return 0;
1855 }
1856 DEFINE_DPU_DEBUGFS_SEQ_FOPS(dpu_crtc_debugfs_state);
1857
1858 static int _dpu_crtc_init_debugfs(struct drm_crtc *crtc)
1859 {
1860         struct dpu_crtc *dpu_crtc;
1861         struct dpu_kms *dpu_kms;
1862
1863         static const struct file_operations debugfs_status_fops = {
1864                 .open =         _dpu_debugfs_status_open,
1865                 .read =         seq_read,
1866                 .llseek =       seq_lseek,
1867                 .release =      single_release,
1868         };
1869
1870         if (!crtc)
1871                 return -EINVAL;
1872         dpu_crtc = to_dpu_crtc(crtc);
1873
1874         dpu_kms = _dpu_crtc_get_kms(crtc);
1875         if (!dpu_kms)
1876                 return -EINVAL;
1877
1878         dpu_crtc->debugfs_root = debugfs_create_dir(dpu_crtc->name,
1879                         crtc->dev->primary->debugfs_root);
1880         if (!dpu_crtc->debugfs_root)
1881                 return -ENOMEM;
1882
1883         /* don't error check these */
1884         debugfs_create_file("status", 0400,
1885                         dpu_crtc->debugfs_root,
1886                         dpu_crtc, &debugfs_status_fops);
1887         debugfs_create_file("state", 0600,
1888                         dpu_crtc->debugfs_root,
1889                         &dpu_crtc->base,
1890                         &dpu_crtc_debugfs_state_fops);
1891
1892         return 0;
1893 }
1894
1895 static void _dpu_crtc_destroy_debugfs(struct drm_crtc *crtc)
1896 {
1897         struct dpu_crtc *dpu_crtc;
1898
1899         if (!crtc)
1900                 return;
1901         dpu_crtc = to_dpu_crtc(crtc);
1902         debugfs_remove_recursive(dpu_crtc->debugfs_root);
1903 }
1904 #else
1905 static int _dpu_crtc_init_debugfs(struct drm_crtc *crtc)
1906 {
1907         return 0;
1908 }
1909
1910 static void _dpu_crtc_destroy_debugfs(struct drm_crtc *crtc)
1911 {
1912 }
1913 #endif /* CONFIG_DEBUG_FS */
1914
1915 static int dpu_crtc_late_register(struct drm_crtc *crtc)
1916 {
1917         return _dpu_crtc_init_debugfs(crtc);
1918 }
1919
1920 static void dpu_crtc_early_unregister(struct drm_crtc *crtc)
1921 {
1922         _dpu_crtc_destroy_debugfs(crtc);
1923 }
1924
1925 static const struct drm_crtc_funcs dpu_crtc_funcs = {
1926         .set_config = drm_atomic_helper_set_config,
1927         .destroy = dpu_crtc_destroy,
1928         .page_flip = drm_atomic_helper_page_flip,
1929         .reset = dpu_crtc_reset,
1930         .atomic_duplicate_state = dpu_crtc_duplicate_state,
1931         .atomic_destroy_state = dpu_crtc_destroy_state,
1932         .late_register = dpu_crtc_late_register,
1933         .early_unregister = dpu_crtc_early_unregister,
1934 };
1935
1936 static const struct drm_crtc_helper_funcs dpu_crtc_helper_funcs = {
1937         .disable = dpu_crtc_disable,
1938         .atomic_enable = dpu_crtc_enable,
1939         .atomic_check = dpu_crtc_atomic_check,
1940         .atomic_begin = dpu_crtc_atomic_begin,
1941         .atomic_flush = dpu_crtc_atomic_flush,
1942 };
1943
1944 /* initialize crtc */
1945 struct drm_crtc *dpu_crtc_init(struct drm_device *dev, struct drm_plane *plane,
1946                                 struct drm_plane *cursor)
1947 {
1948         struct drm_crtc *crtc = NULL;
1949         struct dpu_crtc *dpu_crtc = NULL;
1950         struct msm_drm_private *priv = NULL;
1951         struct dpu_kms *kms = NULL;
1952         int i;
1953
1954         priv = dev->dev_private;
1955         kms = to_dpu_kms(priv->kms);
1956
1957         dpu_crtc = kzalloc(sizeof(*dpu_crtc), GFP_KERNEL);
1958         if (!dpu_crtc)
1959                 return ERR_PTR(-ENOMEM);
1960
1961         crtc = &dpu_crtc->base;
1962         crtc->dev = dev;
1963
1964         mutex_init(&dpu_crtc->crtc_lock);
1965         spin_lock_init(&dpu_crtc->spin_lock);
1966         atomic_set(&dpu_crtc->frame_pending, 0);
1967
1968         mutex_init(&dpu_crtc->rp_lock);
1969         INIT_LIST_HEAD(&dpu_crtc->rp_head);
1970
1971         init_completion(&dpu_crtc->frame_done_comp);
1972
1973         INIT_LIST_HEAD(&dpu_crtc->frame_event_list);
1974
1975         for (i = 0; i < ARRAY_SIZE(dpu_crtc->frame_events); i++) {
1976                 INIT_LIST_HEAD(&dpu_crtc->frame_events[i].list);
1977                 list_add(&dpu_crtc->frame_events[i].list,
1978                                 &dpu_crtc->frame_event_list);
1979                 kthread_init_work(&dpu_crtc->frame_events[i].work,
1980                                 dpu_crtc_frame_event_work);
1981         }
1982
1983         drm_crtc_init_with_planes(dev, crtc, plane, cursor, &dpu_crtc_funcs,
1984                                 NULL);
1985
1986         drm_crtc_helper_add(crtc, &dpu_crtc_helper_funcs);
1987         plane->crtc = crtc;
1988
1989         /* save user friendly CRTC name for later */
1990         snprintf(dpu_crtc->name, DPU_CRTC_NAME_SIZE, "crtc%u", crtc->base.id);
1991
1992         /* initialize event handling */
1993         spin_lock_init(&dpu_crtc->event_lock);
1994
1995         dpu_crtc->phandle = &kms->phandle;
1996
1997         DPU_DEBUG("%s: successfully initialized crtc\n", dpu_crtc->name);
1998         return crtc;
1999 }