drm/amdkfd: CRIU checkpoint and restore queue control stack

[linux-2.6-microblaze.git] / drivers / gpu / drm / amd / amdkfd / kfd_mqd_manager_v9.c

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c
index 4e5932f..87da432 100644 (file)
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c
@@ -108,7 +108,7 @@ static struct kfd_mem_obj *allocate_mqd(struct kfd_dev *kfd,
                mqd_mem_obj = kzalloc(sizeof(struct kfd_mem_obj), GFP_KERNEL);
                if (!mqd_mem_obj)
                        return NULL;
-               retval = amdgpu_amdkfd_alloc_gtt_mem(kfd->kgd,
+               retval = amdgpu_amdkfd_alloc_gtt_mem(kfd->adev,
                        ALIGN(q->ctl_stack_size, PAGE_SIZE) +
                                ALIGN(sizeof(struct v9_mqd), PAGE_SIZE),
                        &(mqd_mem_obj->gtt_mem),
@@ -199,7 +199,7 @@ static int load_mqd(struct mqd_manager *mm, void *mqd,
        /* AQL write pointer counts in 64B packets, PM4/CP counts in dwords. */
        uint32_t wptr_shift = (p->format == KFD_QUEUE_FORMAT_AQL ? 4 : 0);
 
-       return mm->dev->kfd2kgd->hqd_load(mm->dev->kgd, mqd, pipe_id, queue_id,
+       return mm->dev->kfd2kgd->hqd_load(mm->dev->adev, mqd, pipe_id, queue_id,
                                          (uint32_t __user *)p->write_ptr,
                                          wptr_shift, 0, mms);
 }
@@ -208,7 +208,7 @@ static int hiq_load_mqd_kiq(struct mqd_manager *mm, void *mqd,
                            uint32_t pipe_id, uint32_t queue_id,
                            struct queue_properties *p, struct mm_struct *mms)
 {
-       return mm->dev->kfd2kgd->hiq_mqd_load(mm->dev->kgd, mqd, pipe_id,
+       return mm->dev->kfd2kgd->hiq_mqd_load(mm->dev->adev, mqd, pipe_id,
                                              queue_id, p->doorbell_off);
 }
 
@@ -291,7 +291,7 @@ static int destroy_mqd(struct mqd_manager *mm, void *mqd,
                        uint32_t queue_id)
 {
        return mm->dev->kfd2kgd->hqd_destroy
-               (mm->dev->kgd, mqd, type, timeout,
+               (mm->dev->adev, mqd, type, timeout,
                pipe_id, queue_id);
 }
 
@@ -301,7 +301,7 @@ static void free_mqd(struct mqd_manager *mm, void *mqd,
        struct kfd_dev *kfd = mm->dev;
 
        if (mqd_mem_obj->gtt_mem) {
-               amdgpu_amdkfd_free_gtt_mem(kfd->kgd, mqd_mem_obj->gtt_mem);
+               amdgpu_amdkfd_free_gtt_mem(kfd->adev, mqd_mem_obj->gtt_mem);
                kfree(mqd_mem_obj);
        } else {
                kfd_gtt_sa_free(mm->dev, mqd_mem_obj);
@@ -313,7 +313,7 @@ static bool is_occupied(struct mqd_manager *mm, void *mqd,
                        uint32_t queue_id)
 {
        return mm->dev->kfd2kgd->hqd_is_occupied(
-               mm->dev->kgd, queue_address,
+               mm->dev->adev, queue_address,
                pipe_id, queue_id);
 }
 
@@ -340,6 +340,57 @@ static int get_wave_state(struct mqd_manager *mm, void *mqd,
        return 0;
 }
 
+static void get_checkpoint_info(struct mqd_manager *mm, void *mqd, u32 *ctl_stack_size)
+{
+       struct v9_mqd *m = get_mqd(mqd);
+
+       *ctl_stack_size = m->cp_hqd_cntl_stack_size;
+}
+
+static void checkpoint_mqd(struct mqd_manager *mm, void *mqd, void *mqd_dst, void *ctl_stack_dst)
+{
+       struct v9_mqd *m;
+       /* Control stack is located one page after MQD. */
+       void *ctl_stack = (void *)((uintptr_t)mqd + PAGE_SIZE);
+
+       m = get_mqd(mqd);
+
+       memcpy(mqd_dst, m, sizeof(struct v9_mqd));
+       memcpy(ctl_stack_dst, ctl_stack, m->cp_hqd_cntl_stack_size);
+}
+
+static void restore_mqd(struct mqd_manager *mm, void **mqd,
+                       struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
+                       struct queue_properties *qp,
+                       const void *mqd_src,
+                       const void *ctl_stack_src, u32 ctl_stack_size)
+{
+       uint64_t addr;
+       struct v9_mqd *m;
+       void *ctl_stack;
+
+       m = (struct v9_mqd *) mqd_mem_obj->cpu_ptr;
+       addr = mqd_mem_obj->gpu_addr;
+
+       memcpy(m, mqd_src, sizeof(*m));
+
+       *mqd = m;
+       if (gart_addr)
+               *gart_addr = addr;
+
+       /* Control stack is located one page after MQD. */
+       ctl_stack = (void *)((uintptr_t)*mqd + PAGE_SIZE);
+       memcpy(ctl_stack, ctl_stack_src, ctl_stack_size);
+
+       m->cp_hqd_pq_doorbell_control =
+               qp->doorbell_off <<
+                       CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT;
+       pr_debug("cp_hqd_pq_doorbell_control 0x%x\n",
+                               m->cp_hqd_pq_doorbell_control);
+
+       qp->is_active = 0;
+}
+
 static void init_mqd_hiq(struct mqd_manager *mm, void **mqd,
                        struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
                        struct queue_properties *q)
@@ -375,7 +426,7 @@ static int load_mqd_sdma(struct mqd_manager *mm, void *mqd,
                uint32_t pipe_id, uint32_t queue_id,
                struct queue_properties *p, struct mm_struct *mms)
 {
-       return mm->dev->kfd2kgd->hqd_sdma_load(mm->dev->kgd, mqd,
+       return mm->dev->kfd2kgd->hqd_sdma_load(mm->dev->adev, mqd,
                                               (uint32_t __user *)p->write_ptr,
                                               mms);
 }
@@ -418,14 +469,50 @@ static int destroy_mqd_sdma(struct mqd_manager *mm, void *mqd,
                unsigned int timeout, uint32_t pipe_id,
                uint32_t queue_id)
 {
-       return mm->dev->kfd2kgd->hqd_sdma_destroy(mm->dev->kgd, mqd, timeout);
+       return mm->dev->kfd2kgd->hqd_sdma_destroy(mm->dev->adev, mqd, timeout);
 }
 
 static bool is_occupied_sdma(struct mqd_manager *mm, void *mqd,
                uint64_t queue_address, uint32_t pipe_id,
                uint32_t queue_id)
 {
-       return mm->dev->kfd2kgd->hqd_sdma_is_occupied(mm->dev->kgd, mqd);
+       return mm->dev->kfd2kgd->hqd_sdma_is_occupied(mm->dev->adev, mqd);
+}
+
+static void checkpoint_mqd_sdma(struct mqd_manager *mm,
+                               void *mqd,
+                               void *mqd_dst,
+                               void *ctl_stack_dst)
+{
+       struct v9_sdma_mqd *m;
+
+       m = get_sdma_mqd(mqd);
+
+       memcpy(mqd_dst, m, sizeof(struct v9_sdma_mqd));
+}
+
+static void restore_mqd_sdma(struct mqd_manager *mm, void **mqd,
+                            struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr,
+                            struct queue_properties *qp,
+                            const void *mqd_src,
+                            const void *ctl_stack_src, const u32 ctl_stack_size)
+{
+       uint64_t addr;
+       struct v9_sdma_mqd *m;
+
+       m = (struct v9_sdma_mqd *) mqd_mem_obj->cpu_ptr;
+       addr = mqd_mem_obj->gpu_addr;
+
+       memcpy(m, mqd_src, sizeof(*m));
+
+       m->sdmax_rlcx_doorbell_offset =
+               qp->doorbell_off << SDMA0_RLC0_DOORBELL_OFFSET__OFFSET__SHIFT;
+
+       *mqd = m;
+       if (gart_addr)
+               *gart_addr = addr;
+
+       qp->is_active = 0;
 }
 
 #if defined(CONFIG_DEBUG_FS)
@@ -470,6 +557,9 @@ struct mqd_manager *mqd_manager_init_v9(enum KFD_MQD_TYPE type,
                mqd->destroy_mqd = destroy_mqd;
                mqd->is_occupied = is_occupied;
                mqd->get_wave_state = get_wave_state;
+               mqd->get_checkpoint_info = get_checkpoint_info;
+               mqd->checkpoint_mqd = checkpoint_mqd;
+               mqd->restore_mqd = restore_mqd;
                mqd->mqd_size = sizeof(struct v9_mqd);
 #if defined(CONFIG_DEBUG_FS)
                mqd->debugfs_show_mqd = debugfs_show_mqd;
@@ -510,6 +600,8 @@ struct mqd_manager *mqd_manager_init_v9(enum KFD_MQD_TYPE type,
                mqd->update_mqd = update_mqd_sdma;
                mqd->destroy_mqd = destroy_mqd_sdma;
                mqd->is_occupied = is_occupied_sdma;
+               mqd->checkpoint_mqd = checkpoint_mqd_sdma;
+               mqd->restore_mqd = restore_mqd_sdma;
                mqd->mqd_size = sizeof(struct v9_sdma_mqd);
 #if defined(CONFIG_DEBUG_FS)
                mqd->debugfs_show_mqd = debugfs_show_mqd_sdma;