Merge tag 'mm-stable-2022-08-03' of git://git.kernel.org/pub/scm/linux/kernel/git...

[linux-2.6-microblaze.git] / drivers / gpu / drm / amd / amdkfd / kfd_migrate.c

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
index f73e3e3..b059a77 100644 (file)
--- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
@@ -33,6 +33,7 @@
 #include "kfd_priv.h"
 #include "kfd_svm.h"
 #include "kfd_migrate.h"
+#include "kfd_smi_events.h"
 
 #ifdef dev_fmt
 #undef dev_fmt
@@ -402,8 +403,9 @@ out:
 static long
 svm_migrate_vma_to_vram(struct amdgpu_device *adev, struct svm_range *prange,
                        struct vm_area_struct *vma, uint64_t start,
-                       uint64_t end)
+                       uint64_t end, uint32_t trigger)
 {
+       struct kfd_process *p = container_of(prange->svms, struct kfd_process, svms);
        uint64_t npages = (end - start) >> PAGE_SHIFT;
        struct kfd_process_device *pdd;
        struct dma_fence *mfence = NULL;
@@ -430,6 +432,11 @@ svm_migrate_vma_to_vram(struct amdgpu_device *adev, struct svm_range *prange,
        migrate.dst = migrate.src + npages;
        scratch = (dma_addr_t *)(migrate.dst + npages);
 
+       kfd_smi_event_migration_start(adev->kfd.dev, p->lead_thread->pid,
+                                     start >> PAGE_SHIFT, end >> PAGE_SHIFT,
+                                     0, adev->kfd.dev->id, prange->prefetch_loc,
+                                     prange->preferred_loc, trigger);
+
        r = migrate_vma_setup(&migrate);
        if (r) {
                dev_err(adev->dev, "%s: vma setup fail %d range [0x%lx 0x%lx]\n",
@@ -458,6 +465,10 @@ svm_migrate_vma_to_vram(struct amdgpu_device *adev, struct svm_range *prange,
        svm_migrate_copy_done(adev, mfence);
        migrate_vma_finalize(&migrate);
 
+       kfd_smi_event_migration_end(adev->kfd.dev, p->lead_thread->pid,
+                                   start >> PAGE_SHIFT, end >> PAGE_SHIFT,
+                                   0, adev->kfd.dev->id, trigger);
+
        svm_range_dma_unmap(adev->dev, scratch, 0, npages);
        svm_range_free_dma_mappings(prange);
 
@@ -479,6 +490,7 @@ out:
  * @prange: range structure
  * @best_loc: the device to migrate to
  * @mm: the process mm structure
+ * @trigger: reason of migration
  *
  * Context: Process context, caller hold mmap read lock, svms lock, prange lock
  *
@@ -487,7 +499,7 @@ out:
  */
 static int
 svm_migrate_ram_to_vram(struct svm_range *prange, uint32_t best_loc,
-                       struct mm_struct *mm)
+                       struct mm_struct *mm, uint32_t trigger)
 {
        unsigned long addr, start, end;
        struct vm_area_struct *vma;
@@ -524,7 +536,7 @@ svm_migrate_ram_to_vram(struct svm_range *prange, uint32_t best_loc,
                        break;
 
                next = min(vma->vm_end, end);
-               r = svm_migrate_vma_to_vram(adev, prange, vma, addr, next);
+               r = svm_migrate_vma_to_vram(adev, prange, vma, addr, next, trigger);
                if (r < 0) {
                        pr_debug("failed %ld to migrate\n", r);
                        break;
@@ -655,8 +667,10 @@ out_oom:
  */
 static long
 svm_migrate_vma_to_ram(struct amdgpu_device *adev, struct svm_range *prange,
-                      struct vm_area_struct *vma, uint64_t start, uint64_t end)
+                      struct vm_area_struct *vma, uint64_t start, uint64_t end,
+                      uint32_t trigger)
 {
+       struct kfd_process *p = container_of(prange->svms, struct kfd_process, svms);
        uint64_t npages = (end - start) >> PAGE_SHIFT;
        unsigned long upages = npages;
        unsigned long cpages = 0;
@@ -687,6 +701,11 @@ svm_migrate_vma_to_ram(struct amdgpu_device *adev, struct svm_range *prange,
        migrate.dst = migrate.src + npages;
        scratch = (dma_addr_t *)(migrate.dst + npages);
 
+       kfd_smi_event_migration_start(adev->kfd.dev, p->lead_thread->pid,
+                                     start >> PAGE_SHIFT, end >> PAGE_SHIFT,
+                                     adev->kfd.dev->id, 0, prange->prefetch_loc,
+                                     prange->preferred_loc, trigger);
+
        r = migrate_vma_setup(&migrate);
        if (r) {
                dev_err(adev->dev, "%s: vma setup fail %d range [0x%lx 0x%lx]\n",
@@ -717,6 +736,11 @@ svm_migrate_vma_to_ram(struct amdgpu_device *adev, struct svm_range *prange,
 
        svm_migrate_copy_done(adev, mfence);
        migrate_vma_finalize(&migrate);
+
+       kfd_smi_event_migration_end(adev->kfd.dev, p->lead_thread->pid,
+                                   start >> PAGE_SHIFT, end >> PAGE_SHIFT,
+                                   adev->kfd.dev->id, 0, trigger);
+
        svm_range_dma_unmap(adev->dev, scratch, 0, npages);
 
 out_free:
@@ -734,13 +758,15 @@ out:
  * svm_migrate_vram_to_ram - migrate svm range from device to system
  * @prange: range structure
  * @mm: process mm, use current->mm if NULL
+ * @trigger: reason of migration
  *
  * Context: Process context, caller hold mmap read lock, prange->migrate_mutex
  *
  * Return:
  * 0 - OK, otherwise error code
  */
-int svm_migrate_vram_to_ram(struct svm_range *prange, struct mm_struct *mm)
+int svm_migrate_vram_to_ram(struct svm_range *prange, struct mm_struct *mm,
+                           uint32_t trigger)
 {
        struct amdgpu_device *adev;
        struct vm_area_struct *vma;
@@ -781,7 +807,7 @@ int svm_migrate_vram_to_ram(struct svm_range *prange, struct mm_struct *mm)
                }
 
                next = min(vma->vm_end, end);
-               r = svm_migrate_vma_to_ram(adev, prange, vma, addr, next);
+               r = svm_migrate_vma_to_ram(adev, prange, vma, addr, next, trigger);
                if (r < 0) {
                        pr_debug("failed %ld to migrate prange %p\n", r, prange);
                        break;
@@ -804,6 +830,7 @@ int svm_migrate_vram_to_ram(struct svm_range *prange, struct mm_struct *mm)
  * @prange: range structure
  * @best_loc: the device to migrate to
  * @mm: process mm, use current->mm if NULL
+ * @trigger: reason of migration
  *
  * Context: Process context, caller hold mmap read lock, svms lock, prange lock
  *
@@ -812,7 +839,7 @@ int svm_migrate_vram_to_ram(struct svm_range *prange, struct mm_struct *mm)
  */
 static int
 svm_migrate_vram_to_vram(struct svm_range *prange, uint32_t best_loc,
-                        struct mm_struct *mm)
+                        struct mm_struct *mm, uint32_t trigger)
 {
        int r, retries = 3;
 
@@ -824,7 +851,7 @@ svm_migrate_vram_to_vram(struct svm_range *prange, uint32_t best_loc,
        pr_debug("from gpu 0x%x to gpu 0x%x\n", prange->actual_loc, best_loc);
 
        do {
-               r = svm_migrate_vram_to_ram(prange, mm);
+               r = svm_migrate_vram_to_ram(prange, mm, trigger);
                if (r)
                        return r;
        } while (prange->actual_loc && --retries);
@@ -832,17 +859,17 @@ svm_migrate_vram_to_vram(struct svm_range *prange, uint32_t best_loc,
        if (prange->actual_loc)
                return -EDEADLK;
 
-       return svm_migrate_ram_to_vram(prange, best_loc, mm);
+       return svm_migrate_ram_to_vram(prange, best_loc, mm, trigger);
 }
 
 int
 svm_migrate_to_vram(struct svm_range *prange, uint32_t best_loc,
-                   struct mm_struct *mm)
+                   struct mm_struct *mm, uint32_t trigger)
 {
        if  (!prange->actual_loc)
-               return svm_migrate_ram_to_vram(prange, best_loc, mm);
+               return svm_migrate_ram_to_vram(prange, best_loc, mm, trigger);
        else
-               return svm_migrate_vram_to_vram(prange, best_loc, mm);
+               return svm_migrate_vram_to_vram(prange, best_loc, mm, trigger);
 
 }
 
@@ -911,7 +938,7 @@ static vm_fault_t svm_migrate_to_ram(struct vm_fault *vmf)
                goto out_unlock_prange;
        }
 
-       r = svm_migrate_vram_to_ram(prange, mm);
+       r = svm_migrate_vram_to_ram(prange, mm, KFD_MIGRATE_TRIGGER_PAGEFAULT_CPU);
        if (r)
                pr_debug("failed %d migrate 0x%p [0x%lx 0x%lx] to ram\n", r,
                         prange, prange->start, prange->last);
@@ -1000,6 +1027,8 @@ int svm_migrate_init(struct amdgpu_device *adev)
 
        amdgpu_amdkfd_reserve_system_mem(SVM_HMM_PAGE_STRUCT_SIZE(size));
 
+       svm_range_set_max_pages(adev);
+
        pr_info("HMM registered %ldMB device memory\n", size >> 20);
 
        return 0;