spin_lock(&adev->mman.bdev.lru_lock);
list_for_each_entry(bo_base, &vm->idle, vm_status) {
struct amdgpu_bo *bo = bo_base->bo;
+ struct amdgpu_bo *shadow = amdgpu_bo_shadowed(bo);
if (!bo->parent)
continue;
ttm_bo_move_to_lru_tail(&bo->tbo, bo->tbo.resource,
&vm->lru_bulk_move);
- if (bo->shadow)
- ttm_bo_move_to_lru_tail(&bo->shadow->tbo,
- bo->shadow->tbo.resource,
+ if (shadow)
+ ttm_bo_move_to_lru_tail(&shadow->tbo,
+ shadow->tbo.resource,
&vm->lru_bulk_move);
}
spin_unlock(&adev->mman.bdev.lru_lock);
list_for_each_entry_safe(bo_base, tmp, &vm->evicted, vm_status) {
struct amdgpu_bo *bo = bo_base->bo;
+ struct amdgpu_bo *shadow = amdgpu_bo_shadowed(bo);
r = validate(param, bo);
if (r)
return r;
+ if (shadow) {
+ r = validate(param, shadow);
+ if (r)
+ return r;
+ }
if (bo->tbo.type != ttm_bo_type_kernel) {
amdgpu_vm_bo_moved(bo_base);
} else {
- vm->update_funcs->map_table(bo);
+ vm->update_funcs->map_table(to_amdgpu_bo_vm(bo));
amdgpu_vm_bo_relocated(bo_base);
}
}
*
* @adev: amdgpu_device pointer
* @vm: VM to clear BO from
- * @bo: BO to clear
+ * @vmbo: BO to clear
* @immediate: use an immediate update
*
* Root PD needs to be reserved when calling this.
*/
static int amdgpu_vm_clear_bo(struct amdgpu_device *adev,
struct amdgpu_vm *vm,
- struct amdgpu_bo *bo,
+ struct amdgpu_bo_vm *vmbo,
bool immediate)
{
struct ttm_operation_ctx ctx = { true, false };
unsigned level = adev->vm_manager.root_level;
struct amdgpu_vm_update_params params;
- struct amdgpu_bo *ancestor = bo;
+ struct amdgpu_bo *ancestor = &vmbo->bo;
+ struct amdgpu_bo *bo = &vmbo->bo;
unsigned entries, ats_entries;
uint64_t addr;
int r;
if (r)
return r;
- if (bo->shadow) {
- r = ttm_bo_validate(&bo->shadow->tbo, &bo->shadow->placement,
- &ctx);
+ if (vmbo->shadow) {
+ struct amdgpu_bo *shadow = vmbo->shadow;
+
+ r = ttm_bo_validate(&shadow->tbo, &shadow->placement, &ctx);
if (r)
return r;
}
- r = vm->update_funcs->map_table(bo);
+ r = vm->update_funcs->map_table(vmbo);
if (r)
return r;
amdgpu_gmc_get_vm_pde(adev, level, &value, &flags);
}
- r = vm->update_funcs->update(¶ms, bo, addr, 0, ats_entries,
+ r = vm->update_funcs->update(¶ms, vmbo, addr, 0, ats_entries,
value, flags);
if (r)
return r;
}
}
- r = vm->update_funcs->update(¶ms, bo, addr, 0, entries,
+ r = vm->update_funcs->update(¶ms, vmbo, addr, 0, entries,
value, flags);
if (r)
return r;
* @vm: requesting vm
* @level: the page table level
* @immediate: use a immediate update
- * @bo: pointer to the buffer object pointer
+ * @vmbo: pointer to the buffer object pointer
*/
static int amdgpu_vm_pt_create(struct amdgpu_device *adev,
struct amdgpu_vm *vm,
int level, bool immediate,
- struct amdgpu_bo **bo)
+ struct amdgpu_bo_vm **vmbo)
{
struct amdgpu_bo_param bp;
+ struct amdgpu_bo *bo;
+ struct dma_resv *resv;
+ unsigned int num_entries;
int r;
memset(&bp, 0, sizeof(bp));
bp.domain = amdgpu_bo_get_preferred_pin_domain(adev, bp.domain);
bp.flags = AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS |
AMDGPU_GEM_CREATE_CPU_GTT_USWC;
- bp.bo_ptr_size = sizeof(struct amdgpu_bo);
+
+ if (level < AMDGPU_VM_PTB)
+ num_entries = amdgpu_vm_num_entries(adev, level);
+ else
+ num_entries = 0;
+
+ bp.bo_ptr_size = struct_size((*vmbo), entries, num_entries);
+
if (vm->use_cpu_for_update)
bp.flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
if (vm->root.base.bo)
bp.resv = vm->root.base.bo->tbo.base.resv;
- r = amdgpu_bo_create(adev, &bp, bo);
+ r = amdgpu_bo_create_vm(adev, &bp, vmbo);
if (r)
return r;
- if (vm->is_compute_context && (adev->flags & AMD_IS_APU))
+ bo = &(*vmbo)->bo;
+ if (vm->is_compute_context && (adev->flags & AMD_IS_APU)) {
+ (*vmbo)->shadow = NULL;
return 0;
+ }
if (!bp.resv)
- WARN_ON(dma_resv_lock((*bo)->tbo.base.resv,
+ WARN_ON(dma_resv_lock(bo->tbo.base.resv,
NULL));
- r = amdgpu_bo_create_shadow(adev, bp.size, *bo);
+ resv = bp.resv;
+ memset(&bp, 0, sizeof(bp));
+ bp.size = amdgpu_vm_bo_size(adev, level);
+ bp.domain = AMDGPU_GEM_DOMAIN_GTT;
+ bp.flags = AMDGPU_GEM_CREATE_CPU_GTT_USWC;
+ bp.type = ttm_bo_type_kernel;
+ bp.resv = bo->tbo.base.resv;
+ bp.bo_ptr_size = sizeof(struct amdgpu_bo);
- if (!bp.resv)
- dma_resv_unlock((*bo)->tbo.base.resv);
+ r = amdgpu_bo_create(adev, &bp, &(*vmbo)->shadow);
+
+ if (!resv)
+ dma_resv_unlock(bo->tbo.base.resv);
if (r) {
- amdgpu_bo_unref(bo);
+ amdgpu_bo_unref(&bo);
return r;
}
+ (*vmbo)->shadow->parent = amdgpu_bo_ref(bo);
+ amdgpu_bo_add_to_shadow_list((*vmbo)->shadow);
+
return 0;
}
bool immediate)
{
struct amdgpu_vm_pt *entry = cursor->entry;
- struct amdgpu_bo *pt;
+ struct amdgpu_bo *pt_bo;
+ struct amdgpu_bo_vm *pt;
int r;
- if (cursor->level < AMDGPU_VM_PTB && !entry->entries) {
- unsigned num_entries;
-
- num_entries = amdgpu_vm_num_entries(adev, cursor->level);
- entry->entries = kvmalloc_array(num_entries,
- sizeof(*entry->entries),
- GFP_KERNEL | __GFP_ZERO);
- if (!entry->entries)
- return -ENOMEM;
- }
-
- if (entry->base.bo)
+ if (entry->base.bo) {
+ if (cursor->level < AMDGPU_VM_PTB)
+ entry->entries =
+ to_amdgpu_bo_vm(entry->base.bo)->entries;
+ else
+ entry->entries = NULL;
return 0;
+ }
r = amdgpu_vm_pt_create(adev, vm, cursor->level, immediate, &pt);
if (r)
/* Keep a reference to the root directory to avoid
* freeing them up in the wrong order.
*/
- pt->parent = amdgpu_bo_ref(cursor->parent->base.bo);
- amdgpu_vm_bo_base_init(&entry->base, vm, pt);
+ pt_bo = &pt->bo;
+ pt_bo->parent = amdgpu_bo_ref(cursor->parent->base.bo);
+ amdgpu_vm_bo_base_init(&entry->base, vm, pt_bo);
+ if (cursor->level < AMDGPU_VM_PTB)
+ entry->entries = pt->entries;
+ else
+ entry->entries = NULL;
r = amdgpu_vm_clear_bo(adev, vm, pt, immediate);
if (r)
error_free_pt:
amdgpu_bo_unref(&pt->shadow);
- amdgpu_bo_unref(&pt);
+ amdgpu_bo_unref(&pt_bo);
return r;
}
*/
static void amdgpu_vm_free_table(struct amdgpu_vm_pt *entry)
{
+ struct amdgpu_bo *shadow;
+
if (entry->base.bo) {
+ shadow = amdgpu_bo_shadowed(entry->base.bo);
entry->base.bo->vm_bo = NULL;
list_del(&entry->base.vm_status);
- amdgpu_bo_unref(&entry->base.bo->shadow);
+ amdgpu_bo_unref(&shadow);
amdgpu_bo_unref(&entry->base.bo);
}
- kvfree(entry->entries);
entry->entries = NULL;
}
level += params->adev->vm_manager.root_level;
amdgpu_gmc_get_pde_for_bo(entry->base.bo, level, &pt, &flags);
pde = (entry - parent->entries) * 8;
- return vm->update_funcs->update(params, bo, pde, pt, 1, 0, flags);
+ return vm->update_funcs->update(params, to_amdgpu_bo_vm(bo), pde, pt,
+ 1, 0, flags);
}
/**
* Make sure to set the right flags for the PTEs at the desired level.
*/
static void amdgpu_vm_update_flags(struct amdgpu_vm_update_params *params,
- struct amdgpu_bo *bo, unsigned level,
+ struct amdgpu_bo_vm *pt, unsigned int level,
uint64_t pe, uint64_t addr,
- unsigned count, uint32_t incr,
+ unsigned int count, uint32_t incr,
uint64_t flags)
{
flags |= AMDGPU_PTE_EXECUTABLE;
}
- params->vm->update_funcs->update(params, bo, pe, addr, count, incr,
+ params->vm->update_funcs->update(params, pt, pe, addr, count, incr,
flags);
}
nptes, dst, incr, upd_flags,
vm->task_info.pid,
vm->immediate.fence_context);
- amdgpu_vm_update_flags(params, pt, cursor.level,
- pe_start, dst, nptes, incr,
- upd_flags);
+ amdgpu_vm_update_flags(params, to_amdgpu_bo_vm(pt),
+ cursor.level, pe_start, dst,
+ nptes, incr, upd_flags);
pe_start += nptes * 8;
dst += nptes * incr;
* completely covered by the range and so potentially still in use.
*/
while (cursor.pfn < frag_start) {
- amdgpu_vm_free_pts(adev, params->vm, &cursor);
+ /* Make sure previous mapping is freed */
+ if (cursor.entry->base.bo) {
+ params->table_freed = true;
+ amdgpu_vm_free_pts(adev, params->vm, &cursor);
+ }
amdgpu_vm_pt_next(adev, &cursor);
- params->table_freed = true;
}
} else if (frag >= shift) {
bo = gem_to_amdgpu_bo(gobj);
}
mem = bo->tbo.resource;
- if (mem->mem_type == TTM_PL_TT)
+ if (mem->mem_type == TTM_PL_TT ||
+ mem->mem_type == AMDGPU_PL_PREEMPT)
pages_addr = bo->tbo.ttm->dma_address;
}
struct amdgpu_vm_bo_base *bo_base;
/* shadow bo doesn't have bo base, its validation needs its parent */
- if (bo->parent && bo->parent->shadow == bo)
+ if (bo->parent && (amdgpu_bo_shadowed(bo->parent) == bo))
bo = bo->parent;
for (bo_base = bo->vm_bo; bo_base; bo_base = bo_base->next) {
*/
int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm, u32 pasid)
{
- struct amdgpu_bo *root;
+ struct amdgpu_bo *root_bo;
+ struct amdgpu_bo_vm *root;
int r, i;
vm->va = RB_ROOT_CACHED;
false, &root);
if (r)
goto error_free_delayed;
-
- r = amdgpu_bo_reserve(root, true);
+ root_bo = &root->bo;
+ r = amdgpu_bo_reserve(root_bo, true);
if (r)
goto error_free_root;
- r = dma_resv_reserve_shared(root->tbo.base.resv, 1);
+ r = dma_resv_reserve_shared(root_bo->tbo.base.resv, 1);
if (r)
goto error_unreserve;
- amdgpu_vm_bo_base_init(&vm->root.base, vm, root);
+ amdgpu_vm_bo_base_init(&vm->root.base, vm, root_bo);
r = amdgpu_vm_clear_bo(adev, vm, root, false);
if (r)
amdgpu_bo_unreserve(vm->root.base.bo);
error_free_root:
- amdgpu_bo_unref(&vm->root.base.bo->shadow);
- amdgpu_bo_unref(&vm->root.base.bo);
+ amdgpu_bo_unref(&root->shadow);
+ amdgpu_bo_unref(&root_bo);
vm->root.base.bo = NULL;
error_free_delayed:
*/
if (pte_support_ats != vm->pte_support_ats) {
vm->pte_support_ats = pte_support_ats;
- r = amdgpu_vm_clear_bo(adev, vm, vm->root.base.bo, false);
+ r = amdgpu_vm_clear_bo(adev, vm,
+ to_amdgpu_bo_vm(vm->root.base.bo),
+ false);
if (r)
goto free_idr;
}
}
/* Free the shadow bo for compute VM */
- amdgpu_bo_unref(&vm->root.base.bo->shadow);
+ amdgpu_bo_unref(&to_amdgpu_bo_vm(vm->root.base.bo)->shadow);
if (pasid)
vm->pasid = pasid;
projects / linux-2.6-microblaze.git / blobdiff