From 7420f482ea5163bf6dae39a5c7628d5397cd6307 Mon Sep 17 00:00:00 2001 From: Oded Gabbay Date: Thu, 15 Mar 2018 10:08:35 +0200 Subject: drm/amdkfd: add missing include of mm.h This patch fixes kernel build in ARCH=frv Signed-off-by: Oded Gabbay --- drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h | 1 + 1 file changed, 1 insertion(+) (limited to 'drivers/gpu/drm/amd/amdgpu') diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h index d7509b706b26..ad7292fc454c 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h @@ -26,6 +26,7 @@ #define AMDGPU_AMDKFD_H_INCLUDED #include +#include #include #include #include -- cgit v1.2.1 From 5b21d3e5fd2110df5a57a508188308351e698a5a Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Thu, 15 Mar 2018 17:27:40 -0400 Subject: drm/amdgpu: Move KFD-specific fields into struct amdgpu_vm MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove struct amdkfd_vm and move the fields into struct amdgpu_vm. This will allow turning a VM created by a DRM render node into a KFD VM. v2: Removed vm_context field Signed-off-by: Felix Kuehling Reviewed-by: Christian König Signed-off-by: Oded Gabbay --- drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h | 21 ------ drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c | 88 +++++++++++------------- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h | 9 +++ 3 files changed, 50 insertions(+), 68 deletions(-) (limited to 'drivers/gpu/drm/amd/amdgpu') diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h index ad7292fc454c..6ba3dd689875 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h @@ -93,27 +93,6 @@ struct amdkfd_process_info { struct amdgpu_amdkfd_fence *eviction_fence; }; -/* struct amdkfd_vm - - * For Memory Eviction KGD requires a mechanism to keep track of all KFD BOs - * belonging to a KFD process. All the VMs belonging to the same process point - * to the same amdkfd_process_info. - */ -struct amdkfd_vm { - /* Keep base as the first parameter for pointer compatibility between - * amdkfd_vm and amdgpu_vm. - */ - struct amdgpu_vm base; - - /* List node in amdkfd_process_info.vm_list_head*/ - struct list_head vm_list_node; - - struct amdgpu_device *adev; - /* Points to the KFD process VM info*/ - struct amdkfd_process_info *process_info; - - uint64_t pd_phys_addr; -}; - int amdgpu_amdkfd_init(void); void amdgpu_amdkfd_fini(void); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c index a12a1654e124..64c33acb6cd8 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c @@ -333,9 +333,9 @@ static int amdgpu_amdkfd_validate(void *param, struct amdgpu_bo *bo) * again. Page directories are only updated after updating page * tables. */ -static int vm_validate_pt_pd_bos(struct amdkfd_vm *vm) +static int vm_validate_pt_pd_bos(struct amdgpu_vm *vm) { - struct amdgpu_bo *pd = vm->base.root.base.bo; + struct amdgpu_bo *pd = vm->root.base.bo; struct amdgpu_device *adev = amdgpu_ttm_adev(pd->tbo.bdev); struct amdgpu_vm_parser param; uint64_t addr, flags = AMDGPU_PTE_VALID; @@ -344,7 +344,7 @@ static int vm_validate_pt_pd_bos(struct amdkfd_vm *vm) param.domain = AMDGPU_GEM_DOMAIN_VRAM; param.wait = false; - ret = amdgpu_vm_validate_pt_bos(adev, &vm->base, amdgpu_amdkfd_validate, + ret = amdgpu_vm_validate_pt_bos(adev, vm, amdgpu_amdkfd_validate, ¶m); if (ret) { pr_err("amdgpu: failed to validate PT BOs\n"); @@ -357,11 +357,11 @@ static int vm_validate_pt_pd_bos(struct amdkfd_vm *vm) return ret; } - addr = amdgpu_bo_gpu_offset(vm->base.root.base.bo); + addr = amdgpu_bo_gpu_offset(vm->root.base.bo); amdgpu_gmc_get_vm_pde(adev, -1, &addr, &flags); vm->pd_phys_addr = addr; - if (vm->base.use_cpu_for_update) { + if (vm->use_cpu_for_update) { ret = amdgpu_bo_kmap(pd, NULL); if (ret) { pr_err("amdgpu: failed to kmap PD, ret=%d\n", ret); @@ -415,14 +415,12 @@ static int vm_update_pds(struct amdgpu_vm *vm, struct amdgpu_sync *sync) * 4a. Validate new page tables and directories */ static int add_bo_to_vm(struct amdgpu_device *adev, struct kgd_mem *mem, - struct amdgpu_vm *avm, bool is_aql, + struct amdgpu_vm *vm, bool is_aql, struct kfd_bo_va_list **p_bo_va_entry) { int ret; struct kfd_bo_va_list *bo_va_entry; - struct amdkfd_vm *kvm = container_of(avm, - struct amdkfd_vm, base); - struct amdgpu_bo *pd = avm->root.base.bo; + struct amdgpu_bo *pd = vm->root.base.bo; struct amdgpu_bo *bo = mem->bo; uint64_t va = mem->va; struct list_head *list_bo_va = &mem->bo_va_list; @@ -441,10 +439,10 @@ static int add_bo_to_vm(struct amdgpu_device *adev, struct kgd_mem *mem, return -ENOMEM; pr_debug("\t add VA 0x%llx - 0x%llx to vm %p\n", va, - va + bo_size, avm); + va + bo_size, vm); /* Add BO to VM internal data structures*/ - bo_va_entry->bo_va = amdgpu_vm_bo_add(adev, avm, bo); + bo_va_entry->bo_va = amdgpu_vm_bo_add(adev, vm, bo); if (!bo_va_entry->bo_va) { ret = -EINVAL; pr_err("Failed to add BO object to VM. ret == %d\n", @@ -467,28 +465,28 @@ static int add_bo_to_vm(struct amdgpu_device *adev, struct kgd_mem *mem, * fence, so remove it temporarily. */ amdgpu_amdkfd_remove_eviction_fence(pd, - kvm->process_info->eviction_fence, + vm->process_info->eviction_fence, NULL, NULL); - ret = amdgpu_vm_alloc_pts(adev, avm, va, amdgpu_bo_size(bo)); + ret = amdgpu_vm_alloc_pts(adev, vm, va, amdgpu_bo_size(bo)); if (ret) { pr_err("Failed to allocate pts, err=%d\n", ret); goto err_alloc_pts; } - ret = vm_validate_pt_pd_bos(kvm); + ret = vm_validate_pt_pd_bos(vm); if (ret) { pr_err("validate_pt_pd_bos() failed\n"); goto err_alloc_pts; } /* Add the eviction fence back */ - amdgpu_bo_fence(pd, &kvm->process_info->eviction_fence->base, true); + amdgpu_bo_fence(pd, &vm->process_info->eviction_fence->base, true); return 0; err_alloc_pts: - amdgpu_bo_fence(pd, &kvm->process_info->eviction_fence->base, true); + amdgpu_bo_fence(pd, &vm->process_info->eviction_fence->base, true); amdgpu_vm_bo_rmv(adev, bo_va_entry->bo_va); list_del(&bo_va_entry->bo_list); err_vmadd: @@ -703,7 +701,6 @@ static int unmap_bo_from_gpuvm(struct amdgpu_device *adev, { struct amdgpu_bo_va *bo_va = entry->bo_va; struct amdgpu_vm *vm = bo_va->base.vm; - struct amdkfd_vm *kvm = container_of(vm, struct amdkfd_vm, base); struct amdgpu_bo *pd = vm->root.base.bo; /* Remove eviction fence from PD (and thereby from PTs too as @@ -713,14 +710,14 @@ static int unmap_bo_from_gpuvm(struct amdgpu_device *adev, * trigger the eviction fence. */ amdgpu_amdkfd_remove_eviction_fence(pd, - kvm->process_info->eviction_fence, + vm->process_info->eviction_fence, NULL, NULL); amdgpu_vm_bo_unmap(adev, bo_va, entry->va); amdgpu_vm_clear_freed(adev, vm, &bo_va->last_pt_update); /* Add the eviction fence back */ - amdgpu_bo_fence(pd, &kvm->process_info->eviction_fence->base, true); + amdgpu_bo_fence(pd, &vm->process_info->eviction_fence->base, true); sync_vm_fence(adev, sync, bo_va->last_pt_update); @@ -780,7 +777,7 @@ update_gpuvm_pte_failed: static int process_validate_vms(struct amdkfd_process_info *process_info) { - struct amdkfd_vm *peer_vm; + struct amdgpu_vm *peer_vm; int ret; list_for_each_entry(peer_vm, &process_info->vm_list_head, @@ -796,12 +793,12 @@ static int process_validate_vms(struct amdkfd_process_info *process_info) static int process_update_pds(struct amdkfd_process_info *process_info, struct amdgpu_sync *sync) { - struct amdkfd_vm *peer_vm; + struct amdgpu_vm *peer_vm; int ret; list_for_each_entry(peer_vm, &process_info->vm_list_head, vm_list_node) { - ret = vm_update_pds(&peer_vm->base, sync); + ret = vm_update_pds(peer_vm, sync); if (ret) return ret; } @@ -814,7 +811,7 @@ int amdgpu_amdkfd_gpuvm_create_process_vm(struct kgd_dev *kgd, void **vm, struct dma_fence **ef) { int ret; - struct amdkfd_vm *new_vm; + struct amdgpu_vm *new_vm; struct amdkfd_process_info *info; struct amdgpu_device *adev = get_amdgpu_device(kgd); @@ -823,12 +820,11 @@ int amdgpu_amdkfd_gpuvm_create_process_vm(struct kgd_dev *kgd, void **vm, return -ENOMEM; /* Initialize the VM context, allocate the page directory and zero it */ - ret = amdgpu_vm_init(adev, &new_vm->base, AMDGPU_VM_CONTEXT_COMPUTE, 0); + ret = amdgpu_vm_init(adev, new_vm, AMDGPU_VM_CONTEXT_COMPUTE, 0); if (ret) { pr_err("Failed init vm ret %d\n", ret); goto vm_init_fail; } - new_vm->adev = adev; if (!*process_info) { info = kzalloc(sizeof(*info), GFP_KERNEL); @@ -871,7 +867,7 @@ create_evict_fence_fail: mutex_destroy(&info->lock); kfree(info); alloc_process_info_fail: - amdgpu_vm_fini(adev, &new_vm->base); + amdgpu_vm_fini(adev, new_vm); vm_init_fail: kfree(new_vm); return ret; @@ -881,8 +877,7 @@ vm_init_fail: void amdgpu_amdkfd_gpuvm_destroy_process_vm(struct kgd_dev *kgd, void *vm) { struct amdgpu_device *adev = get_amdgpu_device(kgd); - struct amdkfd_vm *kfd_vm = (struct amdkfd_vm *) vm; - struct amdgpu_vm *avm = &kfd_vm->base; + struct amdgpu_vm *avm = (struct amdgpu_vm *)vm; struct amdgpu_bo *pd; struct amdkfd_process_info *process_info; @@ -896,11 +891,11 @@ void amdgpu_amdkfd_gpuvm_destroy_process_vm(struct kgd_dev *kgd, void *vm) amdgpu_bo_fence(pd, NULL, false); amdgpu_bo_unreserve(pd); - process_info = kfd_vm->process_info; + process_info = avm->process_info; mutex_lock(&process_info->lock); process_info->n_vms--; - list_del(&kfd_vm->vm_list_node); + list_del(&avm->vm_list_node); mutex_unlock(&process_info->lock); /* Release per-process resources */ @@ -919,7 +914,7 @@ void amdgpu_amdkfd_gpuvm_destroy_process_vm(struct kgd_dev *kgd, void *vm) uint32_t amdgpu_amdkfd_gpuvm_get_process_page_dir(void *vm) { - struct amdkfd_vm *avm = (struct amdkfd_vm *)vm; + struct amdgpu_vm *avm = (struct amdgpu_vm *)vm; return avm->pd_phys_addr >> AMDGPU_GPU_PAGE_SHIFT; } @@ -930,7 +925,7 @@ int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu( uint64_t *offset, uint32_t flags) { struct amdgpu_device *adev = get_amdgpu_device(kgd); - struct amdkfd_vm *kfd_vm = (struct amdkfd_vm *)vm; + struct amdgpu_vm *avm = (struct amdgpu_vm *)vm; struct amdgpu_bo *bo; int byte_align; u32 alloc_domain; @@ -1010,8 +1005,8 @@ int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu( (*mem)->va = va; (*mem)->domain = alloc_domain; (*mem)->mapped_to_gpu_memory = 0; - (*mem)->process_info = kfd_vm->process_info; - add_kgd_mem_to_kfd_bo_list(*mem, kfd_vm->process_info); + (*mem)->process_info = avm->process_info; + add_kgd_mem_to_kfd_bo_list(*mem, avm->process_info); if (offset) *offset = amdgpu_bo_mmap_offset(bo); @@ -1092,7 +1087,7 @@ int amdgpu_amdkfd_gpuvm_map_memory_to_gpu( struct kgd_dev *kgd, struct kgd_mem *mem, void *vm) { struct amdgpu_device *adev = get_amdgpu_device(kgd); - struct amdkfd_vm *kfd_vm = (struct amdkfd_vm *)vm; + struct amdgpu_vm *avm = (struct amdgpu_vm *)vm; int ret; struct amdgpu_bo *bo; uint32_t domain; @@ -1128,19 +1123,19 @@ int amdgpu_amdkfd_gpuvm_map_memory_to_gpu( if (unlikely(ret)) goto out; - if (check_if_add_bo_to_vm((struct amdgpu_vm *)vm, mem)) { - ret = add_bo_to_vm(adev, mem, (struct amdgpu_vm *)vm, false, + if (check_if_add_bo_to_vm(avm, mem)) { + ret = add_bo_to_vm(adev, mem, avm, false, &bo_va_entry); if (ret) goto add_bo_to_vm_failed; if (mem->aql_queue) { - ret = add_bo_to_vm(adev, mem, (struct amdgpu_vm *)vm, + ret = add_bo_to_vm(adev, mem, avm, true, &bo_va_entry_aql); if (ret) goto add_bo_to_vm_failed_aql; } } else { - ret = vm_validate_pt_pd_bos((struct amdkfd_vm *)vm); + ret = vm_validate_pt_pd_bos(avm); if (unlikely(ret)) goto add_bo_to_vm_failed; } @@ -1184,7 +1179,7 @@ int amdgpu_amdkfd_gpuvm_map_memory_to_gpu( if (!amdgpu_ttm_tt_get_usermm(bo->tbo.ttm) && !bo->pin_count) amdgpu_bo_fence(bo, - &kfd_vm->process_info->eviction_fence->base, + &avm->process_info->eviction_fence->base, true); ret = unreserve_bo_and_vms(&ctx, false, false); @@ -1209,7 +1204,7 @@ int amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu( { struct amdgpu_device *adev = get_amdgpu_device(kgd); struct amdkfd_process_info *process_info = - ((struct amdkfd_vm *)vm)->process_info; + ((struct amdgpu_vm *)vm)->process_info; unsigned long bo_size = mem->bo->tbo.mem.size; struct kfd_bo_va_list *entry; struct bo_vm_reservation_context ctx; @@ -1226,7 +1221,7 @@ int amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu( goto unreserve_out; } - ret = vm_validate_pt_pd_bos((struct amdkfd_vm *)vm); + ret = vm_validate_pt_pd_bos((struct amdgpu_vm *)vm); if (unlikely(ret)) goto unreserve_out; @@ -1368,7 +1363,7 @@ int amdgpu_amdkfd_gpuvm_restore_process_bos(void *info, struct dma_fence **ef) { struct amdgpu_bo_list_entry *pd_bo_list; struct amdkfd_process_info *process_info = info; - struct amdkfd_vm *peer_vm; + struct amdgpu_vm *peer_vm; struct kgd_mem *mem; struct bo_vm_reservation_context ctx; struct amdgpu_amdkfd_fence *new_fence; @@ -1390,8 +1385,7 @@ int amdgpu_amdkfd_gpuvm_restore_process_bos(void *info, struct dma_fence **ef) mutex_lock(&process_info->lock); list_for_each_entry(peer_vm, &process_info->vm_list_head, vm_list_node) - amdgpu_vm_get_pd_bo(&peer_vm->base, &ctx.list, - &pd_bo_list[i++]); + amdgpu_vm_get_pd_bo(peer_vm, &ctx.list, &pd_bo_list[i++]); /* Reserve all BOs and page tables/directory. Add all BOs from * kfd_bo_list to ctx.list @@ -1422,7 +1416,7 @@ int amdgpu_amdkfd_gpuvm_restore_process_bos(void *info, struct dma_fence **ef) /* FIXME: I think this isn't needed */ list_for_each_entry(peer_vm, &process_info->vm_list_head, vm_list_node) { - struct amdgpu_bo *bo = peer_vm->base.root.base.bo; + struct amdgpu_bo *bo = peer_vm->root.base.bo; ttm_bo_wait(&bo->tbo, false, false); } @@ -1491,7 +1485,7 @@ int amdgpu_amdkfd_gpuvm_restore_process_bos(void *info, struct dma_fence **ef) /* Attach eviction fence to PD / PT BOs */ list_for_each_entry(peer_vm, &process_info->vm_list_head, vm_list_node) { - struct amdgpu_bo *bo = peer_vm->base.root.base.bo; + struct amdgpu_bo *bo = peer_vm->root.base.bo; amdgpu_bo_fence(bo, &process_info->eviction_fence->base, true); } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h index e9841518343e..4dfdefb7de1f 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h @@ -207,6 +207,15 @@ struct amdgpu_vm { /* Limit non-retry fault storms */ unsigned int fault_credit; + + /* Points to the KFD process VM info */ + struct amdkfd_process_info *process_info; + + /* List node in amdkfd_process_info.vm_list_head */ + struct list_head vm_list_node; + + /* Valid while the PD is reserved or fenced */ + uint64_t pd_phys_addr; }; struct amdgpu_vm_manager { -- cgit v1.2.1 From 3486625bbfb629621f8a2402e2b24e7ab0a86ef4 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Thu, 15 Mar 2018 17:27:41 -0400 Subject: drm/amdgpu: Fix initial validation of PD BO for KFD VMs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Make sure the PD BO is valid and attach the eviction fence during VM creation. This ensures that the pd_phys_address is actually valid and an eviction that would invalidate it triggers a KFD process eviction like it should. v2: Use uninterruptible waiting in initial PD validation Signed-off-by: Felix Kuehling Acked-by: Christian König Signed-off-by: Oded Gabbay --- drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) (limited to 'drivers/gpu/drm/amd/amdgpu') diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c index 64c33acb6cd8..cafe8a571636 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c @@ -812,7 +812,7 @@ int amdgpu_amdkfd_gpuvm_create_process_vm(struct kgd_dev *kgd, void **vm, { int ret; struct amdgpu_vm *new_vm; - struct amdkfd_process_info *info; + struct amdkfd_process_info *info = NULL; struct amdgpu_device *adev = get_amdgpu_device(kgd); new_vm = kzalloc(sizeof(*new_vm), GFP_KERNEL); @@ -851,6 +851,23 @@ int amdgpu_amdkfd_gpuvm_create_process_vm(struct kgd_dev *kgd, void **vm, new_vm->process_info = *process_info; + /* Validate page directory and attach eviction fence */ + ret = amdgpu_bo_reserve(new_vm->root.base.bo, true); + if (ret) + goto reserve_pd_fail; + ret = vm_validate_pt_pd_bos(new_vm); + if (ret) { + pr_err("validate_pt_pd_bos() failed\n"); + goto validate_pd_fail; + } + ret = ttm_bo_wait(&new_vm->root.base.bo->tbo, false, false); + if (ret) + goto wait_pd_fail; + amdgpu_bo_fence(new_vm->root.base.bo, + &new_vm->process_info->eviction_fence->base, true); + amdgpu_bo_unreserve(new_vm->root.base.bo); + + /* Update process info */ mutex_lock(&new_vm->process_info->lock); list_add_tail(&new_vm->vm_list_node, &(new_vm->process_info->vm_list_head)); @@ -863,6 +880,10 @@ int amdgpu_amdkfd_gpuvm_create_process_vm(struct kgd_dev *kgd, void **vm, return ret; +wait_pd_fail: +validate_pd_fail: + amdgpu_bo_unreserve(new_vm->root.base.bo); +reserve_pd_fail: create_evict_fence_fail: mutex_destroy(&info->lock); kfree(info); -- cgit v1.2.1 From b236fa1d339670cc997b68c31be57855bbabc126 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Thu, 15 Mar 2018 17:27:42 -0400 Subject: drm/amdgpu: Add helper to turn an existing VM into a compute VM MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit v2: Removed updating and checking of vm->vm_context v3: Enable amdgpu_vm_clear_bo in amdgpu_vm_make_compute Signed-off-by: Felix Kuehling Reviewed-by: Christian König Signed-off-by: Oded Gabbay --- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 67 ++++++++++++++++++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h | 1 + 2 files changed, 68 insertions(+) (limited to 'drivers/gpu/drm/amd/amdgpu') diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c index 24474294c92a..ea39ccf288ed 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c @@ -2461,6 +2461,73 @@ error_free_sched_entity: return r; } +/** + * amdgpu_vm_make_compute - Turn a GFX VM into a compute VM + * + * This only works on GFX VMs that don't have any BOs added and no + * page tables allocated yet. + * + * Changes the following VM parameters: + * - use_cpu_for_update + * - pte_supports_ats + * - pasid (old PASID is released, because compute manages its own PASIDs) + * + * Reinitializes the page directory to reflect the changed ATS + * setting. May leave behind an unused shadow BO for the page + * directory when switching from SDMA updates to CPU updates. + * + * Returns 0 for success, -errno for errors. + */ +int amdgpu_vm_make_compute(struct amdgpu_device *adev, struct amdgpu_vm *vm) +{ + bool pte_support_ats = (adev->asic_type == CHIP_RAVEN); + int r; + + r = amdgpu_bo_reserve(vm->root.base.bo, true); + if (r) + return r; + + /* Sanity checks */ + if (!RB_EMPTY_ROOT(&vm->va.rb_root) || vm->root.entries) { + r = -EINVAL; + goto error; + } + + /* Check if PD needs to be reinitialized and do it before + * changing any other state, in case it fails. + */ + if (pte_support_ats != vm->pte_support_ats) { + r = amdgpu_vm_clear_bo(adev, vm, vm->root.base.bo, + adev->vm_manager.root_level, + pte_support_ats); + if (r) + goto error; + } + + /* Update VM state */ + vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode & + AMDGPU_VM_USE_CPU_FOR_COMPUTE); + vm->pte_support_ats = pte_support_ats; + DRM_DEBUG_DRIVER("VM update mode is %s\n", + vm->use_cpu_for_update ? "CPU" : "SDMA"); + WARN_ONCE((vm->use_cpu_for_update & !amdgpu_vm_is_large_bar(adev)), + "CPU update of VM recommended only for large BAR system\n"); + + if (vm->pasid) { + unsigned long flags; + + spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags); + idr_remove(&adev->vm_manager.pasid_idr, vm->pasid); + spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags); + + vm->pasid = 0; + } + +error: + amdgpu_bo_unreserve(vm->root.base.bo); + return r; +} + /** * amdgpu_vm_free_levels - free PD/PT levels * diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h index 4dfdefb7de1f..30f080364c97 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h @@ -260,6 +260,7 @@ void amdgpu_vm_manager_init(struct amdgpu_device *adev); void amdgpu_vm_manager_fini(struct amdgpu_device *adev); int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm, int vm_context, unsigned int pasid); +int amdgpu_vm_make_compute(struct amdgpu_device *adev, struct amdgpu_vm *vm); void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm); bool amdgpu_vm_pasid_fault_credit(struct amdgpu_device *adev, unsigned int pasid); -- cgit v1.2.1 From ede0dd86f45adf2b7083bb161f6bc81da5fe2bad Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Thu, 15 Mar 2018 17:27:43 -0400 Subject: drm/amdgpu: Add kfd2kgd interface to acquire an existing VM MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This allows acquiring an existing VM from a render node FD to use it for a compute process. Such VMs get destroyed when the original file descriptor is released. Added a callback from amdgpu_vm_fini to handle KFD VM destruction correctly in this case. v2: * Removed vm->vm_context check in amdgpu_amdkfd_gpuvm_destroy_cb, check vm->process_info earlier instead Signed-off-by: Felix Kuehling Acked-by: Christian König Signed-off-by: Oded Gabbay --- drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h | 6 + drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c | 166 +++++++++++++++------- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 3 + 5 files changed, 122 insertions(+), 55 deletions(-) (limited to 'drivers/gpu/drm/amd/amdgpu') diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h index 6ba3dd689875..c2c2bea731e0 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h @@ -145,6 +145,12 @@ uint64_t amdgpu_amdkfd_get_vram_usage(struct kgd_dev *kgd); int amdgpu_amdkfd_gpuvm_create_process_vm(struct kgd_dev *kgd, void **vm, void **process_info, struct dma_fence **ef); +int amdgpu_amdkfd_gpuvm_acquire_process_vm(struct kgd_dev *kgd, + struct file *filp, + void **vm, void **process_info, + struct dma_fence **ef); +void amdgpu_amdkfd_gpuvm_destroy_cb(struct amdgpu_device *adev, + struct amdgpu_vm *vm); void amdgpu_amdkfd_gpuvm_destroy_process_vm(struct kgd_dev *kgd, void *vm); uint32_t amdgpu_amdkfd_gpuvm_get_process_page_dir(void *vm); int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu( diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c index 7485c376b90e..ea54e53172b9 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c @@ -205,6 +205,7 @@ static const struct kfd2kgd_calls kfd2kgd = { .get_cu_info = get_cu_info, .get_vram_usage = amdgpu_amdkfd_get_vram_usage, .create_process_vm = amdgpu_amdkfd_gpuvm_create_process_vm, + .acquire_process_vm = amdgpu_amdkfd_gpuvm_acquire_process_vm, .destroy_process_vm = amdgpu_amdkfd_gpuvm_destroy_process_vm, .get_process_page_dir = amdgpu_amdkfd_gpuvm_get_process_page_dir, .set_vm_context_page_table_base = set_vm_context_page_table_base, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c index 7be453494423..89264c9a5e9f 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c @@ -165,6 +165,7 @@ static const struct kfd2kgd_calls kfd2kgd = { .get_cu_info = get_cu_info, .get_vram_usage = amdgpu_amdkfd_get_vram_usage, .create_process_vm = amdgpu_amdkfd_gpuvm_create_process_vm, + .acquire_process_vm = amdgpu_amdkfd_gpuvm_acquire_process_vm, .destroy_process_vm = amdgpu_amdkfd_gpuvm_destroy_process_vm, .get_process_page_dir = amdgpu_amdkfd_gpuvm_get_process_page_dir, .set_vm_context_page_table_base = set_vm_context_page_table_base, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c index cafe8a571636..1d6e1479da38 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c @@ -806,32 +806,16 @@ static int process_update_pds(struct amdkfd_process_info *process_info, return 0; } -int amdgpu_amdkfd_gpuvm_create_process_vm(struct kgd_dev *kgd, void **vm, - void **process_info, - struct dma_fence **ef) +static int init_kfd_vm(struct amdgpu_vm *vm, void **process_info, + struct dma_fence **ef) { - int ret; - struct amdgpu_vm *new_vm; struct amdkfd_process_info *info = NULL; - struct amdgpu_device *adev = get_amdgpu_device(kgd); - - new_vm = kzalloc(sizeof(*new_vm), GFP_KERNEL); - if (!new_vm) - return -ENOMEM; - - /* Initialize the VM context, allocate the page directory and zero it */ - ret = amdgpu_vm_init(adev, new_vm, AMDGPU_VM_CONTEXT_COMPUTE, 0); - if (ret) { - pr_err("Failed init vm ret %d\n", ret); - goto vm_init_fail; - } + int ret; if (!*process_info) { info = kzalloc(sizeof(*info), GFP_KERNEL); - if (!info) { - ret = -ENOMEM; - goto alloc_process_info_fail; - } + if (!info) + return -ENOMEM; mutex_init(&info->lock); INIT_LIST_HEAD(&info->vm_list_head); @@ -842,6 +826,7 @@ int amdgpu_amdkfd_gpuvm_create_process_vm(struct kgd_dev *kgd, void **vm, current->mm); if (!info->eviction_fence) { pr_err("Failed to create eviction fence\n"); + ret = -ENOMEM; goto create_evict_fence_fail; } @@ -849,77 +834,137 @@ int amdgpu_amdkfd_gpuvm_create_process_vm(struct kgd_dev *kgd, void **vm, *ef = dma_fence_get(&info->eviction_fence->base); } - new_vm->process_info = *process_info; + vm->process_info = *process_info; /* Validate page directory and attach eviction fence */ - ret = amdgpu_bo_reserve(new_vm->root.base.bo, true); + ret = amdgpu_bo_reserve(vm->root.base.bo, true); if (ret) goto reserve_pd_fail; - ret = vm_validate_pt_pd_bos(new_vm); + ret = vm_validate_pt_pd_bos(vm); if (ret) { pr_err("validate_pt_pd_bos() failed\n"); goto validate_pd_fail; } - ret = ttm_bo_wait(&new_vm->root.base.bo->tbo, false, false); + ret = ttm_bo_wait(&vm->root.base.bo->tbo, false, false); if (ret) goto wait_pd_fail; - amdgpu_bo_fence(new_vm->root.base.bo, - &new_vm->process_info->eviction_fence->base, true); - amdgpu_bo_unreserve(new_vm->root.base.bo); + amdgpu_bo_fence(vm->root.base.bo, + &vm->process_info->eviction_fence->base, true); + amdgpu_bo_unreserve(vm->root.base.bo); /* Update process info */ - mutex_lock(&new_vm->process_info->lock); - list_add_tail(&new_vm->vm_list_node, - &(new_vm->process_info->vm_list_head)); - new_vm->process_info->n_vms++; - mutex_unlock(&new_vm->process_info->lock); - - *vm = (void *) new_vm; - - pr_debug("Created process vm %p\n", *vm); + mutex_lock(&vm->process_info->lock); + list_add_tail(&vm->vm_list_node, + &(vm->process_info->vm_list_head)); + vm->process_info->n_vms++; + mutex_unlock(&vm->process_info->lock); - return ret; + return 0; wait_pd_fail: validate_pd_fail: - amdgpu_bo_unreserve(new_vm->root.base.bo); + amdgpu_bo_unreserve(vm->root.base.bo); reserve_pd_fail: + vm->process_info = NULL; + if (info) { + /* Two fence references: one in info and one in *ef */ + dma_fence_put(&info->eviction_fence->base); + dma_fence_put(*ef); + *ef = NULL; + *process_info = NULL; create_evict_fence_fail: - mutex_destroy(&info->lock); - kfree(info); -alloc_process_info_fail: + mutex_destroy(&info->lock); + kfree(info); + } + return ret; +} + +int amdgpu_amdkfd_gpuvm_create_process_vm(struct kgd_dev *kgd, void **vm, + void **process_info, + struct dma_fence **ef) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + struct amdgpu_vm *new_vm; + int ret; + + new_vm = kzalloc(sizeof(*new_vm), GFP_KERNEL); + if (!new_vm) + return -ENOMEM; + + /* Initialize AMDGPU part of the VM */ + ret = amdgpu_vm_init(adev, new_vm, AMDGPU_VM_CONTEXT_COMPUTE, 0); + if (ret) { + pr_err("Failed init vm ret %d\n", ret); + goto amdgpu_vm_init_fail; + } + + /* Initialize KFD part of the VM and process info */ + ret = init_kfd_vm(new_vm, process_info, ef); + if (ret) + goto init_kfd_vm_fail; + + *vm = (void *) new_vm; + + return 0; + +init_kfd_vm_fail: amdgpu_vm_fini(adev, new_vm); -vm_init_fail: +amdgpu_vm_init_fail: kfree(new_vm); return ret; - } -void amdgpu_amdkfd_gpuvm_destroy_process_vm(struct kgd_dev *kgd, void *vm) +int amdgpu_amdkfd_gpuvm_acquire_process_vm(struct kgd_dev *kgd, + struct file *filp, + void **vm, void **process_info, + struct dma_fence **ef) { struct amdgpu_device *adev = get_amdgpu_device(kgd); - struct amdgpu_vm *avm = (struct amdgpu_vm *)vm; - struct amdgpu_bo *pd; - struct amdkfd_process_info *process_info; + struct drm_file *drm_priv = filp->private_data; + struct amdgpu_fpriv *drv_priv = drm_priv->driver_priv; + struct amdgpu_vm *avm = &drv_priv->vm; + int ret; - if (WARN_ON(!kgd || !vm)) + /* Already a compute VM? */ + if (avm->process_info) + return -EINVAL; + + /* Convert VM into a compute VM */ + ret = amdgpu_vm_make_compute(adev, avm); + if (ret) + return ret; + + /* Initialize KFD part of the VM and process info */ + ret = init_kfd_vm(avm, process_info, ef); + if (ret) + return ret; + + *vm = (void *)avm; + + return 0; +} + +void amdgpu_amdkfd_gpuvm_destroy_cb(struct amdgpu_device *adev, + struct amdgpu_vm *vm) +{ + struct amdkfd_process_info *process_info = vm->process_info; + struct amdgpu_bo *pd = vm->root.base.bo; + + if (!process_info) return; - pr_debug("Destroying process vm %p\n", vm); /* Release eviction fence from PD */ - pd = avm->root.base.bo; amdgpu_bo_reserve(pd, false); amdgpu_bo_fence(pd, NULL, false); amdgpu_bo_unreserve(pd); - process_info = avm->process_info; - + /* Update process info */ mutex_lock(&process_info->lock); process_info->n_vms--; - list_del(&avm->vm_list_node); + list_del(&vm->vm_list_node); mutex_unlock(&process_info->lock); - /* Release per-process resources */ + /* Release per-process resources when last compute VM is destroyed */ if (!process_info->n_vms) { WARN_ON(!list_empty(&process_info->kfd_bo_list)); @@ -927,6 +972,17 @@ void amdgpu_amdkfd_gpuvm_destroy_process_vm(struct kgd_dev *kgd, void *vm) mutex_destroy(&process_info->lock); kfree(process_info); } +} + +void amdgpu_amdkfd_gpuvm_destroy_process_vm(struct kgd_dev *kgd, void *vm) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + struct amdgpu_vm *avm = (struct amdgpu_vm *)vm; + + if (WARN_ON(!kgd || !vm)) + return; + + pr_debug("Destroying process vm %p\n", vm); /* Release the VM context */ amdgpu_vm_fini(adev, avm); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c index ea39ccf288ed..cbf1421c1963 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c @@ -32,6 +32,7 @@ #include #include "amdgpu.h" #include "amdgpu_trace.h" +#include "amdgpu_amdkfd.h" /* * GPUVM @@ -2575,6 +2576,8 @@ void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm) u64 fault; int i, r; + amdgpu_amdkfd_gpuvm_destroy_cb(adev, vm); + /* Clear pending page faults from IH when the VM is destroyed */ while (kfifo_get(&vm->faults, &fault)) amdgpu_ih_clear_fault(adev, fault); -- cgit v1.2.1 From 810955ba712fc5c517b5e999fd69bfd20251effb Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Fri, 23 Mar 2018 15:30:35 -0400 Subject: drm/amdgpu: Fix acquiring VM on large-BAR systems MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On large-BAR systems the VM page tables for compute are accessed by the CPU. Always allow CPU access to the page directory so that it can be used later by the CPU when a VM is converted to a compute VM. Signed-off-by: Felix Kuehling Reviewed-by: Christian König Signed-off-by: Oded Gabbay --- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'drivers/gpu/drm/amd/amdgpu') diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c index cbf1421c1963..da55a78d7380 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c @@ -2406,8 +2406,7 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm, if (vm->use_cpu_for_update) flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; else - flags |= (AMDGPU_GEM_CREATE_NO_CPU_ACCESS | - AMDGPU_GEM_CREATE_SHADOW); + flags |= AMDGPU_GEM_CREATE_SHADOW; size = amdgpu_vm_bo_size(adev, adev->vm_manager.root_level); r = amdgpu_bo_create(adev, size, align, AMDGPU_GEM_DOMAIN_VRAM, flags, -- cgit v1.2.1