drm/amdgpu: use GFP_NOWAIT for memory allocations

In the critical submission path memory allocations can't wait for
reclaim since that can potentially wait for submissions to finish.

Finally clean that up and mark most memory allocations in the critical
path with GFP_NOWAIT. The only exception left is the dma_fence_array()
used when no VMID is available, but that will be cleaned up later on.

Signed-off-by: Christian König <christian.koenig@amd.com>
Acked-by: Srinivasan Shanmugam <srinivasan.shanmugam@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
index 2ac6d4fa0601983f1b1881bc12de4f93d4cdd101..d2ec4130a316ddb13fe4c59f1f84fefc337cbc81 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
@@ -491,7 +491,7 @@ static int vm_update_pds(struct amdgpu_vm *vm, struct amdgpu_sync *sync)
        if (ret)
                return ret;
 
-       return amdgpu_sync_fence(sync, vm->last_update);
+       return amdgpu_sync_fence(sync, vm->last_update, GFP_KERNEL);
 }
 
 static uint64_t get_pte_flags(struct amdgpu_device *adev, struct kgd_mem *mem)
@@ -1249,7 +1249,7 @@ static int unmap_bo_from_gpuvm(struct kgd_mem *mem,
 
        (void)amdgpu_vm_clear_freed(adev, vm, &bo_va->last_pt_update);
 
-       (void)amdgpu_sync_fence(sync, bo_va->last_pt_update);
+       (void)amdgpu_sync_fence(sync, bo_va->last_pt_update, GFP_KERNEL);
 
        return 0;
 }
@@ -1273,7 +1273,7 @@ static int update_gpuvm_pte(struct kgd_mem *mem,
                return ret;
        }
 
-       return amdgpu_sync_fence(sync, bo_va->last_pt_update);
+       return amdgpu_sync_fence(sync, bo_va->last_pt_update, GFP_KERNEL);
 }
 
 static int map_bo_to_gpuvm(struct kgd_mem *mem,
@@ -2913,7 +2913,7 @@ int amdgpu_amdkfd_gpuvm_restore_process_bos(void *info, struct dma_fence __rcu *
                }
                dma_resv_for_each_fence(&cursor, bo->tbo.base.resv,
                                        DMA_RESV_USAGE_KERNEL, fence) {
-                       ret = amdgpu_sync_fence(&sync_obj, fence);
+                       ret = amdgpu_sync_fence(&sync_obj, fence, GFP_KERNEL);
                        if (ret) {
                                pr_debug("Memory eviction: Sync BO fence failed. Try again\n");
                                goto validate_map_fail;

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
index 5cc5f59e30184ff956823d8ff44040e211ec93eb..4a5b406601fa20c789edc9095ca1813aa78081c1 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
@@ -428,7 +428,7 @@ static int amdgpu_cs_p2_dependencies(struct amdgpu_cs_parser *p,
                        dma_fence_put(old);
                }
 
-               r = amdgpu_sync_fence(&p->sync, fence);
+               r = amdgpu_sync_fence(&p->sync, fence, GFP_KERNEL);
                dma_fence_put(fence);
                if (r)
                        return r;
@@ -450,7 +450,7 @@ static int amdgpu_syncobj_lookup_and_add(struct amdgpu_cs_parser *p,
                return r;
        }
 
-       r = amdgpu_sync_fence(&p->sync, fence);
+       r = amdgpu_sync_fence(&p->sync, fence, GFP_KERNEL);
        dma_fence_put(fence);
        return r;
 }
@@ -1124,7 +1124,8 @@ static int amdgpu_cs_vm_handling(struct amdgpu_cs_parser *p)
        if (r)
                return r;
 
-       r = amdgpu_sync_fence(&p->sync, fpriv->prt_va->last_pt_update);
+       r = amdgpu_sync_fence(&p->sync, fpriv->prt_va->last_pt_update,
+                             GFP_KERNEL);
        if (r)
                return r;
 
@@ -1135,7 +1136,8 @@ static int amdgpu_cs_vm_handling(struct amdgpu_cs_parser *p)
                if (r)
                        return r;
 
-               r = amdgpu_sync_fence(&p->sync, bo_va->last_pt_update);
+               r = amdgpu_sync_fence(&p->sync, bo_va->last_pt_update,
+                                     GFP_KERNEL);
                if (r)
                        return r;
        }
@@ -1154,7 +1156,8 @@ static int amdgpu_cs_vm_handling(struct amdgpu_cs_parser *p)
                if (r)
                        return r;
 
-               r = amdgpu_sync_fence(&p->sync, bo_va->last_pt_update);
+               r = amdgpu_sync_fence(&p->sync, bo_va->last_pt_update,
+                                     GFP_KERNEL);
                if (r)
                        return r;
        }
@@ -1167,7 +1170,7 @@ static int amdgpu_cs_vm_handling(struct amdgpu_cs_parser *p)
        if (r)
                return r;
 
-       r = amdgpu_sync_fence(&p->sync, vm->last_update);
+       r = amdgpu_sync_fence(&p->sync, vm->last_update, GFP_KERNEL);
        if (r)
                return r;
 
@@ -1248,7 +1251,8 @@ static int amdgpu_cs_sync_rings(struct amdgpu_cs_parser *p)
                        continue;
                }
 
-               r = amdgpu_sync_fence(&p->gang_leader->explicit_sync, fence);
+               r = amdgpu_sync_fence(&p->gang_leader->explicit_sync, fence,
+                                     GFP_KERNEL);
                dma_fence_put(fence);
                if (r)
                        return r;

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.c
index 8e712a11aba5d23459aba35ff8d58b9022efc447..56d27cea052ed2a7b77a2fc1f494317762df5cf9 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.c
@@ -209,7 +209,7 @@ static int amdgpu_vmid_grab_idle(struct amdgpu_ring *ring,
                return 0;
        }
 
-       fences = kmalloc_array(id_mgr->num_ids, sizeof(void *), GFP_KERNEL);
+       fences = kmalloc_array(id_mgr->num_ids, sizeof(void *), GFP_NOWAIT);
        if (!fences)
                return -ENOMEM;
 
@@ -326,7 +326,8 @@ static int amdgpu_vmid_grab_reserved(struct amdgpu_vm *vm,
        /* Good we can use this VMID. Remember this submission as
        * user of the VMID.
        */
-       r = amdgpu_sync_fence(&(*id)->active, &job->base.s_fence->finished);
+       r = amdgpu_sync_fence(&(*id)->active, &job->base.s_fence->finished,
+                             GFP_NOWAIT);
        if (r)
                return r;
 
@@ -385,7 +386,8 @@ static int amdgpu_vmid_grab_used(struct amdgpu_vm *vm,
                 * user of the VMID.
                 */
                r = amdgpu_sync_fence(&(*id)->active,
-                                     &job->base.s_fence->finished);
+                                     &job->base.s_fence->finished,
+                                     GFP_NOWAIT);
                if (r)
                        return r;
 
@@ -437,7 +439,8 @@ int amdgpu_vmid_grab(struct amdgpu_vm *vm, struct amdgpu_ring *ring,
 
                        /* Remember this submission as user of the VMID */
                        r = amdgpu_sync_fence(&id->active,
-                                             &job->base.s_fence->finished);
+                                             &job->base.s_fence->finished,
+                                             GFP_NOWAIT);
                        if (r)
                                goto error;
 

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
index 709c11cbeabd885549078871ba4b132cba8d6288..b4eef396d419cd7f8afad0afe56b01a911d54bf2 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
@@ -1336,14 +1336,14 @@ int amdgpu_mes_ctx_map_meta_data(struct amdgpu_device *adev,
                DRM_ERROR("failed to do vm_bo_update on meta data\n");
                goto error_del_bo_va;
        }
-       amdgpu_sync_fence(&sync, bo_va->last_pt_update);
+       amdgpu_sync_fence(&sync, bo_va->last_pt_update, GFP_KERNEL);
 
        r = amdgpu_vm_update_pdes(adev, vm, false);
        if (r) {
                DRM_ERROR("failed to update pdes on meta data\n");
                goto error_del_bo_va;
        }
-       amdgpu_sync_fence(&sync, vm->last_update);
+       amdgpu_sync_fence(&sync, vm->last_update, GFP_KERNEL);
 
        amdgpu_sync_wait(&sync, false);
        drm_exec_fini(&exec);

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c
index c586ab4c911bfeb0681cba5ef23aaea5cc8d46cf..86c17a8946f5dfedb521802fa40e88d146480d13 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c
@@ -152,7 +152,8 @@ static bool amdgpu_sync_add_later(struct amdgpu_sync *sync, struct dma_fence *f)
  *
  * Add the fence to the sync object.
  */
-int amdgpu_sync_fence(struct amdgpu_sync *sync, struct dma_fence *f)
+int amdgpu_sync_fence(struct amdgpu_sync *sync, struct dma_fence *f,
+                     gfp_t flags)
 {
        struct amdgpu_sync_entry *e;
 
@@ -162,7 +163,7 @@ int amdgpu_sync_fence(struct amdgpu_sync *sync, struct dma_fence *f)
        if (amdgpu_sync_add_later(sync, f))
                return 0;
 
-       e = kmem_cache_alloc(amdgpu_sync_slab, GFP_KERNEL);
+       e = kmem_cache_alloc(amdgpu_sync_slab, flags);
        if (!e)
                return -ENOMEM;
 
@@ -249,7 +250,7 @@ int amdgpu_sync_resv(struct amdgpu_device *adev, struct amdgpu_sync *sync,
                        struct dma_fence *tmp = dma_fence_chain_contained(f);
 
                        if (amdgpu_sync_test_fence(adev, mode, owner, tmp)) {
-                               r = amdgpu_sync_fence(sync, f);
+                               r = amdgpu_sync_fence(sync, f, GFP_KERNEL);
                                dma_fence_put(f);
                                if (r)
                                        return r;
@@ -281,7 +282,7 @@ int amdgpu_sync_kfd(struct amdgpu_sync *sync, struct dma_resv *resv)
                if (fence_owner != AMDGPU_FENCE_OWNER_KFD)
                        continue;
 
-               r = amdgpu_sync_fence(sync, f);
+               r = amdgpu_sync_fence(sync, f, GFP_KERNEL);
                if (r)
                        break;
        }
@@ -388,7 +389,7 @@ int amdgpu_sync_clone(struct amdgpu_sync *source, struct amdgpu_sync *clone)
        hash_for_each_safe(source->fences, i, tmp, e, node) {
                f = e->fence;
                if (!dma_fence_is_signaled(f)) {
-                       r = amdgpu_sync_fence(clone, f);
+                       r = amdgpu_sync_fence(clone, f, GFP_KERNEL);
                        if (r)
                                return r;
                } else {

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.h
index e3272dce798d79843e02a98b1c4100a6fda5074e..1504f5e7fc46598f7595250c0eefb38a6d5177cd 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.h
@@ -47,7 +47,8 @@ struct amdgpu_sync {
 };
 
 void amdgpu_sync_create(struct amdgpu_sync *sync);
-int amdgpu_sync_fence(struct amdgpu_sync *sync, struct dma_fence *f);
+int amdgpu_sync_fence(struct amdgpu_sync *sync, struct dma_fence *f,
+                     gfp_t flags);
 int amdgpu_sync_resv(struct amdgpu_device *adev, struct amdgpu_sync *sync,
                     struct dma_resv *resv, enum amdgpu_sync_mode mode,
                     void *owner);