drm/amdgpu: Add support to change mtype for 2nd part of gart BOs on GFX9

This change prepares for a workaround in amdkfd for a GFX9 HW bug. It
requires the control stack memory of compute queues, which is allocated
from the second page of MQD gart BOs, to have mtype NC, rather than
the default UC.

Signed-off-by: Yong Zhao <yong.zhao@amd.com>
Reviewed-by: Christian König <christian.koenig@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
index dfd22db13fb1e6d990bec90a19a233ce38392157..cc3b067e1ec6ac447ddae08cfe522bea4e59e070 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
@@ -834,6 +834,45 @@ static void amdgpu_ttm_tt_unpin_userptr(struct ttm_tt *ttm)
        sg_free_table(ttm->sg);
 }
 
+int amdgpu_ttm_gart_bind(struct amdgpu_device *adev,
+                               struct ttm_buffer_object *tbo,
+                               uint64_t flags)
+{
+       struct amdgpu_bo *abo = ttm_to_amdgpu_bo(tbo);
+       struct ttm_tt *ttm = tbo->ttm;
+       struct amdgpu_ttm_tt *gtt = (void *)ttm;
+       int r;
+
+       if (abo->flags & AMDGPU_GEM_CREATE_MQD_GFX9) {
+               uint64_t page_idx = 1;
+
+               r = amdgpu_gart_bind(adev, gtt->offset, page_idx,
+                               ttm->pages, gtt->ttm.dma_address, flags);
+               if (r)
+                       goto gart_bind_fail;
+
+               /* Patch mtype of the second part BO */
+               flags &=  ~AMDGPU_PTE_MTYPE_MASK;
+               flags |= AMDGPU_PTE_MTYPE(AMDGPU_MTYPE_NC);
+
+               r = amdgpu_gart_bind(adev,
+                               gtt->offset + (page_idx << PAGE_SHIFT),
+                               ttm->num_pages - page_idx,
+                               &ttm->pages[page_idx],
+                               &(gtt->ttm.dma_address[page_idx]), flags);
+       } else {
+               r = amdgpu_gart_bind(adev, gtt->offset, ttm->num_pages,
+                                    ttm->pages, gtt->ttm.dma_address, flags);
+       }
+
+gart_bind_fail:
+       if (r)
+               DRM_ERROR("failed to bind %lu pages at 0x%08llX\n",
+                         ttm->num_pages, gtt->offset);
+
+       return r;
+}
+
 static int amdgpu_ttm_backend_bind(struct ttm_tt *ttm,
                                   struct ttm_mem_reg *bo_mem)
 {
@@ -907,8 +946,7 @@ int amdgpu_ttm_alloc_gart(struct ttm_buffer_object *bo)
 
        flags = amdgpu_ttm_tt_pte_flags(adev, bo->ttm, &tmp);
        gtt->offset = (u64)tmp.start << PAGE_SHIFT;
-       r = amdgpu_gart_bind(adev, gtt->offset, bo->ttm->num_pages,
-                            bo->ttm->pages, gtt->ttm.dma_address, flags);
+       r = amdgpu_ttm_gart_bind(adev, bo, flags);
        if (unlikely(r)) {
                ttm_bo_mem_put(bo, &tmp);
                return r;
@@ -925,19 +963,15 @@ int amdgpu_ttm_alloc_gart(struct ttm_buffer_object *bo)
 int amdgpu_ttm_recover_gart(struct ttm_buffer_object *tbo)
 {
        struct amdgpu_device *adev = amdgpu_ttm_adev(tbo->bdev);
-       struct amdgpu_ttm_tt *gtt = (void *)tbo->ttm;
        uint64_t flags;
        int r;
 
-       if (!gtt)
+       if (!tbo->ttm)
                return 0;
 
-       flags = amdgpu_ttm_tt_pte_flags(adev, &gtt->ttm.ttm, &tbo->mem);
-       r = amdgpu_gart_bind(adev, gtt->offset, gtt->ttm.ttm.num_pages,
-                            gtt->ttm.ttm.pages, gtt->ttm.dma_address, flags);
-       if (r)
-               DRM_ERROR("failed to bind %lu pages at 0x%08llX\n",
-                         gtt->ttm.ttm.num_pages, gtt->offset);
+       flags = amdgpu_ttm_tt_pte_flags(adev, tbo->ttm, &tbo->mem);
+       r = amdgpu_ttm_gart_bind(adev, tbo, flags);
+
        return r;
 }
 

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
index 30f080364c97b5109785419e98151262cf687848..4cf678684a1254338c29ed74d621f0963ec3ba8f 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
@@ -75,11 +75,12 @@ struct amdgpu_bo_list_entry;
 /* PDE Block Fragment Size for VEGA10 */
 #define AMDGPU_PDE_BFS(a)      ((uint64_t)a << 59)
 
-/* VEGA10 only */
+
+/* For GFX9 */
 #define AMDGPU_PTE_MTYPE(a)    ((uint64_t)a << 57)
 #define AMDGPU_PTE_MTYPE_MASK  AMDGPU_PTE_MTYPE(3ULL)
 
-/* For Raven */
+#define AMDGPU_MTYPE_NC 0
 #define AMDGPU_MTYPE_CC 2
 
 #define AMDGPU_PTE_DEFAULT_ATC  (AMDGPU_PTE_SYSTEM      \

diff --git a/include/uapi/drm/amdgpu_drm.h b/include/uapi/drm/amdgpu_drm.h
index 081d25640b64871d13e999c381a2552f630bb27b..78b4dd89fcb4d8b53a1f706de3f888d90462daa9 100644 (file)
--- a/include/uapi/drm/amdgpu_drm.h
+++ b/include/uapi/drm/amdgpu_drm.h
@@ -101,6 +101,10 @@ extern "C" {
 #define AMDGPU_GEM_CREATE_VM_ALWAYS_VALID      (1 << 6)
 /* Flag that BO sharing will be explicitly synchronized */
 #define AMDGPU_GEM_CREATE_EXPLICIT_SYNC                (1 << 7)
+/* Flag that indicates allocating MQD gart on GFX9, where the mtype
+ * for the second page onward should be set to NC.
+ */
+#define AMDGPU_GEM_CREATE_MQD_GFX9             (1 << 8)
 
 struct drm_amdgpu_gem_create_in  {
        /** the requested memory size */