drm/amd/amdgpu : Use the MES INV_TLBS API for tlb invalidation on gfx12

From MES version 0x81, it provide the new API INV_TLBS that support
invalidate tlbs with PASID.

Signed-off-by: Shaoyun Liu <shaoyun.liu@amd.com>
Reviewed-by: Alex Deucher <alexander.deucher@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h
index c0d2c195fe2ed4db09ab6369c621e20440b98986..489a4a0f0610598e3ecc23471a4ca913e07b91ba 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h
@@ -280,6 +280,13 @@ struct mes_reset_queue_input {
        bool                               is_kq;
 };
 
+struct mes_inv_tlbs_pasid_input {
+       uint32_t        xcc_id;
+       uint16_t        pasid;
+       uint8_t         hub_id;
+       uint8_t         flush_type;
+};
+
 enum mes_misc_opcode {
        MES_MISC_OP_WRITE_REG,
        MES_MISC_OP_READ_REG,
@@ -367,6 +374,9 @@ struct amdgpu_mes_funcs {
 
        int (*reset_hw_queue)(struct amdgpu_mes *mes,
                              struct mes_reset_queue_input *input);
+
+       int (*invalidate_tlbs_pasid)(struct amdgpu_mes *mes,
+                             struct mes_inv_tlbs_pasid_input *input);
 };
 
 #define amdgpu_mes_kiq_hw_init(adev) (adev)->mes.kiq_hw_init((adev))

diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c
index a0a5367f9dc40e560e2922dcd179a3b981af4830..9ba055ddc00f6554c76773a90753df634b7609d0 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c
@@ -336,6 +336,22 @@ static void gmc_v12_0_flush_gpu_tlb_pasid(struct amdgpu_device *adev,
        uint16_t queried;
        int vmid, i;
 
+       if (adev->enable_uni_mes && adev->mes.ring[AMDGPU_MES_SCHED_PIPE].sched.ready &&
+           (adev->mes.sched_version & AMDGPU_MES_VERSION_MASK) >= 0x81) {
+               struct mes_inv_tlbs_pasid_input input = {0};
+               input.pasid = pasid;
+               input.flush_type = flush_type;
+               input.hub_id = AMDGPU_GFXHUB(0);
+               /* MES will invalidate all gc_hub for the device from master */
+               adev->mes.funcs->invalidate_tlbs_pasid(&adev->mes, &input);
+               if (all_hub) {
+                       /* Only need to invalidate mm_hub now, gfx12 only support one mmhub */
+                       input.hub_id = AMDGPU_MMHUB0(0);
+                       adev->mes.funcs->invalidate_tlbs_pasid(&adev->mes, &input);
+               }
+               return;
+       }
+
        for (vmid = 1; vmid < 16; vmid++) {
                bool valid;
 

diff --git a/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c b/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c
index 6b222630f3fa1d3b047122254f46bc31c06a3f3d..cd5c966cee957de0bf15df4e359be4a8930040fe 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/mes_v12_0.c
@@ -108,6 +108,7 @@ static const char *mes_v12_0_opcodes[] = {
        "SET_SE_MODE",
        "SET_GANG_SUBMIT",
        "SET_HW_RSRC_1",
+       "INVALIDATE_TLBS",
 };
 
 static const char *mes_v12_0_misc_opcodes[] = {
@@ -879,6 +880,46 @@ static int mes_v12_0_reset_hw_queue(struct amdgpu_mes *mes,
                        offsetof(union MESAPI__RESET, api_status));
 }
 
+static int mes_v12_inv_tlb_convert_hub_id(uint8_t id)
+{
+       /*
+        * MES doesn't support invalidate gc_hub on slave xcc individually
+        * master xcc will invalidate all gc_hub for the partition
+        */
+       if (AMDGPU_IS_GFXHUB(id))
+               return 0;
+       else if (AMDGPU_IS_MMHUB0(id))
+               return 1;
+       else
+               return -EINVAL;
+
+}
+
+static int mes_v12_0_inv_tlbs_pasid(struct amdgpu_mes *mes,
+                                   struct mes_inv_tlbs_pasid_input *input)
+{
+       union MESAPI__INV_TLBS mes_inv_tlbs;
+
+       memset(&mes_inv_tlbs, 0, sizeof(mes_inv_tlbs));
+
+       mes_inv_tlbs.header.type = MES_API_TYPE_SCHEDULER;
+       mes_inv_tlbs.header.opcode = MES_SCH_API_INV_TLBS;
+       mes_inv_tlbs.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
+
+       mes_inv_tlbs.invalidate_tlbs.inv_sel = 0;
+       mes_inv_tlbs.invalidate_tlbs.flush_type = input->flush_type;
+       mes_inv_tlbs.invalidate_tlbs.inv_sel_id = input->pasid;
+
+       /*convert amdgpu_mes_hub_id to mes expected hub_id */
+       mes_inv_tlbs.invalidate_tlbs.hub_id = mes_v12_inv_tlb_convert_hub_id(input->hub_id);
+       if (mes_inv_tlbs.invalidate_tlbs.hub_id < 0)
+               return -EINVAL;
+       return mes_v12_0_submit_pkt_and_poll_completion(mes, AMDGPU_MES_KIQ_PIPE,
+                       &mes_inv_tlbs, sizeof(mes_inv_tlbs),
+                       offsetof(union MESAPI__INV_TLBS, api_status));
+
+}
+
 static const struct amdgpu_mes_funcs mes_v12_0_funcs = {
        .add_hw_queue = mes_v12_0_add_hw_queue,
        .remove_hw_queue = mes_v12_0_remove_hw_queue,
@@ -888,6 +929,7 @@ static const struct amdgpu_mes_funcs mes_v12_0_funcs = {
        .resume_gang = mes_v12_0_resume_gang,
        .misc_op = mes_v12_0_misc_op,
        .reset_hw_queue = mes_v12_0_reset_hw_queue,
+       .invalidate_tlbs_pasid = mes_v12_0_inv_tlbs_pasid,
 };
 
 static int mes_v12_0_allocate_ucode_buffer(struct amdgpu_device *adev,