drm/amdgpu: Reduce dequeue retry timeout for gfx9 family

Dequeue retry timeout controls the interval between checks for unmet
conditions. On MI series, reduce this from 0x40 to 0x1 (~ 1 uS). The
cost of additional bandwidth consumed by CP when polling memory
shouldn't be substantial.

Signed-off-by: Harish Kasiviswanathan <Harish.Kasiviswanathan@amd.com>
Reviewed-by: Jonathan Kim <jonathan.kim@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_aldebaran.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_aldebaran.c
index 6e861d08d0441c5404d9d0fb87eb56ad42a12572..7e9f7a280c1b9823cb7f4f54e85539a0bf721fd4 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_aldebaran.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_aldebaran.c
@@ -189,7 +189,7 @@ const struct kfd2kgd_calls aldebaran_kfd2kgd = {
        .set_address_watch = kgd_gfx_aldebaran_set_address_watch,
        .clear_address_watch = kgd_gfx_v9_clear_address_watch,
        .get_iq_wait_times = kgd_gfx_v9_get_iq_wait_times,
-       .build_grace_period_packet_info = kgd_gfx_v9_build_grace_period_packet_info,
+       .build_dequeue_wait_counts_packet_info = kgd_gfx_v9_build_dequeue_wait_counts_packet_info,
        .program_trap_handler_settings = kgd_gfx_v9_program_trap_handler_settings,
        .hqd_get_pq_addr = kgd_gfx_v9_hqd_get_pq_addr,
        .hqd_reset = kgd_gfx_v9_hqd_reset,

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_arcturus.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_arcturus.c
index c820418e8ccdf9790caeca733e48eab708df211a..ffbaa8bc5eea9ea2ee23ec595eedc08ea97666ab 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_arcturus.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_arcturus.c
@@ -415,7 +415,7 @@ const struct kfd2kgd_calls arcturus_kfd2kgd = {
        .set_address_watch = kgd_gfx_v9_set_address_watch,
        .clear_address_watch = kgd_gfx_v9_clear_address_watch,
        .get_iq_wait_times = kgd_gfx_v9_get_iq_wait_times,
-       .build_grace_period_packet_info = kgd_gfx_v9_build_grace_period_packet_info,
+       .build_dequeue_wait_counts_packet_info = kgd_gfx_v9_build_dequeue_wait_counts_packet_info,
        .get_cu_occupancy = kgd_gfx_v9_get_cu_occupancy,
        .program_trap_handler_settings = kgd_gfx_v9_program_trap_handler_settings,
        .hqd_get_pq_addr = kgd_gfx_v9_hqd_get_pq_addr,

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gc_9_4_3.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gc_9_4_3.c
index 0c09984775980dabe4d2ba58b6524be0d1bd7ced..89a45a9218f3f859ded33c7c42d2a37ba26153ff 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gc_9_4_3.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gc_9_4_3.c
@@ -541,8 +541,8 @@ const struct kfd2kgd_calls gc_9_4_3_kfd2kgd = {
        .get_cu_occupancy = kgd_gfx_v9_get_cu_occupancy,
        .program_trap_handler_settings =
                                kgd_gfx_v9_program_trap_handler_settings,
-       .build_grace_period_packet_info =
-                               kgd_gfx_v9_build_grace_period_packet_info,
+       .build_dequeue_wait_counts_packet_info =
+                               kgd_gfx_v9_build_dequeue_wait_counts_packet_info,
        .get_iq_wait_times = kgd_gfx_v9_get_iq_wait_times,
        .enable_debug_trap = kgd_aldebaran_enable_debug_trap,
        .disable_debug_trap = kgd_gfx_v9_4_3_disable_debug_trap,

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c
index 2887b6f3eaa227bf50a6c5e036886aa3d9ff8751..04ef0ca105414a1c797bc894501451624f03d95c 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c
@@ -1021,25 +1021,25 @@ void kgd_gfx_v10_get_iq_wait_times(struct amdgpu_device *adev,
        *wait_times = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_IQ_WAIT_TIME2));
 }
 
-void kgd_gfx_v10_build_grace_period_packet_info(struct amdgpu_device *adev,
+void kgd_gfx_v10_build_dequeue_wait_counts_packet_info(struct amdgpu_device *adev,
                                                uint32_t wait_times,
-                                               uint32_t grace_period,
+                                               uint32_t sch_wave,
+                                               uint32_t que_sleep,
                                                uint32_t *reg_offset,
                                                uint32_t *reg_data)
 {
        *reg_data = wait_times;
 
-       /*
-        * The CP cannont handle a 0 grace period input and will result in
-        * an infinite grace period being set so set to 1 to prevent this.
-        */
-       if (grace_period == 0)
-               grace_period = 1;
-
-       *reg_data = REG_SET_FIELD(*reg_data,
-                       CP_IQ_WAIT_TIME2,
-                       SCH_WAVE,
-                       grace_period);
+       if (sch_wave)
+               *reg_data = REG_SET_FIELD(*reg_data,
+                               CP_IQ_WAIT_TIME2,
+                               SCH_WAVE,
+                               sch_wave);
+       if (que_sleep)
+               *reg_data = REG_SET_FIELD(*reg_data,
+                               CP_IQ_WAIT_TIME2,
+                               QUE_SLEEP,
+                               que_sleep);
 
        *reg_offset = SOC15_REG_OFFSET(GC, 0, mmCP_IQ_WAIT_TIME2);
 }
@@ -1115,7 +1115,7 @@ const struct kfd2kgd_calls gfx_v10_kfd2kgd = {
        .set_address_watch = kgd_gfx_v10_set_address_watch,
        .clear_address_watch = kgd_gfx_v10_clear_address_watch,
        .get_iq_wait_times = kgd_gfx_v10_get_iq_wait_times,
-       .build_grace_period_packet_info = kgd_gfx_v10_build_grace_period_packet_info,
+       .build_dequeue_wait_counts_packet_info = kgd_gfx_v10_build_dequeue_wait_counts_packet_info,
        .program_trap_handler_settings = program_trap_handler_settings,
        .hqd_get_pq_addr = kgd_gfx_v10_hqd_get_pq_addr,
        .hqd_reset = kgd_gfx_v10_hqd_reset,

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.h
index db577c2a847abfcbfda29883ffe0df76929aba34..a4c607c881783a61182931591dbedde9ee941395 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.h
@@ -51,9 +51,10 @@ uint32_t kgd_gfx_v10_clear_address_watch(struct amdgpu_device *adev,
 void kgd_gfx_v10_get_iq_wait_times(struct amdgpu_device *adev,
                                uint32_t *wait_times,
                                uint32_t inst);
-void kgd_gfx_v10_build_grace_period_packet_info(struct amdgpu_device *adev,
+void kgd_gfx_v10_build_dequeue_wait_counts_packet_info(struct amdgpu_device *adev,
                                               uint32_t wait_times,
-                                              uint32_t grace_period,
+                                              uint32_t sch_wave,
+                                              uint32_t que_sleep,
                                               uint32_t *reg_offset,
                                               uint32_t *reg_data);
 uint64_t kgd_gfx_v10_hqd_get_pq_addr(struct amdgpu_device *adev,

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10_3.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10_3.c
index ac9ad505f9d720ae07492599d89e0e8244dc9a6e..6d08bc2781a397d07ed89cc7632f429aca05709b 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10_3.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10_3.c
@@ -673,7 +673,7 @@ const struct kfd2kgd_calls gfx_v10_3_kfd2kgd = {
        .set_vm_context_page_table_base = set_vm_context_page_table_base_v10_3,
        .program_trap_handler_settings = program_trap_handler_settings_v10_3,
        .get_iq_wait_times = kgd_gfx_v10_get_iq_wait_times,
-       .build_grace_period_packet_info = kgd_gfx_v10_build_grace_period_packet_info,
+       .build_dequeue_wait_counts_packet_info = kgd_gfx_v10_build_dequeue_wait_counts_packet_info,
        .enable_debug_trap = kgd_gfx_v10_enable_debug_trap,
        .disable_debug_trap = kgd_gfx_v10_disable_debug_trap,
        .validate_trap_override_request = kgd_gfx_v10_validate_trap_override_request,

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c
index 84135eb906606b32917b9036187cb58fcbb1c380..088d09cc7a724c057c28761d4e296873acb58944 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c
@@ -1077,25 +1077,25 @@ void kgd_gfx_v9_get_cu_occupancy(struct amdgpu_device *adev,
                                adev->gfx.cu_info.max_waves_per_simd;
 }
 
-void kgd_gfx_v9_build_grace_period_packet_info(struct amdgpu_device *adev,
+void kgd_gfx_v9_build_dequeue_wait_counts_packet_info(struct amdgpu_device *adev,
                uint32_t wait_times,
-               uint32_t grace_period,
+               uint32_t sch_wave,
+               uint32_t que_sleep,
                uint32_t *reg_offset,
                uint32_t *reg_data)
 {
        *reg_data = wait_times;
 
-       /*
-        * The CP cannot handle a 0 grace period input and will result in
-        * an infinite grace period being set so set to 1 to prevent this.
-        */
-       if (grace_period == 0)
-               grace_period = 1;
-
-       *reg_data = REG_SET_FIELD(*reg_data,
-                       CP_IQ_WAIT_TIME2,
-                       SCH_WAVE,
-                       grace_period);
+       if (sch_wave)
+               *reg_data = REG_SET_FIELD(*reg_data,
+                               CP_IQ_WAIT_TIME2,
+                               SCH_WAVE,
+                               sch_wave);
+       if (que_sleep)
+               *reg_data = REG_SET_FIELD(*reg_data,
+                               CP_IQ_WAIT_TIME2,
+                               QUE_SLEEP,
+                               que_sleep);
 
        *reg_offset = SOC15_REG_OFFSET(GC, 0, mmCP_IQ_WAIT_TIME2);
 }
@@ -1255,7 +1255,7 @@ const struct kfd2kgd_calls gfx_v9_kfd2kgd = {
        .set_address_watch = kgd_gfx_v9_set_address_watch,
        .clear_address_watch = kgd_gfx_v9_clear_address_watch,
        .get_iq_wait_times = kgd_gfx_v9_get_iq_wait_times,
-       .build_grace_period_packet_info = kgd_gfx_v9_build_grace_period_packet_info,
+       .build_dequeue_wait_counts_packet_info = kgd_gfx_v9_build_dequeue_wait_counts_packet_info,
        .get_cu_occupancy = kgd_gfx_v9_get_cu_occupancy,
        .program_trap_handler_settings = kgd_gfx_v9_program_trap_handler_settings,
        .hqd_get_pq_addr = kgd_gfx_v9_hqd_get_pq_addr,

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.h
index 90c8fa13d519275fc407e79adef492994d3a4996..704452ca62f8eedf674b001031791240e2a3abc1 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.h
@@ -97,9 +97,10 @@ uint32_t kgd_gfx_v9_clear_address_watch(struct amdgpu_device *adev,
 void kgd_gfx_v9_get_iq_wait_times(struct amdgpu_device *adev,
                                uint32_t *wait_times,
                                uint32_t inst);
-void kgd_gfx_v9_build_grace_period_packet_info(struct amdgpu_device *adev,
+void kgd_gfx_v9_build_dequeue_wait_counts_packet_info(struct amdgpu_device *adev,
                                               uint32_t wait_times,
-                                              uint32_t grace_period,
+                                              uint32_t sch_wave,
+                                              uint32_t que_sleep,
                                               uint32_t *reg_offset,
                                               uint32_t *reg_data);
 uint64_t kgd_gfx_v9_hqd_get_pq_addr(struct amdgpu_device *adev,

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager_v9.c b/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager_v9.c
index b9c611b249e6f4e08176cc001314a22333412fb3..d440df602393403524a0dafc54dfda992b9e4115 100644 (file)
--- a/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager_v9.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager_v9.c
@@ -298,13 +298,14 @@ static int pm_map_queues_v9(struct packet_manager *pm, uint32_t *buffer,
 }
 
 static inline void pm_build_dequeue_wait_counts_packet_info(struct packet_manager *pm,
-                       uint32_t sch_value, uint32_t *reg_offset,
+                       uint32_t sch_value, uint32_t que_sleep, uint32_t *reg_offset,
                        uint32_t *reg_data)
 {
-       pm->dqm->dev->kfd2kgd->build_grace_period_packet_info(
+       pm->dqm->dev->kfd2kgd->build_dequeue_wait_counts_packet_info(
                pm->dqm->dev->adev,
                pm->dqm->wait_times,
                sch_value,
+               que_sleep,
                reg_offset,
                reg_data);
 }
@@ -319,27 +320,43 @@ static int pm_config_dequeue_wait_counts_v9(struct packet_manager *pm,
        uint32_t reg_data = 0;
 
        switch (cmd) {
-       case KFD_DEQUEUE_WAIT_INIT:
-               /* Set CWSR grace period to 1x1000 cycle for GFX9.4.3 APU */
-               if (amdgpu_emu_mode == 0 && pm->dqm->dev->adev->gmc.is_app_apu &&
-                  (KFD_GC_VERSION(pm->dqm->dev) == IP_VERSION(9, 4, 3)))
-                       pm_build_dequeue_wait_counts_packet_info(pm, 1, &reg_offset, &reg_data);
-               else
+       case KFD_DEQUEUE_WAIT_INIT: {
+               uint32_t sch_wave = 0, que_sleep = 0;
+               /* Reduce CP_IQ_WAIT_TIME2.QUE_SLEEP to 0x1 from default 0x40.
+                * On a 1GHz machine this is roughly 1 microsecond, which is
+                * about how long it takes to load data out of memory during
+                * queue connect
+                * QUE_SLEEP: Wait Count for Dequeue Retry.
+                */
+               if (KFD_GC_VERSION(pm->dqm->dev) >= IP_VERSION(9, 4, 1) &&
+                   KFD_GC_VERSION(pm->dqm->dev) < IP_VERSION(10, 0, 0)) {
+                       que_sleep = 1;
+
+                       /* Set CWSR grace period to 1x1000 cycle for GFX9.4.3 APU */
+                       if (amdgpu_emu_mode == 0 && pm->dqm->dev->adev->gmc.is_app_apu &&
+                       (KFD_GC_VERSION(pm->dqm->dev) == IP_VERSION(9, 4, 3)))
+                               sch_wave = 1;
+               } else {
                        return 0;
+               }
+               pm_build_dequeue_wait_counts_packet_info(pm, sch_wave, que_sleep,
+                       &reg_offset, &reg_data);
+
                break;
+       }
        case KFD_DEQUEUE_WAIT_RESET:
-               /* function called only to get reg_offset */
-               pm_build_dequeue_wait_counts_packet_info(pm, 0, &reg_offset, &reg_data);
-               reg_data = pm->dqm->wait_times;
+               /* reg_data would be set to dqm->wait_times */
+               pm_build_dequeue_wait_counts_packet_info(pm, 0, 0, &reg_offset, &reg_data);
                break;
 
        case KFD_DEQUEUE_WAIT_SET_SCH_WAVE:
                /* The CP cannot handle value 0 and it will result in
-                * an infinite grace period being set so set to 1 to prevent this.
+                * an infinite grace period being set so set to 1 to prevent this. Also
+                * avoid debugger API breakage as it sets 0 and expects a low value.
                 */
                if (!value)
                        value = 1;
-               pm_build_dequeue_wait_counts_packet_info(pm, value, &reg_offset, &reg_data);
+               pm_build_dequeue_wait_counts_packet_info(pm, value, 0, &reg_offset, &reg_data);
                break;
        default:
                pr_err("Invalid dequeue wait cmd\n");

diff --git a/drivers/gpu/drm/amd/include/kgd_kfd_interface.h b/drivers/gpu/drm/amd/include/kgd_kfd_interface.h
index 1e8dfa6c0dc82c1355a6c62d860b85c35f34f03a..9aba8596faa7ee2cb6c8071beaa01f4f454f12ee 100644 (file)
--- a/drivers/gpu/drm/amd/include/kgd_kfd_interface.h
+++ b/drivers/gpu/drm/amd/include/kgd_kfd_interface.h
@@ -313,9 +313,10 @@ struct kfd2kgd_calls {
        void (*get_iq_wait_times)(struct amdgpu_device *adev,
                        uint32_t *wait_times,
                        uint32_t inst);
-       void (*build_grace_period_packet_info)(struct amdgpu_device *adev,
+       void (*build_dequeue_wait_counts_packet_info)(struct amdgpu_device *adev,
                        uint32_t wait_times,
-                       uint32_t grace_period,
+                       uint32_t sch_wave,
+                       uint32_t que_sleep,
                        uint32_t *reg_offset,
                        uint32_t *reg_data);
        void (*get_cu_occupancy)(struct amdgpu_device *adev,