drm/amdkfd: add svm range validate timestamp

With xnack on, add validate timestamp in order to handle GPU vm fault
from multiple GPUs.

If GPU retry fault need migrate the range to the best restore location,
use range validate timestamp to record system timestamp after range is
restored to update GPU page table.

Because multiple pages of same range have multiple retry fault, define
AMDGPU_SVM_RANGE_RETRY_FAULT_PENDING to the long time period that
pending retry fault may still comes after page table update, to skip
duplicate retry fault of same range.

If difference between system timestamp and range last validate timestamp
is bigger than AMDGPU_SVM_RANGE_RETRY_FAULT_PENDING, that means the
retry fault is from another GPU, then continue to handle retry fault
recover.

Signed-off-by: Philip Yang <Philip.Yang@amd.com>
Reviewed-by: Felix Kuehling <Felix.Kuehling@amd.com>
Signed-off-by: Felix Kuehling <Felix.Kuehling@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
index 7206e0b7b422b4f198ef0988d071be4def775520..0694211a118b3fe66b72962f6e3a2dfb2fe15f0c 100644 (file)
--- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
@@ -35,6 +35,11 @@
 
 #define AMDGPU_SVM_RANGE_RESTORE_DELAY_MS 1
 
+/* Long enough to ensure no retry fault comes after svm range is restored and
+ * page table is updated.
+ */
+#define AMDGPU_SVM_RANGE_RETRY_FAULT_PENDING   2000
+
 static void svm_range_evict_svm_bo_worker(struct work_struct *work);
 static bool
 svm_range_cpu_invalidate_pagetables(struct mmu_interval_notifier *mni,
@@ -269,6 +274,7 @@ svm_range *svm_range_new(struct svm_range_list *svms, uint64_t start,
        INIT_LIST_HEAD(&prange->deferred_list);
        INIT_LIST_HEAD(&prange->child_list);
        atomic_set(&prange->invalid, 0);
+       prange->validate_timestamp = ktime_to_us(ktime_get());
        mutex_init(&prange->migrate_mutex);
        mutex_init(&prange->lock);
        svm_range_set_default_attributes(&prange->preferred_loc,
@@ -1392,6 +1398,9 @@ unlock_out:
 unreserve_out:
        svm_range_unreserve_bos(&ctx);
 
+       if (!r)
+               prange->validate_timestamp = ktime_to_us(ktime_get());
+
        return r;
 }
 
@@ -2131,6 +2140,7 @@ svm_range_restore_pages(struct amdgpu_device *adev, unsigned int pasid,
        struct svm_range_list *svms;
        struct svm_range *prange;
        struct kfd_process *p;
+       uint64_t timestamp;
        int32_t best_loc, gpuidx;
        int r = 0;
 
@@ -2166,6 +2176,13 @@ svm_range_restore_pages(struct amdgpu_device *adev, unsigned int pasid,
        }
 
        mutex_lock(&prange->migrate_mutex);
+       timestamp = ktime_to_us(ktime_get()) - prange->validate_timestamp;
+       /* skip duplicate vm fault on different pages of same range */
+       if (timestamp < AMDGPU_SVM_RANGE_RETRY_FAULT_PENDING) {
+               pr_debug("svms 0x%p [0x%lx %lx] already restored\n",
+                        svms, prange->start, prange->last);
+               goto out_unlock_range;
+       }
 
        best_loc = svm_range_best_restore_location(prange, adev, &gpuidx);
        if (best_loc == -1) {

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.h b/drivers/gpu/drm/amd/amdkfd/kfd_svm.h
index 11b04a706ff983341429e44addc324aa0a818b14..aa829b3c992dcd2d01f2bea386baef2a94636ec4 100644 (file)
--- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.h
@@ -86,6 +86,7 @@ struct svm_work_list_item {
  * @actual_loc: the actual location, 0 for CPU, or GPU id
  * @granularity:migration granularity, log2 num pages
  * @invalid:    not 0 means cpu page table is invalidated
+ * @validate_timestamp: system timestamp when range is validated
  * @notifier:   register mmu interval notifier
  * @work_item:  deferred work item information
  * @deferred_list: list header used to add range to deferred list
@@ -122,6 +123,7 @@ struct svm_range {
        uint32_t                        actual_loc;
        uint8_t                         granularity;
        atomic_t                        invalid;
+       uint64_t                        validate_timestamp;
        struct mmu_interval_notifier    notifier;
        struct svm_work_list_item       work_item;
        struct list_head                deferred_list;