drm/xe: Fix fault mode invalidation with unbind

Fix fault mode invalidation racing with unbind leading to the
PTE zapping potentially traversing an invalid page-table tree.
Do this by holding the notifier lock across PTE zapping. This
might transfer any contention waiting on the notifier seqlock
read side to the notifier lock read side, but that shouldn't be
a major problem.

At the same time get rid of the open-coded invalidation in the bind
code by relying on the notifier even when the vma bind is not
yet committed.

Finally let userptr invalidation call a dedicated xe_vm function
performing a full invalidation.

Fixes: e8babb280b5e ("drm/xe: Convert multiple bind ops into single job")
Cc: Thomas Hellström <thomas.hellstrom@linux.intel.com>
Cc: Matthew Brost <matthew.brost@intel.com>
Cc: Matthew Auld <matthew.auld@intel.com>
Cc: <stable@vger.kernel.org> # v6.12+
Signed-off-by: Thomas Hellström <thomas.hellstrom@linux.intel.com>
Reviewed-by: Matthew Brost <matthew.brost@intel.com>
Link: https://patchwork.freedesktop.org/patch/msgid/20250228073058.59510-4-thomas.hellstrom@linux.intel.com
(cherry picked from commit 100a5b8dadfca50d91d9a4c9fc01431b42a25cab)
Signed-off-by: Rodrigo Vivi <rodrigo.vivi@intel.com>

diff --git a/drivers/gpu/drm/xe/xe_pt.c b/drivers/gpu/drm/xe/xe_pt.c
index 1ddcc7e79a93ec759aeec9686b536083d5e98010..12a627a23eb45152f1c4d3d24a537e87672426d3 100644 (file)
--- a/drivers/gpu/drm/xe/xe_pt.c
+++ b/drivers/gpu/drm/xe/xe_pt.c
@@ -1213,42 +1213,22 @@ static int vma_check_userptr(struct xe_vm *vm, struct xe_vma *vma,
                return 0;
 
        uvma = to_userptr_vma(vma);
-       notifier_seq = uvma->userptr.notifier_seq;
+       if (xe_pt_userptr_inject_eagain(uvma))
+               xe_vma_userptr_force_invalidate(uvma);
 
-       if (uvma->userptr.initial_bind && !xe_vm_in_fault_mode(vm))
-               return 0;
+       notifier_seq = uvma->userptr.notifier_seq;
 
        if (!mmu_interval_read_retry(&uvma->userptr.notifier,
-                                    notifier_seq) &&
-           !xe_pt_userptr_inject_eagain(uvma))
+                                    notifier_seq))
                return 0;
 
-       if (xe_vm_in_fault_mode(vm)) {
+       if (xe_vm_in_fault_mode(vm))
                return -EAGAIN;
-       } else {
-               spin_lock(&vm->userptr.invalidated_lock);
-               list_move_tail(&uvma->userptr.invalidate_link,
-                              &vm->userptr.invalidated);
-               spin_unlock(&vm->userptr.invalidated_lock);
-
-               if (xe_vm_in_preempt_fence_mode(vm)) {
-                       struct dma_resv_iter cursor;
-                       struct dma_fence *fence;
-                       long err;
-
-                       dma_resv_iter_begin(&cursor, xe_vm_resv(vm),
-                                           DMA_RESV_USAGE_BOOKKEEP);
-                       dma_resv_for_each_fence_unlocked(&cursor, fence)
-                               dma_fence_enable_sw_signaling(fence);
-                       dma_resv_iter_end(&cursor);
-
-                       err = dma_resv_wait_timeout(xe_vm_resv(vm),
-                                                   DMA_RESV_USAGE_BOOKKEEP,
-                                                   false, MAX_SCHEDULE_TIMEOUT);
-                       XE_WARN_ON(err <= 0);
-               }
-       }
 
+       /*
+        * Just continue the operation since exec or rebind worker
+        * will take care of rebinding.
+        */
        return 0;
 }
 

diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c
index d2cd227e4d69448fb78f59420abf87d3f5711dcd..d54aaa5eaff38b46d088f5c86c366b9a7f805a76 100644 (file)
--- a/drivers/gpu/drm/xe/xe_vm.c
+++ b/drivers/gpu/drm/xe/xe_vm.c
@@ -579,51 +579,26 @@ out_unlock_outer:
        trace_xe_vm_rebind_worker_exit(vm);
 }
 
-static bool vma_userptr_invalidate(struct mmu_interval_notifier *mni,
-                                  const struct mmu_notifier_range *range,
-                                  unsigned long cur_seq)
+static void __vma_userptr_invalidate(struct xe_vm *vm, struct xe_userptr_vma *uvma)
 {
-       struct xe_userptr *userptr = container_of(mni, typeof(*userptr), notifier);
-       struct xe_userptr_vma *uvma = container_of(userptr, typeof(*uvma), userptr);
+       struct xe_userptr *userptr = &uvma->userptr;
        struct xe_vma *vma = &uvma->vma;
-       struct xe_vm *vm = xe_vma_vm(vma);
        struct dma_resv_iter cursor;
        struct dma_fence *fence;
        long err;
 
-       xe_assert(vm->xe, xe_vma_is_userptr(vma));
-       trace_xe_vma_userptr_invalidate(vma);
-
-       if (!mmu_notifier_range_blockable(range))
-               return false;
-
-       vm_dbg(&xe_vma_vm(vma)->xe->drm,
-              "NOTIFIER: addr=0x%016llx, range=0x%016llx",
-               xe_vma_start(vma), xe_vma_size(vma));
-
-       down_write(&vm->userptr.notifier_lock);
-       mmu_interval_set_seq(mni, cur_seq);
-
-       /* No need to stop gpu access if the userptr is not yet bound. */
-       if (!userptr->initial_bind) {
-               up_write(&vm->userptr.notifier_lock);
-               return true;
-       }
-
        /*
         * Tell exec and rebind worker they need to repin and rebind this
         * userptr.
         */
        if (!xe_vm_in_fault_mode(vm) &&
-           !(vma->gpuva.flags & XE_VMA_DESTROYED) && vma->tile_present) {
+           !(vma->gpuva.flags & XE_VMA_DESTROYED)) {
                spin_lock(&vm->userptr.invalidated_lock);
                list_move_tail(&userptr->invalidate_link,
                               &vm->userptr.invalidated);
                spin_unlock(&vm->userptr.invalidated_lock);
        }
 
-       up_write(&vm->userptr.notifier_lock);
-
        /*
         * Preempt fences turn into schedule disables, pipeline these.
         * Note that even in fault mode, we need to wait for binds and
@@ -641,11 +616,35 @@ static bool vma_userptr_invalidate(struct mmu_interval_notifier *mni,
                                    false, MAX_SCHEDULE_TIMEOUT);
        XE_WARN_ON(err <= 0);
 
-       if (xe_vm_in_fault_mode(vm)) {
+       if (xe_vm_in_fault_mode(vm) && userptr->initial_bind) {
                err = xe_vm_invalidate_vma(vma);
                XE_WARN_ON(err);
        }
+}
 
+static bool vma_userptr_invalidate(struct mmu_interval_notifier *mni,
+                                  const struct mmu_notifier_range *range,
+                                  unsigned long cur_seq)
+{
+       struct xe_userptr_vma *uvma = container_of(mni, typeof(*uvma), userptr.notifier);
+       struct xe_vma *vma = &uvma->vma;
+       struct xe_vm *vm = xe_vma_vm(vma);
+
+       xe_assert(vm->xe, xe_vma_is_userptr(vma));
+       trace_xe_vma_userptr_invalidate(vma);
+
+       if (!mmu_notifier_range_blockable(range))
+               return false;
+
+       vm_dbg(&xe_vma_vm(vma)->xe->drm,
+              "NOTIFIER: addr=0x%016llx, range=0x%016llx",
+               xe_vma_start(vma), xe_vma_size(vma));
+
+       down_write(&vm->userptr.notifier_lock);
+       mmu_interval_set_seq(mni, cur_seq);
+
+       __vma_userptr_invalidate(vm, uvma);
+       up_write(&vm->userptr.notifier_lock);
        trace_xe_vma_userptr_invalidate_complete(vma);
 
        return true;
@@ -655,6 +654,34 @@ static const struct mmu_interval_notifier_ops vma_userptr_notifier_ops = {
        .invalidate = vma_userptr_invalidate,
 };
 
+#if IS_ENABLED(CONFIG_DRM_XE_USERPTR_INVAL_INJECT)
+/**
+ * xe_vma_userptr_force_invalidate() - force invalidate a userptr
+ * @uvma: The userptr vma to invalidate
+ *
+ * Perform a forced userptr invalidation for testing purposes.
+ */
+void xe_vma_userptr_force_invalidate(struct xe_userptr_vma *uvma)
+{
+       struct xe_vm *vm = xe_vma_vm(&uvma->vma);
+
+       /* Protect against concurrent userptr pinning */
+       lockdep_assert_held(&vm->lock);
+       /* Protect against concurrent notifiers */
+       lockdep_assert_held(&vm->userptr.notifier_lock);
+       /*
+        * Protect against concurrent instances of this function and
+        * the critical exec sections
+        */
+       xe_vm_assert_held(vm);
+
+       if (!mmu_interval_read_retry(&uvma->userptr.notifier,
+                                    uvma->userptr.notifier_seq))
+               uvma->userptr.notifier_seq -= 2;
+       __vma_userptr_invalidate(vm, uvma);
+}
+#endif
+
 int xe_vm_userptr_pin(struct xe_vm *vm)
 {
        struct xe_userptr_vma *uvma, *next;

diff --git a/drivers/gpu/drm/xe/xe_vm.h b/drivers/gpu/drm/xe/xe_vm.h
index 256a837c2704af7de88b54edf674d97fb6a24a47..b882bfb31bd05dc05507cdc79c9419ee1a84ab9a 100644 (file)
--- a/drivers/gpu/drm/xe/xe_vm.h
+++ b/drivers/gpu/drm/xe/xe_vm.h
@@ -279,4 +279,12 @@ struct xe_vm_snapshot *xe_vm_snapshot_capture(struct xe_vm *vm);
 void xe_vm_snapshot_capture_delayed(struct xe_vm_snapshot *snap);
 void xe_vm_snapshot_print(struct xe_vm_snapshot *snap, struct drm_printer *p);
 void xe_vm_snapshot_free(struct xe_vm_snapshot *snap);
+
+#if IS_ENABLED(CONFIG_DRM_XE_USERPTR_INVAL_INJECT)
+void xe_vma_userptr_force_invalidate(struct xe_userptr_vma *uvma);
+#else
+static inline void xe_vma_userptr_force_invalidate(struct xe_userptr_vma *uvma)
+{
+}
+#endif
 #endif

diff --git a/drivers/gpu/drm/xe/xe_vm_types.h b/drivers/gpu/drm/xe/xe_vm_types.h
index 7f9a303e51d896f5a89415ca7e88b871d5cbcf9c..d2511819cdf43041af57f9adbbb36bfe8122b555 100644 (file)
--- a/drivers/gpu/drm/xe/xe_vm_types.h
+++ b/drivers/gpu/drm/xe/xe_vm_types.h
@@ -227,8 +227,8 @@ struct xe_vm {
                 * up for revalidation. Protected from access with the
                 * @invalidated_lock. Removing items from the list
                 * additionally requires @lock in write mode, and adding
-                * items to the list requires the @userptr.notifer_lock in
-                * write mode.
+                * items to the list requires either the @userptr.notifer_lock in
+                * write mode, OR @lock in write mode.
                 */
                struct list_head invalidated;
        } userptr;