habanalabs: abort waiting user threads upon error

User should close the FD when being notified about an error, after
which a device reset takes place.

However, if the user has pending threads that wait for completions,
the device release won't be called and eventually the watchdog timeout
will expire, leading to hard reset and killing the user process.

To avoid it, abort such waiting threads right after the error
notification, and block following waiting operations.

Signed-off-by: Tomer Tayar <ttayar@habana.ai>
Reviewed-by: Oded Gabbay <ogabbay@kernel.org>
Signed-off-by: Oded Gabbay <ogabbay@kernel.org>

diff --git a/drivers/misc/habanalabs/common/command_submission.c b/drivers/misc/habanalabs/common/command_submission.c
index ea0e5101c10ed6ea9f7e71e0c63ec09c941b55ae..cf3b82efc65cd1669e4a8dc63792040078aa6835 100644 (file)
--- a/drivers/misc/habanalabs/common/command_submission.c
+++ b/drivers/misc/habanalabs/common/command_submission.c
@@ -1117,6 +1117,27 @@ void hl_release_pending_user_interrupts(struct hl_device *hdev)
        wake_pending_user_interrupt_threads(interrupt);
 }
 
+static void force_complete_cs(struct hl_device *hdev)
+{
+       struct hl_cs *cs;
+
+       spin_lock(&hdev->cs_mirror_lock);
+
+       list_for_each_entry(cs, &hdev->cs_mirror_list, mirror_node) {
+               cs->fence->error = -EIO;
+               complete_all(&cs->fence->completion);
+       }
+
+       spin_unlock(&hdev->cs_mirror_lock);
+}
+
+void hl_abort_waitings_for_completion(struct hl_device *hdev)
+{
+       force_complete_cs(hdev);
+       force_complete_multi_cs(hdev);
+       hl_release_pending_user_interrupts(hdev);
+}
+
 static void job_wq_completion(struct work_struct *work)
 {
        struct hl_cs_job *job = container_of(work, struct hl_cs_job,
@@ -3489,14 +3510,15 @@ static int hl_interrupt_wait_ioctl(struct hl_fpriv *hpriv, void *data)
 
 int hl_wait_ioctl(struct hl_fpriv *hpriv, void *data)
 {
+       struct hl_device *hdev = hpriv->hdev;
        union hl_wait_cs_args *args = data;
        u32 flags = args->in.flags;
        int rc;
 
-       /* If the device is not operational, no point in waiting for any command submission or
-        * user interrupt
+       /* If the device is not operational, or if an error has happened and user should release the
+        * device, there is no point in waiting for any command submission or user interrupt.
         */
-       if (!hl_device_operational(hpriv->hdev, NULL))
+       if (!hl_device_operational(hpriv->hdev, NULL) || hdev->reset_info.watchdog_active)
                return -EBUSY;
 
        if (flags & HL_WAIT_CS_FLAGS_INTERRUPT)

diff --git a/drivers/misc/habanalabs/common/device.c b/drivers/misc/habanalabs/common/device.c
index 1453f2ec72d9e4332d707be62a79d22d7369788d..92721111b652044b7b3e8c2e4e6a2f81df552211 100644 (file)
--- a/drivers/misc/habanalabs/common/device.c
+++ b/drivers/misc/habanalabs/common/device.c
@@ -1865,6 +1865,8 @@ out:
 
        hl_ctx_put(ctx);
 
+       hl_abort_waitings_for_completion(hdev);
+
        return 0;
 
 device_reset:

diff --git a/drivers/misc/habanalabs/common/habanalabs.h b/drivers/misc/habanalabs/common/habanalabs.h
index 9e42d0e9ce33054860d556b5e345a9655d3b0c65..7fb45610ad0c7d79c326cff672f95e55ba7364b2 100644 (file)
--- a/drivers/misc/habanalabs/common/habanalabs.h
+++ b/drivers/misc/habanalabs/common/habanalabs.h
@@ -3791,6 +3791,7 @@ void hl_dec_fini(struct hl_device *hdev);
 void hl_dec_ctx_fini(struct hl_ctx *ctx);
 
 void hl_release_pending_user_interrupts(struct hl_device *hdev);
+void hl_abort_waitings_for_completion(struct hl_device *hdev);
 int hl_cs_signal_sob_wraparound_handler(struct hl_device *hdev, u32 q_idx,
                        struct hl_hw_sob **hw_sob, u32 count, bool encaps_sig);