habanalabs: fix possible deadlock in cache invl failure

Currently there is a deadlock in driver in scenarios where MMU
cache invalidation fails. The issue is basically device reset
being performed without releasing the MMU mutex.
The solution is to skip device reset as it is not necessary.
In addition we introduce a slight code refactor that prints the
invalidation error from a single location.

Signed-off-by: Ofir Bitton <obitton@habana.ai>
Reviewed-by: Oded Gabbay <ogabbay@kernel.org>
Signed-off-by: Oded Gabbay <ogabbay@kernel.org>

diff --git a/drivers/misc/habanalabs/common/command_buffer.c b/drivers/misc/habanalabs/common/command_buffer.c
index 71910f7809bd0bb397642b460d75f649b8a8562c..c591f0487272c0fe862c13e260786c02167ae6ca 100644 (file)
--- a/drivers/misc/habanalabs/common/command_buffer.c
+++ b/drivers/misc/habanalabs/common/command_buffer.c
@@ -80,14 +80,13 @@ static int cb_map_mem(struct hl_ctx *ctx, struct hl_cb *cb)
                offset += va_block->size;
        }
 
-       hdev->asic_funcs->mmu_invalidate_cache(hdev, false,
-               MMU_OP_USERPTR | MMU_OP_SKIP_LOW_CACHE_INV);
+       rc = hl_mmu_invalidate_cache(hdev, false, MMU_OP_USERPTR | MMU_OP_SKIP_LOW_CACHE_INV);
 
        mutex_unlock(&ctx->mmu_lock);
 
        cb->is_mmu_mapped = true;
 
-       return 0;
+       return rc;
 
 err_va_umap:
        list_for_each_entry(va_block, &cb->va_block_list, node) {
@@ -98,7 +97,7 @@ err_va_umap:
                offset -= va_block->size;
        }
 
-       hdev->asic_funcs->mmu_invalidate_cache(hdev, true, MMU_OP_USERPTR);
+       rc = hl_mmu_invalidate_cache(hdev, true, MMU_OP_USERPTR);
 
        mutex_unlock(&ctx->mmu_lock);
 
@@ -127,7 +126,7 @@ static void cb_unmap_mem(struct hl_ctx *ctx, struct hl_cb *cb)
                                        "Failed to unmap CB's va 0x%llx\n",
                                        va_block->start);
 
-       hdev->asic_funcs->mmu_invalidate_cache(hdev, true, MMU_OP_USERPTR);
+       hl_mmu_invalidate_cache(hdev, true, MMU_OP_USERPTR);
 
        mutex_unlock(&ctx->mmu_lock);
 

diff --git a/drivers/misc/habanalabs/common/habanalabs.h b/drivers/misc/habanalabs/common/habanalabs.h
index 612a9f461b38b23fedcd1a6c87da6830300f179e..406ca50f192aea709e8b474c4d361f2603db66ad 100644 (file)
--- a/drivers/misc/habanalabs/common/habanalabs.h
+++ b/drivers/misc/habanalabs/common/habanalabs.h
@@ -2993,6 +2993,9 @@ int hl_mmu_unmap_page(struct hl_ctx *ctx, u64 virt_addr, u32 page_size,
 int hl_mmu_map_contiguous(struct hl_ctx *ctx, u64 virt_addr,
                                        u64 phys_addr, u32 size);
 int hl_mmu_unmap_contiguous(struct hl_ctx *ctx, u64 virt_addr, u32 size);
+int hl_mmu_invalidate_cache(struct hl_device *hdev, bool is_hard, u32 flags);
+int hl_mmu_invalidate_cache_range(struct hl_device *hdev, bool is_hard,
+                                       u32 flags, u32 asid, u64 va, u64 size);
 void hl_mmu_swap_out(struct hl_ctx *ctx);
 void hl_mmu_swap_in(struct hl_ctx *ctx);
 int hl_mmu_if_set_funcs(struct hl_device *hdev);

diff --git a/drivers/misc/habanalabs/common/memory.c b/drivers/misc/habanalabs/common/memory.c
index 530f8b4fadd2db5a026f082c8eeffd048f5766ae..315594e96dcd10ae4fbfae5e4b650783989fa960 100644 (file)
--- a/drivers/misc/habanalabs/common/memory.c
+++ b/drivers/misc/habanalabs/common/memory.c
@@ -1201,18 +1201,13 @@ static int map_device_va(struct hl_ctx *ctx, struct hl_mem_in *args,
                goto map_err;
        }
 
-       rc = hdev->asic_funcs->mmu_invalidate_cache_range(hdev, false,
-               *vm_type | MMU_OP_SKIP_LOW_CACHE_INV,
-               ctx->asid, ret_vaddr, phys_pg_pack->total_size);
+       rc = hl_mmu_invalidate_cache_range(hdev, false, *vm_type | MMU_OP_SKIP_LOW_CACHE_INV,
+                               ctx->asid, ret_vaddr, phys_pg_pack->total_size);
 
        mutex_unlock(&ctx->mmu_lock);
 
-       if (rc) {
-               dev_err(hdev->dev,
-                       "mapping handle %u failed due to MMU cache invalidation\n",
-                       handle);
+       if (rc)
                goto map_err;
-       }
 
        ret_vaddr += phys_pg_pack->offset;
 
@@ -1350,9 +1345,8 @@ static int unmap_device_va(struct hl_ctx *ctx, struct hl_mem_in *args,
         * at the loop end rather than for each iteration
         */
        if (!ctx_free)
-               rc = hdev->asic_funcs->mmu_invalidate_cache_range(hdev, true,
-                               *vm_type, ctx->asid, vaddr,
-                               phys_pg_pack->total_size);
+               rc = hl_mmu_invalidate_cache_range(hdev, true, *vm_type, ctx->asid, vaddr,
+                                                       phys_pg_pack->total_size);
 
        mutex_unlock(&ctx->mmu_lock);
 
@@ -1365,11 +1359,6 @@ static int unmap_device_va(struct hl_ctx *ctx, struct hl_mem_in *args,
        if (!ctx_free) {
                int tmp_rc;
 
-               if (rc)
-                       dev_err(hdev->dev,
-                               "unmapping vaddr 0x%llx failed due to MMU cache invalidation\n",
-                               vaddr);
-
                tmp_rc = add_va_block(hdev, va_range, vaddr,
                                        vaddr + phys_pg_pack->total_size - 1);
                if (tmp_rc) {
@@ -2640,8 +2629,8 @@ void hl_vm_ctx_fini(struct hl_ctx *ctx)
        mutex_lock(&ctx->mmu_lock);
 
        /* invalidate the cache once after the unmapping loop */
-       hdev->asic_funcs->mmu_invalidate_cache(hdev, true, MMU_OP_USERPTR);
-       hdev->asic_funcs->mmu_invalidate_cache(hdev, true, MMU_OP_PHYS_PACK);
+       hl_mmu_invalidate_cache(hdev, true, MMU_OP_USERPTR);
+       hl_mmu_invalidate_cache(hdev, true, MMU_OP_PHYS_PACK);
 
        mutex_unlock(&ctx->mmu_lock);
 

diff --git a/drivers/misc/habanalabs/common/mmu/mmu.c b/drivers/misc/habanalabs/common/mmu/mmu.c
index aa96917f62e57bc78c3462dcbf0466173009cbc9..9153a1f551752ef9abf95799e3fb0eaf5e4d8966 100644 (file)
--- a/drivers/misc/habanalabs/common/mmu/mmu.c
+++ b/drivers/misc/habanalabs/common/mmu/mmu.c
@@ -637,3 +637,28 @@ u64 hl_mmu_descramble_addr(struct hl_device *hdev, u64 addr)
 {
        return addr;
 }
+
+int hl_mmu_invalidate_cache(struct hl_device *hdev, bool is_hard, u32 flags)
+{
+       int rc;
+
+       rc = hdev->asic_funcs->mmu_invalidate_cache(hdev, is_hard, flags);
+       if (rc)
+               dev_err_ratelimited(hdev->dev, "MMU cache invalidation failed\n");
+
+       return rc;
+}
+
+int hl_mmu_invalidate_cache_range(struct hl_device *hdev, bool is_hard,
+                                       u32 flags, u32 asid, u64 va, u64 size)
+{
+       int rc;
+
+       rc = hdev->asic_funcs->mmu_invalidate_cache_range(hdev, is_hard, flags,
+                                                               asid, va, size);
+       if (rc)
+               dev_err_ratelimited(hdev->dev, "MMU cache range invalidation failed\n");
+
+       return rc;
+}
+

diff --git a/drivers/misc/habanalabs/gaudi/gaudi.c b/drivers/misc/habanalabs/gaudi/gaudi.c
index 465540d064b6957a994f85e4d777588d88a8b0bd..b101a46076b89c9e7d8804f3d58af01679f9de00 100644 (file)
--- a/drivers/misc/habanalabs/gaudi/gaudi.c
+++ b/drivers/misc/habanalabs/gaudi/gaudi.c
@@ -8366,12 +8366,6 @@ static int gaudi_mmu_invalidate_cache(struct hl_device *hdev, bool is_hard,
 
        WREG32(mmSTLB_INV_SET, 0);
 
-       if (rc) {
-               dev_err_ratelimited(hdev->dev,
-                                       "MMU cache invalidation timeout\n");
-               hl_device_reset(hdev, HL_DRV_RESET_HARD);
-       }
-
        return rc;
 }
 

diff --git a/drivers/misc/habanalabs/goya/goya.c b/drivers/misc/habanalabs/goya/goya.c
index 2347de2f426aec1f9f41f92bebf16990d8a7e4ba..5e6998d21adbfa87970a0084d14f4b75edc5cbef 100644 (file)
--- a/drivers/misc/habanalabs/goya/goya.c
+++ b/drivers/misc/habanalabs/goya/goya.c
@@ -5258,12 +5258,6 @@ static int goya_mmu_invalidate_cache(struct hl_device *hdev, bool is_hard,
                1000,
                timeout_usec);
 
-       if (rc) {
-               dev_err_ratelimited(hdev->dev,
-                                       "MMU cache invalidation timeout\n");
-               hl_device_reset(hdev, HL_DRV_RESET_HARD);
-       }
-
        return rc;
 }