arm64: KVM: Add invalidate_icache_range helper

We currently tightly couple dcache clean with icache invalidation,
but KVM could do without the initial flush to PoU, as we've
already flushed things to PoC.

Let's introduce invalidate_icache_range which is limited to
invalidating the icache from the linear mapping (and thus
has none of the userspace fault handling complexity), and
wire it in KVM instead of flush_icache_range.

Reviewed-by: Christoffer Dall <christoffer.dall@linaro.org>
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Christoffer Dall <christoffer.dall@linaro.org>

diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h
index aef72d886677758c76d6b932c863893df7c67b53..0884e1fdfd30321344a86d1825b49a4aa2e841a3 100644 (file)
--- a/arch/arm64/include/asm/assembler.h
+++ b/arch/arm64/include/asm/assembler.h
@@ -387,6 +387,27 @@ alternative_endif
        dsb     \domain
        .endm
 
+/*
+ * Macro to perform an instruction cache maintenance for the interval
+ * [start, end)
+ *
+ *     start, end:     virtual addresses describing the region
+ *     label:          A label to branch to on user fault.
+ *     Corrupts:       tmp1, tmp2
+ */
+       .macro invalidate_icache_by_line start, end, tmp1, tmp2, label
+       icache_line_size \tmp1, \tmp2
+       sub     \tmp2, \tmp1, #1
+       bic     \tmp2, \start, \tmp2
+9997:
+USER(\label, ic        ivau, \tmp2)                    // invalidate I line PoU
+       add     \tmp2, \tmp2, \tmp1
+       cmp     \tmp2, \end
+       b.lo    9997b
+       dsb     ish
+       isb
+       .endm
+
 /*
  * reset_pmuserenr_el0 - reset PMUSERENR_EL0 if PMUv3 present
  */

diff --git a/arch/arm64/include/asm/cacheflush.h b/arch/arm64/include/asm/cacheflush.h
index 955130762a3c6acc09f3ee76574f7cefc5097b4c..bef9f418f08986830e68f6bc2c41c88eb5333a29 100644 (file)
--- a/arch/arm64/include/asm/cacheflush.h
+++ b/arch/arm64/include/asm/cacheflush.h
@@ -52,6 +52,12 @@
  *             - start  - virtual start address
  *             - end    - virtual end address
  *
+ *     invalidate_icache_range(start, end)
+ *
+ *             Invalidate the I-cache in the region described by start, end.
+ *             - start  - virtual start address
+ *             - end    - virtual end address
+ *
  *     __flush_cache_user_range(start, end)
  *
  *             Ensure coherency between the I-cache and the D-cache in the
@@ -66,6 +72,7 @@
  *             - size   - region size
  */
 extern void flush_icache_range(unsigned long start, unsigned long end);
+extern int  invalidate_icache_range(unsigned long start, unsigned long end);
 extern void __flush_dcache_area(void *addr, size_t len);
 extern void __inval_dcache_area(void *addr, size_t len);
 extern void __clean_dcache_area_poc(void *addr, size_t len);

diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_mmu.h
index 8034b96fb3a412f108899d38f4875bba89836d89..56b3e03c85e7479b073ad6d1b230e6c117609273 100644 (file)
--- a/arch/arm64/include/asm/kvm_mmu.h
+++ b/arch/arm64/include/asm/kvm_mmu.h
@@ -250,8 +250,8 @@ static inline void __invalidate_icache_guest_page(struct kvm_vcpu *vcpu,
                /* PIPT or VPIPT at EL2 (see comment in __kvm_tlb_flush_vmid_ipa) */
                void *va = page_address(pfn_to_page(pfn));
 
-               flush_icache_range((unsigned long)va,
-                                  (unsigned long)va + size);
+               invalidate_icache_range((unsigned long)va,
+                                       (unsigned long)va + size);
        }
 }
 

diff --git a/arch/arm64/mm/cache.S b/arch/arm64/mm/cache.S
index 7f1dbe962cf581c88eb488f01636f78996b4ee43..bedd23da83f4ed82fa624e532748bc78bac31887 100644 (file)
--- a/arch/arm64/mm/cache.S
+++ b/arch/arm64/mm/cache.S
@@ -60,16 +60,7 @@ user_alt 9f, "dc cvau, x4",  "dc civac, x4",  ARM64_WORKAROUND_CLEAN_CACHE
        b.lo    1b
        dsb     ish
 
-       icache_line_size x2, x3
-       sub     x3, x2, #1
-       bic     x4, x0, x3
-1:
-USER(9f, ic    ivau, x4        )               // invalidate I line PoU
-       add     x4, x4, x2
-       cmp     x4, x1
-       b.lo    1b
-       dsb     ish
-       isb
+       invalidate_icache_by_line x0, x1, x2, x3, 9f
        mov     x0, #0
 1:
        uaccess_ttbr0_disable x1
@@ -80,6 +71,27 @@ USER(9f, ic  ivau, x4        )               // invalidate I line PoU
 ENDPROC(flush_icache_range)
 ENDPROC(__flush_cache_user_range)
 
+/*
+ *     invalidate_icache_range(start,end)
+ *
+ *     Ensure that the I cache is invalid within specified region.
+ *
+ *     - start   - virtual start address of region
+ *     - end     - virtual end address of region
+ */
+ENTRY(invalidate_icache_range)
+       uaccess_ttbr0_enable x2, x3
+
+       invalidate_icache_by_line x0, x1, x2, x3, 2f
+       mov     x0, xzr
+1:
+       uaccess_ttbr0_disable x1
+       ret
+2:
+       mov     x0, #-EFAULT
+       b       1b
+ENDPROC(invalidate_icache_range)
+
 /*
  *     __flush_dcache_area(kaddr, size)
  *