KVM: arm64: Create each pKVM hyp vcpu after its corresponding host vcpu

Instead of creating and initializing _all_ hyp vcpus in pKVM when
the first host vcpu runs for the first time, initialize _each_
hyp vcpu in conjunction with its corresponding host vcpu.

Some of the host vcpu state (e.g., system registers and traps
values) is not initialized until the first time the host vcpu is
run. Therefore, initializing a hyp vcpu before its corresponding
host vcpu has run for the first time might not view the complete
host state of these vcpus.

Additionally, this behavior is inline with non-protected modes.

Acked-by: Will Deacon <will@kernel.org>
Reviewed-by: Marc Zyngier <maz@kernel.org>
Signed-off-by: Fuad Tabba <tabba@google.com>
Link: https://lore.kernel.org/r/20250314111832.4137161-5-tabba@google.com
Signed-off-by: Oliver Upton <oliver.upton@linux.dev>

diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
index 51754a354b7a468c854bd09242df134dbe2b56ac..a7db58d1ef5804236d397fcc0909be1c047b583c 100644 (file)
--- a/arch/arm64/include/asm/kvm_host.h
+++ b/arch/arm64/include/asm/kvm_host.h
@@ -873,6 +873,8 @@ struct kvm_vcpu_arch {
 #define VCPU_INITIALIZED       __vcpu_single_flag(cflags, BIT(0))
 /* SVE config completed */
 #define VCPU_SVE_FINALIZED     __vcpu_single_flag(cflags, BIT(1))
+/* pKVM VCPU setup completed */
+#define VCPU_PKVM_FINALIZED    __vcpu_single_flag(cflags, BIT(2))
 
 /* Exception pending */
 #define PENDING_EXCEPTION      __vcpu_single_flag(iflags, BIT(0))

diff --git a/arch/arm64/include/asm/kvm_pkvm.h b/arch/arm64/include/asm/kvm_pkvm.h
index eb65f12e81d90939de45f0c6e59f0777a1f3f78f..abd693ce5b93e628c74bf187d0bdfbc2ba687ffb 100644 (file)
--- a/arch/arm64/include/asm/kvm_pkvm.h
+++ b/arch/arm64/include/asm/kvm_pkvm.h
@@ -19,6 +19,7 @@
 int pkvm_init_host_vm(struct kvm *kvm);
 int pkvm_create_hyp_vm(struct kvm *kvm);
 void pkvm_destroy_hyp_vm(struct kvm *kvm);
+int pkvm_create_hyp_vcpu(struct kvm_vcpu *vcpu);
 
 /*
  * This functions as an allow-list of protected VM capabilities.

diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c
index b8e55a441282f57cb2d4bc55a43b41d5b774dfdd..cd7a808eeb64f7bb3ac3c2467386ed8d4af6ead2 100644 (file)
--- a/arch/arm64/kvm/arm.c
+++ b/arch/arm64/kvm/arm.c
@@ -833,6 +833,10 @@ int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu)
                ret = pkvm_create_hyp_vm(kvm);
                if (ret)
                        return ret;
+
+               ret = pkvm_create_hyp_vcpu(vcpu);
+               if (ret)
+                       return ret;
        }
 
        mutex_lock(&kvm->arch.config_lock);

diff --git a/arch/arm64/kvm/hyp/include/nvhe/pkvm.h b/arch/arm64/kvm/hyp/include/nvhe/pkvm.h
index e42bf68c88482c207951f9fc69756d907b69f39e..ce31d3b73603b0b79584824ed8a0f7c92af3257c 100644 (file)
--- a/arch/arm64/kvm/hyp/include/nvhe/pkvm.h
+++ b/arch/arm64/kvm/hyp/include/nvhe/pkvm.h
@@ -43,12 +43,6 @@ struct pkvm_hyp_vm {
        struct hyp_pool pool;
        hyp_spinlock_t lock;
 
-       /*
-        * The number of vcpus initialized and ready to run.
-        * Modifying this is protected by 'vm_table_lock'.
-        */
-       unsigned int nr_vcpus;
-
        /* Array of the hyp vCPU structures for this VM. */
        struct pkvm_hyp_vcpu *vcpus[];
 };

diff --git a/arch/arm64/kvm/hyp/nvhe/pkvm.c b/arch/arm64/kvm/hyp/nvhe/pkvm.c
index c34a6da5fbecc12e54da8c2f5571c8cdc56bfb29..99f22c1f10c995768e0eaca40d2992adcd2a1b20 100644 (file)
--- a/arch/arm64/kvm/hyp/nvhe/pkvm.c
+++ b/arch/arm64/kvm/hyp/nvhe/pkvm.c
@@ -245,10 +245,12 @@ struct pkvm_hyp_vcpu *pkvm_load_hyp_vcpu(pkvm_handle_t handle,
 
        hyp_spin_lock(&vm_table_lock);
        hyp_vm = get_vm_by_handle(handle);
-       if (!hyp_vm || hyp_vm->nr_vcpus <= vcpu_idx)
+       if (!hyp_vm || hyp_vm->kvm.created_vcpus <= vcpu_idx)
                goto unlock;
 
        hyp_vcpu = hyp_vm->vcpus[vcpu_idx];
+       if (!hyp_vcpu)
+               goto unlock;
 
        /* Ensure vcpu isn't loaded on more than one cpu simultaneously. */
        if (unlikely(hyp_vcpu->loaded_hyp_vcpu)) {
@@ -367,8 +369,14 @@ static void unpin_host_vcpus(struct pkvm_hyp_vcpu *hyp_vcpus[],
 {
        int i;
 
-       for (i = 0; i < nr_vcpus; i++)
-               unpin_host_vcpu(hyp_vcpus[i]->host_vcpu);
+       for (i = 0; i < nr_vcpus; i++) {
+               struct pkvm_hyp_vcpu *hyp_vcpu = hyp_vcpus[i];
+
+               if (!hyp_vcpu)
+                       continue;
+
+               unpin_host_vcpu(hyp_vcpu->host_vcpu);
+       }
 }
 
 static void init_pkvm_hyp_vm(struct kvm *host_kvm, struct pkvm_hyp_vm *hyp_vm,
@@ -392,24 +400,18 @@ static void pkvm_vcpu_init_sve(struct pkvm_hyp_vcpu *hyp_vcpu, struct kvm_vcpu *
 
 static int init_pkvm_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu,
                              struct pkvm_hyp_vm *hyp_vm,
-                             struct kvm_vcpu *host_vcpu,
-                             unsigned int vcpu_idx)
+                             struct kvm_vcpu *host_vcpu)
 {
        int ret = 0;
 
        if (hyp_pin_shared_mem(host_vcpu, host_vcpu + 1))
                return -EBUSY;
 
-       if (host_vcpu->vcpu_idx != vcpu_idx) {
-               ret = -EINVAL;
-               goto done;
-       }
-
        hyp_vcpu->host_vcpu = host_vcpu;
 
        hyp_vcpu->vcpu.kvm = &hyp_vm->kvm;
        hyp_vcpu->vcpu.vcpu_id = READ_ONCE(host_vcpu->vcpu_id);
-       hyp_vcpu->vcpu.vcpu_idx = vcpu_idx;
+       hyp_vcpu->vcpu.vcpu_idx = READ_ONCE(host_vcpu->vcpu_idx);
 
        hyp_vcpu->vcpu.arch.hw_mmu = &hyp_vm->kvm.arch.mmu;
        hyp_vcpu->vcpu.arch.cflags = READ_ONCE(host_vcpu->arch.cflags);
@@ -647,27 +649,28 @@ int __pkvm_init_vcpu(pkvm_handle_t handle, struct kvm_vcpu *host_vcpu,
                goto unlock;
        }
 
-       idx = hyp_vm->nr_vcpus;
+       ret = init_pkvm_hyp_vcpu(hyp_vcpu, hyp_vm, host_vcpu);
+       if (ret)
+               goto unlock;
+
+       idx = hyp_vcpu->vcpu.vcpu_idx;
        if (idx >= hyp_vm->kvm.created_vcpus) {
                ret = -EINVAL;
                goto unlock;
        }
 
-       ret = init_pkvm_hyp_vcpu(hyp_vcpu, hyp_vm, host_vcpu, idx);
-       if (ret)
+       if (hyp_vm->vcpus[idx]) {
+               ret = -EINVAL;
                goto unlock;
+       }
 
        hyp_vm->vcpus[idx] = hyp_vcpu;
-       hyp_vm->nr_vcpus++;
 unlock:
        hyp_spin_unlock(&vm_table_lock);
 
-       if (ret) {
+       if (ret)
                unmap_donated_memory(hyp_vcpu, sizeof(*hyp_vcpu));
-               return ret;
-       }
-
-       return 0;
+       return ret;
 }
 
 static void
@@ -714,11 +717,17 @@ int __pkvm_teardown_vm(pkvm_handle_t handle)
        mc = &host_kvm->arch.pkvm.teardown_mc;
        stage2_mc = &host_kvm->arch.pkvm.stage2_teardown_mc;
        reclaim_pgtable_pages(hyp_vm, stage2_mc);
-       unpin_host_vcpus(hyp_vm->vcpus, hyp_vm->nr_vcpus);
+       unpin_host_vcpus(hyp_vm->vcpus, hyp_vm->kvm.created_vcpus);
 
-       for (idx = 0; idx < hyp_vm->nr_vcpus; ++idx) {
+       /* Push the metadata pages to the teardown memcache */
+       for (idx = 0; idx < hyp_vm->kvm.created_vcpus; ++idx) {
                struct pkvm_hyp_vcpu *hyp_vcpu = hyp_vm->vcpus[idx];
-               struct kvm_hyp_memcache *vcpu_mc = &hyp_vcpu->vcpu.arch.pkvm_memcache;
+               struct kvm_hyp_memcache *vcpu_mc;
+
+               if (!hyp_vcpu)
+                       continue;
+
+               vcpu_mc = &hyp_vcpu->vcpu.arch.pkvm_memcache;
 
                while (vcpu_mc->nr_pages) {
                        void *addr = pop_hyp_memcache(vcpu_mc, hyp_phys_to_virt);

diff --git a/arch/arm64/kvm/pkvm.c b/arch/arm64/kvm/pkvm.c
index f297ccdaef64f1757763fbee36440e7888f90a11..0f89157d31fd6fbaacc3dc80f823524fe242686f 100644 (file)
--- a/arch/arm64/kvm/pkvm.c
+++ b/arch/arm64/kvm/pkvm.c
@@ -128,7 +128,9 @@ static int __pkvm_create_hyp_vcpu(struct kvm_vcpu *vcpu)
                return -ENOMEM;
 
        ret = kvm_call_hyp_nvhe(__pkvm_init_vcpu, handle, vcpu, hyp_vcpu);
-       if (ret)
+       if (!ret)
+               vcpu_set_flag(vcpu, VCPU_PKVM_FINALIZED);
+       else
                free_pages_exact(hyp_vcpu, hyp_vcpu_sz);
 
        return ret;
@@ -147,9 +149,7 @@ static int __pkvm_create_hyp_vcpu(struct kvm_vcpu *vcpu)
 static int __pkvm_create_hyp_vm(struct kvm *host_kvm)
 {
        size_t pgd_sz, hyp_vm_sz;
-       struct kvm_vcpu *host_vcpu;
        void *pgd, *hyp_vm;
-       unsigned long idx;
        int ret;
 
        if (host_kvm->created_vcpus < 1)
@@ -185,17 +185,7 @@ static int __pkvm_create_hyp_vm(struct kvm *host_kvm)
        host_kvm->arch.pkvm.stage2_teardown_mc.flags |= HYP_MEMCACHE_ACCOUNT_STAGE2;
        kvm_account_pgtable_pages(pgd, pgd_sz / PAGE_SIZE);
 
-       kvm_for_each_vcpu(idx, host_vcpu, host_kvm) {
-               ret = __pkvm_create_hyp_vcpu(host_vcpu);
-               if (ret)
-                       goto destroy_vm;
-       }
-
        return 0;
-
-destroy_vm:
-       __pkvm_destroy_hyp_vm(host_kvm);
-       return ret;
 free_vm:
        free_pages_exact(hyp_vm, hyp_vm_sz);
 free_pgd:
@@ -215,6 +205,18 @@ int pkvm_create_hyp_vm(struct kvm *host_kvm)
        return ret;
 }
 
+int pkvm_create_hyp_vcpu(struct kvm_vcpu *vcpu)
+{
+       int ret = 0;
+
+       mutex_lock(&vcpu->kvm->arch.config_lock);
+       if (!vcpu_get_flag(vcpu, VCPU_PKVM_FINALIZED))
+               ret = __pkvm_create_hyp_vcpu(vcpu);
+       mutex_unlock(&vcpu->kvm->arch.config_lock);
+
+       return ret;
+}
+
 void pkvm_destroy_hyp_vm(struct kvm *host_kvm)
 {
        mutex_lock(&host_kvm->arch.config_lock);