riscv: sched: defer restoring Vector context for user

User will use its Vector registers only after the kernel really returns
to the userspace. So we can delay restoring Vector registers as long as
we are still running in kernel mode. So, add a thread flag to indicates
the need of restoring Vector and do the restore at the last
arch-specific exit-to-user hook. This save the context restoring cost
when we switch over multiple processes that run V in kernel mode. For
example, if the kernel performs a context swicth from A->B->C, and
returns to C's userspace, then there is no need to restore B's
V-register.

Besides, this also prevents us from repeatedly restoring V context when
executing kernel-mode Vector multiple times.

The cost of this is that we must disable preemption and mark vector as
busy during vstate_{save,restore}. Because then the V context will not
get restored back immediately when a trap-causing context switch happens
in the middle of vstate_{save,restore}.

Signed-off-by: Andy Chiu <andy.chiu@sifive.com>
Acked-by: Conor Dooley <conor.dooley@microchip.com>
Tested-by: Björn Töpel <bjorn@rivosinc.com>
Tested-by: Lad Prabhakar <prabhakar.mahadev-lad.rj@bp.renesas.com>
Link: https://lore.kernel.org/r/20240115055929.4736-5-andy.chiu@sifive.com
Signed-off-by: Palmer Dabbelt <palmer@rivosinc.com>

diff --git a/arch/riscv/include/asm/entry-common.h b/arch/riscv/include/asm/entry-common.h
index 7ab5e34318c85fe05df525a5f80a49d25051bcd7..19023c430a9b587e0c24e0ceb07c7a444f05e557 100644 (file)
--- a/arch/riscv/include/asm/entry-common.h
+++ b/arch/riscv/include/asm/entry-common.h
@@ -4,6 +4,23 @@
 #define _ASM_RISCV_ENTRY_COMMON_H
 
 #include <asm/stacktrace.h>
+#include <asm/thread_info.h>
+#include <asm/vector.h>
+
+static inline void arch_exit_to_user_mode_prepare(struct pt_regs *regs,
+                                                 unsigned long ti_work)
+{
+       if (ti_work & _TIF_RISCV_V_DEFER_RESTORE) {
+               clear_thread_flag(TIF_RISCV_V_DEFER_RESTORE);
+               /*
+                * We are already called with irq disabled, so go without
+                * keeping track of riscv_v_flags.
+                */
+               riscv_v_vstate_restore(current, regs);
+       }
+}
+
+#define arch_exit_to_user_mode_prepare arch_exit_to_user_mode_prepare
 
 void handle_page_fault(struct pt_regs *regs);
 void handle_break(struct pt_regs *regs);

diff --git a/arch/riscv/include/asm/thread_info.h b/arch/riscv/include/asm/thread_info.h
index 574779900bfb339eeb446e49f4aae119fe382ae3..1047a97ddbc8a9164240f21fcdfb99796ec6538b 100644 (file)
--- a/arch/riscv/include/asm/thread_info.h
+++ b/arch/riscv/include/asm/thread_info.h
@@ -103,12 +103,14 @@ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src);
 #define TIF_NOTIFY_SIGNAL      9       /* signal notifications exist */
 #define TIF_UPROBE             10      /* uprobe breakpoint or singlestep */
 #define TIF_32BIT              11      /* compat-mode 32bit process */
+#define TIF_RISCV_V_DEFER_RESTORE      12 /* restore Vector before returing to user */
 
 #define _TIF_NOTIFY_RESUME     (1 << TIF_NOTIFY_RESUME)
 #define _TIF_SIGPENDING                (1 << TIF_SIGPENDING)
 #define _TIF_NEED_RESCHED      (1 << TIF_NEED_RESCHED)
 #define _TIF_NOTIFY_SIGNAL     (1 << TIF_NOTIFY_SIGNAL)
 #define _TIF_UPROBE            (1 << TIF_UPROBE)
+#define _TIF_RISCV_V_DEFER_RESTORE     (1 << TIF_RISCV_V_DEFER_RESTORE)
 
 #define _TIF_WORK_MASK \
        (_TIF_NOTIFY_RESUME | _TIF_SIGPENDING | _TIF_NEED_RESCHED | \

diff --git a/arch/riscv/include/asm/vector.h b/arch/riscv/include/asm/vector.h
index 71af3404fda140ede0e38571fedb6a0cd6df39ab..961c4e3d1b620f36c7b05b7e1eb67f85e4183e1c 100644 (file)
--- a/arch/riscv/include/asm/vector.h
+++ b/arch/riscv/include/asm/vector.h
@@ -193,6 +193,15 @@ static inline void riscv_v_vstate_restore(struct task_struct *task,
        }
 }
 
+static inline void riscv_v_vstate_set_restore(struct task_struct *task,
+                                             struct pt_regs *regs)
+{
+       if ((regs->status & SR_VS) != SR_VS_OFF) {
+               set_tsk_thread_flag(task, TIF_RISCV_V_DEFER_RESTORE);
+               riscv_v_vstate_on(regs);
+       }
+}
+
 static inline void __switch_to_vector(struct task_struct *prev,
                                      struct task_struct *next)
 {
@@ -200,7 +209,7 @@ static inline void __switch_to_vector(struct task_struct *prev,
 
        regs = task_pt_regs(prev);
        riscv_v_vstate_save(prev, regs);
-       riscv_v_vstate_restore(next, task_pt_regs(next));
+       riscv_v_vstate_set_restore(next, task_pt_regs(next));
 }
 
 void riscv_v_vstate_ctrl_init(struct task_struct *tsk);

diff --git a/arch/riscv/kernel/kernel_mode_vector.c b/arch/riscv/kernel/kernel_mode_vector.c
index 2fc145edae3ddf5f8d15c0b78c8213406116c548..8422c881f4529d9b2158f5c78e3f64edd0ef069a 100644 (file)
--- a/arch/riscv/kernel/kernel_mode_vector.c
+++ b/arch/riscv/kernel/kernel_mode_vector.c
@@ -117,7 +117,7 @@ void kernel_vector_end(void)
        if (WARN_ON(!has_vector()))
                return;
 
-       riscv_v_vstate_restore(current, task_pt_regs(current));
+       riscv_v_vstate_set_restore(current, task_pt_regs(current));
 
        riscv_v_disable();
 

diff --git a/arch/riscv/kernel/process.c b/arch/riscv/kernel/process.c
index 4a1275db114608e1993d4647355cb41ff5028111..36993f408de4a45d16787771eb831da434371f1d 100644 (file)
--- a/arch/riscv/kernel/process.c
+++ b/arch/riscv/kernel/process.c
@@ -171,6 +171,7 @@ void flush_thread(void)
        riscv_v_vstate_off(task_pt_regs(current));
        kfree(current->thread.vstate.datap);
        memset(&current->thread.vstate, 0, sizeof(struct __riscv_v_ext_state));
+       clear_tsk_thread_flag(current, TIF_RISCV_V_DEFER_RESTORE);
 #endif
 }
 
@@ -187,6 +188,7 @@ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src)
        *dst = *src;
        /* clear entire V context, including datap for a new task */
        memset(&dst->thread.vstate, 0, sizeof(struct __riscv_v_ext_state));
+       clear_tsk_thread_flag(dst, TIF_RISCV_V_DEFER_RESTORE);
 
        return 0;
 }

diff --git a/arch/riscv/kernel/ptrace.c b/arch/riscv/kernel/ptrace.c
index 2afe460de16a62ba21cf3c7db4a96c5ec8f7d3a5..7b93bcbdf9fab25057f70d1377843bbdcab48479 100644 (file)
--- a/arch/riscv/kernel/ptrace.c
+++ b/arch/riscv/kernel/ptrace.c
@@ -99,8 +99,11 @@ static int riscv_vr_get(struct task_struct *target,
         * Ensure the vector registers have been saved to the memory before
         * copying them to membuf.
         */
-       if (target == current)
+       if (target == current) {
+               get_cpu_vector_context();
                riscv_v_vstate_save(current, task_pt_regs(current));
+               put_cpu_vector_context();
+       }
 
        ptrace_vstate.vstart = vstate->vstart;
        ptrace_vstate.vl = vstate->vl;

diff --git a/arch/riscv/kernel/signal.c b/arch/riscv/kernel/signal.c
index 88b6220b260879ee75ac6a6824def025b004041b..aca4a12c84162ccbfa09566cea2322386417500f 100644 (file)
--- a/arch/riscv/kernel/signal.c
+++ b/arch/riscv/kernel/signal.c
@@ -86,7 +86,10 @@ static long save_v_state(struct pt_regs *regs, void __user **sc_vec)
        /* datap is designed to be 16 byte aligned for better performance */
        WARN_ON(unlikely(!IS_ALIGNED((unsigned long)datap, 16)));
 
+       get_cpu_vector_context();
        riscv_v_vstate_save(current, regs);
+       put_cpu_vector_context();
+
        /* Copy everything of vstate but datap. */
        err = __copy_to_user(&state->v_state, &current->thread.vstate,
                             offsetof(struct __riscv_v_ext_state, datap));
@@ -134,7 +137,7 @@ static long __restore_v_state(struct pt_regs *regs, void __user *sc_vec)
        if (unlikely(err))
                return err;
 
-       riscv_v_vstate_restore(current, regs);
+       riscv_v_vstate_set_restore(current, regs);
 
        return err;
 }

diff --git a/arch/riscv/kernel/vector.c b/arch/riscv/kernel/vector.c
index 578b6292487e1bb5e32309ee6874b8ba7a0c8315..66e8c6ab09d257c8f50cd8d966c1ac7e0b2a1517 100644 (file)
--- a/arch/riscv/kernel/vector.c
+++ b/arch/riscv/kernel/vector.c
@@ -167,7 +167,7 @@ bool riscv_v_first_use_handler(struct pt_regs *regs)
                return true;
        }
        riscv_v_vstate_on(regs);
-       riscv_v_vstate_restore(current, regs);
+       riscv_v_vstate_set_restore(current, regs);
        return true;
 }