sched_ext: Move disable machinery into scx_sched

Because disable can be triggered from any place and the scheduler cannot be
trusted, disable path uses an irq_work to bounce and a kthread_work which is
executed on an RT helper kthread to perform disable. These must be per
scheduler instance to guarantee forward progress. Move them into scx_sched.

- If an scx_sched is accessible, its helper kthread is always valid making
  the `helper` check in schedule_scx_disable_work() unnecessary. As the
  function becomes trivial after the removal of the test, inline it.

- scx_create_rt_helper() has only one user - creation of the disable helper
  kthread. Inline it into scx_alloc_and_add_sched().

Signed-off-by: Tejun Heo <tj@kernel.org>
Reviewed-by: Andrea Righi <arighi@nvidia.com>
Acked-by: Changwoo Min <changwoo@igalia.com>

diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c
index 86cad4722b60c46dc634b142e9f69ceba29723de..a21390c9e9772b543f9ae7d7a2254084c154e7b8 100644 (file)
--- a/kernel/sched/ext.c
+++ b/kernel/sched/ext.c
@@ -851,6 +851,10 @@ struct scx_sched {
        struct scx_exit_info    *exit_info;
 
        struct kobject          kobj;
+
+       struct kthread_worker   *helper;
+       struct irq_work         error_irq_work;
+       struct kthread_work     disable_work;
        struct rcu_work         rcu_work;
 };
 
@@ -1024,7 +1028,6 @@ static DEFINE_SPINLOCK(scx_tasks_lock);
 static LIST_HEAD(scx_tasks);
 
 /* ops enable/disable */
-static struct kthread_worker *scx_helper;
 static DEFINE_MUTEX(scx_enable_mutex);
 DEFINE_STATIC_KEY_FALSE(__scx_enabled);
 DEFINE_STATIC_PERCPU_RWSEM(scx_fork_rwsem);
@@ -4418,6 +4421,7 @@ static void scx_sched_free_rcu_work(struct work_struct *work)
        struct scx_dispatch_q *dsq;
        int node;
 
+       kthread_stop(sch->helper->task);
        free_percpu(sch->event_stats_cpu);
 
        for_each_node_state(node, N_POSSIBLE)
@@ -4740,7 +4744,7 @@ static const char *scx_exit_reason(enum scx_exit_kind kind)
 
 static void scx_disable_workfn(struct kthread_work *work)
 {
-       struct scx_sched *sch = scx_root;
+       struct scx_sched *sch = container_of(work, struct scx_sched, disable_work);
        struct scx_exit_info *ei = sch->exit_info;
        struct scx_task_iter sti;
        struct task_struct *p;
@@ -4886,20 +4890,6 @@ done:
        scx_bypass(false);
 }
 
-static DEFINE_KTHREAD_WORK(scx_disable_work, scx_disable_workfn);
-
-static void schedule_scx_disable_work(void)
-{
-       struct kthread_worker *helper = READ_ONCE(scx_helper);
-
-       /*
-        * We may be called spuriously before the first bpf_sched_ext_reg(). If
-        * scx_helper isn't set up yet, there's nothing to do.
-        */
-       if (helper)
-               kthread_queue_work(helper, &scx_disable_work);
-}
-
 static void scx_disable(enum scx_exit_kind kind)
 {
        int none = SCX_EXIT_NONE;
@@ -4912,7 +4902,7 @@ static void scx_disable(enum scx_exit_kind kind)
        sch = rcu_dereference(scx_root);
        if (sch) {
                atomic_try_cmpxchg(&sch->exit_kind, &none, kind);
-               schedule_scx_disable_work();
+               kthread_queue_work(sch->helper, &sch->disable_work);
        }
        rcu_read_unlock();
 }
@@ -5214,16 +5204,15 @@ static void scx_dump_state(struct scx_exit_info *ei, size_t dump_len)
 
 static void scx_error_irq_workfn(struct irq_work *irq_work)
 {
-       struct scx_exit_info *ei = scx_root->exit_info;
+       struct scx_sched *sch = container_of(irq_work, struct scx_sched, error_irq_work);
+       struct scx_exit_info *ei = sch->exit_info;
 
        if (ei->kind >= SCX_EXIT_ERROR)
-               scx_dump_state(ei, scx_root->ops.exit_dump_len);
+               scx_dump_state(ei, sch->ops.exit_dump_len);
 
-       schedule_scx_disable_work();
+       kthread_queue_work(sch->helper, &sch->disable_work);
 }
 
-static DEFINE_IRQ_WORK(scx_error_irq_work, scx_error_irq_workfn);
-
 static __printf(3, 4) void __scx_exit(enum scx_exit_kind kind, s64 exit_code,
                                      const char *fmt, ...)
 {
@@ -5250,17 +5239,7 @@ static __printf(3, 4) void __scx_exit(enum scx_exit_kind kind, s64 exit_code,
        ei->kind = kind;
        ei->reason = scx_exit_reason(ei->kind);
 
-       irq_work_queue(&scx_error_irq_work);
-}
-
-static struct kthread_worker *scx_create_rt_helper(const char *name)
-{
-       struct kthread_worker *helper;
-
-       helper = kthread_run_worker(0, name);
-       if (helper)
-               sched_set_fifo(helper->task);
-       return helper;
+       irq_work_queue(&scx_root->error_irq_work);
 }
 
 static struct scx_sched *scx_alloc_and_add_sched(struct sched_ext_ops *ops)
@@ -5306,17 +5285,26 @@ static struct scx_sched *scx_alloc_and_add_sched(struct sched_ext_ops *ops)
        if (!sch->event_stats_cpu)
                goto err_free_gdsqs;
 
+       sch->helper = kthread_run_worker(0, "sched_ext_helper");
+       if (!sch->helper)
+               goto err_free_event_stats;
+       sched_set_fifo(sch->helper->task);
+
        atomic_set(&sch->exit_kind, SCX_EXIT_NONE);
+       init_irq_work(&sch->error_irq_work, scx_error_irq_workfn);
+       kthread_init_work(&sch->disable_work, scx_disable_workfn);
        sch->ops = *ops;
        ops->priv = sch;
 
        sch->kobj.kset = scx_kset;
        ret = kobject_init_and_add(&sch->kobj, &scx_ktype, NULL, "root");
        if (ret < 0)
-               goto err_free_event_stats;
+               goto err_stop_helper;
 
        return sch;
 
+err_stop_helper:
+       kthread_stop(sch->helper->task);
 err_free_event_stats:
        free_percpu(sch->event_stats_cpu);
 err_free_gdsqs:
@@ -5394,14 +5382,6 @@ static int scx_enable(struct sched_ext_ops *ops, struct bpf_link *link)
 
        mutex_lock(&scx_enable_mutex);
 
-       if (!scx_helper) {
-               WRITE_ONCE(scx_helper, scx_create_rt_helper("sched_ext_helper"));
-               if (!scx_helper) {
-                       ret = -ENOMEM;
-                       goto err_unlock;
-               }
-       }
-
        if (scx_enable_state() != SCX_DISABLED) {
                ret = -EBUSY;
                goto err_unlock;
@@ -5630,7 +5610,7 @@ err_disable:
         * completion. sch's base reference will be put by bpf_scx_unreg().
         */
        scx_error("scx_enable() failed (%d)", ret);
-       kthread_flush_work(&scx_disable_work);
+       kthread_flush_work(&sch->disable_work);
        return 0;
 }
 
@@ -5783,7 +5763,7 @@ static void bpf_scx_unreg(void *kdata, struct bpf_link *link)
        struct scx_sched *sch = ops->priv;
 
        scx_disable(SCX_EXIT_UNREG);
-       kthread_flush_work(&scx_disable_work);
+       kthread_flush_work(&sch->disable_work);
        kobject_put(&sch->kobj);
 }
 
@@ -5906,10 +5886,7 @@ static struct bpf_struct_ops bpf_sched_ext_ops = {
 
 static void sysrq_handle_sched_ext_reset(u8 key)
 {
-       if (scx_helper)
-               scx_disable(SCX_EXIT_SYSRQ);
-       else
-               pr_info("sched_ext: BPF scheduler not yet used\n");
+       scx_disable(SCX_EXIT_SYSRQ);
 }
 
 static const struct sysrq_key_op sysrq_sched_ext_reset_op = {