sched_ext: Remove scx_ops_enq_* static_keys

scx_ops_enq_last/exiting/migration_disabled are used to encode the
corresponding SCX_OPS_ flags into static_keys. These flags aren't hot enough
for static_key usage to make any meaningful difference and are made
static_keys mostly because there was no reason not to. However, global
static_keys can't work with the planned hierarchical multiple scheduler
support. Remove the static_keys and test the ops flags directly.

In repeated hackbench runs before and after static_keys removal on an AMD
Ryzen 3900X, I couldn't tell any measurable performance difference.

Signed-off-by: Tejun Heo <tj@kernel.org>
Acked-by: Changwoo Min <changwoo@igalia.com>
Acked-by: Andrea Righi <arighi@nvidia.com>

diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c
index 022343518f991c405152dac64678597670634c09..1e685e77b5e41af13125f8abb14cfc0217a08ca4 100644 (file)
--- a/kernel/sched/ext.c
+++ b/kernel/sched/ext.c
@@ -924,9 +924,6 @@ static struct sched_ext_ops scx_ops;
 static bool scx_warned_zero_slice;
 
 DEFINE_STATIC_KEY_FALSE(scx_ops_allow_queued_wakeup);
-static DEFINE_STATIC_KEY_FALSE(scx_ops_enq_last);
-static DEFINE_STATIC_KEY_FALSE(scx_ops_enq_exiting);
-static DEFINE_STATIC_KEY_FALSE(scx_ops_enq_migration_disabled);
 static DEFINE_STATIC_KEY_FALSE(scx_ops_cpu_preempt);
 
 static struct static_key_false scx_has_op[SCX_OPI_END] =
@@ -2144,14 +2141,14 @@ static void do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags,
                goto direct;
 
        /* see %SCX_OPS_ENQ_EXITING */
-       if (!static_branch_unlikely(&scx_ops_enq_exiting) &&
+       if (!(scx_ops.flags & SCX_OPS_ENQ_EXITING) &&
            unlikely(p->flags & PF_EXITING)) {
                __scx_add_event(SCX_EV_ENQ_SKIP_EXITING, 1);
                goto local;
        }
 
        /* see %SCX_OPS_ENQ_MIGRATION_DISABLED */
-       if (!static_branch_unlikely(&scx_ops_enq_migration_disabled) &&
+       if (!(scx_ops.flags & SCX_OPS_ENQ_MIGRATION_DISABLED) &&
            is_migration_disabled(p)) {
                __scx_add_event(SCX_EV_ENQ_SKIP_MIGRATION_DISABLED, 1);
                goto local;
@@ -3022,8 +3019,8 @@ no_tasks:
         * Didn't find another task to run. Keep running @prev unless
         * %SCX_OPS_ENQ_LAST is in effect.
         */
-       if (prev_on_rq && (!static_branch_unlikely(&scx_ops_enq_last) ||
-            scx_rq_bypassing(rq))) {
+       if (prev_on_rq &&
+           (!(scx_ops.flags & SCX_OPS_ENQ_LAST) || scx_rq_bypassing(rq))) {
                rq->scx.flags |= SCX_RQ_BAL_KEEP;
                __scx_add_event(SCX_EV_DISPATCH_KEEP_LAST, 1);
                goto has_tasks;
@@ -3228,7 +3225,7 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p,
                 * which should trigger an explicit follow-up scheduling event.
                 */
                if (sched_class_above(&ext_sched_class, next->sched_class)) {
-                       WARN_ON_ONCE(!static_branch_unlikely(&scx_ops_enq_last));
+                       WARN_ON_ONCE(!(scx_ops.flags & SCX_OPS_ENQ_LAST));
                        do_enqueue_task(rq, p, SCX_ENQ_LAST, -1);
                } else {
                        do_enqueue_task(rq, p, 0, -1);
@@ -4728,9 +4725,6 @@ static void scx_disable_workfn(struct kthread_work *work)
        for (i = SCX_OPI_BEGIN; i < SCX_OPI_END; i++)
                static_branch_disable(&scx_has_op[i]);
        static_branch_disable(&scx_ops_allow_queued_wakeup);
-       static_branch_disable(&scx_ops_enq_last);
-       static_branch_disable(&scx_ops_enq_exiting);
-       static_branch_disable(&scx_ops_enq_migration_disabled);
        static_branch_disable(&scx_ops_cpu_preempt);
        scx_idle_disable();
        synchronize_rcu();
@@ -5372,12 +5366,6 @@ static int scx_enable(struct sched_ext_ops *ops, struct bpf_link *link)
 
        if (ops->flags & SCX_OPS_ALLOW_QUEUED_WAKEUP)
                static_branch_enable(&scx_ops_allow_queued_wakeup);
-       if (ops->flags & SCX_OPS_ENQ_LAST)
-               static_branch_enable(&scx_ops_enq_last);
-       if (ops->flags & SCX_OPS_ENQ_EXITING)
-               static_branch_enable(&scx_ops_enq_exiting);
-       if (ops->flags & SCX_OPS_ENQ_MIGRATION_DISABLED)
-               static_branch_enable(&scx_ops_enq_migration_disabled);
        if (scx_ops.cpu_acquire || scx_ops.cpu_release)
                static_branch_enable(&scx_ops_cpu_preempt);