cgroup: move rstat base stat objects into their own struct

This non-functional change serves as preparation for moving to
subsystem-based rstat trees. The base stats are not an actual subsystem,
but in future commits they will have exclusive rstat trees just as other
subsystems will.

Moving the base stat objects into a new struct allows the cgroup_rstat_cpu
struct to become more compact since it now only contains the minimum amount
of pointers needed for rstat participation. Subsystems will (in future
commits) make use of the compact cgroup_rstat_cpu struct while avoiding the
memory overhead of the base stat objects which they will not use.

An instance of the new struct cgroup_rstat_base_cpu was placed on the
cgroup struct so it can retain ownership of these base stats common to all
cgroups. A helper function was added for looking up the cpu-specific base
stats of a given cgroup. Finally, initialization and variable names were
adjusted where applicable.

Signed-off-by: JP Kobryn <inwardvessel@gmail.com>
Signed-off-by: Tejun Heo <tj@kernel.org>

diff --git a/include/linux/cgroup-defs.h b/include/linux/cgroup-defs.h
index 5bc8f55c8cca1458ae28c41663fcf4d665ffaded..1bf2e8db6dacf48f1864b3a46925972c98bae27f 100644 (file)
--- a/include/linux/cgroup-defs.h
+++ b/include/linux/cgroup-defs.h
@@ -344,10 +344,29 @@ struct cgroup_base_stat {
  * frequency decreases the cost of each read.
  *
  * This struct hosts both the fields which implement the above -
- * updated_children and updated_next - and the fields which track basic
- * resource statistics on top of it - bsync, bstat and last_bstat.
+ * updated_children and updated_next.
  */
 struct cgroup_rstat_cpu {
+       /*
+        * Child cgroups with stat updates on this cpu since the last read
+        * are linked on the parent's ->updated_children through
+        * ->updated_next.
+        *
+        * In addition to being more compact, singly-linked list pointing
+        * to the cgroup makes it unnecessary for each per-cpu struct to
+        * point back to the associated cgroup.
+        *
+        * Protected by per-cpu cgroup_rstat_cpu_lock.
+        */
+       struct cgroup *updated_children;        /* terminated by self cgroup */
+       struct cgroup *updated_next;            /* NULL iff not on the list */
+};
+
+/*
+ * This struct hosts the fields which track basic resource statistics on
+ * top of it - bsync, bstat and last_bstat.
+ */
+struct cgroup_rstat_base_cpu {
        /*
         * ->bsync protects ->bstat.  These are the only fields which get
         * updated in the hot path.
@@ -374,20 +393,6 @@ struct cgroup_rstat_cpu {
         * deltas to propagate to the per-cpu subtree_bstat.
         */
        struct cgroup_base_stat last_subtree_bstat;
-
-       /*
-        * Child cgroups with stat updates on this cpu since the last read
-        * are linked on the parent's ->updated_children through
-        * ->updated_next.
-        *
-        * In addition to being more compact, singly-linked list pointing
-        * to the cgroup makes it unnecessary for each per-cpu struct to
-        * point back to the associated cgroup.
-        *
-        * Protected by per-cpu cgroup_rstat_cpu_lock.
-        */
-       struct cgroup *updated_children;        /* terminated by self cgroup */
-       struct cgroup *updated_next;            /* NULL iff not on the list */
 };
 
 struct cgroup_freezer_state {
@@ -518,6 +523,7 @@ struct cgroup {
 
        /* per-cpu recursive resource statistics */
        struct cgroup_rstat_cpu __percpu *rstat_cpu;
+       struct cgroup_rstat_base_cpu __percpu *rstat_base_cpu;
        struct list_head rstat_css_list;
 
        /*

diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c
index 49d6222059977a1670307da6c15a924179faaaa3..e1f0284cbbb8e15a90c97fe7764e01bdb7b96aa7 100644 (file)
--- a/kernel/cgroup/cgroup.c
+++ b/kernel/cgroup/cgroup.c
@@ -161,10 +161,14 @@ static struct static_key_true *cgroup_subsys_on_dfl_key[] = {
 };
 #undef SUBSYS
 
-static DEFINE_PER_CPU(struct cgroup_rstat_cpu, cgrp_dfl_root_rstat_cpu);
+static DEFINE_PER_CPU(struct cgroup_rstat_cpu, root_rstat_cpu);
+static DEFINE_PER_CPU(struct cgroup_rstat_base_cpu, root_rstat_base_cpu);
 
 /* the default hierarchy */
-struct cgroup_root cgrp_dfl_root = { .cgrp.rstat_cpu = &cgrp_dfl_root_rstat_cpu };
+struct cgroup_root cgrp_dfl_root = {
+       .cgrp.rstat_cpu = &root_rstat_cpu,
+       .cgrp.rstat_base_cpu = &root_rstat_base_cpu,
+};
 EXPORT_SYMBOL_GPL(cgrp_dfl_root);
 
 /*

diff --git a/kernel/cgroup/rstat.c b/kernel/cgroup/rstat.c
index b2239156b7def9bc8d796d3b5af9915b59c5a21c..918931f68de35a218b29a7a554f72611c338f80f 100644 (file)
--- a/kernel/cgroup/rstat.c
+++ b/kernel/cgroup/rstat.c
@@ -19,6 +19,12 @@ static struct cgroup_rstat_cpu *cgroup_rstat_cpu(struct cgroup *cgrp, int cpu)
        return per_cpu_ptr(cgrp->rstat_cpu, cpu);
 }
 
+static struct cgroup_rstat_base_cpu *cgroup_rstat_base_cpu(
+               struct cgroup *cgrp, int cpu)
+{
+       return per_cpu_ptr(cgrp->rstat_base_cpu, cpu);
+}
+
 /*
  * Helper functions for rstat per CPU lock (cgroup_rstat_cpu_lock).
  *
@@ -352,12 +358,22 @@ int cgroup_rstat_init(struct cgroup *cgrp)
                        return -ENOMEM;
        }
 
+       if (!cgrp->rstat_base_cpu) {
+               cgrp->rstat_base_cpu = alloc_percpu(struct cgroup_rstat_base_cpu);
+               if (!cgrp->rstat_cpu) {
+                       free_percpu(cgrp->rstat_cpu);
+                       return -ENOMEM;
+               }
+       }
+
        /* ->updated_children list is self terminated */
        for_each_possible_cpu(cpu) {
                struct cgroup_rstat_cpu *rstatc = cgroup_rstat_cpu(cgrp, cpu);
+               struct cgroup_rstat_base_cpu *rstatbc =
+                       cgroup_rstat_base_cpu(cgrp, cpu);
 
                rstatc->updated_children = cgrp;
-               u64_stats_init(&rstatc->bsync);
+               u64_stats_init(&rstatbc->bsync);
        }
 
        return 0;
@@ -380,6 +396,8 @@ void cgroup_rstat_exit(struct cgroup *cgrp)
 
        free_percpu(cgrp->rstat_cpu);
        cgrp->rstat_cpu = NULL;
+       free_percpu(cgrp->rstat_base_cpu);
+       cgrp->rstat_base_cpu = NULL;
 }
 
 void __init cgroup_rstat_boot(void)
@@ -420,9 +438,9 @@ static void cgroup_base_stat_sub(struct cgroup_base_stat *dst_bstat,
 
 static void cgroup_base_stat_flush(struct cgroup *cgrp, int cpu)
 {
-       struct cgroup_rstat_cpu *rstatc = cgroup_rstat_cpu(cgrp, cpu);
+       struct cgroup_rstat_base_cpu *rstatbc = cgroup_rstat_base_cpu(cgrp, cpu);
        struct cgroup *parent = cgroup_parent(cgrp);
-       struct cgroup_rstat_cpu *prstatc;
+       struct cgroup_rstat_base_cpu *prstatbc;
        struct cgroup_base_stat delta;
        unsigned seq;
 
@@ -432,15 +450,15 @@ static void cgroup_base_stat_flush(struct cgroup *cgrp, int cpu)
 
        /* fetch the current per-cpu values */
        do {
-               seq = __u64_stats_fetch_begin(&rstatc->bsync);
-               delta = rstatc->bstat;
-       } while (__u64_stats_fetch_retry(&rstatc->bsync, seq));
+               seq = __u64_stats_fetch_begin(&rstatbc->bsync);
+               delta = rstatbc->bstat;
+       } while (__u64_stats_fetch_retry(&rstatbc->bsync, seq));
 
        /* propagate per-cpu delta to cgroup and per-cpu global statistics */
-       cgroup_base_stat_sub(&delta, &rstatc->last_bstat);
+       cgroup_base_stat_sub(&delta, &rstatbc->last_bstat);
        cgroup_base_stat_add(&cgrp->bstat, &delta);
-       cgroup_base_stat_add(&rstatc->last_bstat, &delta);
-       cgroup_base_stat_add(&rstatc->subtree_bstat, &delta);
+       cgroup_base_stat_add(&rstatbc->last_bstat, &delta);
+       cgroup_base_stat_add(&rstatbc->subtree_bstat, &delta);
 
        /* propagate cgroup and per-cpu global delta to parent (unless that's root) */
        if (cgroup_parent(parent)) {
@@ -449,73 +467,73 @@ static void cgroup_base_stat_flush(struct cgroup *cgrp, int cpu)
                cgroup_base_stat_add(&parent->bstat, &delta);
                cgroup_base_stat_add(&cgrp->last_bstat, &delta);
 
-               delta = rstatc->subtree_bstat;
-               prstatc = cgroup_rstat_cpu(parent, cpu);
-               cgroup_base_stat_sub(&delta, &rstatc->last_subtree_bstat);
-               cgroup_base_stat_add(&prstatc->subtree_bstat, &delta);
-               cgroup_base_stat_add(&rstatc->last_subtree_bstat, &delta);
+               delta = rstatbc->subtree_bstat;
+               prstatbc = cgroup_rstat_base_cpu(parent, cpu);
+               cgroup_base_stat_sub(&delta, &rstatbc->last_subtree_bstat);
+               cgroup_base_stat_add(&prstatbc->subtree_bstat, &delta);
+               cgroup_base_stat_add(&rstatbc->last_subtree_bstat, &delta);
        }
 }
 
-static struct cgroup_rstat_cpu *
+static struct cgroup_rstat_base_cpu *
 cgroup_base_stat_cputime_account_begin(struct cgroup *cgrp, unsigned long *flags)
 {
-       struct cgroup_rstat_cpu *rstatc;
+       struct cgroup_rstat_base_cpu *rstatbc;
 
-       rstatc = get_cpu_ptr(cgrp->rstat_cpu);
-       *flags = u64_stats_update_begin_irqsave(&rstatc->bsync);
-       return rstatc;
+       rstatbc = get_cpu_ptr(cgrp->rstat_base_cpu);
+       *flags = u64_stats_update_begin_irqsave(&rstatbc->bsync);
+       return rstatbc;
 }
 
 static void cgroup_base_stat_cputime_account_end(struct cgroup *cgrp,
-                                                struct cgroup_rstat_cpu *rstatc,
+                                                struct cgroup_rstat_base_cpu *rstatbc,
                                                 unsigned long flags)
 {
-       u64_stats_update_end_irqrestore(&rstatc->bsync, flags);
+       u64_stats_update_end_irqrestore(&rstatbc->bsync, flags);
        cgroup_rstat_updated(cgrp, smp_processor_id());
-       put_cpu_ptr(rstatc);
+       put_cpu_ptr(rstatbc);
 }
 
 void __cgroup_account_cputime(struct cgroup *cgrp, u64 delta_exec)
 {
-       struct cgroup_rstat_cpu *rstatc;
+       struct cgroup_rstat_base_cpu *rstatbc;
        unsigned long flags;
 
-       rstatc = cgroup_base_stat_cputime_account_begin(cgrp, &flags);
-       rstatc->bstat.cputime.sum_exec_runtime += delta_exec;
-       cgroup_base_stat_cputime_account_end(cgrp, rstatc, flags);
+       rstatbc = cgroup_base_stat_cputime_account_begin(cgrp, &flags);
+       rstatbc->bstat.cputime.sum_exec_runtime += delta_exec;
+       cgroup_base_stat_cputime_account_end(cgrp, rstatbc, flags);
 }
 
 void __cgroup_account_cputime_field(struct cgroup *cgrp,
                                    enum cpu_usage_stat index, u64 delta_exec)
 {
-       struct cgroup_rstat_cpu *rstatc;
+       struct cgroup_rstat_base_cpu *rstatbc;
        unsigned long flags;
 
-       rstatc = cgroup_base_stat_cputime_account_begin(cgrp, &flags);
+       rstatbc = cgroup_base_stat_cputime_account_begin(cgrp, &flags);
 
        switch (index) {
        case CPUTIME_NICE:
-               rstatc->bstat.ntime += delta_exec;
+               rstatbc->bstat.ntime += delta_exec;
                fallthrough;
        case CPUTIME_USER:
-               rstatc->bstat.cputime.utime += delta_exec;
+               rstatbc->bstat.cputime.utime += delta_exec;
                break;
        case CPUTIME_SYSTEM:
        case CPUTIME_IRQ:
        case CPUTIME_SOFTIRQ:
-               rstatc->bstat.cputime.stime += delta_exec;
+               rstatbc->bstat.cputime.stime += delta_exec;
                break;
 #ifdef CONFIG_SCHED_CORE
        case CPUTIME_FORCEIDLE:
-               rstatc->bstat.forceidle_sum += delta_exec;
+               rstatbc->bstat.forceidle_sum += delta_exec;
                break;
 #endif
        default:
                break;
        }
 
-       cgroup_base_stat_cputime_account_end(cgrp, rstatc, flags);
+       cgroup_base_stat_cputime_account_end(cgrp, rstatbc, flags);
 }
 
 /*