cpufreq: Avoid using inconsistent policy->min and policy->max

Since cpufreq_driver_resolve_freq() can run in parallel with
cpufreq_set_policy() and there is no synchronization between them,
the former may access policy->min and policy->max while the latter
is updating them and it may see intermediate values of them due
to the way the update is carried out.  Also the compiler is free
to apply any optimizations it wants both to the stores in
cpufreq_set_policy() and to the loads in cpufreq_driver_resolve_freq()
which may result in additional inconsistencies.

To address this, use WRITE_ONCE() when updating policy->min and
policy->max in cpufreq_set_policy() and use READ_ONCE() for reading
them in cpufreq_driver_resolve_freq().  Moreover, rearrange the update
in cpufreq_set_policy() to avoid storing intermediate values in
policy->min and policy->max with the help of the observation that
their new values are expected to be properly ordered upfront.

Also modify cpufreq_driver_resolve_freq() to take the possible reverse
ordering of policy->min and policy->max, which may happen depending on
the ordering of operations when this function and cpufreq_set_policy()
run concurrently, into account by always honoring the max when it
turns out to be less than the min (in case it comes from thermal
throttling or similar).

Fixes: 151717690694 ("cpufreq: Make policy min/max hard requirements")
Cc: 5.16+ <stable@vger.kernel.org> # 5.16+
Signed-off-by: Rafael J. Wysocki <rafael.j.wysocki@intel.com>
Reviewed-by: Christian Loehle <christian.loehle@arm.com>
Acked-by: Viresh Kumar <viresh.kumar@linaro.org>
Link: https://patch.msgid.link/5907080.DvuYhMxLoT@rjwysocki.net

diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c
index 3841c9da6cac3624acc05511d19e3d0bcda6f89a..acf19b0042bb6aa8d02900e812459a9f22a9384e 100644 (file)
--- a/drivers/cpufreq/cpufreq.c
+++ b/drivers/cpufreq/cpufreq.c
@@ -540,8 +540,6 @@ static unsigned int __resolve_freq(struct cpufreq_policy *policy,
 {
        unsigned int idx;
 
-       target_freq = clamp_val(target_freq, policy->min, policy->max);
-
        if (!policy->freq_table)
                return target_freq;
 
@@ -565,7 +563,22 @@ static unsigned int __resolve_freq(struct cpufreq_policy *policy,
 unsigned int cpufreq_driver_resolve_freq(struct cpufreq_policy *policy,
                                         unsigned int target_freq)
 {
-       return __resolve_freq(policy, target_freq, CPUFREQ_RELATION_LE);
+       unsigned int min = READ_ONCE(policy->min);
+       unsigned int max = READ_ONCE(policy->max);
+
+       /*
+        * If this function runs in parallel with cpufreq_set_policy(), it may
+        * read policy->min before the update and policy->max after the update
+        * or the other way around, so there is no ordering guarantee.
+        *
+        * Resolve this by always honoring the max (in case it comes from
+        * thermal throttling or similar).
+        */
+       if (unlikely(min > max))
+               min = max;
+
+       return __resolve_freq(policy, clamp_val(target_freq, min, max),
+                             CPUFREQ_RELATION_LE);
 }
 EXPORT_SYMBOL_GPL(cpufreq_driver_resolve_freq);
 
@@ -2384,6 +2397,7 @@ int __cpufreq_driver_target(struct cpufreq_policy *policy,
        if (cpufreq_disabled())
                return -ENODEV;
 
+       target_freq = clamp_val(target_freq, policy->min, policy->max);
        target_freq = __resolve_freq(policy, target_freq, relation);
 
        pr_debug("target for CPU %u: %u kHz, relation %u, requested %u kHz\n",
@@ -2708,11 +2722,15 @@ static int cpufreq_set_policy(struct cpufreq_policy *policy,
         * Resolve policy min/max to available frequencies. It ensures
         * no frequency resolution will neither overshoot the requested maximum
         * nor undershoot the requested minimum.
+        *
+        * Avoid storing intermediate values in policy->max or policy->min and
+        * compiler optimizations around them because they may be accessed
+        * concurrently by cpufreq_driver_resolve_freq() during the update.
         */
-       policy->min = new_data.min;
-       policy->max = new_data.max;
-       policy->min = __resolve_freq(policy, policy->min, CPUFREQ_RELATION_L);
-       policy->max = __resolve_freq(policy, policy->max, CPUFREQ_RELATION_H);
+       WRITE_ONCE(policy->max, __resolve_freq(policy, new_data.max, CPUFREQ_RELATION_H));
+       new_data.min = __resolve_freq(policy, new_data.min, CPUFREQ_RELATION_L);
+       WRITE_ONCE(policy->min, new_data.min > policy->max ? policy->max : new_data.min);
+
        trace_cpu_frequency_limits(policy);
 
        cpufreq_update_pressure(policy);