time: Replace __get_cpu_var uses

Convert uses of __get_cpu_var for creating a address from a percpu
offset to this_cpu_ptr.

The two cases where get_cpu_var is used to actually access a percpu
variable are changed to use this_cpu_read/raw_cpu_read.

Reviewed-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Christoph Lameter <cl@linux.com>
Signed-off-by: Tejun Heo <tj@kernel.org>

diff --git a/drivers/clocksource/dummy_timer.c b/drivers/clocksource/dummy_timer.c
index ad357254172890ff4170c2a32634f57ad831b26e..31990600fcff3fc5bacb22f0d03a1e5cde1e2d4a 100644 (file)
--- a/drivers/clocksource/dummy_timer.c
+++ b/drivers/clocksource/dummy_timer.c
@@ -28,7 +28,7 @@ static void dummy_timer_set_mode(enum clock_event_mode mode,
 static void dummy_timer_setup(void)
 {
        int cpu = smp_processor_id();
-       struct clock_event_device *evt = __this_cpu_ptr(&dummy_timer_evt);
+       struct clock_event_device *evt = raw_cpu_ptr(&dummy_timer_evt);
 
        evt->name       = "dummy_timer";
        evt->features   = CLOCK_EVT_FEAT_PERIODIC |

diff --git a/kernel/irq_work.c b/kernel/irq_work.c
index e6bcbe756663abd64adf9c416eddee7a91c9a2c9..345d19edcdaed81a5c0194fb9256f3b113b7f402 100644 (file)
--- a/kernel/irq_work.c
+++ b/kernel/irq_work.c
@@ -95,11 +95,11 @@ bool irq_work_queue(struct irq_work *work)
 
        /* If the work is "lazy", handle it from next tick if any */
        if (work->flags & IRQ_WORK_LAZY) {
-               if (llist_add(&work->llnode, &__get_cpu_var(lazy_list)) &&
+               if (llist_add(&work->llnode, this_cpu_ptr(&lazy_list)) &&
                    tick_nohz_tick_stopped())
                        arch_irq_work_raise();
        } else {
-               if (llist_add(&work->llnode, &__get_cpu_var(raised_list)))
+               if (llist_add(&work->llnode, this_cpu_ptr(&raised_list)))
                        arch_irq_work_raise();
        }
 
@@ -113,8 +113,8 @@ bool irq_work_needs_cpu(void)
 {
        struct llist_head *raised, *lazy;
 
-       raised = &__get_cpu_var(raised_list);
-       lazy = &__get_cpu_var(lazy_list);
+       raised = this_cpu_ptr(&raised_list);
+       lazy = this_cpu_ptr(&lazy_list);
        if (llist_empty(raised) && llist_empty(lazy))
                return false;
 
@@ -166,8 +166,8 @@ static void irq_work_run_list(struct llist_head *list)
  */
 void irq_work_run(void)
 {
-       irq_work_run_list(&__get_cpu_var(raised_list));
-       irq_work_run_list(&__get_cpu_var(lazy_list));
+       irq_work_run_list(this_cpu_ptr(&raised_list));
+       irq_work_run_list(this_cpu_ptr(&lazy_list));
 }
 EXPORT_SYMBOL_GPL(irq_work_run);
 

diff --git a/kernel/sched/clock.c b/kernel/sched/clock.c
index 3ef6451e972ed06571b94a4d19a11d78edd39bf6..c27e4f8f4879eba57937d9a36387ef04a1bcaacf 100644 (file)
--- a/kernel/sched/clock.c
+++ b/kernel/sched/clock.c
@@ -134,7 +134,7 @@ static DEFINE_PER_CPU_SHARED_ALIGNED(struct sched_clock_data, sched_clock_data);
 
 static inline struct sched_clock_data *this_scd(void)
 {
-       return &__get_cpu_var(sched_clock_data);
+       return this_cpu_ptr(&sched_clock_data);
 }
 
 static inline struct sched_clock_data *cpu_sdc(int cpu)

diff --git a/kernel/softirq.c b/kernel/softirq.c
index 5918d227730f33d7daec6dfab1e8ef30cd653abf..2d44b5714fe6a80515bc7705374c1270255d74dc 100644 (file)
--- a/kernel/softirq.c
+++ b/kernel/softirq.c
@@ -485,7 +485,7 @@ static void tasklet_action(struct softirq_action *a)
        local_irq_disable();
        list = __this_cpu_read(tasklet_vec.head);
        __this_cpu_write(tasklet_vec.head, NULL);
-       __this_cpu_write(tasklet_vec.tail, &__get_cpu_var(tasklet_vec).head);
+       __this_cpu_write(tasklet_vec.tail, this_cpu_ptr(&tasklet_vec.head));
        local_irq_enable();
 
        while (list) {
@@ -521,7 +521,7 @@ static void tasklet_hi_action(struct softirq_action *a)
        local_irq_disable();
        list = __this_cpu_read(tasklet_hi_vec.head);
        __this_cpu_write(tasklet_hi_vec.head, NULL);
-       __this_cpu_write(tasklet_hi_vec.tail, &__get_cpu_var(tasklet_hi_vec).head);
+       __this_cpu_write(tasklet_hi_vec.tail, this_cpu_ptr(&tasklet_hi_vec.head));
        local_irq_enable();
 
        while (list) {

diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c
index 1c2fe7de28427c6393b61374b5871971fcffc240..5f2229ba53d671aac8fe6a2c30ff453ef0512863 100644 (file)
--- a/kernel/time/hrtimer.c
+++ b/kernel/time/hrtimer.c
@@ -1144,7 +1144,7 @@ static void __hrtimer_init(struct hrtimer *timer, clockid_t clock_id,
 
        memset(timer, 0, sizeof(struct hrtimer));
 
-       cpu_base = &__raw_get_cpu_var(hrtimer_bases);
+       cpu_base = raw_cpu_ptr(&hrtimer_bases);
 
        if (clock_id == CLOCK_REALTIME && mode != HRTIMER_MODE_ABS)
                clock_id = CLOCK_MONOTONIC;
@@ -1187,7 +1187,7 @@ int hrtimer_get_res(const clockid_t which_clock, struct timespec *tp)
        struct hrtimer_cpu_base *cpu_base;
        int base = hrtimer_clockid_to_base(which_clock);
 
-       cpu_base = &__raw_get_cpu_var(hrtimer_bases);
+       cpu_base = raw_cpu_ptr(&hrtimer_bases);
        *tp = ktime_to_timespec(cpu_base->clock_base[base].resolution);
 
        return 0;
@@ -1376,7 +1376,7 @@ static void __hrtimer_peek_ahead_timers(void)
        if (!hrtimer_hres_active())
                return;
 
-       td = &__get_cpu_var(tick_cpu_device);
+       td = this_cpu_ptr(&tick_cpu_device);
        if (td && td->evtdev)
                hrtimer_interrupt(td->evtdev);
 }

diff --git a/kernel/time/tick-broadcast.c b/kernel/time/tick-broadcast.c
index 64c5990fd500b86e86e9d0a92d86fa7b7b45a0b1..066f0ec05e487396315356df0ea04c8563ffa390 100644 (file)
--- a/kernel/time/tick-broadcast.c
+++ b/kernel/time/tick-broadcast.c
@@ -554,7 +554,7 @@ int tick_resume_broadcast_oneshot(struct clock_event_device *bc)
 void tick_check_oneshot_broadcast_this_cpu(void)
 {
        if (cpumask_test_cpu(smp_processor_id(), tick_broadcast_oneshot_mask)) {
-               struct tick_device *td = &__get_cpu_var(tick_cpu_device);
+               struct tick_device *td = this_cpu_ptr(&tick_cpu_device);
 
                /*
                 * We might be in the middle of switching over from

diff --git a/kernel/time/tick-common.c b/kernel/time/tick-common.c
index 0a0608edeb2665e88d2cb1c917b4cf4f112a4e8e..decfb5f6edb04ebf257e3df669c0e10828455fb6 100644 (file)
--- a/kernel/time/tick-common.c
+++ b/kernel/time/tick-common.c
@@ -224,7 +224,7 @@ static void tick_setup_device(struct tick_device *td,
 
 void tick_install_replacement(struct clock_event_device *newdev)
 {
-       struct tick_device *td = &__get_cpu_var(tick_cpu_device);
+       struct tick_device *td = this_cpu_ptr(&tick_cpu_device);
        int cpu = smp_processor_id();
 
        clockevents_exchange_device(td->evtdev, newdev);
@@ -374,14 +374,14 @@ void tick_shutdown(unsigned int *cpup)
 
 void tick_suspend(void)
 {
-       struct tick_device *td = &__get_cpu_var(tick_cpu_device);
+       struct tick_device *td = this_cpu_ptr(&tick_cpu_device);
 
        clockevents_shutdown(td->evtdev);
 }
 
 void tick_resume(void)
 {
-       struct tick_device *td = &__get_cpu_var(tick_cpu_device);
+       struct tick_device *td = this_cpu_ptr(&tick_cpu_device);
        int broadcast = tick_resume_broadcast();
 
        clockevents_set_mode(td->evtdev, CLOCK_EVT_MODE_RESUME);

diff --git a/kernel/time/tick-oneshot.c b/kernel/time/tick-oneshot.c
index 824109060a33808b7f5d15d13ececbd298534859..7ce740e78e1b506b155c07e3ac50a9a96e6b262d 100644 (file)
--- a/kernel/time/tick-oneshot.c
+++ b/kernel/time/tick-oneshot.c
@@ -59,7 +59,7 @@ void tick_setup_oneshot(struct clock_event_device *newdev,
  */
 int tick_switch_to_oneshot(void (*handler)(struct clock_event_device *))
 {
-       struct tick_device *td = &__get_cpu_var(tick_cpu_device);
+       struct tick_device *td = this_cpu_ptr(&tick_cpu_device);
        struct clock_event_device *dev = td->evtdev;
 
        if (!dev || !(dev->features & CLOCK_EVT_FEAT_ONESHOT) ||

diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c
index 99aa6ee3908fbbff83923b2a8bb4c72b06e3f281..73f90932282b88fa386a096dd27b8112f462f061 100644 (file)
--- a/kernel/time/tick-sched.c
+++ b/kernel/time/tick-sched.c
@@ -205,7 +205,7 @@ static void tick_nohz_restart_sched_tick(struct tick_sched *ts, ktime_t now);
  */
 void __tick_nohz_full_check(void)
 {
-       struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
+       struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
 
        if (tick_nohz_full_cpu(smp_processor_id())) {
                if (ts->tick_stopped && !is_idle_task(current)) {
@@ -545,7 +545,7 @@ static ktime_t tick_nohz_stop_sched_tick(struct tick_sched *ts,
        unsigned long seq, last_jiffies, next_jiffies, delta_jiffies;
        ktime_t last_update, expires, ret = { .tv64 = 0 };
        unsigned long rcu_delta_jiffies;
-       struct clock_event_device *dev = __get_cpu_var(tick_cpu_device).evtdev;
+       struct clock_event_device *dev = __this_cpu_read(tick_cpu_device.evtdev);
        u64 time_delta;
 
        time_delta = timekeeping_max_deferment();
@@ -813,7 +813,7 @@ void tick_nohz_idle_enter(void)
 
        local_irq_disable();
 
-       ts = &__get_cpu_var(tick_cpu_sched);
+       ts = this_cpu_ptr(&tick_cpu_sched);
        ts->inidle = 1;
        __tick_nohz_idle_enter(ts);
 
@@ -831,7 +831,7 @@ EXPORT_SYMBOL_GPL(tick_nohz_idle_enter);
  */
 void tick_nohz_irq_exit(void)
 {
-       struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
+       struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
 
        if (ts->inidle)
                __tick_nohz_idle_enter(ts);
@@ -846,7 +846,7 @@ void tick_nohz_irq_exit(void)
  */
 ktime_t tick_nohz_get_sleep_length(void)
 {
-       struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
+       struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
 
        return ts->sleep_length;
 }
@@ -959,7 +959,7 @@ static int tick_nohz_reprogram(struct tick_sched *ts, ktime_t now)
  */
 static void tick_nohz_handler(struct clock_event_device *dev)
 {
-       struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
+       struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
        struct pt_regs *regs = get_irq_regs();
        ktime_t now = ktime_get();
 
@@ -979,7 +979,7 @@ static void tick_nohz_handler(struct clock_event_device *dev)
  */
 static void tick_nohz_switch_to_nohz(void)
 {
-       struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
+       struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
        ktime_t next;
 
        if (!tick_nohz_enabled)
@@ -1115,7 +1115,7 @@ early_param("skew_tick", skew_tick);
  */
 void tick_setup_sched_timer(void)
 {
-       struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
+       struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
        ktime_t now = ktime_get();
 
        /*
@@ -1184,7 +1184,7 @@ void tick_clock_notify(void)
  */
 void tick_oneshot_notify(void)
 {
-       struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
+       struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
 
        set_bit(0, &ts->check_clocks);
 }
@@ -1199,7 +1199,7 @@ void tick_oneshot_notify(void)
  */
 int tick_check_oneshot_change(int allow_nohz)
 {
-       struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
+       struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
 
        if (!test_and_clear_bit(0, &ts->check_clocks))
                return 0;

diff --git a/kernel/time/timer.c b/kernel/time/timer.c
index aca5dfe2fa3de5df0738b6dc495b2e186483a593..04d8ed8399b03f88a74025a37f4f23248f07e082 100644 (file)
--- a/kernel/time/timer.c
+++ b/kernel/time/timer.c
@@ -655,7 +655,7 @@ static inline void debug_assert_init(struct timer_list *timer)
 static void do_init_timer(struct timer_list *timer, unsigned int flags,
                          const char *name, struct lock_class_key *key)
 {
-       struct tvec_base *base = __raw_get_cpu_var(tvec_bases);
+       struct tvec_base *base = raw_cpu_read(tvec_bases);
 
        timer->entry.next = NULL;
        timer->base = (void *)((unsigned long)base | flags);