x86: Replace __get_cpu_var uses

__get_cpu_var() is used for multiple purposes in the kernel source. One of
them is address calculation via the form &__get_cpu_var(x).  This calculates
the address for the instance of the percpu variable of the current processor
based on an offset.

Other use cases are for storing and retrieving data from the current
processors percpu area.  __get_cpu_var() can be used as an lvalue when
writing data or on the right side of an assignment.

__get_cpu_var() is defined as :

#define __get_cpu_var(var) (*this_cpu_ptr(&(var)))

__get_cpu_var() always only does an address determination. However, store
and retrieve operations could use a segment prefix (or global register on
other platforms) to avoid the address calculation.

this_cpu_write() and this_cpu_read() can directly take an offset into a
percpu area and use optimized assembly code to read and write per cpu
variables.

This patch converts __get_cpu_var into either an explicit address
calculation using this_cpu_ptr() or into a use of this_cpu operations that
use the offset.  Thereby address calculations are avoided and less registers
are used when code is generated.

Transformations done to __get_cpu_var()

1. Determine the address of the percpu instance of the current processor.

	DEFINE_PER_CPU(int, y);
	int *x = &__get_cpu_var(y);

    Converts to

	int *x = this_cpu_ptr(&y);

2. Same as #1 but this time an array structure is involved.

	DEFINE_PER_CPU(int, y[20]);
	int *x = __get_cpu_var(y);

    Converts to

	int *x = this_cpu_ptr(y);

3. Retrieve the content of the current processors instance of a per cpu
variable.

	DEFINE_PER_CPU(int, y);
	int x = __get_cpu_var(y)

   Converts to

	int x = __this_cpu_read(y);

4. Retrieve the content of a percpu struct

	DEFINE_PER_CPU(struct mystruct, y);
	struct mystruct x = __get_cpu_var(y);

   Converts to

	memcpy(&x, this_cpu_ptr(&y), sizeof(x));

5. Assignment to a per cpu variable

	DEFINE_PER_CPU(int, y)
	__get_cpu_var(y) = x;

   Converts to

	__this_cpu_write(y, x);

6. Increment/Decrement etc of a per cpu variable

	DEFINE_PER_CPU(int, y);
	__get_cpu_var(y)++

   Converts to

	__this_cpu_inc(y)

Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: x86@kernel.org
Acked-by: H. Peter Anvin <hpa@linux.intel.com>
Acked-by: Ingo Molnar <mingo@kernel.org>
Signed-off-by: Christoph Lameter <cl@linux.com>
Signed-off-by: Tejun Heo <tj@kernel.org>

diff --git a/arch/x86/include/asm/debugreg.h b/arch/x86/include/asm/debugreg.h
index 4b528a970bd43f0ceed423d4f01819e251d47284..61fd18b83b6c6af777f2cdd2a63d943089de55e1 100644 (file)
--- a/arch/x86/include/asm/debugreg.h
+++ b/arch/x86/include/asm/debugreg.h
@@ -97,11 +97,11 @@ extern void hw_breakpoint_restore(void);
 DECLARE_PER_CPU(int, debug_stack_usage);
 static inline void debug_stack_usage_inc(void)
 {
-       __get_cpu_var(debug_stack_usage)++;
+       __this_cpu_inc(debug_stack_usage);
 }
 static inline void debug_stack_usage_dec(void)
 {
-       __get_cpu_var(debug_stack_usage)--;
+       __this_cpu_dec(debug_stack_usage);
 }
 int is_debug_stack(unsigned long addr);
 void debug_stack_set_zero(void);

diff --git a/arch/x86/include/asm/uv/uv_hub.h b/arch/x86/include/asm/uv/uv_hub.h
index c63e925fd6b7382900dcd6805082f354a20934de..bb84cfd5a1a1a1075a9136e47168ce7aa96cdf78 100644 (file)
--- a/arch/x86/include/asm/uv/uv_hub.h
+++ b/arch/x86/include/asm/uv/uv_hub.h
@@ -164,7 +164,7 @@ struct uv_hub_info_s {
 };
 
 DECLARE_PER_CPU(struct uv_hub_info_s, __uv_hub_info);
-#define uv_hub_info            (&__get_cpu_var(__uv_hub_info))
+#define uv_hub_info            this_cpu_ptr(&__uv_hub_info)
 #define uv_cpu_hub_info(cpu)   (&per_cpu(__uv_hub_info, cpu))
 
 /*

diff --git a/arch/x86/kernel/apb_timer.c b/arch/x86/kernel/apb_timer.c
index af5b08ab3b712836e09d7eb9f6cd79919035d22f..5972b108f15a739a06b47276a05d86713d08a3a2 100644 (file)
--- a/arch/x86/kernel/apb_timer.c
+++ b/arch/x86/kernel/apb_timer.c
@@ -146,7 +146,7 @@ static inline int is_apbt_capable(void)
 static int __init apbt_clockevent_register(void)
 {
        struct sfi_timer_table_entry *mtmr;
-       struct apbt_dev *adev = &__get_cpu_var(cpu_apbt_dev);
+       struct apbt_dev *adev = this_cpu_ptr(&cpu_apbt_dev);
 
        mtmr = sfi_get_mtmr(APBT_CLOCKEVENT0_NUM);
        if (mtmr == NULL) {
@@ -200,7 +200,7 @@ void apbt_setup_secondary_clock(void)
        if (!cpu)
                return;
 
-       adev = &__get_cpu_var(cpu_apbt_dev);
+       adev = this_cpu_ptr(&cpu_apbt_dev);
        if (!adev->timer) {
                adev->timer = dw_apb_clockevent_init(cpu, adev->name,
                        APBT_CLOCKEVENT_RATING, adev_virt_addr(adev),

diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c
index 67760275544b4f548f98c5b18c098d481637995d..00853b254ab09c36afdb2a681bd9c68037ac17dc 100644 (file)
--- a/arch/x86/kernel/apic/apic.c
+++ b/arch/x86/kernel/apic/apic.c
@@ -561,7 +561,7 @@ static DEFINE_PER_CPU(struct clock_event_device, lapic_events);
  */
 static void setup_APIC_timer(void)
 {
-       struct clock_event_device *levt = &__get_cpu_var(lapic_events);
+       struct clock_event_device *levt = this_cpu_ptr(&lapic_events);
 
        if (this_cpu_has(X86_FEATURE_ARAT)) {
                lapic_clockevent.features &= ~CLOCK_EVT_FEAT_C3STOP;
@@ -696,7 +696,7 @@ calibrate_by_pmtimer(long deltapm, long *delta, long *deltatsc)
 
 static int __init calibrate_APIC_clock(void)
 {
-       struct clock_event_device *levt = &__get_cpu_var(lapic_events);
+       struct clock_event_device *levt = this_cpu_ptr(&lapic_events);
        void (*real_handler)(struct clock_event_device *dev);
        unsigned long deltaj;
        long delta, deltatsc;

diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index e4ab2b42bd6f469528c73cac4ac0fb186b5bdbd3..5666eb9568fce54d71439a0f7b524bd9a688067b 100644 (file)
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -1198,9 +1198,9 @@ DEFINE_PER_CPU(int, debug_stack_usage);
 
 int is_debug_stack(unsigned long addr)
 {
-       return __get_cpu_var(debug_stack_usage) ||
-               (addr <= __get_cpu_var(debug_stack_addr) &&
-                addr > (__get_cpu_var(debug_stack_addr) - DEBUG_STKSZ));
+       return __this_cpu_read(debug_stack_usage) ||
+               (addr <= __this_cpu_read(debug_stack_addr) &&
+                addr > (__this_cpu_read(debug_stack_addr) - DEBUG_STKSZ));
 }
 NOKPROBE_SYMBOL(is_debug_stack);
 

diff --git a/arch/x86/kernel/cpu/mcheck/mce-inject.c b/arch/x86/kernel/cpu/mcheck/mce-inject.c
index 5ac2d1fb28bc590bd9403aea70d8641d3b05e03b..4cfba4371a71f28c4615610475e3eec9e8d9790c 100644 (file)
--- a/arch/x86/kernel/cpu/mcheck/mce-inject.c
+++ b/arch/x86/kernel/cpu/mcheck/mce-inject.c
@@ -83,7 +83,7 @@ static DEFINE_MUTEX(mce_inject_mutex);
 static int mce_raise_notify(unsigned int cmd, struct pt_regs *regs)
 {
        int cpu = smp_processor_id();
-       struct mce *m = &__get_cpu_var(injectm);
+       struct mce *m = this_cpu_ptr(&injectm);
        if (!cpumask_test_cpu(cpu, mce_inject_cpumask))
                return NMI_DONE;
        cpumask_clear_cpu(cpu, mce_inject_cpumask);
@@ -97,7 +97,7 @@ static int mce_raise_notify(unsigned int cmd, struct pt_regs *regs)
 static void mce_irq_ipi(void *info)
 {
        int cpu = smp_processor_id();
-       struct mce *m = &__get_cpu_var(injectm);
+       struct mce *m = this_cpu_ptr(&injectm);
 
        if (cpumask_test_cpu(cpu, mce_inject_cpumask) &&
                        m->inject_flags & MCJ_EXCEPTION) {
@@ -109,7 +109,7 @@ static void mce_irq_ipi(void *info)
 /* Inject mce on current CPU */
 static int raise_local(void)
 {
-       struct mce *m = &__get_cpu_var(injectm);
+       struct mce *m = this_cpu_ptr(&injectm);
        int context = MCJ_CTX(m->inject_flags);
        int ret = 0;
        int cpu = m->extcpu;

diff --git a/arch/x86/kernel/cpu/mcheck/mce.c b/arch/x86/kernel/cpu/mcheck/mce.c
index bd9ccda8087ff5f46ab30bc89225b8912ceaac40..61a9668cebfde8d0ca22ce7dd5bde07f99a2e65f 100644 (file)
--- a/arch/x86/kernel/cpu/mcheck/mce.c
+++ b/arch/x86/kernel/cpu/mcheck/mce.c
@@ -400,7 +400,7 @@ static u64 mce_rdmsrl(u32 msr)
 
                if (offset < 0)
                        return 0;
-               return *(u64 *)((char *)&__get_cpu_var(injectm) + offset);
+               return *(u64 *)((char *)this_cpu_ptr(&injectm) + offset);
        }
 
        if (rdmsrl_safe(msr, &v)) {
@@ -422,7 +422,7 @@ static void mce_wrmsrl(u32 msr, u64 v)
                int offset = msr_to_offset(msr);
 
                if (offset >= 0)
-                       *(u64 *)((char *)&__get_cpu_var(injectm) + offset) = v;
+                       *(u64 *)((char *)this_cpu_ptr(&injectm) + offset) = v;
                return;
        }
        wrmsrl(msr, v);
@@ -478,7 +478,7 @@ static DEFINE_PER_CPU(struct mce_ring, mce_ring);
 /* Runs with CPU affinity in workqueue */
 static int mce_ring_empty(void)
 {
-       struct mce_ring *r = &__get_cpu_var(mce_ring);
+       struct mce_ring *r = this_cpu_ptr(&mce_ring);
 
        return r->start == r->end;
 }
@@ -490,7 +490,7 @@ static int mce_ring_get(unsigned long *pfn)
 
        *pfn = 0;
        get_cpu();
-       r = &__get_cpu_var(mce_ring);
+       r = this_cpu_ptr(&mce_ring);
        if (r->start == r->end)
                goto out;
        *pfn = r->ring[r->start];
@@ -504,7 +504,7 @@ out:
 /* Always runs in MCE context with preempt off */
 static int mce_ring_add(unsigned long pfn)
 {
-       struct mce_ring *r = &__get_cpu_var(mce_ring);
+       struct mce_ring *r = this_cpu_ptr(&mce_ring);
        unsigned next;
 
        next = (r->end + 1) % MCE_RING_SIZE;
@@ -526,7 +526,7 @@ int mce_available(struct cpuinfo_x86 *c)
 static void mce_schedule_work(void)
 {
        if (!mce_ring_empty())
-               schedule_work(&__get_cpu_var(mce_work));
+               schedule_work(this_cpu_ptr(&mce_work));
 }
 
 DEFINE_PER_CPU(struct irq_work, mce_irq_work);
@@ -551,7 +551,7 @@ static void mce_report_event(struct pt_regs *regs)
                return;
        }
 
-       irq_work_queue(&__get_cpu_var(mce_irq_work));
+       irq_work_queue(this_cpu_ptr(&mce_irq_work));
 }
 
 /*
@@ -1045,7 +1045,7 @@ void do_machine_check(struct pt_regs *regs, long error_code)
 
        mce_gather_info(&m, regs);
 
-       final = &__get_cpu_var(mces_seen);
+       final = this_cpu_ptr(&mces_seen);
        *final = m;
 
        memset(valid_banks, 0, sizeof(valid_banks));
@@ -1278,22 +1278,22 @@ static unsigned long (*mce_adjust_timer)(unsigned long interval) =
 
 static int cmc_error_seen(void)
 {
-       unsigned long *v = &__get_cpu_var(mce_polled_error);
+       unsigned long *v = this_cpu_ptr(&mce_polled_error);
 
        return test_and_clear_bit(0, v);
 }
 
 static void mce_timer_fn(unsigned long data)
 {
-       struct timer_list *t = &__get_cpu_var(mce_timer);
+       struct timer_list *t = this_cpu_ptr(&mce_timer);
        unsigned long iv;
        int notify;
 
        WARN_ON(smp_processor_id() != data);
 
-       if (mce_available(__this_cpu_ptr(&cpu_info))) {
+       if (mce_available(this_cpu_ptr(&cpu_info))) {
                machine_check_poll(MCP_TIMESTAMP,
-                               &__get_cpu_var(mce_poll_banks));
+                               this_cpu_ptr(&mce_poll_banks));
                mce_intel_cmci_poll();
        }
 
@@ -1323,7 +1323,7 @@ static void mce_timer_fn(unsigned long data)
  */
 void mce_timer_kick(unsigned long interval)
 {
-       struct timer_list *t = &__get_cpu_var(mce_timer);
+       struct timer_list *t = this_cpu_ptr(&mce_timer);
        unsigned long when = jiffies + interval;
        unsigned long iv = __this_cpu_read(mce_next_interval);
 
@@ -1659,7 +1659,7 @@ static void mce_start_timer(unsigned int cpu, struct timer_list *t)
 
 static void __mcheck_cpu_init_timer(void)
 {
-       struct timer_list *t = &__get_cpu_var(mce_timer);
+       struct timer_list *t = this_cpu_ptr(&mce_timer);
        unsigned int cpu = smp_processor_id();
 
        setup_timer(t, mce_timer_fn, cpu);
@@ -1702,8 +1702,8 @@ void mcheck_cpu_init(struct cpuinfo_x86 *c)
        __mcheck_cpu_init_generic();
        __mcheck_cpu_init_vendor(c);
        __mcheck_cpu_init_timer();
-       INIT_WORK(&__get_cpu_var(mce_work), mce_process_work);
-       init_irq_work(&__get_cpu_var(mce_irq_work), &mce_irq_work_cb);
+       INIT_WORK(this_cpu_ptr(&mce_work), mce_process_work);
+       init_irq_work(this_cpu_ptr(&mce_irq_work), &mce_irq_work_cb);
 }
 
 /*
@@ -1955,7 +1955,7 @@ static struct miscdevice mce_chrdev_device = {
 static void __mce_disable_bank(void *arg)
 {
        int bank = *((int *)arg);
-       __clear_bit(bank, __get_cpu_var(mce_poll_banks));
+       __clear_bit(bank, this_cpu_ptr(mce_poll_banks));
        cmci_disable_bank(bank);
 }
 
@@ -2065,7 +2065,7 @@ static void mce_syscore_shutdown(void)
 static void mce_syscore_resume(void)
 {
        __mcheck_cpu_init_generic();
-       __mcheck_cpu_init_vendor(__this_cpu_ptr(&cpu_info));
+       __mcheck_cpu_init_vendor(raw_cpu_ptr(&cpu_info));
 }
 
 static struct syscore_ops mce_syscore_ops = {
@@ -2080,7 +2080,7 @@ static struct syscore_ops mce_syscore_ops = {
 
 static void mce_cpu_restart(void *data)
 {
-       if (!mce_available(__this_cpu_ptr(&cpu_info)))
+       if (!mce_available(raw_cpu_ptr(&cpu_info)))
                return;
        __mcheck_cpu_init_generic();
        __mcheck_cpu_init_timer();
@@ -2096,14 +2096,14 @@ static void mce_restart(void)
 /* Toggle features for corrected errors */
 static void mce_disable_cmci(void *data)
 {
-       if (!mce_available(__this_cpu_ptr(&cpu_info)))
+       if (!mce_available(raw_cpu_ptr(&cpu_info)))
                return;
        cmci_clear();
 }
 
 static void mce_enable_ce(void *all)
 {
-       if (!mce_available(__this_cpu_ptr(&cpu_info)))
+       if (!mce_available(raw_cpu_ptr(&cpu_info)))
                return;
        cmci_reenable();
        cmci_recheck();
@@ -2336,7 +2336,7 @@ static void mce_disable_cpu(void *h)
        unsigned long action = *(unsigned long *)h;
        int i;
 
-       if (!mce_available(__this_cpu_ptr(&cpu_info)))
+       if (!mce_available(raw_cpu_ptr(&cpu_info)))
                return;
 
        if (!(action & CPU_TASKS_FROZEN))
@@ -2354,7 +2354,7 @@ static void mce_reenable_cpu(void *h)
        unsigned long action = *(unsigned long *)h;
        int i;
 
-       if (!mce_available(__this_cpu_ptr(&cpu_info)))
+       if (!mce_available(raw_cpu_ptr(&cpu_info)))
                return;
 
        if (!(action & CPU_TASKS_FROZEN))

diff --git a/arch/x86/kernel/cpu/mcheck/mce_amd.c b/arch/x86/kernel/cpu/mcheck/mce_amd.c
index 1e49f8f41276edac73dfc8cbea3417693c6aedd2..5d4999f95aec54c70f237183fedd89ed10cad832 100644 (file)
--- a/arch/x86/kernel/cpu/mcheck/mce_amd.c
+++ b/arch/x86/kernel/cpu/mcheck/mce_amd.c
@@ -310,7 +310,7 @@ static void amd_threshold_interrupt(void)
                         * event.
                         */
                        machine_check_poll(MCP_TIMESTAMP,
-                                       &__get_cpu_var(mce_poll_banks));
+                                       this_cpu_ptr(&mce_poll_banks));
 
                        if (high & MASK_OVERFLOW_HI) {
                                rdmsrl(address, m.misc);

diff --git a/arch/x86/kernel/cpu/mcheck/mce_intel.c b/arch/x86/kernel/cpu/mcheck/mce_intel.c
index 3bdb95ae8c430fa8bacc76a9f644c4abce8ec35e..b3c97bafc1238fedd30f1ec0a3bab2ba7ee05cb8 100644 (file)
--- a/arch/x86/kernel/cpu/mcheck/mce_intel.c
+++ b/arch/x86/kernel/cpu/mcheck/mce_intel.c
@@ -86,7 +86,7 @@ void mce_intel_cmci_poll(void)
 {
        if (__this_cpu_read(cmci_storm_state) == CMCI_STORM_NONE)
                return;
-       machine_check_poll(MCP_TIMESTAMP, &__get_cpu_var(mce_banks_owned));
+       machine_check_poll(MCP_TIMESTAMP, this_cpu_ptr(&mce_banks_owned));
 }
 
 void mce_intel_hcpu_update(unsigned long cpu)
@@ -145,7 +145,7 @@ static void cmci_storm_disable_banks(void)
        u64 val;
 
        raw_spin_lock_irqsave(&cmci_discover_lock, flags);
-       owned = __get_cpu_var(mce_banks_owned);
+       owned = this_cpu_ptr(mce_banks_owned);
        for_each_set_bit(bank, owned, MAX_NR_BANKS) {
                rdmsrl(MSR_IA32_MCx_CTL2(bank), val);
                val &= ~MCI_CTL2_CMCI_EN;
@@ -195,7 +195,7 @@ static void intel_threshold_interrupt(void)
 {
        if (cmci_storm_detect())
                return;
-       machine_check_poll(MCP_TIMESTAMP, &__get_cpu_var(mce_banks_owned));
+       machine_check_poll(MCP_TIMESTAMP, this_cpu_ptr(&mce_banks_owned));
        mce_notify_irq();
 }
 
@@ -206,7 +206,7 @@ static void intel_threshold_interrupt(void)
  */
 static void cmci_discover(int banks)
 {
-       unsigned long *owned = (void *)&__get_cpu_var(mce_banks_owned);
+       unsigned long *owned = (void *)this_cpu_ptr(&mce_banks_owned);
        unsigned long flags;
        int i;
        int bios_wrong_thresh = 0;
@@ -228,7 +228,7 @@ static void cmci_discover(int banks)
                /* Already owned by someone else? */
                if (val & MCI_CTL2_CMCI_EN) {
                        clear_bit(i, owned);
-                       __clear_bit(i, __get_cpu_var(mce_poll_banks));
+                       __clear_bit(i, this_cpu_ptr(mce_poll_banks));
                        continue;
                }
 
@@ -252,7 +252,7 @@ static void cmci_discover(int banks)
                /* Did the enable bit stick? -- the bank supports CMCI */
                if (val & MCI_CTL2_CMCI_EN) {
                        set_bit(i, owned);
-                       __clear_bit(i, __get_cpu_var(mce_poll_banks));
+                       __clear_bit(i, this_cpu_ptr(mce_poll_banks));
                        /*
                         * We are able to set thresholds for some banks that
                         * had a threshold of 0. This means the BIOS has not
@@ -263,7 +263,7 @@ static void cmci_discover(int banks)
                                        (val & MCI_CTL2_CMCI_THRESHOLD_MASK))
                                bios_wrong_thresh = 1;
                } else {
-                       WARN_ON(!test_bit(i, __get_cpu_var(mce_poll_banks)));
+                       WARN_ON(!test_bit(i, this_cpu_ptr(mce_poll_banks)));
                }
        }
        raw_spin_unlock_irqrestore(&cmci_discover_lock, flags);
@@ -284,10 +284,10 @@ void cmci_recheck(void)
        unsigned long flags;
        int banks;
 
-       if (!mce_available(__this_cpu_ptr(&cpu_info)) || !cmci_supported(&banks))
+       if (!mce_available(raw_cpu_ptr(&cpu_info)) || !cmci_supported(&banks))
                return;
        local_irq_save(flags);
-       machine_check_poll(MCP_TIMESTAMP, &__get_cpu_var(mce_banks_owned));
+       machine_check_poll(MCP_TIMESTAMP, this_cpu_ptr(&mce_banks_owned));
        local_irq_restore(flags);
 }
 
@@ -296,12 +296,12 @@ static void __cmci_disable_bank(int bank)
 {
        u64 val;
 
-       if (!test_bit(bank, __get_cpu_var(mce_banks_owned)))
+       if (!test_bit(bank, this_cpu_ptr(mce_banks_owned)))
                return;
        rdmsrl(MSR_IA32_MCx_CTL2(bank), val);
        val &= ~MCI_CTL2_CMCI_EN;
        wrmsrl(MSR_IA32_MCx_CTL2(bank), val);
-       __clear_bit(bank, __get_cpu_var(mce_banks_owned));
+       __clear_bit(bank, this_cpu_ptr(mce_banks_owned));
 }
 
 /*

diff --git a/arch/x86/kernel/cpu/perf_event.c b/arch/x86/kernel/cpu/perf_event.c
index 2879ecdaac430c62710db3c90b326e6e25f60acf..5cd2b7967370310022109556844972d6de95de5c 100644 (file)
--- a/arch/x86/kernel/cpu/perf_event.c
+++ b/arch/x86/kernel/cpu/perf_event.c
@@ -487,7 +487,7 @@ static int __x86_pmu_event_init(struct perf_event *event)
 
 void x86_pmu_disable_all(void)
 {
-       struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
+       struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
        int idx;
 
        for (idx = 0; idx < x86_pmu.num_counters; idx++) {
@@ -505,7 +505,7 @@ void x86_pmu_disable_all(void)
 
 static void x86_pmu_disable(struct pmu *pmu)
 {
-       struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
+       struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 
        if (!x86_pmu_initialized())
                return;
@@ -522,7 +522,7 @@ static void x86_pmu_disable(struct pmu *pmu)
 
 void x86_pmu_enable_all(int added)
 {
-       struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
+       struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
        int idx;
 
        for (idx = 0; idx < x86_pmu.num_counters; idx++) {
@@ -869,7 +869,7 @@ static void x86_pmu_start(struct perf_event *event, int flags);
 
 static void x86_pmu_enable(struct pmu *pmu)
 {
-       struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
+       struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
        struct perf_event *event;
        struct hw_perf_event *hwc;
        int i, added = cpuc->n_added;
@@ -1020,7 +1020,7 @@ void x86_pmu_enable_event(struct perf_event *event)
  */
 static int x86_pmu_add(struct perf_event *event, int flags)
 {
-       struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
+       struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
        struct hw_perf_event *hwc;
        int assign[X86_PMC_IDX_MAX];
        int n, n0, ret;
@@ -1071,7 +1071,7 @@ out:
 
 static void x86_pmu_start(struct perf_event *event, int flags)
 {
-       struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
+       struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
        int idx = event->hw.idx;
 
        if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED)))
@@ -1150,7 +1150,7 @@ void perf_event_print_debug(void)
 
 void x86_pmu_stop(struct perf_event *event, int flags)
 {
-       struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
+       struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
        struct hw_perf_event *hwc = &event->hw;
 
        if (__test_and_clear_bit(hwc->idx, cpuc->active_mask)) {
@@ -1172,7 +1172,7 @@ void x86_pmu_stop(struct perf_event *event, int flags)
 
 static void x86_pmu_del(struct perf_event *event, int flags)
 {
-       struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
+       struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
        int i;
 
        /*
@@ -1227,7 +1227,7 @@ int x86_pmu_handle_irq(struct pt_regs *regs)
        int idx, handled = 0;
        u64 val;
 
-       cpuc = &__get_cpu_var(cpu_hw_events);
+       cpuc = this_cpu_ptr(&cpu_hw_events);
 
        /*
         * Some chipsets need to unmask the LVTPC in a particular spot
@@ -1636,7 +1636,7 @@ static void x86_pmu_cancel_txn(struct pmu *pmu)
  */
 static int x86_pmu_commit_txn(struct pmu *pmu)
 {
-       struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
+       struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
        int assign[X86_PMC_IDX_MAX];
        int n, ret;
 
@@ -1995,7 +1995,7 @@ static unsigned long get_segment_base(unsigned int segment)
                if (idx > GDT_ENTRIES)
                        return 0;
 
-               desc = __this_cpu_ptr(&gdt_page.gdt[0]);
+               desc = raw_cpu_ptr(gdt_page.gdt);
        }
 
        return get_desc_base(desc + idx);

diff --git a/arch/x86/kernel/cpu/perf_event_amd.c b/arch/x86/kernel/cpu/perf_event_amd.c
index beeb7cc0704400aaec6d62479331bfe089bef7eb..28926311aac1833b72a932e7108916e25bf4804d 100644 (file)
--- a/arch/x86/kernel/cpu/perf_event_amd.c
+++ b/arch/x86/kernel/cpu/perf_event_amd.c
@@ -699,7 +699,7 @@ __init int amd_pmu_init(void)
 
 void amd_pmu_enable_virt(void)
 {
-       struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
+       struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 
        cpuc->perf_ctr_virt_mask = 0;
 
@@ -711,7 +711,7 @@ EXPORT_SYMBOL_GPL(amd_pmu_enable_virt);
 
 void amd_pmu_disable_virt(void)
 {
-       struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
+       struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 
        /*
         * We only mask out the Host-only bit so that host-only counting works

diff --git a/arch/x86/kernel/cpu/perf_event_intel.c b/arch/x86/kernel/cpu/perf_event_intel.c
index 2502d0d9d246a1fe63c57070176b2bcc04cad3bb..6f80accf137d4dfb72a6b221c5c058a77f80017f 100644 (file)
--- a/arch/x86/kernel/cpu/perf_event_intel.c
+++ b/arch/x86/kernel/cpu/perf_event_intel.c
@@ -1045,7 +1045,7 @@ static inline bool intel_pmu_needs_lbr_smpl(struct perf_event *event)
 
 static void intel_pmu_disable_all(void)
 {
-       struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
+       struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 
        wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, 0);
 
@@ -1058,7 +1058,7 @@ static void intel_pmu_disable_all(void)
 
 static void intel_pmu_enable_all(int added)
 {
-       struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
+       struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 
        intel_pmu_pebs_enable_all();
        intel_pmu_lbr_enable_all();
@@ -1092,7 +1092,7 @@ static void intel_pmu_enable_all(int added)
  */
 static void intel_pmu_nhm_workaround(void)
 {
-       struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
+       struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
        static const unsigned long nhm_magic[4] = {
                0x4300B5,
                0x4300D2,
@@ -1191,7 +1191,7 @@ static inline bool event_is_checkpointed(struct perf_event *event)
 static void intel_pmu_disable_event(struct perf_event *event)
 {
        struct hw_perf_event *hwc = &event->hw;
-       struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
+       struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 
        if (unlikely(hwc->idx == INTEL_PMC_IDX_FIXED_BTS)) {
                intel_pmu_disable_bts();
@@ -1255,7 +1255,7 @@ static void intel_pmu_enable_fixed(struct hw_perf_event *hwc)
 static void intel_pmu_enable_event(struct perf_event *event)
 {
        struct hw_perf_event *hwc = &event->hw;
-       struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
+       struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 
        if (unlikely(hwc->idx == INTEL_PMC_IDX_FIXED_BTS)) {
                if (!__this_cpu_read(cpu_hw_events.enabled))
@@ -1349,7 +1349,7 @@ static int intel_pmu_handle_irq(struct pt_regs *regs)
        u64 status;
        int handled;
 
-       cpuc = &__get_cpu_var(cpu_hw_events);
+       cpuc = this_cpu_ptr(&cpu_hw_events);
 
        /*
         * No known reason to not always do late ACK,
@@ -1781,7 +1781,7 @@ EXPORT_SYMBOL_GPL(perf_guest_get_msrs);
 
 static struct perf_guest_switch_msr *intel_guest_get_msrs(int *nr)
 {
-       struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
+       struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
        struct perf_guest_switch_msr *arr = cpuc->guest_switch_msrs;
 
        arr[0].msr = MSR_CORE_PERF_GLOBAL_CTRL;
@@ -1802,7 +1802,7 @@ static struct perf_guest_switch_msr *intel_guest_get_msrs(int *nr)
 
 static struct perf_guest_switch_msr *core_guest_get_msrs(int *nr)
 {
-       struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
+       struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
        struct perf_guest_switch_msr *arr = cpuc->guest_switch_msrs;
        int idx;
 
@@ -1836,7 +1836,7 @@ static void core_pmu_enable_event(struct perf_event *event)
 
 static void core_pmu_enable_all(int added)
 {
-       struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
+       struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
        int idx;
 
        for (idx = 0; idx < x86_pmu.num_counters; idx++) {

diff --git a/arch/x86/kernel/cpu/perf_event_intel_ds.c b/arch/x86/kernel/cpu/perf_event_intel_ds.c
index 696ade311ded7d01103323d159cff621b46d4a89..7b786b369789eefd82ea01c6f4b0d3abc8763fc9 100644 (file)
--- a/arch/x86/kernel/cpu/perf_event_intel_ds.c
+++ b/arch/x86/kernel/cpu/perf_event_intel_ds.c
@@ -475,7 +475,7 @@ void intel_pmu_enable_bts(u64 config)
 
 void intel_pmu_disable_bts(void)
 {
-       struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
+       struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
        unsigned long debugctlmsr;
 
        if (!cpuc->ds)
@@ -492,7 +492,7 @@ void intel_pmu_disable_bts(void)
 
 int intel_pmu_drain_bts_buffer(void)
 {
-       struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
+       struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
        struct debug_store *ds = cpuc->ds;
        struct bts_record {
                u64     from;
@@ -712,7 +712,7 @@ struct event_constraint *intel_pebs_constraints(struct perf_event *event)
 
 void intel_pmu_pebs_enable(struct perf_event *event)
 {
-       struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
+       struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
        struct hw_perf_event *hwc = &event->hw;
 
        hwc->config &= ~ARCH_PERFMON_EVENTSEL_INT;
@@ -727,7 +727,7 @@ void intel_pmu_pebs_enable(struct perf_event *event)
 
 void intel_pmu_pebs_disable(struct perf_event *event)
 {
-       struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
+       struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
        struct hw_perf_event *hwc = &event->hw;
 
        cpuc->pebs_enabled &= ~(1ULL << hwc->idx);
@@ -745,7 +745,7 @@ void intel_pmu_pebs_disable(struct perf_event *event)
 
 void intel_pmu_pebs_enable_all(void)
 {
-       struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
+       struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 
        if (cpuc->pebs_enabled)
                wrmsrl(MSR_IA32_PEBS_ENABLE, cpuc->pebs_enabled);
@@ -753,7 +753,7 @@ void intel_pmu_pebs_enable_all(void)
 
 void intel_pmu_pebs_disable_all(void)
 {
-       struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
+       struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 
        if (cpuc->pebs_enabled)
                wrmsrl(MSR_IA32_PEBS_ENABLE, 0);
@@ -761,7 +761,7 @@ void intel_pmu_pebs_disable_all(void)
 
 static int intel_pmu_pebs_fixup_ip(struct pt_regs *regs)
 {
-       struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
+       struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
        unsigned long from = cpuc->lbr_entries[0].from;
        unsigned long old_to, to = cpuc->lbr_entries[0].to;
        unsigned long ip = regs->ip;
@@ -868,7 +868,7 @@ static void __intel_pmu_pebs_event(struct perf_event *event,
         * We cast to the biggest pebs_record but are careful not to
         * unconditionally access the 'extra' entries.
         */
-       struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
+       struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
        struct pebs_record_hsw *pebs = __pebs;
        struct perf_sample_data data;
        struct pt_regs regs;
@@ -957,7 +957,7 @@ static void __intel_pmu_pebs_event(struct perf_event *event,
 
 static void intel_pmu_drain_pebs_core(struct pt_regs *iregs)
 {
-       struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
+       struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
        struct debug_store *ds = cpuc->ds;
        struct perf_event *event = cpuc->events[0]; /* PMC0 only */
        struct pebs_record_core *at, *top;
@@ -998,7 +998,7 @@ static void intel_pmu_drain_pebs_core(struct pt_regs *iregs)
 
 static void intel_pmu_drain_pebs_nhm(struct pt_regs *iregs)
 {
-       struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
+       struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
        struct debug_store *ds = cpuc->ds;
        struct perf_event *event = NULL;
        void *at, *top;

diff --git a/arch/x86/kernel/cpu/perf_event_intel_lbr.c b/arch/x86/kernel/cpu/perf_event_intel_lbr.c
index 9dd2459a4c738d99bb4a75ebf51062bcbf6b21de..ebb0d3144551342422dc03e78886bb9baad1fac0 100644 (file)
--- a/arch/x86/kernel/cpu/perf_event_intel_lbr.c
+++ b/arch/x86/kernel/cpu/perf_event_intel_lbr.c
@@ -133,7 +133,7 @@ static void intel_pmu_lbr_filter(struct cpu_hw_events *cpuc);
 static void __intel_pmu_lbr_enable(void)
 {
        u64 debugctl;
-       struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
+       struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 
        if (cpuc->lbr_sel)
                wrmsrl(MSR_LBR_SELECT, cpuc->lbr_sel->config);
@@ -183,7 +183,7 @@ void intel_pmu_lbr_reset(void)
 
 void intel_pmu_lbr_enable(struct perf_event *event)
 {
-       struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
+       struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 
        if (!x86_pmu.lbr_nr)
                return;
@@ -203,7 +203,7 @@ void intel_pmu_lbr_enable(struct perf_event *event)
 
 void intel_pmu_lbr_disable(struct perf_event *event)
 {
-       struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
+       struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 
        if (!x86_pmu.lbr_nr)
                return;
@@ -220,7 +220,7 @@ void intel_pmu_lbr_disable(struct perf_event *event)
 
 void intel_pmu_lbr_enable_all(void)
 {
-       struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
+       struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 
        if (cpuc->lbr_users)
                __intel_pmu_lbr_enable();
@@ -228,7 +228,7 @@ void intel_pmu_lbr_enable_all(void)
 
 void intel_pmu_lbr_disable_all(void)
 {
-       struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
+       struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 
        if (cpuc->lbr_users)
                __intel_pmu_lbr_disable();
@@ -332,7 +332,7 @@ static void intel_pmu_lbr_read_64(struct cpu_hw_events *cpuc)
 
 void intel_pmu_lbr_read(void)
 {
-       struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
+       struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 
        if (!cpuc->lbr_users)
                return;

diff --git a/arch/x86/kernel/cpu/perf_event_intel_rapl.c b/arch/x86/kernel/cpu/perf_event_intel_rapl.c
index 619f7699487aa1ec60f5a2687bede8e19d0a2c7a..d64f275fe274a14df186eb31a247f04d683058eb 100644 (file)
--- a/arch/x86/kernel/cpu/perf_event_intel_rapl.c
+++ b/arch/x86/kernel/cpu/perf_event_intel_rapl.c
@@ -135,7 +135,7 @@ static inline u64 rapl_scale(u64 v)
         * or use ldexp(count, -32).
         * Watts = Joules/Time delta
         */
-       return v << (32 - __get_cpu_var(rapl_pmu)->hw_unit);
+       return v << (32 - __this_cpu_read(rapl_pmu->hw_unit));
 }
 
 static u64 rapl_event_update(struct perf_event *event)
@@ -187,7 +187,7 @@ static void rapl_stop_hrtimer(struct rapl_pmu *pmu)
 
 static enum hrtimer_restart rapl_hrtimer_handle(struct hrtimer *hrtimer)
 {
-       struct rapl_pmu *pmu = __get_cpu_var(rapl_pmu);
+       struct rapl_pmu *pmu = __this_cpu_read(rapl_pmu);
        struct perf_event *event;
        unsigned long flags;
 
@@ -234,7 +234,7 @@ static void __rapl_pmu_event_start(struct rapl_pmu *pmu,
 
 static void rapl_pmu_event_start(struct perf_event *event, int mode)
 {
-       struct rapl_pmu *pmu = __get_cpu_var(rapl_pmu);
+       struct rapl_pmu *pmu = __this_cpu_read(rapl_pmu);
        unsigned long flags;
 
        spin_lock_irqsave(&pmu->lock, flags);
@@ -244,7 +244,7 @@ static void rapl_pmu_event_start(struct perf_event *event, int mode)
 
 static void rapl_pmu_event_stop(struct perf_event *event, int mode)
 {
-       struct rapl_pmu *pmu = __get_cpu_var(rapl_pmu);
+       struct rapl_pmu *pmu = __this_cpu_read(rapl_pmu);
        struct hw_perf_event *hwc = &event->hw;
        unsigned long flags;
 
@@ -278,7 +278,7 @@ static void rapl_pmu_event_stop(struct perf_event *event, int mode)
 
 static int rapl_pmu_event_add(struct perf_event *event, int mode)
 {
-       struct rapl_pmu *pmu = __get_cpu_var(rapl_pmu);
+       struct rapl_pmu *pmu = __this_cpu_read(rapl_pmu);
        struct hw_perf_event *hwc = &event->hw;
        unsigned long flags;
 
@@ -696,7 +696,7 @@ static int __init rapl_pmu_init(void)
                return -1;
        }
 
-       pmu = __get_cpu_var(rapl_pmu);
+       pmu = __this_cpu_read(rapl_pmu);
 
        pr_info("RAPL PMU detected, hw unit 2^-%d Joules,"
                " API unit is 2^-32 Joules,"

diff --git a/arch/x86/kernel/cpu/perf_event_knc.c b/arch/x86/kernel/cpu/perf_event_knc.c
index 838fa8772c628822cd3bd26b6949af47886fbd4f..5b0c232d1ee62e3e50223ba1d8ac27364e2d1ad5 100644 (file)
--- a/arch/x86/kernel/cpu/perf_event_knc.c
+++ b/arch/x86/kernel/cpu/perf_event_knc.c
@@ -217,7 +217,7 @@ static int knc_pmu_handle_irq(struct pt_regs *regs)
        int bit, loops;
        u64 status;
 
-       cpuc = &__get_cpu_var(cpu_hw_events);
+       cpuc = this_cpu_ptr(&cpu_hw_events);
 
        knc_pmu_disable_all();
 

diff --git a/arch/x86/kernel/cpu/perf_event_p4.c b/arch/x86/kernel/cpu/perf_event_p4.c
index 5d466b7d8609814355c90e2cf09ced760debd458..f2e56783af3d83b89de8fa39ae40693cbac2f375 100644 (file)
--- a/arch/x86/kernel/cpu/perf_event_p4.c
+++ b/arch/x86/kernel/cpu/perf_event_p4.c
@@ -915,7 +915,7 @@ static inline void p4_pmu_disable_event(struct perf_event *event)
 
 static void p4_pmu_disable_all(void)
 {
-       struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
+       struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
        int idx;
 
        for (idx = 0; idx < x86_pmu.num_counters; idx++) {
@@ -984,7 +984,7 @@ static void p4_pmu_enable_event(struct perf_event *event)
 
 static void p4_pmu_enable_all(int added)
 {
-       struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
+       struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
        int idx;
 
        for (idx = 0; idx < x86_pmu.num_counters; idx++) {
@@ -1004,7 +1004,7 @@ static int p4_pmu_handle_irq(struct pt_regs *regs)
        int idx, handled = 0;
        u64 val;
 
-       cpuc = &__get_cpu_var(cpu_hw_events);
+       cpuc = this_cpu_ptr(&cpu_hw_events);
 
        for (idx = 0; idx < x86_pmu.num_counters; idx++) {
                int overflow;

diff --git a/arch/x86/kernel/hw_breakpoint.c b/arch/x86/kernel/hw_breakpoint.c
index 5f9cf20cdb68025ae7175860fba5abab136ef9f7..3d5fb509bdebc300af21de5079b7ed61c16a54e4 100644 (file)
--- a/arch/x86/kernel/hw_breakpoint.c
+++ b/arch/x86/kernel/hw_breakpoint.c
@@ -108,7 +108,7 @@ int arch_install_hw_breakpoint(struct perf_event *bp)
        int i;
 
        for (i = 0; i < HBP_NUM; i++) {
-               struct perf_event **slot = &__get_cpu_var(bp_per_reg[i]);
+               struct perf_event **slot = this_cpu_ptr(&bp_per_reg[i]);
 
                if (!*slot) {
                        *slot = bp;
@@ -122,7 +122,7 @@ int arch_install_hw_breakpoint(struct perf_event *bp)
        set_debugreg(info->address, i);
        __this_cpu_write(cpu_debugreg[i], info->address);
 
-       dr7 = &__get_cpu_var(cpu_dr7);
+       dr7 = this_cpu_ptr(&cpu_dr7);
        *dr7 |= encode_dr7(i, info->len, info->type);
 
        set_debugreg(*dr7, 7);
@@ -146,7 +146,7 @@ void arch_uninstall_hw_breakpoint(struct perf_event *bp)
        int i;
 
        for (i = 0; i < HBP_NUM; i++) {
-               struct perf_event **slot = &__get_cpu_var(bp_per_reg[i]);
+               struct perf_event **slot = this_cpu_ptr(&bp_per_reg[i]);
 
                if (*slot == bp) {
                        *slot = NULL;
@@ -157,7 +157,7 @@ void arch_uninstall_hw_breakpoint(struct perf_event *bp)
        if (WARN_ONCE(i == HBP_NUM, "Can't find any breakpoint slot"))
                return;
 
-       dr7 = &__get_cpu_var(cpu_dr7);
+       dr7 = this_cpu_ptr(&cpu_dr7);
        *dr7 &= ~__encode_dr7(i, info->len, info->type);
 
        set_debugreg(*dr7, 7);

diff --git a/arch/x86/kernel/irq_64.c b/arch/x86/kernel/irq_64.c
index 4d1c746892eb9a9aae2e38dedfec88868f347cff..e4b503d5558c5d435eddc3c3c5be8c55f1de558d 100644 (file)
--- a/arch/x86/kernel/irq_64.c
+++ b/arch/x86/kernel/irq_64.c
@@ -52,13 +52,13 @@ static inline void stack_overflow_check(struct pt_regs *regs)
            regs->sp <= curbase + THREAD_SIZE)
                return;
 
-       irq_stack_top = (u64)__get_cpu_var(irq_stack_union.irq_stack) +
+       irq_stack_top = (u64)this_cpu_ptr(irq_stack_union.irq_stack) +
                        STACK_TOP_MARGIN;
-       irq_stack_bottom = (u64)__get_cpu_var(irq_stack_ptr);
+       irq_stack_bottom = (u64)__this_cpu_read(irq_stack_ptr);
        if (regs->sp >= irq_stack_top && regs->sp <= irq_stack_bottom)
                return;
 
-       oist = &__get_cpu_var(orig_ist);
+       oist = this_cpu_ptr(&orig_ist);
        estack_top = (u64)oist->ist[0] - EXCEPTION_STKSZ + STACK_TOP_MARGIN;
        estack_bottom = (u64)oist->ist[N_EXCEPTION_STACKS - 1];
        if (regs->sp >= estack_top && regs->sp <= estack_bottom)

diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c
index 3dd8e2c4d74a9ed4a124baf36a83de78be1be02d..2b68102dbbebe4d5d3074bd2dc24a4f6b7b4fd26 100644 (file)
--- a/arch/x86/kernel/kvm.c
+++ b/arch/x86/kernel/kvm.c
@@ -243,9 +243,9 @@ u32 kvm_read_and_reset_pf_reason(void)
 {
        u32 reason = 0;
 
-       if (__get_cpu_var(apf_reason).enabled) {
-               reason = __get_cpu_var(apf_reason).reason;
-               __get_cpu_var(apf_reason).reason = 0;
+       if (__this_cpu_read(apf_reason.enabled)) {
+               reason = __this_cpu_read(apf_reason.reason);
+               __this_cpu_write(apf_reason.reason, 0);
        }
 
        return reason;
@@ -318,7 +318,7 @@ static void kvm_guest_apic_eoi_write(u32 reg, u32 val)
         * there's no need for lock or memory barriers.
         * An optimization barrier is implied in apic write.
         */
-       if (__test_and_clear_bit(KVM_PV_EOI_BIT, &__get_cpu_var(kvm_apic_eoi)))
+       if (__test_and_clear_bit(KVM_PV_EOI_BIT, this_cpu_ptr(&kvm_apic_eoi)))
                return;
        apic_write(APIC_EOI, APIC_EOI_ACK);
 }
@@ -329,13 +329,13 @@ void kvm_guest_cpu_init(void)
                return;
 
        if (kvm_para_has_feature(KVM_FEATURE_ASYNC_PF) && kvmapf) {
-               u64 pa = slow_virt_to_phys(&__get_cpu_var(apf_reason));
+               u64 pa = slow_virt_to_phys(this_cpu_ptr(&apf_reason));
 
 #ifdef CONFIG_PREEMPT
                pa |= KVM_ASYNC_PF_SEND_ALWAYS;
 #endif
                wrmsrl(MSR_KVM_ASYNC_PF_EN, pa | KVM_ASYNC_PF_ENABLED);
-               __get_cpu_var(apf_reason).enabled = 1;
+               __this_cpu_write(apf_reason.enabled, 1);
                printk(KERN_INFO"KVM setup async PF for cpu %d\n",
                       smp_processor_id());
        }
@@ -344,8 +344,8 @@ void kvm_guest_cpu_init(void)
                unsigned long pa;
                /* Size alignment is implied but just to make it explicit. */
                BUILD_BUG_ON(__alignof__(kvm_apic_eoi) < 4);
-               __get_cpu_var(kvm_apic_eoi) = 0;
-               pa = slow_virt_to_phys(&__get_cpu_var(kvm_apic_eoi))
+               __this_cpu_write(kvm_apic_eoi, 0);
+               pa = slow_virt_to_phys(this_cpu_ptr(&kvm_apic_eoi))
                        | KVM_MSR_ENABLED;
                wrmsrl(MSR_KVM_PV_EOI_EN, pa);
        }
@@ -356,11 +356,11 @@ void kvm_guest_cpu_init(void)
 
 static void kvm_pv_disable_apf(void)
 {
-       if (!__get_cpu_var(apf_reason).enabled)
+       if (!__this_cpu_read(apf_reason.enabled))
                return;
 
        wrmsrl(MSR_KVM_ASYNC_PF_EN, 0);
-       __get_cpu_var(apf_reason).enabled = 0;
+       __this_cpu_write(apf_reason.enabled, 0);
 
        printk(KERN_INFO"Unregister pv shared memory for cpu %d\n",
               smp_processor_id());
@@ -716,7 +716,7 @@ __visible void kvm_lock_spinning(struct arch_spinlock *lock, __ticket_t want)
        if (in_nmi())
                return;
 
-       w = &__get_cpu_var(klock_waiting);
+       w = this_cpu_ptr(&klock_waiting);
        cpu = smp_processor_id();
        start = spin_time_start();
 

diff --git a/arch/x86/kvm/svm.c b/arch/x86/kvm/svm.c
index ddf742768ecf2f823b7e8462a8ccda2c45225f2f..1b0e90658d8d741b7419ff2b5b002f2b1146fe9f 100644 (file)
--- a/arch/x86/kvm/svm.c
+++ b/arch/x86/kvm/svm.c
@@ -670,7 +670,7 @@ static int svm_hardware_enable(void *garbage)
 
        if (static_cpu_has(X86_FEATURE_TSCRATEMSR)) {
                wrmsrl(MSR_AMD64_TSC_RATIO, TSC_RATIO_DEFAULT);
-               __get_cpu_var(current_tsc_ratio) = TSC_RATIO_DEFAULT;
+               __this_cpu_write(current_tsc_ratio, TSC_RATIO_DEFAULT);
        }
 
 
@@ -1312,8 +1312,8 @@ static void svm_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
                rdmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]);
 
        if (static_cpu_has(X86_FEATURE_TSCRATEMSR) &&
-           svm->tsc_ratio != __get_cpu_var(current_tsc_ratio)) {
-               __get_cpu_var(current_tsc_ratio) = svm->tsc_ratio;
+           svm->tsc_ratio != __this_cpu_read(current_tsc_ratio)) {
+               __this_cpu_write(current_tsc_ratio, svm->tsc_ratio);
                wrmsrl(MSR_AMD64_TSC_RATIO, svm->tsc_ratio);
        }
 }

diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c
index bfe11cf124a1ea26cfa8513ceac83d450e13c7ef..36cf28a910b8e42b1c94e6dc35c9b3e15cbb4556 100644 (file)
--- a/arch/x86/kvm/vmx.c
+++ b/arch/x86/kvm/vmx.c
@@ -1601,7 +1601,7 @@ static void reload_tss(void)
        /*
         * VT restores TR but not its size.  Useless.
         */
-       struct desc_ptr *gdt = &__get_cpu_var(host_gdt);
+       struct desc_ptr *gdt = this_cpu_ptr(&host_gdt);
        struct desc_struct *descs;
 
        descs = (void *)gdt->address;
@@ -1647,7 +1647,7 @@ static bool update_transition_efer(struct vcpu_vmx *vmx, int efer_offset)
 
 static unsigned long segment_base(u16 selector)
 {
-       struct desc_ptr *gdt = &__get_cpu_var(host_gdt);
+       struct desc_ptr *gdt = this_cpu_ptr(&host_gdt);
        struct desc_struct *d;
        unsigned long table_base;
        unsigned long v;
@@ -1777,7 +1777,7 @@ static void __vmx_load_host_state(struct vcpu_vmx *vmx)
         */
        if (!user_has_fpu() && !vmx->vcpu.guest_fpu_loaded)
                stts();
-       load_gdt(&__get_cpu_var(host_gdt));
+       load_gdt(this_cpu_ptr(&host_gdt));
 }
 
 static void vmx_load_host_state(struct vcpu_vmx *vmx)
@@ -1807,7 +1807,7 @@ static void vmx_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
        }
 
        if (vmx->loaded_vmcs->cpu != cpu) {
-               struct desc_ptr *gdt = &__get_cpu_var(host_gdt);
+               struct desc_ptr *gdt = this_cpu_ptr(&host_gdt);
                unsigned long sysenter_esp;
 
                kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
@@ -2744,7 +2744,7 @@ static int hardware_enable(void *garbage)
                ept_sync_global();
        }
 
-       native_store_gdt(&__get_cpu_var(host_gdt));
+       native_store_gdt(this_cpu_ptr(&host_gdt));
 
        return 0;
 }

diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 8f1e22d3b286ccf03c90dc3bc28f120fe27a244b..c84ee536f9a35d0ff9bcbc1d6f602282e5a47dd8 100644 (file)
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -1556,7 +1556,7 @@ static int kvm_guest_time_update(struct kvm_vcpu *v)
 
        /* Keep irq disabled to prevent changes to the clock */
        local_irq_save(flags);
-       this_tsc_khz = __get_cpu_var(cpu_tsc_khz);
+       this_tsc_khz = __this_cpu_read(cpu_tsc_khz);
        if (unlikely(this_tsc_khz == 0)) {
                local_irq_restore(flags);
                kvm_make_request(KVM_REQ_CLOCK_UPDATE, v);

diff --git a/arch/x86/mm/kmemcheck/kmemcheck.c b/arch/x86/mm/kmemcheck/kmemcheck.c
index dd89a13f1051adcb18d3565a495a7d34b63b4f5a..b4f2e7e9e90785534e10d017a94e787aa5e8fc03 100644 (file)
--- a/arch/x86/mm/kmemcheck/kmemcheck.c
+++ b/arch/x86/mm/kmemcheck/kmemcheck.c
@@ -140,7 +140,7 @@ static DEFINE_PER_CPU(struct kmemcheck_context, kmemcheck_context);
 
 bool kmemcheck_active(struct pt_regs *regs)
 {
-       struct kmemcheck_context *data = &__get_cpu_var(kmemcheck_context);
+       struct kmemcheck_context *data = this_cpu_ptr(&kmemcheck_context);
 
        return data->balance > 0;
 }
@@ -148,7 +148,7 @@ bool kmemcheck_active(struct pt_regs *regs)
 /* Save an address that needs to be shown/hidden */
 static void kmemcheck_save_addr(unsigned long addr)
 {
-       struct kmemcheck_context *data = &__get_cpu_var(kmemcheck_context);
+       struct kmemcheck_context *data = this_cpu_ptr(&kmemcheck_context);
 
        BUG_ON(data->n_addrs >= ARRAY_SIZE(data->addr));
        data->addr[data->n_addrs++] = addr;
@@ -156,7 +156,7 @@ static void kmemcheck_save_addr(unsigned long addr)
 
 static unsigned int kmemcheck_show_all(void)
 {
-       struct kmemcheck_context *data = &__get_cpu_var(kmemcheck_context);
+       struct kmemcheck_context *data = this_cpu_ptr(&kmemcheck_context);
        unsigned int i;
        unsigned int n;
 
@@ -169,7 +169,7 @@ static unsigned int kmemcheck_show_all(void)
 
 static unsigned int kmemcheck_hide_all(void)
 {
-       struct kmemcheck_context *data = &__get_cpu_var(kmemcheck_context);
+       struct kmemcheck_context *data = this_cpu_ptr(&kmemcheck_context);
        unsigned int i;
        unsigned int n;
 
@@ -185,7 +185,7 @@ static unsigned int kmemcheck_hide_all(void)
  */
 void kmemcheck_show(struct pt_regs *regs)
 {
-       struct kmemcheck_context *data = &__get_cpu_var(kmemcheck_context);
+       struct kmemcheck_context *data = this_cpu_ptr(&kmemcheck_context);
 
        BUG_ON(!irqs_disabled());
 
@@ -226,7 +226,7 @@ void kmemcheck_show(struct pt_regs *regs)
  */
 void kmemcheck_hide(struct pt_regs *regs)
 {
-       struct kmemcheck_context *data = &__get_cpu_var(kmemcheck_context);
+       struct kmemcheck_context *data = this_cpu_ptr(&kmemcheck_context);
        int n;
 
        BUG_ON(!irqs_disabled());
@@ -528,7 +528,7 @@ static void kmemcheck_access(struct pt_regs *regs,
        const uint8_t *insn_primary;
        unsigned int size;
 
-       struct kmemcheck_context *data = &__get_cpu_var(kmemcheck_context);
+       struct kmemcheck_context *data = this_cpu_ptr(&kmemcheck_context);
 
        /* Recursive fault -- ouch. */
        if (data->busy) {

diff --git a/arch/x86/oprofile/nmi_int.c b/arch/x86/oprofile/nmi_int.c
index 379e8bd0deeabf8bb1e839ea6bcb11f0bc1dd12b..1d2e6392f5faaf6f450cdfa598b9b2a734064962 100644 (file)
--- a/arch/x86/oprofile/nmi_int.c
+++ b/arch/x86/oprofile/nmi_int.c
@@ -64,11 +64,11 @@ u64 op_x86_get_ctrl(struct op_x86_model_spec const *model,
 static int profile_exceptions_notify(unsigned int val, struct pt_regs *regs)
 {
        if (ctr_running)
-               model->check_ctrs(regs, &__get_cpu_var(cpu_msrs));
+               model->check_ctrs(regs, this_cpu_ptr(&cpu_msrs));
        else if (!nmi_enabled)
                return NMI_DONE;
        else
-               model->stop(&__get_cpu_var(cpu_msrs));
+               model->stop(this_cpu_ptr(&cpu_msrs));
        return NMI_HANDLED;
 }
 
@@ -91,7 +91,7 @@ static void nmi_cpu_save_registers(struct op_msrs *msrs)
 
 static void nmi_cpu_start(void *dummy)
 {
-       struct op_msrs const *msrs = &__get_cpu_var(cpu_msrs);
+       struct op_msrs const *msrs = this_cpu_ptr(&cpu_msrs);
        if (!msrs->controls)
                WARN_ON_ONCE(1);
        else
@@ -111,7 +111,7 @@ static int nmi_start(void)
 
 static void nmi_cpu_stop(void *dummy)
 {
-       struct op_msrs const *msrs = &__get_cpu_var(cpu_msrs);
+       struct op_msrs const *msrs = this_cpu_ptr(&cpu_msrs);
        if (!msrs->controls)
                WARN_ON_ONCE(1);
        else

diff --git a/arch/x86/platform/uv/uv_time.c b/arch/x86/platform/uv/uv_time.c
index 5c86786bbfd2e363a60c88e78ea73f6fb22753eb..a244237f3cfa834af62fb81eabed244548a50f90 100644 (file)
--- a/arch/x86/platform/uv/uv_time.c
+++ b/arch/x86/platform/uv/uv_time.c
@@ -365,7 +365,7 @@ __setup("uvrtcevt", uv_enable_evt_rtc);
 
 static __init void uv_rtc_register_clockevents(struct work_struct *dummy)
 {
-       struct clock_event_device *ced = &__get_cpu_var(cpu_ced);
+       struct clock_event_device *ced = this_cpu_ptr(&cpu_ced);
 
        *ced = clock_event_device_uv;
        ced->cpumask = cpumask_of(smp_processor_id());

diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c
index c0cb11fb500895f60b91e8ad1af679d878d40903..2628ee5567567060250303128900635e0bb44899 100644 (file)
--- a/arch/x86/xen/enlighten.c
+++ b/arch/x86/xen/enlighten.c
@@ -821,7 +821,7 @@ static void xen_convert_trap_info(const struct desc_ptr *desc,
 
 void xen_copy_trap_info(struct trap_info *traps)
 {
-       const struct desc_ptr *desc = &__get_cpu_var(idt_desc);
+       const struct desc_ptr *desc = this_cpu_ptr(&idt_desc);
 
        xen_convert_trap_info(desc, traps);
 }
@@ -838,7 +838,7 @@ static void xen_load_idt(const struct desc_ptr *desc)
 
        spin_lock(&lock);
 
-       __get_cpu_var(idt_desc) = *desc;
+       memcpy(this_cpu_ptr(&idt_desc), desc, sizeof(idt_desc));
 
        xen_convert_trap_info(desc, traps);
 

diff --git a/arch/x86/xen/multicalls.c b/arch/x86/xen/multicalls.c
index 0d82003e76adab8afe2ef3848f0c4384cd99601e..ea54a08d83012dce01f542e46cbaf453a0881ba5 100644 (file)
--- a/arch/x86/xen/multicalls.c
+++ b/arch/x86/xen/multicalls.c
@@ -54,7 +54,7 @@ DEFINE_PER_CPU(unsigned long, xen_mc_irq_flags);
 
 void xen_mc_flush(void)
 {
-       struct mc_buffer *b = &__get_cpu_var(mc_buffer);
+       struct mc_buffer *b = this_cpu_ptr(&mc_buffer);
        struct multicall_entry *mc;
        int ret = 0;
        unsigned long flags;
@@ -131,7 +131,7 @@ void xen_mc_flush(void)
 
 struct multicall_space __xen_mc_entry(size_t args)
 {
-       struct mc_buffer *b = &__get_cpu_var(mc_buffer);
+       struct mc_buffer *b = this_cpu_ptr(&mc_buffer);
        struct multicall_space ret;
        unsigned argidx = roundup(b->argidx, sizeof(u64));
 
@@ -162,7 +162,7 @@ struct multicall_space __xen_mc_entry(size_t args)
 
 struct multicall_space xen_mc_extend_args(unsigned long op, size_t size)
 {
-       struct mc_buffer *b = &__get_cpu_var(mc_buffer);
+       struct mc_buffer *b = this_cpu_ptr(&mc_buffer);
        struct multicall_space ret = { NULL, NULL };
 
        BUG_ON(preemptible());
@@ -192,7 +192,7 @@ out:
 
 void xen_mc_callback(void (*fn)(void *), void *data)
 {
-       struct mc_buffer *b = &__get_cpu_var(mc_buffer);
+       struct mc_buffer *b = this_cpu_ptr(&mc_buffer);
        struct callback *cb;
 
        if (b->cbidx == MC_BATCH) {

diff --git a/arch/x86/xen/spinlock.c b/arch/x86/xen/spinlock.c
index 0ba5f3b967f0041c5ea6445f898a944008b96c58..23b45eb9a89ce4d56f8a73e181d004b107dc1f81 100644 (file)
--- a/arch/x86/xen/spinlock.c
+++ b/arch/x86/xen/spinlock.c
@@ -109,7 +109,7 @@ static bool xen_pvspin = true;
 __visible void xen_lock_spinning(struct arch_spinlock *lock, __ticket_t want)
 {
        int irq = __this_cpu_read(lock_kicker_irq);
-       struct xen_lock_waiting *w = &__get_cpu_var(lock_waiting);
+       struct xen_lock_waiting *w = this_cpu_ptr(&lock_waiting);
        int cpu = smp_processor_id();
        u64 start;
        unsigned long flags;

diff --git a/arch/x86/xen/time.c b/arch/x86/xen/time.c
index 5718b0b58b60f663a845207af02d930962e96b8f..a1d430b112b33cc23ad07f925fa96325d327127f 100644 (file)
--- a/arch/x86/xen/time.c
+++ b/arch/x86/xen/time.c
@@ -80,7 +80,7 @@ static void get_runstate_snapshot(struct vcpu_runstate_info *res)
 
        BUG_ON(preemptible());
 
-       state = &__get_cpu_var(xen_runstate);
+       state = this_cpu_ptr(&xen_runstate);
 
        /*
         * The runstate info is always updated by the hypervisor on
@@ -123,7 +123,7 @@ static void do_stolen_accounting(void)
 
        WARN_ON(state.state != RUNSTATE_running);
 
-       snap = &__get_cpu_var(xen_runstate_snapshot);
+       snap = this_cpu_ptr(&xen_runstate_snapshot);
 
        /* work out how much time the VCPU has not been runn*ing*  */
        runnable = state.time[RUNSTATE_runnable] - snap->time[RUNSTATE_runnable];
@@ -158,7 +158,7 @@ cycle_t xen_clocksource_read(void)
        cycle_t ret;
 
        preempt_disable_notrace();
-       src = &__get_cpu_var(xen_vcpu)->time;
+       src = this_cpu_ptr(&xen_vcpu->time);
        ret = pvclock_clocksource_read(src);
        preempt_enable_notrace();
        return ret;
@@ -397,7 +397,7 @@ static DEFINE_PER_CPU(struct xen_clock_event_device, xen_clock_events) = { .evt.
 
 static irqreturn_t xen_timer_interrupt(int irq, void *dev_id)
 {
-       struct clock_event_device *evt = &__get_cpu_var(xen_clock_events).evt;
+       struct clock_event_device *evt = this_cpu_ptr(&xen_clock_events.evt);
        irqreturn_t ret;
 
        ret = IRQ_NONE;
@@ -460,7 +460,7 @@ void xen_setup_cpu_clockevents(void)
 {
        BUG_ON(preemptible());
 
-       clockevents_register_device(&__get_cpu_var(xen_clock_events).evt);
+       clockevents_register_device(this_cpu_ptr(&xen_clock_events.evt));
 }
 
 void xen_timer_resume(void)