From 23e9ecd9c87398613ade84d828643201958baab1 Mon Sep 17 00:00:00 2001 From: Nick Alcock Date: Mon, 8 Dec 2014 13:42:37 +0000 Subject: [PATCH] sparc64, vdso: Add gettimeofday() and clock_gettime(). This commit adds gettimeofday() and clock_gettime() entry points to the SPARC64 vDSO: in conjunction with a suitably-modified glibc this provides a speedup to gettimeofday(), time() and some clock_gettime() calls of on the order of 10--15x (the higher figure is for the coarse clock_gettime() clocks). gettimeofday() and clock_gettime() use largely separate code paths: all other approaches with less code duplication (e.g. doing all the work in a struct timespec and only shifting it into a struct timeval before return) turned out slower. Tested on %stick-capable machines only: %tick codepaths untested. Orabug: 20861959 Signed-off-by: Nick Alcock Signed-off-by: Dave Kleikamp (cherry picked from commit e8f9e5cfc9d297c7ebbc459b677ee0b8a3e45154) --- arch/sparc/Kconfig | 5 + arch/sparc/include/asm/clocksource.h | 14 ++ arch/sparc/include/asm/vvar.h | 24 +++ arch/sparc/kernel/Makefile | 1 + arch/sparc/kernel/time_64.c | 7 +- arch/sparc/kernel/vsyscall_gtod.c | 59 ++++++ arch/sparc/vdso/Makefile | 3 +- arch/sparc/vdso/vclock_gettime.c | 256 +++++++++++++++++++++++++++ arch/sparc/vdso/vdso.lds.S | 5 + include/linux/clocksource.h | 4 + 10 files changed, 375 insertions(+), 3 deletions(-) create mode 100644 arch/sparc/include/asm/clocksource.h create mode 100644 arch/sparc/kernel/vsyscall_gtod.c create mode 100644 arch/sparc/vdso/vclock_gettime.c diff --git a/arch/sparc/Kconfig b/arch/sparc/Kconfig index a4c397706f95..c26416ff64c9 100644 --- a/arch/sparc/Kconfig +++ b/arch/sparc/Kconfig @@ -80,6 +80,7 @@ config SPARC64 select NO_BOOTMEM select HAVE_ARCH_AUDITSYSCALL select ARCH_SUPPORTS_ATOMIC_RMW + select ARCH_CLOCKSOURCE_DATA config ARCH_DEFCONFIG string @@ -204,6 +205,10 @@ config GENERIC_CALIBRATE_DELAY bool default y +config GENERIC_TIME_VSYSCALL + bool + default y if SPARC64 + config ARCH_MAY_HAVE_PC_FDC bool default y diff --git a/arch/sparc/include/asm/clocksource.h b/arch/sparc/include/asm/clocksource.h new file mode 100644 index 000000000000..87987885bb6f --- /dev/null +++ b/arch/sparc/include/asm/clocksource.h @@ -0,0 +1,14 @@ +/* SPARC-specific clocksource additions */ + +#ifndef _ASM_SPARC_CLOCKSOURCE_H +#define _ASM_SPARC_CLOCKSOURCE_H + +#define VCLOCK_NONE 0 /* Nothing userspace can do. */ +#define VCLOCK_TICK 1 /* Use %tick. */ +#define VCLOCK_STICK 2 /* Use %stick. */ + +struct arch_clocksource_data { + int vclock_mode; +}; + +#endif /* _ASM_SPARC_CLOCKSOURCE_H */ diff --git a/arch/sparc/include/asm/vvar.h b/arch/sparc/include/asm/vvar.h index ca2da3bef2fd..4981949f4227 100644 --- a/arch/sparc/include/asm/vvar.h +++ b/arch/sparc/include/asm/vvar.h @@ -1,7 +1,31 @@ #ifndef _ASM_SPARC_VVAR_DATA_H #define _ASM_SPARC_VVAR_DATA_H +#include + +struct vsyscall_gtod_data { + seqcount_t seq; + + int vclock_mode; + struct { /* extract of a clocksource struct */ + cycle_t cycle_last; + cycle_t mask; + u32 mult; + u32 shift; + } clock; + /* open coded 'struct timespec' */ + time_t wall_time_sec; + u64 wall_time_snsec; + u64 monotonic_time_snsec; + time_t monotonic_time_sec; + + struct timezone sys_tz; + struct timespec wall_time_coarse; + struct timespec monotonic_time_coarse; +}; + struct vvar_data { + struct vsyscall_gtod_data gtod; }; extern struct vvar_data *vvar_data; diff --git a/arch/sparc/kernel/Makefile b/arch/sparc/kernel/Makefile index 7cf9c6ea3f1f..b8f20ad521c7 100644 --- a/arch/sparc/kernel/Makefile +++ b/arch/sparc/kernel/Makefile @@ -37,6 +37,7 @@ obj-$(CONFIG_SPARC32) += ioport.o obj-y += setup_$(BITS).o obj-y += idprom.o obj-y += sys_sparc_$(BITS).o +obj-y += vsyscall_gtod.o obj-$(CONFIG_SPARC32) += systbls_32.o obj-y += time_$(BITS).o obj-$(CONFIG_SPARC32) += windows.o diff --git a/arch/sparc/kernel/time_64.c b/arch/sparc/kernel/time_64.c index 2e6035c0a8ca..2916c3d02823 100644 --- a/arch/sparc/kernel/time_64.c +++ b/arch/sparc/kernel/time_64.c @@ -583,7 +583,7 @@ static int __init clock_init(void) fs_initcall(clock_init); /* This is gets the master TICK_INT timer going. */ -static unsigned long sparc64_init_timers(void) +static unsigned long sparc64_init_timers(struct arch_clocksource_data *archdata) { struct device_node *dp; unsigned long freq; @@ -600,13 +600,16 @@ static unsigned long sparc64_init_timers(void) /* Hummingbird, aka Ultra-IIe */ tick_ops = &hbtick_operations; freq = of_getintprop_default(dp, "stick-frequency", 0); + archdata->vclock_mode = VCLOCK_NONE; } else { tick_ops = &tick_operations; freq = local_cpu_data().clock_tick; + archdata->vclock_mode = VCLOCK_TICK; } } else { tick_ops = &stick_operations; freq = of_getintprop_default(dp, "stick-frequency", 0); + archdata->vclock_mode = VCLOCK_STICK; } return freq; @@ -790,7 +793,7 @@ static cycle_t clocksource_tick_read(struct clocksource *cs) void __init time_init(void) { - unsigned long freq = sparc64_init_timers(); + unsigned long freq = sparc64_init_timers(&clocksource_tick.archdata); tb_ticks_per_usec = freq / USEC_PER_SEC; diff --git a/arch/sparc/kernel/vsyscall_gtod.c b/arch/sparc/kernel/vsyscall_gtod.c new file mode 100644 index 000000000000..142cdf6ac7e1 --- /dev/null +++ b/arch/sparc/kernel/vsyscall_gtod.c @@ -0,0 +1,59 @@ +/* + * Copyright (C) 2001 Andrea Arcangeli SuSE + * Copyright 2003 Andi Kleen, SuSE Labs. + * + * Thanks to hpa@transmeta.com for some useful hint. + * Special thanks to Ingo Molnar for his early experience with + * a different vsyscall implementation for Linux/IA32 and for the name. + */ + +#include +#include +#include + +#include + +void update_vsyscall_tz(void) +{ + if (unlikely(vvar_data == NULL)) + return; + + vvar_data->gtod.sys_tz = sys_tz; +} + +void update_vsyscall(struct timekeeper *tk) +{ + struct vsyscall_gtod_data *vdata = &vvar_data->gtod; + + if (unlikely(vvar_data == NULL)) + return; + + write_seqcount_begin(&vdata->seq); + vdata->vclock_mode = tk->tkr_mono.clock->archdata.vclock_mode; + vdata->clock.cycle_last = tk->tkr_mono.cycle_last; + vdata->clock.mask = tk->tkr_mono.mask; + vdata->clock.mult = tk->tkr_mono.mult; + vdata->clock.shift = tk->tkr_mono.shift; + + vdata->wall_time_sec = tk->xtime_sec; + vdata->wall_time_snsec = tk->tkr_mono.xtime_nsec; + + vdata->monotonic_time_sec = tk->xtime_sec + + tk->wall_to_monotonic.tv_sec; + vdata->monotonic_time_snsec = tk->tkr_mono.xtime_nsec + + (tk->wall_to_monotonic.tv_nsec << + tk->tkr_mono.shift); + + while (vdata->monotonic_time_snsec >= + (((u64)NSEC_PER_SEC) << tk->tkr_mono.shift)) { + vdata->monotonic_time_snsec -= + ((u64)NSEC_PER_SEC) << tk->tkr_mono.shift; + vdata->monotonic_time_sec++; + } + + vdata->wall_time_coarse.tv_sec = tk->xtime_sec; + vdata->wall_time_coarse.tv_nsec = + (long)(tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift); + + write_seqcount_end(&vdata->seq); +} diff --git a/arch/sparc/vdso/Makefile b/arch/sparc/vdso/Makefile index dd12809235fb..57f326672d82 100644 --- a/arch/sparc/vdso/Makefile +++ b/arch/sparc/vdso/Makefile @@ -7,7 +7,7 @@ KBUILD_CFLAGS += $(DISABLE_LTO) VDSO64-$(CONFIG_SPARC64) := y # files to link into the vdso -vobjs-y := vdso-note.o +vobjs-y := vdso-note.o vclock_gettime.o # files to link into kernel obj-y += vma.o @@ -68,6 +68,7 @@ $(vobjs): KBUILD_CFLAGS += $(CFL) # vDSO code runs in userspace and -pg doesn't help with profiling anyway. # CFLAGS_REMOVE_vdso-note.o = -pg +CFLAGS_REMOVE_vclock_gettime.o = -pg $(obj)/%.so: OBJCOPYFLAGS := -S $(obj)/%.so: $(obj)/%.so.dbg diff --git a/arch/sparc/vdso/vclock_gettime.c b/arch/sparc/vdso/vclock_gettime.c new file mode 100644 index 000000000000..538532eeebb9 --- /dev/null +++ b/arch/sparc/vdso/vclock_gettime.c @@ -0,0 +1,256 @@ +/* + * Copyright 2006 Andi Kleen, SUSE Labs. + * Subject to the GNU Public License, v.2 + * + * Fast user context implementation of clock_gettime, gettimeofday, and time. + * + * The code should have no internal unresolved relocations. + * Check with readelf after changing. + * Also alternative() doesn't work. + */ + +/* Disable profiling for userspace code: */ +#define DISABLE_BRANCH_PROFILING + +#include +#include +#include +#include +#include +#include +#include + +#define TICK_PRIV_BIT (1UL << 63) + +#define SYSCALL_STRING \ + "ta 0x6d;" \ + "bcc,pt %%xcc, 1f;" \ + "sub %%g0, %%o0, %%o0;" \ + "1:" + +#define SYSCALL_CLOBBERS \ + "f0", "f1", "f2", "f3", "f4", "f5", "f6", "f7", \ + "f8", "f9", "f10", "f11", "f12", "f13", "f14", "f15", \ + "f16", "f17", "f18", "f19", "f20", "f21", "f22", "f23", \ + "f24", "f25", "f26", "f27", "f28", "f29", "f30", "f31", \ + "f32", "f34", "f36", "f38", "f40", "f42", "f44", "f46", \ + "f48", "f50", "f52", "f54", "f56", "f58", "f60", "f62", \ + "cc", "memory" + +/* + * Compute the vvar page's address in the process address space, and return it + * as a pointer to the vvar_data. + */ +notrace static inline struct vvar_data * +get_vvar_data(void) +{ + unsigned long ret; + + /* + * This horrible hack avoids taking the address of anything and thus + * generating a reference to a GOT that we don't have. + */ + ret = (unsigned long) current_text_addr(); + ret &= ~(PAGE_SIZE - 1); + ret -= PAGE_SIZE; + + return (struct vvar_data *) ret; +} + +notrace static long +vdso_fallback_gettime(long clock, struct timespec *ts) +{ + register long num __asm__("g1") = __NR_clock_gettime; + register long o0 __asm__("o0") = clock; + register long o1 __asm__("o1") = (long) ts; + + __asm__ __volatile__(SYSCALL_STRING : "=r" (o0) : "r" (num), + "0" (o0), "r" (o1) : SYSCALL_CLOBBERS); + return o0; +} + +notrace static __always_inline long +vdso_fallback_gettimeofday(struct timeval *tv, struct timezone *tz) +{ + register long num __asm__("g1") = __NR_gettimeofday; + register long o0 __asm__("o0") = (long) tv; + register long o1 __asm__("o1") = (long) tz; + + __asm__ __volatile__(SYSCALL_STRING : "=r" (o0) : "r" (num), + "0" (o0), "r" (o1) : SYSCALL_CLOBBERS); + return o0; +} + +notrace static __always_inline long +vread_stick(void) +{ + long ret; + + __asm__ __volatile__("rd %%asr24, %0" + : "=r" (ret)); + + return ret & ~TICK_PRIV_BIT; +} + +notrace static unsigned long long +vread_tick(void) +{ + unsigned long ret; + + __asm__ __volatile__("rd %%tick, %0\n\t" + "mov %0, %0" + : "=r" (ret)); + + return ret & ~TICK_PRIV_BIT; +} + +notrace static inline u64 +vgetsns(struct vsyscall_gtod_data *gtod) +{ + long v; + cycles_t cycles; + + switch (gtod->vclock_mode) { + case VCLOCK_TICK: + cycles = vread_tick(); + break; + case VCLOCK_STICK: + cycles = vread_stick(); + break; + default: + return 0; + } + v = (cycles - gtod->clock.cycle_last) & gtod->clock.mask; + return v * gtod->clock.mult; +} + +notrace static noinline int +do_realtime(struct vsyscall_gtod_data *gtod, struct timespec *ts) +{ + unsigned long seq; + u64 ns; + + ts->tv_nsec = 0; + do { + seq = read_seqcount_begin(>od->seq); + ts->tv_sec = gtod->wall_time_sec; + ns = gtod->wall_time_snsec; + ns += vgetsns(gtod); + ns >>= gtod->clock.shift; + } while (unlikely(read_seqcount_retry(>od->seq, seq))); + + timespec_add_ns(ts, ns); + + return 0; +} + +notrace static noinline int +do_monotonic(struct vsyscall_gtod_data *gtod, struct timespec *ts) +{ + unsigned long seq; + u64 ns; + + ts->tv_nsec = 0; + do { + seq = read_seqcount_begin(>od->seq); + ts->tv_sec = gtod->monotonic_time_sec; + ns = gtod->monotonic_time_snsec; + ns += vgetsns(gtod); + ns >>= gtod->clock.shift; + } while (unlikely(read_seqcount_retry(>od->seq, seq))); + + timespec_add_ns(ts, ns); + + return 0; +} + +notrace static noinline int +do_realtime_coarse(struct vsyscall_gtod_data *gtod, struct timespec *ts) +{ + unsigned long seq; + do { + seq = read_seqcount_begin(>od->seq); + ts->tv_sec = gtod->wall_time_coarse.tv_sec; + ts->tv_nsec = gtod->wall_time_coarse.tv_nsec; + } while (unlikely(read_seqcount_retry(>od->seq, seq))); + return 0; +} + +notrace static noinline int +do_monotonic_coarse(struct vsyscall_gtod_data *gtod, struct timespec *ts) +{ + unsigned long seq; + do { + seq = read_seqcount_begin(>od->seq); + ts->tv_sec = gtod->monotonic_time_coarse.tv_sec; + ts->tv_nsec = gtod->monotonic_time_coarse.tv_nsec; + } while (unlikely(read_seqcount_retry(>od->seq, seq))); + + return 0; +} + +notrace int +__vdso_clock_gettime(clockid_t clock, struct timespec *ts) +{ + struct vvar_data *vvd = get_vvar_data(); + struct vsyscall_gtod_data *gtod = &vvd->gtod; + + switch (clock) { + case CLOCK_REALTIME: + if (unlikely(gtod->vclock_mode == VCLOCK_NONE)) + break; + return do_realtime(gtod, ts); + case CLOCK_MONOTONIC: + if (unlikely(gtod->vclock_mode == VCLOCK_NONE)) + break; + return do_monotonic(gtod, ts); + case CLOCK_REALTIME_COARSE: + return do_realtime_coarse(gtod, ts); + case CLOCK_MONOTONIC_COARSE: + return do_monotonic_coarse(gtod, ts); + } + /* + * Unknown clock ID ? Fall back to the syscall. + */ + return vdso_fallback_gettime(clock, ts); +} +int +clock_gettime(clockid_t, struct timespec *) + __attribute__((weak, alias("__vdso_clock_gettime"))); + +notrace int +__vdso_gettimeofday(struct timeval *tv, struct timezone *tz) +{ + struct vvar_data *vvd = get_vvar_data(); + struct vsyscall_gtod_data *gtod = &vvd->gtod; + + if (likely(gtod->vclock_mode != VCLOCK_NONE)) { + if (likely(tv != NULL)) { + union tstv_t { + struct timespec ts; + struct timeval tv; + } *tstv = (union tstv_t *) tv; + do_realtime(gtod, &tstv->ts); + /* + * Assign before dividing to ensure that the division is + * done in the type of tv_usec, not tv_nsec. + * + * There cannot be > 1 billion usec in a second: + * do_realtime() has already distributed such overflow + * into tv_sec. So we can assign it to an int safely. + */ + tstv->tv.tv_usec = tstv->ts.tv_nsec; + tstv->tv.tv_usec /= 1000; + } + if (unlikely(tz != NULL)) { + /* Avoid memcpy. Some old compilers fail to inline it */ + tz->tz_minuteswest = gtod->sys_tz.tz_minuteswest; + tz->tz_dsttime = gtod->sys_tz.tz_dsttime; + } + return 0; + } + return vdso_fallback_gettimeofday(tv, tz); +} +int +gettimeofday(struct timeval *, struct timezone *) + __attribute__((weak, alias("__vdso_gettimeofday"))); diff --git a/arch/sparc/vdso/vdso.lds.S b/arch/sparc/vdso/vdso.lds.S index 773ba8ae48aa..f3caa29a331c 100644 --- a/arch/sparc/vdso/vdso.lds.S +++ b/arch/sparc/vdso/vdso.lds.S @@ -15,6 +15,11 @@ */ VERSION { LINUX_2.6 { + global: + clock_gettime; + __vdso_clock_gettime; + gettimeofday; + __vdso_gettimeofday; local: *; }; } diff --git a/include/linux/clocksource.h b/include/linux/clocksource.h index d27d0152271f..17b38199b6db 100644 --- a/include/linux/clocksource.h +++ b/include/linux/clocksource.h @@ -25,6 +25,10 @@ struct module; #include #endif +#ifdef CONFIG_ARCH_CLOCKSOURCE_DATA +#include +#endif + /** * struct clocksource - hardware abstraction for a free running counter * Provides mostly state-free accessors to the underlying hardware. -- 2.50.1