]> www.infradead.org Git - users/jedix/linux-maple.git/commitdiff
sparc64, vdso: Add gettimeofday() and clock_gettime().
authorNick Alcock <nick.alcock@oracle.com>
Mon, 8 Dec 2014 13:42:37 +0000 (13:42 +0000)
committerAllen Pais <allen.pais@oracle.com>
Tue, 15 Sep 2015 12:09:05 +0000 (17:39 +0530)
This commit adds gettimeofday() and clock_gettime() entry points to the SPARC64
vDSO: in conjunction with a suitably-modified glibc this provides a speedup to
gettimeofday(), time() and some clock_gettime() calls of on the order of
10--15x (the higher figure is for the coarse clock_gettime() clocks).

gettimeofday() and clock_gettime() use largely separate code paths: all
other approaches with less code duplication (e.g. doing all the work in
a struct timespec and only shifting it into a struct timeval before return)
turned out slower.

Tested on %stick-capable machines only: %tick codepaths untested.

Orabug: 20861959
Signed-off-by: Nick Alcock <nick.alcock@oracle.com>
Signed-off-by: Dave Kleikamp <dave.kleikamp@oracle.com>
(cherry picked from commit e8f9e5cfc9d297c7ebbc459b677ee0b8a3e45154)

arch/sparc/Kconfig
arch/sparc/include/asm/clocksource.h [new file with mode: 0644]
arch/sparc/include/asm/vvar.h
arch/sparc/kernel/Makefile
arch/sparc/kernel/time_64.c
arch/sparc/kernel/vsyscall_gtod.c [new file with mode: 0644]
arch/sparc/vdso/Makefile
arch/sparc/vdso/vclock_gettime.c [new file with mode: 0644]
arch/sparc/vdso/vdso.lds.S
include/linux/clocksource.h

index a4c397706f952d6d9fa824f495ba1253c9386154..c26416ff64c9ef4dba0d0af9398ad871e3f077e5 100644 (file)
@@ -80,6 +80,7 @@ config SPARC64
        select NO_BOOTMEM
        select HAVE_ARCH_AUDITSYSCALL
        select ARCH_SUPPORTS_ATOMIC_RMW
+       select ARCH_CLOCKSOURCE_DATA
 
 config ARCH_DEFCONFIG
        string
@@ -204,6 +205,10 @@ config GENERIC_CALIBRATE_DELAY
        bool
        default y
 
+config GENERIC_TIME_VSYSCALL
+       bool
+       default y if SPARC64
+
 config ARCH_MAY_HAVE_PC_FDC
        bool
        default y
diff --git a/arch/sparc/include/asm/clocksource.h b/arch/sparc/include/asm/clocksource.h
new file mode 100644 (file)
index 0000000..8798788
--- /dev/null
@@ -0,0 +1,14 @@
+/* SPARC-specific clocksource additions */
+
+#ifndef _ASM_SPARC_CLOCKSOURCE_H
+#define _ASM_SPARC_CLOCKSOURCE_H
+
+#define VCLOCK_NONE   0  /* Nothing userspace can do. */
+#define VCLOCK_TICK   1  /* Use %tick.  */
+#define VCLOCK_STICK  2  /* Use %stick. */
+
+struct arch_clocksource_data {
+       int vclock_mode;
+};
+
+#endif /* _ASM_SPARC_CLOCKSOURCE_H */
index ca2da3bef2fde2533b3cfe9e3e166405eb7ca3ff..4981949f4227359b294074ccb21ced20ab677200 100644 (file)
@@ -1,7 +1,31 @@
 #ifndef _ASM_SPARC_VVAR_DATA_H
 #define _ASM_SPARC_VVAR_DATA_H
 
+#include <linux/clocksource.h>
+
+struct vsyscall_gtod_data {
+       seqcount_t      seq;
+
+       int vclock_mode;
+       struct { /* extract of a clocksource struct */
+               cycle_t cycle_last;
+               cycle_t mask;
+               u32     mult;
+               u32     shift;
+       } clock;
+       /* open coded 'struct timespec' */
+       time_t          wall_time_sec;
+       u64             wall_time_snsec;
+       u64             monotonic_time_snsec;
+       time_t          monotonic_time_sec;
+
+       struct timezone sys_tz;
+       struct timespec wall_time_coarse;
+       struct timespec monotonic_time_coarse;
+};
+
 struct vvar_data {
+       struct vsyscall_gtod_data gtod;
 };
 
 extern struct vvar_data *vvar_data;
index 7cf9c6ea3f1f210c0856351e47d1a4252913667b..b8f20ad521c78ce448340798a0089e76c739793f 100644 (file)
@@ -37,6 +37,7 @@ obj-$(CONFIG_SPARC32)   += ioport.o
 obj-y                   += setup_$(BITS).o
 obj-y                   += idprom.o
 obj-y                   += sys_sparc_$(BITS).o
+obj-y                   += vsyscall_gtod.o
 obj-$(CONFIG_SPARC32)   += systbls_32.o
 obj-y                   += time_$(BITS).o
 obj-$(CONFIG_SPARC32)   += windows.o
index 2e6035c0a8ca91032e8fcf751241ac1dfe406cf8..2916c3d02823c1cbc565ddcb9fbe19938eb65bd6 100644 (file)
@@ -583,7 +583,7 @@ static int __init clock_init(void)
 fs_initcall(clock_init);
 
 /* This is gets the master TICK_INT timer going. */
-static unsigned long sparc64_init_timers(void)
+static unsigned long sparc64_init_timers(struct arch_clocksource_data *archdata)
 {
        struct device_node *dp;
        unsigned long freq;
@@ -600,13 +600,16 @@ static unsigned long sparc64_init_timers(void)
                        /* Hummingbird, aka Ultra-IIe */
                        tick_ops = &hbtick_operations;
                        freq = of_getintprop_default(dp, "stick-frequency", 0);
+                       archdata->vclock_mode = VCLOCK_NONE;
                } else {
                        tick_ops = &tick_operations;
                        freq = local_cpu_data().clock_tick;
+                       archdata->vclock_mode = VCLOCK_TICK;
                }
        } else {
                tick_ops = &stick_operations;
                freq = of_getintprop_default(dp, "stick-frequency", 0);
+               archdata->vclock_mode = VCLOCK_STICK;
        }
 
        return freq;
@@ -790,7 +793,7 @@ static cycle_t clocksource_tick_read(struct clocksource *cs)
 
 void __init time_init(void)
 {
-       unsigned long freq = sparc64_init_timers();
+       unsigned long freq = sparc64_init_timers(&clocksource_tick.archdata);
 
        tb_ticks_per_usec = freq / USEC_PER_SEC;
 
diff --git a/arch/sparc/kernel/vsyscall_gtod.c b/arch/sparc/kernel/vsyscall_gtod.c
new file mode 100644 (file)
index 0000000..142cdf6
--- /dev/null
@@ -0,0 +1,59 @@
+/*
+ *  Copyright (C) 2001 Andrea Arcangeli <andrea@suse.de> SuSE
+ *  Copyright 2003 Andi Kleen, SuSE Labs.
+ *
+ *  Thanks to hpa@transmeta.com for some useful hint.
+ *  Special thanks to Ingo Molnar for his early experience with
+ *  a different vsyscall implementation for Linux/IA32 and for the name.
+ */
+
+#include <linux/seqlock.h>
+#include <linux/time.h>
+#include <linux/timekeeper_internal.h>
+
+#include <asm/vvar.h>
+
+void update_vsyscall_tz(void)
+{
+       if (unlikely(vvar_data == NULL))
+               return;
+
+       vvar_data->gtod.sys_tz = sys_tz;
+}
+
+void update_vsyscall(struct timekeeper *tk)
+{
+       struct vsyscall_gtod_data *vdata = &vvar_data->gtod;
+
+       if (unlikely(vvar_data == NULL))
+               return;
+
+       write_seqcount_begin(&vdata->seq);
+       vdata->vclock_mode = tk->tkr_mono.clock->archdata.vclock_mode;
+       vdata->clock.cycle_last = tk->tkr_mono.cycle_last;
+       vdata->clock.mask = tk->tkr_mono.mask;
+       vdata->clock.mult = tk->tkr_mono.mult;
+       vdata->clock.shift = tk->tkr_mono.shift;
+
+       vdata->wall_time_sec = tk->xtime_sec;
+       vdata->wall_time_snsec = tk->tkr_mono.xtime_nsec;
+
+       vdata->monotonic_time_sec = tk->xtime_sec +
+                                   tk->wall_to_monotonic.tv_sec;
+       vdata->monotonic_time_snsec = tk->tkr_mono.xtime_nsec +
+                                     (tk->wall_to_monotonic.tv_nsec <<
+                                      tk->tkr_mono.shift);
+
+       while (vdata->monotonic_time_snsec >=
+              (((u64)NSEC_PER_SEC) << tk->tkr_mono.shift)) {
+               vdata->monotonic_time_snsec -=
+                               ((u64)NSEC_PER_SEC) << tk->tkr_mono.shift;
+               vdata->monotonic_time_sec++;
+       }
+
+       vdata->wall_time_coarse.tv_sec = tk->xtime_sec;
+       vdata->wall_time_coarse.tv_nsec =
+                       (long)(tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift);
+
+       write_seqcount_end(&vdata->seq);
+}
index dd12809235fbcd48ad2070c7ce380c59a0e3dd26..57f326672d824120dad283aa3e1fb8111aaecbf7 100644 (file)
@@ -7,7 +7,7 @@ KBUILD_CFLAGS += $(DISABLE_LTO)
 VDSO64-$(CONFIG_SPARC64)       := y
 
 # files to link into the vdso
-vobjs-y := vdso-note.o
+vobjs-y := vdso-note.o vclock_gettime.o
 
 # files to link into kernel
 obj-y                          += vma.o
@@ -68,6 +68,7 @@ $(vobjs): KBUILD_CFLAGS += $(CFL)
 # vDSO code runs in userspace and -pg doesn't help with profiling anyway.
 #
 CFLAGS_REMOVE_vdso-note.o = -pg
+CFLAGS_REMOVE_vclock_gettime.o = -pg
 
 $(obj)/%.so: OBJCOPYFLAGS := -S
 $(obj)/%.so: $(obj)/%.so.dbg
diff --git a/arch/sparc/vdso/vclock_gettime.c b/arch/sparc/vdso/vclock_gettime.c
new file mode 100644 (file)
index 0000000..538532e
--- /dev/null
@@ -0,0 +1,256 @@
+/*
+ * Copyright 2006 Andi Kleen, SUSE Labs.
+ * Subject to the GNU Public License, v.2
+ *
+ * Fast user context implementation of clock_gettime, gettimeofday, and time.
+ *
+ * The code should have no internal unresolved relocations.
+ * Check with readelf after changing.
+ * Also alternative() doesn't work.
+ */
+
+/* Disable profiling for userspace code: */
+#define DISABLE_BRANCH_PROFILING
+
+#include <linux/kernel.h>
+#include <linux/time.h>
+#include <linux/string.h>
+#include <asm/vvar.h>
+#include <asm/timex.h>
+#include <asm/unistd.h>
+#include <asm/io.h>
+
+#define TICK_PRIV_BIT  (1UL << 63)
+
+#define SYSCALL_STRING                                                 \
+       "ta     0x6d;"                                                  \
+       "bcc,pt %%xcc, 1f;"                                             \
+       "sub    %%g0, %%o0, %%o0;"                                      \
+       "1:"
+
+#define SYSCALL_CLOBBERS                                               \
+       "f0", "f1", "f2", "f3", "f4", "f5", "f6", "f7",                 \
+       "f8", "f9", "f10", "f11", "f12", "f13", "f14", "f15",           \
+       "f16", "f17", "f18", "f19", "f20", "f21", "f22", "f23",         \
+       "f24", "f25", "f26", "f27", "f28", "f29", "f30", "f31",         \
+       "f32", "f34", "f36", "f38", "f40", "f42", "f44", "f46",         \
+       "f48", "f50", "f52", "f54", "f56", "f58", "f60", "f62",         \
+       "cc", "memory"
+
+/*
+ * Compute the vvar page's address in the process address space, and return it
+ * as a pointer to the vvar_data.
+ */
+notrace static inline struct vvar_data *
+get_vvar_data(void)
+{
+       unsigned long ret;
+
+       /*
+        * This horrible hack avoids taking the address of anything and thus
+        * generating a reference to a GOT that we don't have.
+        */
+       ret = (unsigned long) current_text_addr();
+       ret &= ~(PAGE_SIZE - 1);
+       ret -= PAGE_SIZE;
+
+       return (struct vvar_data *) ret;
+}
+
+notrace static long
+vdso_fallback_gettime(long clock, struct timespec *ts)
+{
+       register long num __asm__("g1") = __NR_clock_gettime;
+       register long o0 __asm__("o0") = clock;
+       register long o1 __asm__("o1") = (long) ts;
+
+       __asm__ __volatile__(SYSCALL_STRING : "=r" (o0) : "r" (num),
+                            "0" (o0), "r" (o1) : SYSCALL_CLOBBERS);
+       return o0;
+}
+
+notrace static __always_inline long
+vdso_fallback_gettimeofday(struct timeval *tv, struct timezone *tz)
+{
+       register long num __asm__("g1") = __NR_gettimeofday;
+       register long o0 __asm__("o0") = (long) tv;
+       register long o1 __asm__("o1") = (long) tz;
+
+       __asm__ __volatile__(SYSCALL_STRING : "=r" (o0) : "r" (num),
+                            "0" (o0), "r" (o1) : SYSCALL_CLOBBERS);
+       return o0;
+}
+
+notrace static __always_inline long
+vread_stick(void)
+{
+       long ret;
+
+       __asm__ __volatile__("rd        %%asr24, %0"
+                            : "=r" (ret));
+
+       return ret & ~TICK_PRIV_BIT;
+}
+
+notrace static unsigned long long
+vread_tick(void)
+{
+       unsigned long ret;
+
+       __asm__ __volatile__("rd        %%tick, %0\n\t"
+                            "mov       %0, %0"
+                            : "=r" (ret));
+
+       return ret & ~TICK_PRIV_BIT;
+}
+
+notrace static inline u64
+vgetsns(struct vsyscall_gtod_data *gtod)
+{
+       long v;
+       cycles_t cycles;
+
+       switch (gtod->vclock_mode) {
+       case VCLOCK_TICK:
+               cycles = vread_tick();
+               break;
+       case VCLOCK_STICK:
+               cycles = vread_stick();
+               break;
+       default:
+               return 0;
+       }
+       v = (cycles - gtod->clock.cycle_last) & gtod->clock.mask;
+       return v * gtod->clock.mult;
+}
+
+notrace static noinline int
+do_realtime(struct vsyscall_gtod_data *gtod, struct timespec *ts)
+{
+       unsigned long seq;
+       u64 ns;
+
+       ts->tv_nsec = 0;
+       do {
+               seq = read_seqcount_begin(&gtod->seq);
+               ts->tv_sec = gtod->wall_time_sec;
+               ns = gtod->wall_time_snsec;
+               ns += vgetsns(gtod);
+               ns >>= gtod->clock.shift;
+       } while (unlikely(read_seqcount_retry(&gtod->seq, seq)));
+
+       timespec_add_ns(ts, ns);
+
+       return 0;
+}
+
+notrace static noinline int
+do_monotonic(struct vsyscall_gtod_data *gtod, struct timespec *ts)
+{
+       unsigned long seq;
+       u64 ns;
+
+       ts->tv_nsec = 0;
+       do {
+               seq = read_seqcount_begin(&gtod->seq);
+               ts->tv_sec = gtod->monotonic_time_sec;
+               ns = gtod->monotonic_time_snsec;
+               ns += vgetsns(gtod);
+               ns >>= gtod->clock.shift;
+       } while (unlikely(read_seqcount_retry(&gtod->seq, seq)));
+
+       timespec_add_ns(ts, ns);
+
+       return 0;
+}
+
+notrace static noinline int
+do_realtime_coarse(struct vsyscall_gtod_data *gtod, struct timespec *ts)
+{
+       unsigned long seq;
+       do {
+               seq = read_seqcount_begin(&gtod->seq);
+               ts->tv_sec = gtod->wall_time_coarse.tv_sec;
+               ts->tv_nsec = gtod->wall_time_coarse.tv_nsec;
+       } while (unlikely(read_seqcount_retry(&gtod->seq, seq)));
+       return 0;
+}
+
+notrace static noinline int
+do_monotonic_coarse(struct vsyscall_gtod_data *gtod, struct timespec *ts)
+{
+       unsigned long seq;
+       do {
+               seq = read_seqcount_begin(&gtod->seq);
+               ts->tv_sec = gtod->monotonic_time_coarse.tv_sec;
+               ts->tv_nsec = gtod->monotonic_time_coarse.tv_nsec;
+       } while (unlikely(read_seqcount_retry(&gtod->seq, seq)));
+
+       return 0;
+}
+
+notrace int
+__vdso_clock_gettime(clockid_t clock, struct timespec *ts)
+{
+       struct vvar_data *vvd = get_vvar_data();
+       struct vsyscall_gtod_data *gtod = &vvd->gtod;
+
+       switch (clock) {
+       case CLOCK_REALTIME:
+               if (unlikely(gtod->vclock_mode == VCLOCK_NONE))
+                       break;
+               return do_realtime(gtod, ts);
+       case CLOCK_MONOTONIC:
+               if (unlikely(gtod->vclock_mode == VCLOCK_NONE))
+                       break;
+               return do_monotonic(gtod, ts);
+       case CLOCK_REALTIME_COARSE:
+               return do_realtime_coarse(gtod, ts);
+       case CLOCK_MONOTONIC_COARSE:
+               return do_monotonic_coarse(gtod, ts);
+       }
+       /*
+        * Unknown clock ID ? Fall back to the syscall.
+        */
+       return vdso_fallback_gettime(clock, ts);
+}
+int
+clock_gettime(clockid_t, struct timespec *)
+       __attribute__((weak, alias("__vdso_clock_gettime")));
+
+notrace int
+__vdso_gettimeofday(struct timeval *tv, struct timezone *tz)
+{
+       struct vvar_data *vvd = get_vvar_data();
+       struct vsyscall_gtod_data *gtod = &vvd->gtod;
+
+       if (likely(gtod->vclock_mode != VCLOCK_NONE)) {
+               if (likely(tv != NULL)) {
+                       union tstv_t {
+                               struct timespec ts;
+                               struct timeval tv;
+                       } *tstv = (union tstv_t *) tv;
+                       do_realtime(gtod, &tstv->ts);
+                       /*
+                        * Assign before dividing to ensure that the division is
+                        * done in the type of tv_usec, not tv_nsec.
+                        *
+                        * There cannot be > 1 billion usec in a second:
+                        * do_realtime() has already distributed such overflow
+                        * into tv_sec.  So we can assign it to an int safely.
+                        */
+                       tstv->tv.tv_usec = tstv->ts.tv_nsec;
+                       tstv->tv.tv_usec /= 1000;
+               }
+               if (unlikely(tz != NULL)) {
+                       /* Avoid memcpy. Some old compilers fail to inline it */
+                       tz->tz_minuteswest = gtod->sys_tz.tz_minuteswest;
+                       tz->tz_dsttime = gtod->sys_tz.tz_dsttime;
+               }
+               return 0;
+       }
+       return vdso_fallback_gettimeofday(tv, tz);
+}
+int
+gettimeofday(struct timeval *, struct timezone *)
+       __attribute__((weak, alias("__vdso_gettimeofday")));
index 773ba8ae48aa6e4d891e490f8bb0633053d349fc..f3caa29a331c58175b67ea60d7ac15cd467fe5ff 100644 (file)
  */
 VERSION {
        LINUX_2.6 {
+       global:
+               clock_gettime;
+               __vdso_clock_gettime;
+               gettimeofday;
+               __vdso_gettimeofday;
        local: *;
        };
 }
index d27d0152271f9e8b487a48a9f2d74f51fe9a58a5..17b38199b6db04cb3f338da24269af43cd2ed816 100644 (file)
@@ -25,6 +25,10 @@ struct module;
 #include <asm/clocksource.h>
 #endif
 
+#ifdef CONFIG_ARCH_CLOCKSOURCE_DATA
+#include <asm/clocksource.h>
+#endif
+
 /**
  * struct clocksource - hardware abstraction for a free running counter
  *     Provides mostly state-free accessors to the underlying hardware.