linux/arch/x86/kernel/vsyscall_64.c
<<
>>
Prefs
   1/*
   2 *  Copyright (C) 2001 Andrea Arcangeli <andrea@suse.de> SuSE
   3 *  Copyright 2003 Andi Kleen, SuSE Labs.
   4 *
   5 *  Thanks to hpa@transmeta.com for some useful hint.
   6 *  Special thanks to Ingo Molnar for his early experience with
   7 *  a different vsyscall implementation for Linux/IA32 and for the name.
   8 *
   9 *  vsyscall 1 is located at -10Mbyte, vsyscall 2 is located
  10 *  at virtual address -10Mbyte+1024bytes etc... There are at max 4
  11 *  vsyscalls. One vsyscall can reserve more than 1 slot to avoid
  12 *  jumping out of line if necessary. We cannot add more with this
  13 *  mechanism because older kernels won't return -ENOSYS.
  14 *  If we want more than four we need a vDSO.
  15 *
  16 *  Note: the concept clashes with user mode linux. If you use UML and
  17 *  want per guest time just set the kernel.vsyscall64 sysctl to 0.
  18 */
  19
  20/* Disable profiling for userspace code: */
  21#define DISABLE_BRANCH_PROFILING
  22
  23#include <linux/time.h>
  24#include <linux/init.h>
  25#include <linux/kernel.h>
  26#include <linux/timer.h>
  27#include <linux/seqlock.h>
  28#include <linux/jiffies.h>
  29#include <linux/sysctl.h>
  30#include <linux/clocksource.h>
  31#include <linux/getcpu.h>
  32#include <linux/cpu.h>
  33#include <linux/smp.h>
  34#include <linux/notifier.h>
  35
  36#include <asm/vsyscall.h>
  37#include <asm/pgtable.h>
  38#include <asm/page.h>
  39#include <asm/unistd.h>
  40#include <asm/fixmap.h>
  41#include <asm/errno.h>
  42#include <asm/io.h>
  43#include <asm/segment.h>
  44#include <asm/desc.h>
  45#include <asm/topology.h>
  46#include <asm/vgtod.h>
  47
  48#define __vsyscall(nr) \
  49                __attribute__ ((unused, __section__(".vsyscall_" #nr))) notrace
  50#define __syscall_clobber "r11","cx","memory"
  51
  52/*
  53 * vsyscall_gtod_data contains data that is :
  54 * - readonly from vsyscalls
  55 * - written by timer interrupt or systcl (/proc/sys/kernel/vsyscall64)
  56 * Try to keep this structure as small as possible to avoid cache line ping pongs
  57 */
  58int __vgetcpu_mode __section_vgetcpu_mode;
  59
  60struct vsyscall_gtod_data __vsyscall_gtod_data __section_vsyscall_gtod_data =
  61{
  62        .lock = SEQLOCK_UNLOCKED,
  63        .sysctl_enabled = 1,
  64};
  65
  66void update_vsyscall_tz(void)
  67{
  68        unsigned long flags;
  69
  70        write_seqlock_irqsave(&vsyscall_gtod_data.lock, flags);
  71        /* sys_tz has changed */
  72        vsyscall_gtod_data.sys_tz = sys_tz;
  73        write_sequnlock_irqrestore(&vsyscall_gtod_data.lock, flags);
  74}
  75
  76void update_vsyscall(struct timespec *wall_time, struct timespec *wtm,
  77                        struct clocksource *clock, u32 mult)
  78{
  79        unsigned long flags;
  80
  81        write_seqlock_irqsave(&vsyscall_gtod_data.lock, flags);
  82        /* copy vsyscall data */
  83        vsyscall_gtod_data.clock.vread = clock->vread;
  84        vsyscall_gtod_data.clock.cycle_last = clock->cycle_last;
  85        vsyscall_gtod_data.clock.mask = clock->mask;
  86        vsyscall_gtod_data.clock.mult = mult;
  87        vsyscall_gtod_data.clock.shift = clock->shift;
  88        vsyscall_gtod_data.wall_time_sec = wall_time->tv_sec;
  89        vsyscall_gtod_data.wall_time_nsec = wall_time->tv_nsec;
  90        vsyscall_gtod_data.wall_to_monotonic = *wtm;
  91        vsyscall_gtod_data.wall_time_coarse = __current_kernel_time();
  92        write_sequnlock_irqrestore(&vsyscall_gtod_data.lock, flags);
  93}
  94
  95/* RED-PEN may want to readd seq locking, but then the variable should be
  96 * write-once.
  97 */
  98static __always_inline void do_get_tz(struct timezone * tz)
  99{
 100        *tz = __vsyscall_gtod_data.sys_tz;
 101}
 102
 103static __always_inline int gettimeofday(struct timeval *tv, struct timezone *tz)
 104{
 105        int ret;
 106        asm volatile("syscall"
 107                : "=a" (ret)
 108                : "0" (__NR_gettimeofday),"D" (tv),"S" (tz)
 109                : __syscall_clobber );
 110        return ret;
 111}
 112
 113static __always_inline long time_syscall(long *t)
 114{
 115        long secs;
 116        asm volatile("syscall"
 117                : "=a" (secs)
 118                : "0" (__NR_time),"D" (t) : __syscall_clobber);
 119        return secs;
 120}
 121
 122static __always_inline void do_vgettimeofday(struct timeval * tv)
 123{
 124        cycle_t now, base, mask, cycle_delta;
 125        unsigned seq;
 126        unsigned long mult, shift, nsec;
 127        cycle_t (*vread)(void);
 128        do {
 129                seq = read_seqbegin(&__vsyscall_gtod_data.lock);
 130
 131                vread = __vsyscall_gtod_data.clock.vread;
 132                if (unlikely(!__vsyscall_gtod_data.sysctl_enabled || !vread)) {
 133                        gettimeofday(tv,NULL);
 134                        return;
 135                }
 136
 137                now = vread();
 138                base = __vsyscall_gtod_data.clock.cycle_last;
 139                mask = __vsyscall_gtod_data.clock.mask;
 140                mult = __vsyscall_gtod_data.clock.mult;
 141                shift = __vsyscall_gtod_data.clock.shift;
 142
 143                tv->tv_sec = __vsyscall_gtod_data.wall_time_sec;
 144                nsec = __vsyscall_gtod_data.wall_time_nsec;
 145        } while (read_seqretry(&__vsyscall_gtod_data.lock, seq));
 146
 147        /* calculate interval: */
 148        cycle_delta = (now - base) & mask;
 149        /* convert to nsecs: */
 150        nsec += (cycle_delta * mult) >> shift;
 151
 152        while (nsec >= NSEC_PER_SEC) {
 153                tv->tv_sec += 1;
 154                nsec -= NSEC_PER_SEC;
 155        }
 156        tv->tv_usec = nsec / NSEC_PER_USEC;
 157}
 158
 159int __vsyscall(0) vgettimeofday(struct timeval * tv, struct timezone * tz)
 160{
 161        if (tv)
 162                do_vgettimeofday(tv);
 163        if (tz)
 164                do_get_tz(tz);
 165        return 0;
 166}
 167
 168/* This will break when the xtime seconds get inaccurate, but that is
 169 * unlikely */
 170time_t __vsyscall(1) vtime(time_t *t)
 171{
 172        unsigned seq;
 173        time_t result;
 174        if (unlikely(!__vsyscall_gtod_data.sysctl_enabled))
 175                return time_syscall(t);
 176
 177        do {
 178                seq = read_seqbegin(&__vsyscall_gtod_data.lock);
 179
 180                result = __vsyscall_gtod_data.wall_time_sec;
 181
 182        } while (read_seqretry(&__vsyscall_gtod_data.lock, seq));
 183
 184        if (t)
 185                *t = result;
 186        return result;
 187}
 188
 189/* Fast way to get current CPU and node.
 190   This helps to do per node and per CPU caches in user space.
 191   The result is not guaranteed without CPU affinity, but usually
 192   works out because the scheduler tries to keep a thread on the same
 193   CPU.
 194
 195   tcache must point to a two element sized long array.
 196   All arguments can be NULL. */
 197long __vsyscall(2)
 198vgetcpu(unsigned *cpu, unsigned *node, struct getcpu_cache *tcache)
 199{
 200        unsigned int p;
 201        unsigned long j = 0;
 202
 203        /* Fast cache - only recompute value once per jiffies and avoid
 204           relatively costly rdtscp/cpuid otherwise.
 205           This works because the scheduler usually keeps the process
 206           on the same CPU and this syscall doesn't guarantee its
 207           results anyways.
 208           We do this here because otherwise user space would do it on
 209           its own in a likely inferior way (no access to jiffies).
 210           If you don't like it pass NULL. */
 211        if (tcache && tcache->blob[0] == (j = __jiffies)) {
 212                p = tcache->blob[1];
 213        } else if (__vgetcpu_mode == VGETCPU_RDTSCP) {
 214                /* Load per CPU data from RDTSCP */
 215                native_read_tscp(&p);
 216        } else {
 217                /* Load per CPU data from GDT */
 218                asm("lsl %1,%0" : "=r" (p) : "r" (__PER_CPU_SEG));
 219        }
 220        if (tcache) {
 221                tcache->blob[0] = j;
 222                tcache->blob[1] = p;
 223        }
 224        if (cpu)
 225                *cpu = p & 0xfff;
 226        if (node)
 227                *node = p >> 12;
 228        return 0;
 229}
 230
 231static long __vsyscall(3) venosys_1(void)
 232{
 233        return -ENOSYS;
 234}
 235
 236#ifdef CONFIG_SYSCTL
 237static ctl_table kernel_table2[] = {
 238        { .procname = "vsyscall64",
 239          .data = &vsyscall_gtod_data.sysctl_enabled, .maxlen = sizeof(int),
 240          .mode = 0644,
 241          .proc_handler = proc_dointvec },
 242        {}
 243};
 244
 245static ctl_table kernel_root_table2[] = {
 246        { .procname = "kernel", .mode = 0555,
 247          .child = kernel_table2 },
 248        {}
 249};
 250#endif
 251
 252/* Assume __initcall executes before all user space. Hopefully kmod
 253   doesn't violate that. We'll find out if it does. */
 254static void __cpuinit vsyscall_set_cpu(int cpu)
 255{
 256        unsigned long d;
 257        unsigned long node = 0;
 258#ifdef CONFIG_NUMA
 259        node = cpu_to_node(cpu);
 260#endif
 261        if (cpu_has(&cpu_data(cpu), X86_FEATURE_RDTSCP))
 262                write_rdtscp_aux((node << 12) | cpu);
 263
 264        /* Store cpu number in limit so that it can be loaded quickly
 265           in user space in vgetcpu.
 266           12 bits for the CPU and 8 bits for the node. */
 267        d = 0x0f40000000000ULL;
 268        d |= cpu;
 269        d |= (node & 0xf) << 12;
 270        d |= (node >> 4) << 48;
 271        write_gdt_entry(get_cpu_gdt_table(cpu), GDT_ENTRY_PER_CPU, &d, DESCTYPE_S);
 272}
 273
 274static void __cpuinit cpu_vsyscall_init(void *arg)
 275{
 276        /* preemption should be already off */
 277        vsyscall_set_cpu(raw_smp_processor_id());
 278}
 279
 280static int __cpuinit
 281cpu_vsyscall_notifier(struct notifier_block *n, unsigned long action, void *arg)
 282{
 283        long cpu = (long)arg;
 284        if (action == CPU_ONLINE || action == CPU_ONLINE_FROZEN)
 285                smp_call_function_single(cpu, cpu_vsyscall_init, NULL, 1);
 286        return NOTIFY_DONE;
 287}
 288
 289void __init map_vsyscall(void)
 290{
 291        extern char __vsyscall_0;
 292        unsigned long physaddr_page0 = __pa_symbol(&__vsyscall_0);
 293
 294        /* Note that VSYSCALL_MAPPED_PAGES must agree with the code below. */
 295        __set_fixmap(VSYSCALL_FIRST_PAGE, physaddr_page0, PAGE_KERNEL_VSYSCALL);
 296}
 297
 298static int __init vsyscall_init(void)
 299{
 300        BUG_ON(((unsigned long) &vgettimeofday !=
 301                        VSYSCALL_ADDR(__NR_vgettimeofday)));
 302        BUG_ON((unsigned long) &vtime != VSYSCALL_ADDR(__NR_vtime));
 303        BUG_ON((VSYSCALL_ADDR(0) != __fix_to_virt(VSYSCALL_FIRST_PAGE)));
 304        BUG_ON((unsigned long) &vgetcpu != VSYSCALL_ADDR(__NR_vgetcpu));
 305#ifdef CONFIG_SYSCTL
 306        register_sysctl_table(kernel_root_table2);
 307#endif
 308        on_each_cpu(cpu_vsyscall_init, NULL, 1);
 309        /* notifier priority > KVM */
 310        hotcpu_notifier(cpu_vsyscall_notifier, 30);
 311        return 0;
 312}
 313
 314__initcall(vsyscall_init);
 315
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.