linux/kernel/time/tick-sched.c
<<
>>
Prefs
   1/*
   2 *  linux/kernel/time/tick-sched.c
   3 *
   4 *  Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de>
   5 *  Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar
   6 *  Copyright(C) 2006-2007  Timesys Corp., Thomas Gleixner
   7 *
   8 *  No idle tick implementation for low and high resolution timers
   9 *
  10 *  Started by: Thomas Gleixner and Ingo Molnar
  11 *
  12 *  Distribute under GPLv2.
  13 */
  14#include <linux/cpu.h>
  15#include <linux/err.h>
  16#include <linux/hrtimer.h>
  17#include <linux/interrupt.h>
  18#include <linux/kernel_stat.h>
  19#include <linux/percpu.h>
  20#include <linux/profile.h>
  21#include <linux/sched.h>
  22#include <linux/tick.h>
  23#include <linux/module.h>
  24
  25#include <asm/irq_regs.h>
  26
  27#include "tick-internal.h"
  28
  29/*
  30 * Per cpu nohz control structure
  31 */
  32static DEFINE_PER_CPU(struct tick_sched, tick_cpu_sched);
  33
  34/*
  35 * The time, when the last jiffy update happened. Protected by xtime_lock.
  36 */
  37static ktime_t last_jiffies_update;
  38
  39struct tick_sched *tick_get_tick_sched(int cpu)
  40{
  41        return &per_cpu(tick_cpu_sched, cpu);
  42}
  43
  44/*
  45 * Must be called with interrupts disabled !
  46 */
  47static void tick_do_update_jiffies64(ktime_t now)
  48{
  49        unsigned long ticks = 0;
  50        ktime_t delta;
  51
  52        /*
  53         * Do a quick check without holding xtime_lock:
  54         */
  55        delta = ktime_sub(now, last_jiffies_update);
  56        if (delta.tv64 < tick_period.tv64)
  57                return;
  58
  59        /* Reevalute with xtime_lock held */
  60        write_seqlock(&xtime_lock);
  61
  62        delta = ktime_sub(now, last_jiffies_update);
  63        if (delta.tv64 >= tick_period.tv64) {
  64
  65                delta = ktime_sub(delta, tick_period);
  66                last_jiffies_update = ktime_add(last_jiffies_update,
  67                                                tick_period);
  68
  69                /* Slow path for long timeouts */
  70                if (unlikely(delta.tv64 >= tick_period.tv64)) {
  71                        s64 incr = ktime_to_ns(tick_period);
  72
  73                        ticks = ktime_divns(delta, incr);
  74
  75                        last_jiffies_update = ktime_add_ns(last_jiffies_update,
  76                                                           incr * ticks);
  77                }
  78                do_timer(++ticks);
  79
  80                /* Keep the tick_next_period variable up to date */
  81                tick_next_period = ktime_add(last_jiffies_update, tick_period);
  82        }
  83        write_sequnlock(&xtime_lock);
  84}
  85
  86/*
  87 * Initialize and return retrieve the jiffies update.
  88 */
  89static ktime_t tick_init_jiffy_update(void)
  90{
  91        ktime_t period;
  92
  93        write_seqlock(&xtime_lock);
  94        /* Did we start the jiffies update yet ? */
  95        if (last_jiffies_update.tv64 == 0)
  96                last_jiffies_update = tick_next_period;
  97        period = last_jiffies_update;
  98        write_sequnlock(&xtime_lock);
  99        return period;
 100}
 101
 102/*
 103 * NOHZ - aka dynamic tick functionality
 104 */
 105#ifdef CONFIG_NO_HZ
 106/*
 107 * NO HZ enabled ?
 108 */
 109static int tick_nohz_enabled __read_mostly  = 1;
 110
 111/*
 112 * Enable / Disable tickless mode
 113 */
 114static int __init setup_tick_nohz(char *str)
 115{
 116        if (!strcmp(str, "off"))
 117                tick_nohz_enabled = 0;
 118        else if (!strcmp(str, "on"))
 119                tick_nohz_enabled = 1;
 120        else
 121                return 0;
 122        return 1;
 123}
 124
 125__setup("nohz=", setup_tick_nohz);
 126
 127/**
 128 * tick_nohz_update_jiffies - update jiffies when idle was interrupted
 129 *
 130 * Called from interrupt entry when the CPU was idle
 131 *
 132 * In case the sched_tick was stopped on this CPU, we have to check if jiffies
 133 * must be updated. Otherwise an interrupt handler could use a stale jiffy
 134 * value. We do this unconditionally on any cpu, as we don't know whether the
 135 * cpu, which has the update task assigned is in a long sleep.
 136 */
 137void tick_nohz_update_jiffies(void)
 138{
 139        int cpu = smp_processor_id();
 140        struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
 141        unsigned long flags;
 142        ktime_t now;
 143
 144        if (!ts->tick_stopped)
 145                return;
 146
 147        cpu_clear(cpu, nohz_cpu_mask);
 148        now = ktime_get();
 149        ts->idle_waketime = now;
 150
 151        local_irq_save(flags);
 152        tick_do_update_jiffies64(now);
 153        local_irq_restore(flags);
 154
 155        touch_softlockup_watchdog();
 156}
 157
 158static void tick_nohz_stop_idle(int cpu)
 159{
 160        struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
 161
 162        if (ts->idle_active) {
 163                ktime_t now, delta;
 164                now = ktime_get();
 165                delta = ktime_sub(now, ts->idle_entrytime);
 166                ts->idle_lastupdate = now;
 167                ts->idle_sleeptime = ktime_add(ts->idle_sleeptime, delta);
 168                ts->idle_active = 0;
 169
 170                sched_clock_idle_wakeup_event(0);
 171        }
 172}
 173
 174static ktime_t tick_nohz_start_idle(struct tick_sched *ts)
 175{
 176        ktime_t now, delta;
 177
 178        now = ktime_get();
 179        if (ts->idle_active) {
 180                delta = ktime_sub(now, ts->idle_entrytime);
 181                ts->idle_lastupdate = now;
 182                ts->idle_sleeptime = ktime_add(ts->idle_sleeptime, delta);
 183        }
 184        ts->idle_entrytime = now;
 185        ts->idle_active = 1;
 186        sched_clock_idle_sleep_event();
 187        return now;
 188}
 189
 190u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time)
 191{
 192        struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
 193
 194        if (!tick_nohz_enabled)
 195                return -1;
 196
 197        if (ts->idle_active)
 198                *last_update_time = ktime_to_us(ts->idle_lastupdate);
 199        else
 200                *last_update_time = ktime_to_us(ktime_get());
 201
 202        return ktime_to_us(ts->idle_sleeptime);
 203}
 204EXPORT_SYMBOL_GPL(get_cpu_idle_time_us);
 205
 206/**
 207 * tick_nohz_stop_sched_tick - stop the idle tick from the idle task
 208 *
 209 * When the next event is more than a tick into the future, stop the idle tick
 210 * Called either from the idle loop or from irq_exit() when an idle period was
 211 * just interrupted by an interrupt which did not cause a reschedule.
 212 */
 213void tick_nohz_stop_sched_tick(int inidle)
 214{
 215        unsigned long seq, last_jiffies, next_jiffies, delta_jiffies, flags;
 216        struct tick_sched *ts;
 217        ktime_t last_update, expires, now;
 218        struct clock_event_device *dev = __get_cpu_var(tick_cpu_device).evtdev;
 219        int cpu;
 220
 221        local_irq_save(flags);
 222
 223        cpu = smp_processor_id();
 224        ts = &per_cpu(tick_cpu_sched, cpu);
 225        now = tick_nohz_start_idle(ts);
 226
 227        /*
 228         * If this cpu is offline and it is the one which updates
 229         * jiffies, then give up the assignment and let it be taken by
 230         * the cpu which runs the tick timer next. If we don't drop
 231         * this here the jiffies might be stale and do_timer() never
 232         * invoked.
 233         */
 234        if (unlikely(!cpu_online(cpu))) {
 235                if (cpu == tick_do_timer_cpu)
 236                        tick_do_timer_cpu = TICK_DO_TIMER_NONE;
 237        }
 238
 239        if (unlikely(ts->nohz_mode == NOHZ_MODE_INACTIVE))
 240                goto end;
 241
 242        if (!inidle && !ts->inidle)
 243                goto end;
 244
 245        ts->inidle = 1;
 246
 247        if (need_resched())
 248                goto end;
 249
 250        if (unlikely(local_softirq_pending())) {
 251                static int ratelimit;
 252
 253                if (ratelimit < 10) {
 254                        printk(KERN_ERR "NOHZ: local_softirq_pending %02x\n",
 255                               local_softirq_pending());
 256                        ratelimit++;
 257                }
 258                goto end;
 259        }
 260
 261        ts->idle_calls++;
 262        /* Read jiffies and the time when jiffies were updated last */
 263        do {
 264                seq = read_seqbegin(&xtime_lock);
 265                last_update = last_jiffies_update;
 266                last_jiffies = jiffies;
 267        } while (read_seqretry(&xtime_lock, seq));
 268
 269        /* Get the next timer wheel timer */
 270        next_jiffies = get_next_timer_interrupt(last_jiffies);
 271        delta_jiffies = next_jiffies - last_jiffies;
 272
 273        if (rcu_needs_cpu(cpu) || printk_needs_cpu(cpu))
 274                delta_jiffies = 1;
 275        /*
 276         * Do not stop the tick, if we are only one off
 277         * or if the cpu is required for rcu
 278         */
 279        if (!ts->tick_stopped && delta_jiffies == 1)
 280                goto out;
 281
 282        /* Schedule the tick, if we are at least one jiffie off */
 283        if ((long)delta_jiffies >= 1) {
 284
 285                if (delta_jiffies > 1)
 286                        cpu_set(cpu, nohz_cpu_mask);
 287                /*
 288                 * nohz_stop_sched_tick can be called several times before
 289                 * the nohz_restart_sched_tick is called. This happens when
 290                 * interrupts arrive which do not cause a reschedule. In the
 291                 * first call we save the current tick time, so we can restart
 292                 * the scheduler tick in nohz_restart_sched_tick.
 293                 */
 294                if (!ts->tick_stopped) {
 295                        if (select_nohz_load_balancer(1)) {
 296                                /*
 297                                 * sched tick not stopped!
 298                                 */
 299                                cpu_clear(cpu, nohz_cpu_mask);
 300                                goto out;
 301                        }
 302
 303                        ts->idle_tick = hrtimer_get_expires(&ts->sched_timer);
 304                        ts->tick_stopped = 1;
 305                        ts->idle_jiffies = last_jiffies;
 306                        rcu_enter_nohz();
 307                }
 308
 309                /*
 310                 * If this cpu is the one which updates jiffies, then
 311                 * give up the assignment and let it be taken by the
 312                 * cpu which runs the tick timer next, which might be
 313                 * this cpu as well. If we don't drop this here the
 314                 * jiffies might be stale and do_timer() never
 315                 * invoked.
 316                 */
 317                if (cpu == tick_do_timer_cpu)
 318                        tick_do_timer_cpu = TICK_DO_TIMER_NONE;
 319
 320                ts->idle_sleeps++;
 321
 322                /*
 323                 * delta_jiffies >= NEXT_TIMER_MAX_DELTA signals that
 324                 * there is no timer pending or at least extremly far
 325                 * into the future (12 days for HZ=1000). In this case
 326                 * we simply stop the tick timer:
 327                 */
 328                if (unlikely(delta_jiffies >= NEXT_TIMER_MAX_DELTA)) {
 329                        ts->idle_expires.tv64 = KTIME_MAX;
 330                        if (ts->nohz_mode == NOHZ_MODE_HIGHRES)
 331                                hrtimer_cancel(&ts->sched_timer);
 332                        goto out;
 333                }
 334
 335                /*
 336                 * calculate the expiry time for the next timer wheel
 337                 * timer
 338                 */
 339                expires = ktime_add_ns(last_update, tick_period.tv64 *
 340                                       delta_jiffies);
 341                ts->idle_expires = expires;
 342
 343                if (ts->nohz_mode == NOHZ_MODE_HIGHRES) {
 344                        hrtimer_start(&ts->sched_timer, expires,
 345                                      HRTIMER_MODE_ABS);
 346                        /* Check, if the timer was already in the past */
 347                        if (hrtimer_active(&ts->sched_timer))
 348                                goto out;
 349                } else if (!tick_program_event(expires, 0))
 350                                goto out;
 351                /*
 352                 * We are past the event already. So we crossed a
 353                 * jiffie boundary. Update jiffies and raise the
 354                 * softirq.
 355                 */
 356                tick_do_update_jiffies64(ktime_get());
 357                cpu_clear(cpu, nohz_cpu_mask);
 358        }
 359        raise_softirq_irqoff(TIMER_SOFTIRQ);
 360out:
 361        ts->next_jiffies = next_jiffies;
 362        ts->last_jiffies = last_jiffies;
 363        ts->sleep_length = ktime_sub(dev->next_event, now);
 364end:
 365        local_irq_restore(flags);
 366}
 367
 368/**
 369 * tick_nohz_get_sleep_length - return the length of the current sleep
 370 *
 371 * Called from power state control code with interrupts disabled
 372 */
 373ktime_t tick_nohz_get_sleep_length(void)
 374{
 375        struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
 376
 377        return ts->sleep_length;
 378}
 379
 380static void tick_nohz_restart(struct tick_sched *ts, ktime_t now)
 381{
 382        hrtimer_cancel(&ts->sched_timer);
 383        hrtimer_set_expires(&ts->sched_timer, ts->idle_tick);
 384
 385        while (1) {
 386                /* Forward the time to expire in the future */
 387                hrtimer_forward(&ts->sched_timer, now, tick_period);
 388
 389                if (ts->nohz_mode == NOHZ_MODE_HIGHRES) {
 390                        hrtimer_start_expires(&ts->sched_timer,
 391                                      HRTIMER_MODE_ABS);
 392                        /* Check, if the timer was already in the past */
 393                        if (hrtimer_active(&ts->sched_timer))
 394                                break;
 395                } else {
 396                        if (!tick_program_event(
 397                                hrtimer_get_expires(&ts->sched_timer), 0))
 398                                break;
 399                }
 400                /* Update jiffies and reread time */
 401                tick_do_update_jiffies64(now);
 402                now = ktime_get();
 403        }
 404}
 405
 406/**
 407 * tick_nohz_restart_sched_tick - restart the idle tick from the idle task
 408 *
 409 * Restart the idle tick when the CPU is woken up from idle
 410 */
 411void tick_nohz_restart_sched_tick(void)
 412{
 413        int cpu = smp_processor_id();
 414        struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
 415        unsigned long ticks;
 416        ktime_t now;
 417
 418        local_irq_disable();
 419        tick_nohz_stop_idle(cpu);
 420
 421        if (!ts->inidle || !ts->tick_stopped) {
 422                ts->inidle = 0;
 423                local_irq_enable();
 424                return;
 425        }
 426
 427        ts->inidle = 0;
 428
 429        rcu_exit_nohz();
 430
 431        /* Update jiffies first */
 432        select_nohz_load_balancer(0);
 433        now = ktime_get();
 434        tick_do_update_jiffies64(now);
 435        cpu_clear(cpu, nohz_cpu_mask);
 436
 437        /*
 438         * We stopped the tick in idle. Update process times would miss the
 439         * time we slept as update_process_times does only a 1 tick
 440         * accounting. Enforce that this is accounted to idle !
 441         */
 442        ticks = jiffies - ts->idle_jiffies;
 443        /*
 444         * We might be one off. Do not randomly account a huge number of ticks!
 445         */
 446        if (ticks && ticks < LONG_MAX) {
 447                add_preempt_count(HARDIRQ_OFFSET);
 448                account_system_time(current, HARDIRQ_OFFSET,
 449                                    jiffies_to_cputime(ticks));
 450                sub_preempt_count(HARDIRQ_OFFSET);
 451        }
 452
 453        touch_softlockup_watchdog();
 454        /*
 455         * Cancel the scheduled timer and restore the tick
 456         */
 457        ts->tick_stopped  = 0;
 458        ts->idle_exittime = now;
 459
 460        tick_nohz_restart(ts, now);
 461
 462        local_irq_enable();
 463}
 464
 465static int tick_nohz_reprogram(struct tick_sched *ts, ktime_t now)
 466{
 467        hrtimer_forward(&ts->sched_timer, now, tick_period);
 468        return tick_program_event(hrtimer_get_expires(&ts->sched_timer), 0);
 469}
 470
 471/*
 472 * The nohz low res interrupt handler
 473 */
 474static void tick_nohz_handler(struct clock_event_device *dev)
 475{
 476        struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
 477        struct pt_regs *regs = get_irq_regs();
 478        int cpu = smp_processor_id();
 479        ktime_t now = ktime_get();
 480
 481        dev->next_event.tv64 = KTIME_MAX;
 482
 483        /*
 484         * Check if the do_timer duty was dropped. We don't care about
 485         * concurrency: This happens only when the cpu in charge went
 486         * into a long sleep. If two cpus happen to assign themself to
 487         * this duty, then the jiffies update is still serialized by
 488         * xtime_lock.
 489         */
 490        if (unlikely(tick_do_timer_cpu == TICK_DO_TIMER_NONE))
 491                tick_do_timer_cpu = cpu;
 492
 493        /* Check, if the jiffies need an update */
 494        if (tick_do_timer_cpu == cpu)
 495                tick_do_update_jiffies64(now);
 496
 497        /*
 498         * When we are idle and the tick is stopped, we have to touch
 499         * the watchdog as we might not schedule for a really long
 500         * time. This happens on complete idle SMP systems while
 501         * waiting on the login prompt. We also increment the "start
 502         * of idle" jiffy stamp so the idle accounting adjustment we
 503         * do when we go busy again does not account too much ticks.
 504         */
 505        if (ts->tick_stopped) {
 506                touch_softlockup_watchdog();
 507                ts->idle_jiffies++;
 508        }
 509
 510        update_process_times(user_mode(regs));
 511        profile_tick(CPU_PROFILING);
 512
 513        while (tick_nohz_reprogram(ts, now)) {
 514                now = ktime_get();
 515                tick_do_update_jiffies64(now);
 516        }
 517}
 518
 519/**
 520 * tick_nohz_switch_to_nohz - switch to nohz mode
 521 */
 522static void tick_nohz_switch_to_nohz(void)
 523{
 524        struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
 525        ktime_t next;
 526
 527        if (!tick_nohz_enabled)
 528                return;
 529
 530        local_irq_disable();
 531        if (tick_switch_to_oneshot(tick_nohz_handler)) {
 532                local_irq_enable();
 533                return;
 534        }
 535
 536        ts->nohz_mode = NOHZ_MODE_LOWRES;
 537
 538        /*
 539         * Recycle the hrtimer in ts, so we can share the
 540         * hrtimer_forward with the highres code.
 541         */
 542        hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
 543        /* Get the next period */
 544        next = tick_init_jiffy_update();
 545
 546        for (;;) {
 547                hrtimer_set_expires(&ts->sched_timer, next);
 548                if (!tick_program_event(next, 0))
 549                        break;
 550                next = ktime_add(next, tick_period);
 551        }
 552        local_irq_enable();
 553
 554        printk(KERN_INFO "Switched to NOHz mode on CPU #%d\n",
 555               smp_processor_id());
 556}
 557
 558/*
 559 * When NOHZ is enabled and the tick is stopped, we need to kick the
 560 * tick timer from irq_enter() so that the jiffies update is kept
 561 * alive during long running softirqs. That's ugly as hell, but
 562 * correctness is key even if we need to fix the offending softirq in
 563 * the first place.
 564 *
 565 * Note, this is different to tick_nohz_restart. We just kick the
 566 * timer and do not touch the other magic bits which need to be done
 567 * when idle is left.
 568 */
 569static void tick_nohz_kick_tick(int cpu)
 570{
 571#if 0
 572        /* Switch back to 2.6.27 behaviour */
 573
 574        struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
 575        ktime_t delta, now;
 576
 577        if (!ts->tick_stopped)
 578                return;
 579
 580        /*
 581         * Do not touch the tick device, when the next expiry is either
 582         * already reached or less/equal than the tick period.
 583         */
 584        now = ktime_get();
 585        delta = ktime_sub(hrtimer_get_expires(&ts->sched_timer), now);
 586        if (delta.tv64 <= tick_period.tv64)
 587                return;
 588
 589        tick_nohz_restart(ts, now);
 590#endif
 591}
 592
 593#else
 594
 595static inline void tick_nohz_switch_to_nohz(void) { }
 596
 597#endif /* NO_HZ */
 598
 599/*
 600 * Called from irq_enter to notify about the possible interruption of idle()
 601 */
 602void tick_check_idle(int cpu)
 603{
 604        tick_check_oneshot_broadcast(cpu);
 605#ifdef CONFIG_NO_HZ
 606        tick_nohz_stop_idle(cpu);
 607        tick_nohz_update_jiffies();
 608        tick_nohz_kick_tick(cpu);
 609#endif
 610}
 611
 612/*
 613 * High resolution timer specific code
 614 */
 615#ifdef CONFIG_HIGH_RES_TIMERS
 616/*
 617 * We rearm the timer until we get disabled by the idle code.
 618 * Called with interrupts disabled and timer->base->cpu_base->lock held.
 619 */
 620static enum hrtimer_restart tick_sched_timer(struct hrtimer *timer)
 621{
 622        struct tick_sched *ts =
 623                container_of(timer, struct tick_sched, sched_timer);
 624        struct pt_regs *regs = get_irq_regs();
 625        ktime_t now = ktime_get();
 626        int cpu = smp_processor_id();
 627
 628#ifdef CONFIG_NO_HZ
 629        /*
 630         * Check if the do_timer duty was dropped. We don't care about
 631         * concurrency: This happens only when the cpu in charge went
 632         * into a long sleep. If two cpus happen to assign themself to
 633         * this duty, then the jiffies update is still serialized by
 634         * xtime_lock.
 635         */
 636        if (unlikely(tick_do_timer_cpu == TICK_DO_TIMER_NONE))
 637                tick_do_timer_cpu = cpu;
 638#endif
 639
 640        /* Check, if the jiffies need an update */
 641        if (tick_do_timer_cpu == cpu)
 642                tick_do_update_jiffies64(now);
 643
 644        /*
 645         * Do not call, when we are not in irq context and have
 646         * no valid regs pointer
 647         */
 648        if (regs) {
 649                /*
 650                 * When we are idle and the tick is stopped, we have to touch
 651                 * the watchdog as we might not schedule for a really long
 652                 * time. This happens on complete idle SMP systems while
 653                 * waiting on the login prompt. We also increment the "start of
 654                 * idle" jiffy stamp so the idle accounting adjustment we do
 655                 * when we go busy again does not account too much ticks.
 656                 */
 657                if (ts->tick_stopped) {
 658                        touch_softlockup_watchdog();
 659                        ts->idle_jiffies++;
 660                }
 661                update_process_times(user_mode(regs));
 662                profile_tick(CPU_PROFILING);
 663        }
 664
 665        hrtimer_forward(timer, now, tick_period);
 666
 667        return HRTIMER_RESTART;
 668}
 669
 670/**
 671 * tick_setup_sched_timer - setup the tick emulation timer
 672 */
 673void tick_setup_sched_timer(void)
 674{
 675        struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
 676        ktime_t now = ktime_get();
 677        u64 offset;
 678
 679        /*
 680         * Emulate tick processing via per-CPU hrtimers:
 681         */
 682        hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
 683        ts->sched_timer.function = tick_sched_timer;
 684        ts->sched_timer.cb_mode = HRTIMER_CB_IRQSAFE_PERCPU;
 685
 686        /* Get the next period (per cpu) */
 687        hrtimer_set_expires(&ts->sched_timer, tick_init_jiffy_update());
 688        offset = ktime_to_ns(tick_period) >> 1;
 689        do_div(offset, num_possible_cpus());
 690        offset *= smp_processor_id();
 691        hrtimer_add_expires_ns(&ts->sched_timer, offset);
 692
 693        for (;;) {
 694                hrtimer_forward(&ts->sched_timer, now, tick_period);
 695                hrtimer_start_expires(&ts->sched_timer, HRTIMER_MODE_ABS);
 696                /* Check, if the timer was already in the past */
 697                if (hrtimer_active(&ts->sched_timer))
 698                        break;
 699                now = ktime_get();
 700        }
 701
 702#ifdef CONFIG_NO_HZ
 703        if (tick_nohz_enabled)
 704                ts->nohz_mode = NOHZ_MODE_HIGHRES;
 705#endif
 706}
 707#endif /* HIGH_RES_TIMERS */
 708
 709#if defined CONFIG_NO_HZ || defined CONFIG_HIGH_RES_TIMERS
 710void tick_cancel_sched_timer(int cpu)
 711{
 712        struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
 713
 714# ifdef CONFIG_HIGH_RES_TIMERS
 715        if (ts->sched_timer.base)
 716                hrtimer_cancel(&ts->sched_timer);
 717# endif
 718
 719        ts->nohz_mode = NOHZ_MODE_INACTIVE;
 720}
 721#endif
 722
 723/**
 724 * Async notification about clocksource changes
 725 */
 726void tick_clock_notify(void)
 727{
 728        int cpu;
 729
 730        for_each_possible_cpu(cpu)
 731                set_bit(0, &per_cpu(tick_cpu_sched, cpu).check_clocks);
 732}
 733
 734/*
 735 * Async notification about clock event changes
 736 */
 737void tick_oneshot_notify(void)
 738{
 739        struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
 740
 741        set_bit(0, &ts->check_clocks);
 742}
 743
 744/**
 745 * Check, if a change happened, which makes oneshot possible.
 746 *
 747 * Called cyclic from the hrtimer softirq (driven by the timer
 748 * softirq) allow_nohz signals, that we can switch into low-res nohz
 749 * mode, because high resolution timers are disabled (either compile
 750 * or runtime).
 751 */
 752int tick_check_oneshot_change(int allow_nohz)
 753{
 754        struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
 755
 756        if (!test_and_clear_bit(0, &ts->check_clocks))
 757                return 0;
 758
 759        if (ts->nohz_mode != NOHZ_MODE_INACTIVE)
 760                return 0;
 761
 762        if (!timekeeping_valid_for_hres() || !tick_is_oneshot_available())
 763                return 0;
 764
 765        if (!allow_nohz)
 766                return 1;
 767
 768        tick_nohz_switch_to_nohz();
 769        return 0;
 770}
 771