linux/kernel/time/tick-sched.c
<<
>>
Prefs
   1/*
   2 *  linux/kernel/time/tick-sched.c
   3 *
   4 *  Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de>
   5 *  Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar
   6 *  Copyright(C) 2006-2007  Timesys Corp., Thomas Gleixner
   7 *
   8 *  No idle tick implementation for low and high resolution timers
   9 *
  10 *  Started by: Thomas Gleixner and Ingo Molnar
  11 *
  12 *  Distribute under GPLv2.
  13 */
  14#include <linux/cpu.h>
  15#include <linux/err.h>
  16#include <linux/hrtimer.h>
  17#include <linux/interrupt.h>
  18#include <linux/kernel_stat.h>
  19#include <linux/percpu.h>
  20#include <linux/profile.h>
  21#include <linux/sched.h>
  22#include <linux/module.h>
  23#include <linux/irq_work.h>
  24
  25#include <asm/irq_regs.h>
  26
  27#include "tick-internal.h"
  28
  29/*
  30 * Per cpu nohz control structure
  31 */
  32DEFINE_PER_CPU(struct tick_sched, tick_cpu_sched);
  33
  34/*
  35 * The time, when the last jiffy update happened. Protected by jiffies_lock.
  36 */
  37static ktime_t last_jiffies_update;
  38
  39struct tick_sched *tick_get_tick_sched(int cpu)
  40{
  41        return &per_cpu(tick_cpu_sched, cpu);
  42}
  43
  44/*
  45 * Must be called with interrupts disabled !
  46 */
  47static void tick_do_update_jiffies64(ktime_t now)
  48{
  49        unsigned long ticks = 0;
  50        ktime_t delta;
  51
  52        /*
  53         * Do a quick check without holding jiffies_lock:
  54         */
  55        delta = ktime_sub(now, last_jiffies_update);
  56        if (delta.tv64 < tick_period.tv64)
  57                return;
  58
  59        /* Reevalute with jiffies_lock held */
  60        write_seqlock(&jiffies_lock);
  61
  62        delta = ktime_sub(now, last_jiffies_update);
  63        if (delta.tv64 >= tick_period.tv64) {
  64
  65                delta = ktime_sub(delta, tick_period);
  66                last_jiffies_update = ktime_add(last_jiffies_update,
  67                                                tick_period);
  68
  69                /* Slow path for long timeouts */
  70                if (unlikely(delta.tv64 >= tick_period.tv64)) {
  71                        s64 incr = ktime_to_ns(tick_period);
  72
  73                        ticks = ktime_divns(delta, incr);
  74
  75                        last_jiffies_update = ktime_add_ns(last_jiffies_update,
  76                                                           incr * ticks);
  77                }
  78                do_timer(++ticks);
  79
  80                /* Keep the tick_next_period variable up to date */
  81                tick_next_period = ktime_add(last_jiffies_update, tick_period);
  82        }
  83        write_sequnlock(&jiffies_lock);
  84}
  85
  86/*
  87 * Initialize and return retrieve the jiffies update.
  88 */
  89static ktime_t tick_init_jiffy_update(void)
  90{
  91        ktime_t period;
  92
  93        write_seqlock(&jiffies_lock);
  94        /* Did we start the jiffies update yet ? */
  95        if (last_jiffies_update.tv64 == 0)
  96                last_jiffies_update = tick_next_period;
  97        period = last_jiffies_update;
  98        write_sequnlock(&jiffies_lock);
  99        return period;
 100}
 101
 102
 103static void tick_sched_do_timer(ktime_t now)
 104{
 105        int cpu = smp_processor_id();
 106
 107#ifdef CONFIG_NO_HZ
 108        /*
 109         * Check if the do_timer duty was dropped. We don't care about
 110         * concurrency: This happens only when the cpu in charge went
 111         * into a long sleep. If two cpus happen to assign themself to
 112         * this duty, then the jiffies update is still serialized by
 113         * jiffies_lock.
 114         */
 115        if (unlikely(tick_do_timer_cpu == TICK_DO_TIMER_NONE))
 116                tick_do_timer_cpu = cpu;
 117#endif
 118
 119        /* Check, if the jiffies need an update */
 120        if (tick_do_timer_cpu == cpu)
 121                tick_do_update_jiffies64(now);
 122}
 123
 124static void tick_sched_handle(struct tick_sched *ts, struct pt_regs *regs)
 125{
 126#ifdef CONFIG_NO_HZ
 127        /*
 128         * When we are idle and the tick is stopped, we have to touch
 129         * the watchdog as we might not schedule for a really long
 130         * time. This happens on complete idle SMP systems while
 131         * waiting on the login prompt. We also increment the "start of
 132         * idle" jiffy stamp so the idle accounting adjustment we do
 133         * when we go busy again does not account too much ticks.
 134         */
 135        if (ts->tick_stopped) {
 136                touch_softlockup_watchdog();
 137                if (is_idle_task(current))
 138                        ts->idle_jiffies++;
 139        }
 140#endif
 141        update_process_times(user_mode(regs));
 142        profile_tick(CPU_PROFILING);
 143}
 144
 145/*
 146 * NOHZ - aka dynamic tick functionality
 147 */
 148#ifdef CONFIG_NO_HZ
 149/*
 150 * NO HZ enabled ?
 151 */
 152int tick_nohz_enabled __read_mostly  = 1;
 153
 154/*
 155 * Enable / Disable tickless mode
 156 */
 157static int __init setup_tick_nohz(char *str)
 158{
 159        if (!strcmp(str, "off"))
 160                tick_nohz_enabled = 0;
 161        else if (!strcmp(str, "on"))
 162                tick_nohz_enabled = 1;
 163        else
 164                return 0;
 165        return 1;
 166}
 167
 168__setup("nohz=", setup_tick_nohz);
 169
 170/**
 171 * tick_nohz_update_jiffies - update jiffies when idle was interrupted
 172 *
 173 * Called from interrupt entry when the CPU was idle
 174 *
 175 * In case the sched_tick was stopped on this CPU, we have to check if jiffies
 176 * must be updated. Otherwise an interrupt handler could use a stale jiffy
 177 * value. We do this unconditionally on any cpu, as we don't know whether the
 178 * cpu, which has the update task assigned is in a long sleep.
 179 */
 180static void tick_nohz_update_jiffies(ktime_t now)
 181{
 182        int cpu = smp_processor_id();
 183        struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
 184        unsigned long flags;
 185
 186        ts->idle_waketime = now;
 187
 188        local_irq_save(flags);
 189        tick_do_update_jiffies64(now);
 190        local_irq_restore(flags);
 191
 192        touch_softlockup_watchdog();
 193}
 194
 195/*
 196 * Updates the per cpu time idle statistics counters
 197 */
 198static void
 199update_ts_time_stats(int cpu, struct tick_sched *ts, ktime_t now, u64 *last_update_time)
 200{
 201        ktime_t delta;
 202
 203        if (ts->idle_active) {
 204                delta = ktime_sub(now, ts->idle_entrytime);
 205                if (nr_iowait_cpu(cpu) > 0)
 206                        ts->iowait_sleeptime = ktime_add(ts->iowait_sleeptime, delta);
 207                else
 208                        ts->idle_sleeptime = ktime_add(ts->idle_sleeptime, delta);
 209                ts->idle_entrytime = now;
 210        }
 211
 212        if (last_update_time)
 213                *last_update_time = ktime_to_us(now);
 214
 215}
 216
 217static void tick_nohz_stop_idle(int cpu, ktime_t now)
 218{
 219        struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
 220
 221        update_ts_time_stats(cpu, ts, now, NULL);
 222        ts->idle_active = 0;
 223
 224        sched_clock_idle_wakeup_event(0);
 225}
 226
 227static ktime_t tick_nohz_start_idle(int cpu, struct tick_sched *ts)
 228{
 229        ktime_t now = ktime_get();
 230
 231        ts->idle_entrytime = now;
 232        ts->idle_active = 1;
 233        sched_clock_idle_sleep_event();
 234        return now;
 235}
 236
 237/**
 238 * get_cpu_idle_time_us - get the total idle time of a cpu
 239 * @cpu: CPU number to query
 240 * @last_update_time: variable to store update time in. Do not update
 241 * counters if NULL.
 242 *
 243 * Return the cummulative idle time (since boot) for a given
 244 * CPU, in microseconds.
 245 *
 246 * This time is measured via accounting rather than sampling,
 247 * and is as accurate as ktime_get() is.
 248 *
 249 * This function returns -1 if NOHZ is not enabled.
 250 */
 251u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time)
 252{
 253        struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
 254        ktime_t now, idle;
 255
 256        if (!tick_nohz_enabled)
 257                return -1;
 258
 259        now = ktime_get();
 260        if (last_update_time) {
 261                update_ts_time_stats(cpu, ts, now, last_update_time);
 262                idle = ts->idle_sleeptime;
 263        } else {
 264                if (ts->idle_active && !nr_iowait_cpu(cpu)) {
 265                        ktime_t delta = ktime_sub(now, ts->idle_entrytime);
 266
 267                        idle = ktime_add(ts->idle_sleeptime, delta);
 268                } else {
 269                        idle = ts->idle_sleeptime;
 270                }
 271        }
 272
 273        return ktime_to_us(idle);
 274
 275}
 276EXPORT_SYMBOL_GPL(get_cpu_idle_time_us);
 277
 278/**
 279 * get_cpu_iowait_time_us - get the total iowait time of a cpu
 280 * @cpu: CPU number to query
 281 * @last_update_time: variable to store update time in. Do not update
 282 * counters if NULL.
 283 *
 284 * Return the cummulative iowait time (since boot) for a given
 285 * CPU, in microseconds.
 286 *
 287 * This time is measured via accounting rather than sampling,
 288 * and is as accurate as ktime_get() is.
 289 *
 290 * This function returns -1 if NOHZ is not enabled.
 291 */
 292u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_time)
 293{
 294        struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
 295        ktime_t now, iowait;
 296
 297        if (!tick_nohz_enabled)
 298                return -1;
 299
 300        now = ktime_get();
 301        if (last_update_time) {
 302                update_ts_time_stats(cpu, ts, now, last_update_time);
 303                iowait = ts->iowait_sleeptime;
 304        } else {
 305                if (ts->idle_active && nr_iowait_cpu(cpu) > 0) {
 306                        ktime_t delta = ktime_sub(now, ts->idle_entrytime);
 307
 308                        iowait = ktime_add(ts->iowait_sleeptime, delta);
 309                } else {
 310                        iowait = ts->iowait_sleeptime;
 311                }
 312        }
 313
 314        return ktime_to_us(iowait);
 315}
 316EXPORT_SYMBOL_GPL(get_cpu_iowait_time_us);
 317
 318static ktime_t tick_nohz_stop_sched_tick(struct tick_sched *ts,
 319                                         ktime_t now, int cpu)
 320{
 321        unsigned long seq, last_jiffies, next_jiffies, delta_jiffies;
 322        ktime_t last_update, expires, ret = { .tv64 = 0 };
 323        unsigned long rcu_delta_jiffies;
 324        struct clock_event_device *dev = __get_cpu_var(tick_cpu_device).evtdev;
 325        u64 time_delta;
 326
 327        /* Read jiffies and the time when jiffies were updated last */
 328        do {
 329                seq = read_seqbegin(&jiffies_lock);
 330                last_update = last_jiffies_update;
 331                last_jiffies = jiffies;
 332                time_delta = timekeeping_max_deferment();
 333        } while (read_seqretry(&jiffies_lock, seq));
 334
 335        if (rcu_needs_cpu(cpu, &rcu_delta_jiffies) ||
 336            arch_needs_cpu(cpu) || irq_work_needs_cpu()) {
 337                next_jiffies = last_jiffies + 1;
 338                delta_jiffies = 1;
 339        } else {
 340                /* Get the next timer wheel timer */
 341                next_jiffies = get_next_timer_interrupt(last_jiffies);
 342                delta_jiffies = next_jiffies - last_jiffies;
 343                if (rcu_delta_jiffies < delta_jiffies) {
 344                        next_jiffies = last_jiffies + rcu_delta_jiffies;
 345                        delta_jiffies = rcu_delta_jiffies;
 346                }
 347        }
 348        /*
 349         * Do not stop the tick, if we are only one off
 350         * or if the cpu is required for rcu
 351         */
 352        if (!ts->tick_stopped && delta_jiffies == 1)
 353                goto out;
 354
 355        /* Schedule the tick, if we are at least one jiffie off */
 356        if ((long)delta_jiffies >= 1) {
 357
 358                /*
 359                 * If this cpu is the one which updates jiffies, then
 360                 * give up the assignment and let it be taken by the
 361                 * cpu which runs the tick timer next, which might be
 362                 * this cpu as well. If we don't drop this here the
 363                 * jiffies might be stale and do_timer() never
 364                 * invoked. Keep track of the fact that it was the one
 365                 * which had the do_timer() duty last. If this cpu is
 366                 * the one which had the do_timer() duty last, we
 367                 * limit the sleep time to the timekeeping
 368                 * max_deferement value which we retrieved
 369                 * above. Otherwise we can sleep as long as we want.
 370                 */
 371                if (cpu == tick_do_timer_cpu) {
 372                        tick_do_timer_cpu = TICK_DO_TIMER_NONE;
 373                        ts->do_timer_last = 1;
 374                } else if (tick_do_timer_cpu != TICK_DO_TIMER_NONE) {
 375                        time_delta = KTIME_MAX;
 376                        ts->do_timer_last = 0;
 377                } else if (!ts->do_timer_last) {
 378                        time_delta = KTIME_MAX;
 379                }
 380
 381                /*
 382                 * calculate the expiry time for the next timer wheel
 383                 * timer. delta_jiffies >= NEXT_TIMER_MAX_DELTA signals
 384                 * that there is no timer pending or at least extremely
 385                 * far into the future (12 days for HZ=1000). In this
 386                 * case we set the expiry to the end of time.
 387                 */
 388                if (likely(delta_jiffies < NEXT_TIMER_MAX_DELTA)) {
 389                        /*
 390                         * Calculate the time delta for the next timer event.
 391                         * If the time delta exceeds the maximum time delta
 392                         * permitted by the current clocksource then adjust
 393                         * the time delta accordingly to ensure the
 394                         * clocksource does not wrap.
 395                         */
 396                        time_delta = min_t(u64, time_delta,
 397                                           tick_period.tv64 * delta_jiffies);
 398                }
 399
 400                if (time_delta < KTIME_MAX)
 401                        expires = ktime_add_ns(last_update, time_delta);
 402                else
 403                        expires.tv64 = KTIME_MAX;
 404
 405                /* Skip reprogram of event if its not changed */
 406                if (ts->tick_stopped && ktime_equal(expires, dev->next_event))
 407                        goto out;
 408
 409                ret = expires;
 410
 411                /*
 412                 * nohz_stop_sched_tick can be called several times before
 413                 * the nohz_restart_sched_tick is called. This happens when
 414                 * interrupts arrive which do not cause a reschedule. In the
 415                 * first call we save the current tick time, so we can restart
 416                 * the scheduler tick in nohz_restart_sched_tick.
 417                 */
 418                if (!ts->tick_stopped) {
 419                        nohz_balance_enter_idle(cpu);
 420                        calc_load_enter_idle();
 421
 422                        ts->last_tick = hrtimer_get_expires(&ts->sched_timer);
 423                        ts->tick_stopped = 1;
 424                }
 425
 426                /*
 427                 * If the expiration time == KTIME_MAX, then
 428                 * in this case we simply stop the tick timer.
 429                 */
 430                 if (unlikely(expires.tv64 == KTIME_MAX)) {
 431                        if (ts->nohz_mode == NOHZ_MODE_HIGHRES)
 432                                hrtimer_cancel(&ts->sched_timer);
 433                        goto out;
 434                }
 435
 436                if (ts->nohz_mode == NOHZ_MODE_HIGHRES) {
 437                        hrtimer_start(&ts->sched_timer, expires,
 438                                      HRTIMER_MODE_ABS_PINNED);
 439                        /* Check, if the timer was already in the past */
 440                        if (hrtimer_active(&ts->sched_timer))
 441                                goto out;
 442                } else if (!tick_program_event(expires, 0))
 443                                goto out;
 444                /*
 445                 * We are past the event already. So we crossed a
 446                 * jiffie boundary. Update jiffies and raise the
 447                 * softirq.
 448                 */
 449                tick_do_update_jiffies64(ktime_get());
 450        }
 451        raise_softirq_irqoff(TIMER_SOFTIRQ);
 452out:
 453        ts->next_jiffies = next_jiffies;
 454        ts->last_jiffies = last_jiffies;
 455        ts->sleep_length = ktime_sub(dev->next_event, now);
 456
 457        return ret;
 458}
 459
 460static bool can_stop_idle_tick(int cpu, struct tick_sched *ts)
 461{
 462        /*
 463         * If this cpu is offline and it is the one which updates
 464         * jiffies, then give up the assignment and let it be taken by
 465         * the cpu which runs the tick timer next. If we don't drop
 466         * this here the jiffies might be stale and do_timer() never
 467         * invoked.
 468         */
 469        if (unlikely(!cpu_online(cpu))) {
 470                if (cpu == tick_do_timer_cpu)
 471                        tick_do_timer_cpu = TICK_DO_TIMER_NONE;
 472        }
 473
 474        if (unlikely(ts->nohz_mode == NOHZ_MODE_INACTIVE))
 475                return false;
 476
 477        if (need_resched())
 478                return false;
 479
 480        if (unlikely(local_softirq_pending() && cpu_online(cpu))) {
 481                static int ratelimit;
 482
 483                if (ratelimit < 10 &&
 484                    (local_softirq_pending() & SOFTIRQ_STOP_IDLE_MASK)) {
 485                        printk(KERN_ERR "NOHZ: local_softirq_pending %02x\n",
 486                               (unsigned int) local_softirq_pending());
 487                        ratelimit++;
 488                }
 489                return false;
 490        }
 491
 492        return true;
 493}
 494
 495static void __tick_nohz_idle_enter(struct tick_sched *ts)
 496{
 497        ktime_t now, expires;
 498        int cpu = smp_processor_id();
 499
 500        now = tick_nohz_start_idle(cpu, ts);
 501
 502        if (can_stop_idle_tick(cpu, ts)) {
 503                int was_stopped = ts->tick_stopped;
 504
 505                ts->idle_calls++;
 506
 507                expires = tick_nohz_stop_sched_tick(ts, now, cpu);
 508                if (expires.tv64 > 0LL) {
 509                        ts->idle_sleeps++;
 510                        ts->idle_expires = expires;
 511                }
 512
 513                if (!was_stopped && ts->tick_stopped)
 514                        ts->idle_jiffies = ts->last_jiffies;
 515        }
 516}
 517
 518/**
 519 * tick_nohz_idle_enter - stop the idle tick from the idle task
 520 *
 521 * When the next event is more than a tick into the future, stop the idle tick
 522 * Called when we start the idle loop.
 523 *
 524 * The arch is responsible of calling:
 525 *
 526 * - rcu_idle_enter() after its last use of RCU before the CPU is put
 527 *  to sleep.
 528 * - rcu_idle_exit() before the first use of RCU after the CPU is woken up.
 529 */
 530void tick_nohz_idle_enter(void)
 531{
 532        struct tick_sched *ts;
 533
 534        WARN_ON_ONCE(irqs_disabled());
 535
 536        /*
 537         * Update the idle state in the scheduler domain hierarchy
 538         * when tick_nohz_stop_sched_tick() is called from the idle loop.
 539         * State will be updated to busy during the first busy tick after
 540         * exiting idle.
 541         */
 542        set_cpu_sd_state_idle();
 543
 544        local_irq_disable();
 545
 546        ts = &__get_cpu_var(tick_cpu_sched);
 547        /*
 548         * set ts->inidle unconditionally. even if the system did not
 549         * switch to nohz mode the cpu frequency governers rely on the
 550         * update of the idle time accounting in tick_nohz_start_idle().
 551         */
 552        ts->inidle = 1;
 553        __tick_nohz_idle_enter(ts);
 554
 555        local_irq_enable();
 556}
 557EXPORT_SYMBOL_GPL(tick_nohz_idle_enter);
 558
 559/**
 560 * tick_nohz_irq_exit - update next tick event from interrupt exit
 561 *
 562 * When an interrupt fires while we are idle and it doesn't cause
 563 * a reschedule, it may still add, modify or delete a timer, enqueue
 564 * an RCU callback, etc...
 565 * So we need to re-calculate and reprogram the next tick event.
 566 */
 567void tick_nohz_irq_exit(void)
 568{
 569        struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
 570
 571        if (!ts->inidle)
 572                return;
 573
 574        /* Cancel the timer because CPU already waken up from the C-states*/
 575        menu_hrtimer_cancel();
 576        __tick_nohz_idle_enter(ts);
 577}
 578
 579/**
 580 * tick_nohz_get_sleep_length - return the length of the current sleep
 581 *
 582 * Called from power state control code with interrupts disabled
 583 */
 584ktime_t tick_nohz_get_sleep_length(void)
 585{
 586        struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
 587
 588        return ts->sleep_length;
 589}
 590
 591static void tick_nohz_restart(struct tick_sched *ts, ktime_t now)
 592{
 593        hrtimer_cancel(&ts->sched_timer);
 594        hrtimer_set_expires(&ts->sched_timer, ts->last_tick);
 595
 596        while (1) {
 597                /* Forward the time to expire in the future */
 598                hrtimer_forward(&ts->sched_timer, now, tick_period);
 599
 600                if (ts->nohz_mode == NOHZ_MODE_HIGHRES) {
 601                        hrtimer_start_expires(&ts->sched_timer,
 602                                              HRTIMER_MODE_ABS_PINNED);
 603                        /* Check, if the timer was already in the past */
 604                        if (hrtimer_active(&ts->sched_timer))
 605                                break;
 606                } else {
 607                        if (!tick_program_event(
 608                                hrtimer_get_expires(&ts->sched_timer), 0))
 609                                break;
 610                }
 611                /* Reread time and update jiffies */
 612                now = ktime_get();
 613                tick_do_update_jiffies64(now);
 614        }
 615}
 616
 617static void tick_nohz_restart_sched_tick(struct tick_sched *ts, ktime_t now)
 618{
 619        /* Update jiffies first */
 620        tick_do_update_jiffies64(now);
 621        update_cpu_load_nohz();
 622
 623        calc_load_exit_idle();
 624        touch_softlockup_watchdog();
 625        /*
 626         * Cancel the scheduled timer and restore the tick
 627         */
 628        ts->tick_stopped  = 0;
 629        ts->idle_exittime = now;
 630
 631        tick_nohz_restart(ts, now);
 632}
 633
 634static void tick_nohz_account_idle_ticks(struct tick_sched *ts)
 635{
 636#ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
 637        unsigned long ticks;
 638
 639        if (vtime_accounting_enabled())
 640                return;
 641        /*
 642         * We stopped the tick in idle. Update process times would miss the
 643         * time we slept as update_process_times does only a 1 tick
 644         * accounting. Enforce that this is accounted to idle !
 645         */
 646        ticks = jiffies - ts->idle_jiffies;
 647        /*
 648         * We might be one off. Do not randomly account a huge number of ticks!
 649         */
 650        if (ticks && ticks < LONG_MAX)
 651                account_idle_ticks(ticks);
 652#endif
 653}
 654
 655/**
 656 * tick_nohz_idle_exit - restart the idle tick from the idle task
 657 *
 658 * Restart the idle tick when the CPU is woken up from idle
 659 * This also exit the RCU extended quiescent state. The CPU
 660 * can use RCU again after this function is called.
 661 */
 662void tick_nohz_idle_exit(void)
 663{
 664        int cpu = smp_processor_id();
 665        struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
 666        ktime_t now;
 667
 668        local_irq_disable();
 669
 670        WARN_ON_ONCE(!ts->inidle);
 671
 672        ts->inidle = 0;
 673
 674        /* Cancel the timer because CPU already waken up from the C-states*/
 675        menu_hrtimer_cancel();
 676        if (ts->idle_active || ts->tick_stopped)
 677                now = ktime_get();
 678
 679        if (ts->idle_active)
 680                tick_nohz_stop_idle(cpu, now);
 681
 682        if (ts->tick_stopped) {
 683                tick_nohz_restart_sched_tick(ts, now);
 684                tick_nohz_account_idle_ticks(ts);
 685        }
 686
 687        local_irq_enable();
 688}
 689EXPORT_SYMBOL_GPL(tick_nohz_idle_exit);
 690
 691static int tick_nohz_reprogram(struct tick_sched *ts, ktime_t now)
 692{
 693        hrtimer_forward(&ts->sched_timer, now, tick_period);
 694        return tick_program_event(hrtimer_get_expires(&ts->sched_timer), 0);
 695}
 696
 697/*
 698 * The nohz low res interrupt handler
 699 */
 700static void tick_nohz_handler(struct clock_event_device *dev)
 701{
 702        struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
 703        struct pt_regs *regs = get_irq_regs();
 704        ktime_t now = ktime_get();
 705
 706        dev->next_event.tv64 = KTIME_MAX;
 707
 708        tick_sched_do_timer(now);
 709        tick_sched_handle(ts, regs);
 710
 711        while (tick_nohz_reprogram(ts, now)) {
 712                now = ktime_get();
 713                tick_do_update_jiffies64(now);
 714        }
 715}
 716
 717/**
 718 * tick_nohz_switch_to_nohz - switch to nohz mode
 719 */
 720static void tick_nohz_switch_to_nohz(void)
 721{
 722        struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
 723        ktime_t next;
 724
 725        if (!tick_nohz_enabled)
 726                return;
 727
 728        local_irq_disable();
 729        if (tick_switch_to_oneshot(tick_nohz_handler)) {
 730                local_irq_enable();
 731                return;
 732        }
 733
 734        ts->nohz_mode = NOHZ_MODE_LOWRES;
 735
 736        /*
 737         * Recycle the hrtimer in ts, so we can share the
 738         * hrtimer_forward with the highres code.
 739         */
 740        hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
 741        /* Get the next period */
 742        next = tick_init_jiffy_update();
 743
 744        for (;;) {
 745                hrtimer_set_expires(&ts->sched_timer, next);
 746                if (!tick_program_event(next, 0))
 747                        break;
 748                next = ktime_add(next, tick_period);
 749        }
 750        local_irq_enable();
 751}
 752
 753/*
 754 * When NOHZ is enabled and the tick is stopped, we need to kick the
 755 * tick timer from irq_enter() so that the jiffies update is kept
 756 * alive during long running softirqs. That's ugly as hell, but
 757 * correctness is key even if we need to fix the offending softirq in
 758 * the first place.
 759 *
 760 * Note, this is different to tick_nohz_restart. We just kick the
 761 * timer and do not touch the other magic bits which need to be done
 762 * when idle is left.
 763 */
 764static void tick_nohz_kick_tick(int cpu, ktime_t now)
 765{
 766#if 0
 767        /* Switch back to 2.6.27 behaviour */
 768
 769        struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
 770        ktime_t delta;
 771
 772        /*
 773         * Do not touch the tick device, when the next expiry is either
 774         * already reached or less/equal than the tick period.
 775         */
 776        delta = ktime_sub(hrtimer_get_expires(&ts->sched_timer), now);
 777        if (delta.tv64 <= tick_period.tv64)
 778                return;
 779
 780        tick_nohz_restart(ts, now);
 781#endif
 782}
 783
 784static inline void tick_check_nohz(int cpu)
 785{
 786        struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
 787        ktime_t now;
 788
 789        if (!ts->idle_active && !ts->tick_stopped)
 790                return;
 791        now = ktime_get();
 792        if (ts->idle_active)
 793                tick_nohz_stop_idle(cpu, now);
 794        if (ts->tick_stopped) {
 795                tick_nohz_update_jiffies(now);
 796                tick_nohz_kick_tick(cpu, now);
 797        }
 798}
 799
 800#else
 801
 802static inline void tick_nohz_switch_to_nohz(void) { }
 803static inline void tick_check_nohz(int cpu) { }
 804
 805#endif /* NO_HZ */
 806
 807/*
 808 * Called from irq_enter to notify about the possible interruption of idle()
 809 */
 810void tick_check_idle(int cpu)
 811{
 812        tick_check_oneshot_broadcast(cpu);
 813        tick_check_nohz(cpu);
 814}
 815
 816/*
 817 * High resolution timer specific code
 818 */
 819#ifdef CONFIG_HIGH_RES_TIMERS
 820/*
 821 * We rearm the timer until we get disabled by the idle code.
 822 * Called with interrupts disabled.
 823 */
 824static enum hrtimer_restart tick_sched_timer(struct hrtimer *timer)
 825{
 826        struct tick_sched *ts =
 827                container_of(timer, struct tick_sched, sched_timer);
 828        struct pt_regs *regs = get_irq_regs();
 829        ktime_t now = ktime_get();
 830
 831        tick_sched_do_timer(now);
 832
 833        /*
 834         * Do not call, when we are not in irq context and have
 835         * no valid regs pointer
 836         */
 837        if (regs)
 838                tick_sched_handle(ts, regs);
 839
 840        hrtimer_forward(timer, now, tick_period);
 841
 842        return HRTIMER_RESTART;
 843}
 844
 845static int sched_skew_tick;
 846
 847static int __init skew_tick(char *str)
 848{
 849        get_option(&str, &sched_skew_tick);
 850
 851        return 0;
 852}
 853early_param("skew_tick", skew_tick);
 854
 855/**
 856 * tick_setup_sched_timer - setup the tick emulation timer
 857 */
 858void tick_setup_sched_timer(void)
 859{
 860        struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
 861        ktime_t now = ktime_get();
 862
 863        /*
 864         * Emulate tick processing via per-CPU hrtimers:
 865         */
 866        hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
 867        ts->sched_timer.function = tick_sched_timer;
 868
 869        /* Get the next period (per cpu) */
 870        hrtimer_set_expires(&ts->sched_timer, tick_init_jiffy_update());
 871
 872        /* Offset the tick to avert jiffies_lock contention. */
 873        if (sched_skew_tick) {
 874                u64 offset = ktime_to_ns(tick_period) >> 1;
 875                do_div(offset, num_possible_cpus());
 876                offset *= smp_processor_id();
 877                hrtimer_add_expires_ns(&ts->sched_timer, offset);
 878        }
 879
 880        for (;;) {
 881                hrtimer_forward(&ts->sched_timer, now, tick_period);
 882                hrtimer_start_expires(&ts->sched_timer,
 883                                      HRTIMER_MODE_ABS_PINNED);
 884                /* Check, if the timer was already in the past */
 885                if (hrtimer_active(&ts->sched_timer))
 886                        break;
 887                now = ktime_get();
 888        }
 889
 890#ifdef CONFIG_NO_HZ
 891        if (tick_nohz_enabled)
 892                ts->nohz_mode = NOHZ_MODE_HIGHRES;
 893#endif
 894}
 895#endif /* HIGH_RES_TIMERS */
 896
 897#if defined CONFIG_NO_HZ || defined CONFIG_HIGH_RES_TIMERS
 898void tick_cancel_sched_timer(int cpu)
 899{
 900        struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
 901
 902# ifdef CONFIG_HIGH_RES_TIMERS
 903        if (ts->sched_timer.base)
 904                hrtimer_cancel(&ts->sched_timer);
 905# endif
 906
 907        memset(ts, 0, sizeof(*ts));
 908}
 909#endif
 910
 911/**
 912 * Async notification about clocksource changes
 913 */
 914void tick_clock_notify(void)
 915{
 916        int cpu;
 917
 918        for_each_possible_cpu(cpu)
 919                set_bit(0, &per_cpu(tick_cpu_sched, cpu).check_clocks);
 920}
 921
 922/*
 923 * Async notification about clock event changes
 924 */
 925void tick_oneshot_notify(void)
 926{
 927        struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
 928
 929        set_bit(0, &ts->check_clocks);
 930}
 931
 932/**
 933 * Check, if a change happened, which makes oneshot possible.
 934 *
 935 * Called cyclic from the hrtimer softirq (driven by the timer
 936 * softirq) allow_nohz signals, that we can switch into low-res nohz
 937 * mode, because high resolution timers are disabled (either compile
 938 * or runtime).
 939 */
 940int tick_check_oneshot_change(int allow_nohz)
 941{
 942        struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
 943
 944        if (!test_and_clear_bit(0, &ts->check_clocks))
 945                return 0;
 946
 947        if (ts->nohz_mode != NOHZ_MODE_INACTIVE)
 948                return 0;
 949
 950        if (!timekeeping_valid_for_hres() || !tick_is_oneshot_available())
 951                return 0;
 952
 953        if (!allow_nohz)
 954                return 1;
 955
 956        tick_nohz_switch_to_nohz();
 957        return 0;
 958}
 959
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.