linux/kernel/sched/clock.c
<<
n va2ot/spa v3 2ot/formv3 2ota n va2o href="../linux+v34.17/kernel/sched/clock.c">n va2otimg src="../.static/gfx/right.png" alt=">>">n t/spa v3n tspa class="lxr_search">n va ="+search" method="post" onsubmit="return do_search(this);">n va2otinput typ hidden" nam navtarget" on> ">n va2otinput typ text" nam search" id search">n va2otbuttipityp submit">Searchn va2oPrefs3 2ot/a>n t/spa v3va2o ot/divv3va2o otform ac> ="ajax+*" method="post" onsubmit="return false;">n tinput typ hidden" nam ajax_lookup" id ajax_lookup" on> ">nva2o ot/formv3nva2o otdiv class="headingbottim">3 tdiv id file_contents"v
o o1t/a>tspa  class="comment">/*t/spa v3o o2t/a>tspa  class="comment"> * sched_clock for unstable cpu clockst/spa v3o o3t/a>tspa  class="comment"> *t/spa v3o o4t/a>tspa  class="comment"> *  Copyright (C) 2008 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>t/spa v3o o5t/a>tspa  class="comment"> *t/spa v3o o6t/a>tspa  class="comment"> *  Updates and enhancements:t/spa v3o o7t/a>tspa  class="comment"> *    Copyright (C) 2008 Red Hat, Inc. Steven Rostedt <srostedt@redhat.com>t/spa v3o o8t/a>tspa  class="comment"> *t/spa v3o o9t/a>tspa  class="comment"> * Based ipicode by:t/spa v3o "
	 a>tspa  class="comment"> *   Ingo Molnar <mingo@redhat.com>t/spa v3o 11t/a>tspa  class="comment"> *   Guillaume Chazarain <guichaz@gmail.com>t/spa v3o 12t/a>tspa  class="comment"> *t/spa v3o 13t/a>tspa  class="comment"> *t/spa v3o 14t/a>tspa  class="comment"> * What:t/spa v3o 15t/a>tspa  class="comment"> *t/spa v3o 16t/a>tspa  class="comment"> * cpu_clock(i) provides a fast (execu2.ipitime) high resolu2.ipt/spa v3o 17t/a>tspa  class="comment"> * clock with bounded drift between CPUs. The on>
  of cpu_clock(i)t/spa v3o 18t/a>tspa  class="comment"> * is monotipic for constant i. The timestamp returned is in nanoseconds.t/spa v3o 19t/a>tspa  class="comment"> *t/spa v3o 2
	 a>tspa  class="comment"> * ######################### BIG FAT WARNING ##########################t/spa v3o 21t/a>tspa  class="comment"> * # whepicomparing cpu_clock(i) to cpu_clock(j) for i != j, time ca  #t/spa v3o 22t/a>tspa  class="comment"> * # go backwards !!                                                  #t/spa v3o 23t/a>tspa  class="comment"> * ####################################################################t/spa v3o 24t/a>tspa  class="comment"> *t/spa v3o 25t/a>tspa  class="comment"> * There is no strict promise about the base, although it tends to startt/spa v3o 26t/a>tspa  class="comment"> * at 0 ipiboot (but people really shouldn't rely ipithat).t/spa v3o 27t/a>tspa  class="comment"> *t/spa v3o 28t/a>tspa  class="comment"> * cpu_clock(i)       -- ca  be used from any context, including NMI.t/spa v3o 29t/a>tspa  class="comment"> * sched_clock_cpu(i) -- must be used with local IRQs disabled (implied by NMI)t/spa v3o 3
	 a>tspa  class="comment"> * local_clock()      -- is cpu_clock() ipithe current cpu.t/spa v3o 31t/a>tspa  class="comment"> *t/spa v3o 32t/a>tspa  class="comment"> * How:t/spa v3o 33t/a>tspa  class="comment"> *t/spa v3o 34t/a>tspa  class="comment"> * The implementa2.ipieither uses sched_clock() whept/spa v3o 35t/a>tspa  class="comment"> * !CONFIG_HAVE_UNSTABLE_SCHED_CLOCK, which means in that case thet/spa v3o 36t/a>tspa  class="comment"> * sched_clock() is assumed to provide these properties (mostly it meanst/spa v3o 37t/a>tspa  class="comment"> * the architecture provides a globally synchripized highres time source).t/spa v3o 38t/a>tspa  class="comment"> *t/spa v3o 39t/a>tspa  class="comment"> * Otherwise it tries to create a semi stable clock from a mixture of othert/spa v3o 4
	 a>tspa  class="comment"> * clocks, including:t/spa v3o 41t/a>tspa  class="comment"> *t/spa v3o 42t/a>tspa  class="comment"> *  - GTOD (clock monotimic)t/spa v3o 43t/a>tspa  class="comment"> *  - sched_clock()t/spa v3o 44t/a>tspa  class="comment"> *  - explicit idle eventst/spa v3o 45t/a>tspa  class="comment"> *t/spa v3o 46t/a>tspa  class="comment"> * We use GTOD as base and use sched_clock() deltas to improve resolu2.ip. Thet/spa v3o 47t/a>tspa  class="comment"> * deltas are filtered to provide monotipicity and keeping it within apt/spa v3o 48t/a>tspa  class="comment"> * expected window.t/spa v3o 49t/a>tspa  class="comment"> *t/spa v3o 5
	 a>tspa  class="comment"> * Furthermore, explicit sleep and wakeup hooks allow us to account for timet/spa v3o 51t/a>tspa  class="comment"> * that is otherwise invisible (TSC gets stopped).t/spa v3o 52t/a>tspa  class="comment"> *t/spa v3o 53t/a>tspa  class="comment"> *t/spa v3o 54t/a>tspa  class="comment"> * Notes:t/spa v3o 55t/a>tspa  class="comment"> *t/spa v3o 56t/a>tspa  class="comment"> * The !IRQ-safetly of sched_clock() and sched_clock_cpu()icomes from thingst/spa v3o 57t/a>tspa  class="comment"> * like cpufreq interrupts that ca  change the base clock (TSC) multipliert/spa v3o 58t/a>tspa  class="comment"> * and cause funny jumps in time -- although the filtering provided byt/spa v3o 59t/a>tspa  class="comment"> * sched_clock_cpu() should mitigate serious artifacts we ca not rely ipiitt/spa v3o 6
	 a>tspa  class="comment"> * in general since for !CONFIG_HAVE_UNSTABLE_SCHED_CLOCK we fully rely ipt/spa v3o 61t/a>tspa  class="comment"> * sched_clock().t/spa v3o 62t/a>tspa  class="comment"> */t/spa v3o 63t/a>#include <linux/spinlock.ht/a>>3o 64t/a>#include <linux/hardirq.ht/a>>3o 65t/a>#include <linux/export.ht/a>>3o 66t/a>#include <linux/percpu.ht/a>>3o 67t/a>#include <linux/ktime.ht/a>>3o 68t/a>#include <linux/sched.ht/a>>3o 69t/a>3o 7
	 a>tspa  class="comment">/*t/spa v3o 71t/a>tspa  class="comment"> * Scheduler clock - returns current time in nanosec units.t/spa v3o 72t/a>tspa  class="comment"> * This is default implementa2.ip.t/spa v3o 73t/a>tspa  class="comment"> * Architectures and sub-architectures ca  override this.t/spa v3o 74t/a>tspa  class="comment"> */t/spa v3o 75t/a>unsigned long long ta href="+code=__attribute__" class="sref">__attribute__t/a>((ta href="+code=weak" class="sref">weakt/a>)) ta href="+code=sched_clock" class="sref">sched_clockt/a>(void)3o 76t/a>{3o 77t/a>        return (unsigned long long)(ta href="+code=jiffies" class="sref">jiffiest/a> - ta href="+code=INITIAL_JIFFIES" class="sref">INITIAL_JIFFIESt/a>)3o 78t/a>                                        * (ta href="+code=NSEC_PER_SEC" class="sref">NSEC_PER_SECt/a> / ta href="+code=HZ" class="sref">HZt/a>);3o 79t/a>}3o 8
	 a>ta href="+code=EXPORT_SYMBOL_GPL" class="sref">EXPORT_SYMBOL_GPLt/a>(ta href="+code=sched_clock" class="sref">sched_clockt/a>);3o 81t/a>3o 82	 a>ta href="+code=__read_mostly" class="sref">__read_mostlyt/a> int ta href="+code=sched_clock_running" class="sref">sched_clock_runningt/a>;3o 83t/a>3o 84t/a>#ifdef ta href="+code=CONFIG_HAVE_UNSTABLE_SCHED_CLOCK" class="sref">CONFIG_HAVE_UNSTABLE_SCHED_CLOCKt/a>3o 85	 a>ta href="+code=__read_mostly" class="sref">__read_mostlyt/a> int ta href="+code=sched_clock_stable" class="sref">sched_clock_stablet/a>;3o 86t/a>3o 87t/a>struct ta href="+code=sched_clock_data" class="sref">sched_clock_datat/a> {3o 88t/a>        ta href="+code=u64" class="sref">u64t/a>                     ta href="+code=tick_raw" class="sref">tick_rawt/a>;3o 89t/a>        ta href="+code=u64" class="sref">u64t/a>                     ta href="+code=tick_gtod" class="sref">tick_gtodt/a>;3o 90t/a>        ta href="+code=u64" class="sref">u64t/a>                     ta href="+code=clock" class="sref">clockt/a>;3o 91t/a>};3o 92t/a>3o 93t/a>static ta href="+code=DEFINE_PER_CPU_SHARED_ALIGNED" class="sref">DEFINE_PER_CPU_SHARED_ALIGNEDt/a>(struct ta href="+code=sched_clock_data" class="sref">sched_clock_datat/a>, ta href="+code=sched_clock_data" class="sref">sched_clock_datat/a>);3o 94t/a>3o 95t/a>static ta href="+code=inline" class="sref">inlinet/a> struct ta href="+code=sched_clock_data" class="sref">sched_clock_datat/a> *ta href="+code=this_scd" class="sref">this_scdt/a>(void)3o 96t/a>{3o 97t/a>        return &ta href="+code=__get_cpu_var" class="sref">__get_cpu_vart/a>(ta href="+code=sched_clock_data" class="sref">sched_clock_datat/a>);3o 98t/a>}3o 99t/a>3o100t/a>static ta href="+code=inline" class="sref">inlinet/a> struct ta href="+code=sched_clock_data" class="sref">sched_clock_datat/a> *ta href="+code=cpu_sdc" class="sref">cpu_sdct/a>(int ta href="+code=cpu" class="sref">cput/a>)3o101t/a>{3o102t/a>        return &ta href="+code=per_cpu" class="sref">per_cput/a>(ta href="+code=sched_clock_data" class="sref">sched_clock_datat/a>, ta href="+code=cpu" class="sref">cput/a>);3o103t/a>}3o104t/a>3o105t/a>void ta href="+code=sched_clock_init" class="sref">sched_clock_initt/a>(void)3o106t/a>{3o107t/a>        ta href="+code=u64" class="sref">u64t/a> ta href="+code=ktime_now" class="sref">ktime_nowt/a> = ta href="+code=ktime_to_ns" class="sref">ktime_to_nst/a>(ta href="+code=ktime_get" class="sref">ktime_gett/a>());3o108t/a>        int ta href="+code=cpu" class="sref">cput/a>;3o109t/a>3o110t/a>        ta href="+code=for_each_possible_cpu" class="sref">for_each_possible_cput/a>(ta href="+code=cpu" class="sref">cput/a>) {3o111t/a>                struct ta href="+code=sched_clock_data" class="sref">sched_clock_datat/a> *ta href="+code=scd" class="sref">scdt/a> = ta href="+code=cpu_sdc" class="sref">cpu_sdct/a>(ta href="+code=cpu" class="sref">cput/a>);3o112t/a>3o113t/a>                ta href="+code=scd" class="sref">scdt/a>->ta href="+code=tick_raw" class="sref">tick_rawt/a> = 0;3o114t/a>                ta href="+code=scd" class="sref">scdt/a>->ta href="+code=tick_gtod" class="sref">tick_gtodt/a> = ta href="+code=ktime_now" class="sref">ktime_nowt/a>;3o115t/a>                ta href="+code=scd" class="sref">scdt/a>->ta href="+code=clock" class="sref">clockt/a> = ta href="+code=ktime_now" class="sref">ktime_nowt/a>;3o116t/a>        }3o117t/a>3o118t/a>        ta href="+code=sched_clock_running" class="sref">sched_clock_runningt/a> = 1;3o119t/a>}3o12
	 a>3o121t/a>tspa  class="comment">/*t/spa v3o122t/a>tspa  class="comment"> * min, max except they take wrapping into accountt/spa v3o123t/a>tspa  class="comment"> */t/spa v3o124t/a>3o125t/a>static ta href="+code=inline" class="sref">inlinet/a> ta href="+code=u64" class="sref">u64t/a> ta href="+code=wrap_min" class="sref">wrap_mint/a>(ta href="+code=u64" class="sref">u64t/a> ta href="+code=x" class="sref">xt/a>, ta href="+code=u64" class="sref">u64t/a> ta href="+code=y" class="sref">yt/a>)3o126t/a>{3o127t/a>        return (ta href="+code=s64" class="sref">s64t/a>)(ta href="+code=x" class="sref">xt/a> - ta href="+code=y" class="sref">yt/a>) < 0 ? ta href="+code=x" class="sref">xt/a> : ta href="+code=y" class="sref">yt/a>;3o128t/a>}3o129t/a>3o130t/a>static ta href="+code=inline" class="sref">inlinet/a> ta href="+code=u64" class="sref">u64t/a> ta href="+code=wrap_max" class="sref">wrap_maxt/a>(ta href="+code=u64" class="sref">u64t/a> ta href="+code=x" class="sref">xt/a>, ta href="+code=u64" class="sref">u64t/a> ta href="+code=y" class="sref">yt/a>)3o131t/a>{3o132t/a>        return (ta href="+code=s64" class="sref">s64t/a>)(ta href="+code=x" class="sref">xt/a> - ta href="+code=y" class="sref">yt/a>) > 0 ? ta href="+code=x" class="sref">xt/a> : ta href="+code=y" class="sref">yt/a>;3o133t/a>}3o134t/a>3o135t/a>tspa  class="comment">/*t/spa v3o136t/a>tspa  class="comment"> * update the percpu scd from the raw @now on>
 t/spa v3o137t/a>tspa  class="comment"> *t/spa v3o138t/a>tspa  class="comment"> *  - filter out backward mo2.ipt/spa v3o139t/a>tspa  class="comment"> *  - use the GTOD tick on>
  to create a window to filter crazy TSC on>
 st/spa v3o14
	 a>tspa  class="comment"> */t/spa v3o141t/a>static ta href="+code=u64" class="sref">u64t/a> ta href="+code=sched_clock_local" class="sref">sched_clock_localt/a>(struct ta href="+code=sched_clock_data" class="sref">sched_clock_datat/a> *ta href="+code=scd" class="sref">scdt/a>)3o142t/a>{3o143t/a>        ta href="+code=u64" class="sref">u64t/a> ta href="+code=now" class="sref">nowt/a>, ta href="+code=clock" class="sref">clockt/a>, ta href="+code=old_clock" class="sref">old_clockt/a>, ta href="+code=min_clock" class="sref">min_clockt/a>, ta href="+code=max_clock" class="sref">max_clockt/a>;3o144t/a>        ta href="+code=s64" class="sref">s64t/a> ta href="+code=delta" class="sref">deltat/a>;3o145t/a>3o146t/a>ta href="+code=again" class="sref">againt/a>:3o147t/a>        ta href="+code=now" class="sref">nowt/a> = ta href="+code=sched_clock" class="sref">sched_clockt/a>();3o148t/a>        ta href="+code=delta" class="sref">deltat/a> = ta href="+code=now" class="sref">nowt/a> - ta href="+code=scd" class="sref">scdt/a>->ta href="+code=tick_raw" class="sref">tick_rawt/a>;3o149t/a>        if (ta href="+code=unlikely" class="sref">unlikelyt/a>(ta href="+code=delta" class="sref">deltat/a> < 0))3o150t/a>                ta href="+code=delta" class="sref">deltat/a> = 0;3o151t/a>3o152t/a>        ta href="+code=old_clock" class="sref">old_clockt/a> = ta href="+code=scd" class="sref">scdt/a>->ta href="+code=clock" class="sref">clockt/a>;3o153t/a>3o154t/a>        tspa  class="comment">/*t/spa v3o155t/a>tspa  class="comment">         * scd->clock = clamp(scd->tick_gtod + delta,t/spa v3o156t/a>tspa  class="comment">         *                    max(scd->tick_gtod, scd->clock),t/spa v3o157t/a>tspa  class="comment">         *                    scd->tick_gtod + TICK_NSEC);t/spa v3o158t/a>tspa  class="comment">         */t/spa v3o159t/a>3o160t/a>        ta href="+code=clock" class="sref">clockt/a> = ta href="+code=scd" class="sref">scdt/a>->ta href="+code=tick_gtod" class="sref">tick_gtodt/a> + ta href="+code=delta" class="sref">deltat/a>;3o161t/a>        ta href="+code=min_clock" class="sref">min_clockt/a> = ta href="+code=wrap_max" class="sref">wrap_maxt/a>(ta href="+code=scd" class="sref">scdt/a>->ta href="+code=tick_gtod" class="sref">tick_gtodt/a>, ta href="+code=old_clock" class="sref">old_clockt/a>);3o162t/a>        ta href="+code=max_clock" class="sref">max_clockt/a> = ta href="+code=wrap_max" class="sref">wrap_maxt/a>(ta href="+code=old_clock" class="sref">old_clockt/a>, ta href="+code=scd" class="sref">scdt/a>->ta href="+code=tick_gtod" class="sref">tick_gtodt/a> + ta href="+code=TICK_NSEC" class="sref">TICK_NSECt/a>);3o163t/a>3o164t/a>        ta href="+code=clock" class="sref">clockt/a> = ta href="+code=wrap_max" class="sref">wrap_maxt/a>(ta href="+code=clock" class="sref">clockt/a>, ta href="+code=min_clock" class="sref">min_clockt/a>);3o165t/a>        ta href="+code=clock" class="sref">clockt/a> = ta href="+code=wrap_min" class="sref">wrap_mint/a>(ta href="+code=clock" class="sref">clockt/a>, ta href="+code=max_clock" class="sref">max_clockt/a>);3o166t/a>3o167t/a>        if (ta href="+code=cmpxchg64" class="sref">cmpxchg64t/a>(&ta href="+code=scd" class="sref">scdt/a>->ta href="+code=clock" class="sref">clockt/a>, ta href="+code=old_clock" class="sref">old_clockt/a>, ta href="+code=clock" class="sref">clockt/a>) != ta href="+code=old_clock" class="sref">old_clockt/a>)3o168t/a>                goto ta href="+code=again" class="sref">againt/a>;3o169t/a>3o170t/a>        return ta href="+code=clock" class="sref">clockt/a>;3o171t/a>}3o172t/a>3o173t/a>static ta href="+code=u64" class="sref">u64t/a> ta href="+code=sched_clock_remote" class="sref">sched_clock_remotet/a>(struct ta href="+code=sched_clock_data" class="sref">sched_clock_datat/a> *ta href="+code=scd" class="sref">scdt/a>)3o174t/a>{3o175t/a>        struct ta href="+code=sched_clock_data" class="sref">sched_clock_datat/a> *ta href="+code=my_scd" class="sref">my_scdt/a> = ta href="+code=this_scd" class="sref">this_scdt/a>();3o176t/a>        ta href="+code=u64" class="sref">u64t/a> ta href="+code=this_clock" class="sref">this_clockt/a>, ta href="+code=remote_clock" class="sref">remote_clockt/a>;3o177t/a>        ta href="+code=u64" class="sref">u64t/a> *ta href="+code=ptr" class="sref">ptrt/a>, ta href="+code=old_val" class="sref">old_valt/a>, ta href="+code=val" class="sref">valt/a>;3o178t/a>3o179t/a>        ta href="+code=sched_clock_local" class="sref">sched_clock_localt/a>(ta href="+code=my_scd" class="sref">my_scdt/a>);3o18
	 a>ta href="+code=again" class="sref">againt/a>:3o181t/a>        ta href="+code=this_clock" class="sref">this_clockt/a> = ta href="+code=my_scd" class="sref">my_scdt/a>->ta href="+code=clock" class="sref">clockt/a>;3o182t/a>        ta href="+code=remote_clock" class="sref">remote_clockt/a> = ta href="+code=scd" class="sref">scdt/a>->ta href="+code=clock" class="sref">clockt/a>;3o183t/a>3o184t/a>        tspa  class="comment">/*t/spa v3o185t/a>tspa  class="comment">         * Use the opportunity that we have both lockst/spa v3o186t/a>tspa  class="comment">         * taken to couple the two clocks: we take thet/spa v3o187t/a>tspa  class="comment">         * larger time as the latest time for botht/spa v3o188t/a>tspa  class="comment">         * runqueues. (this creates monotipic movement)t/spa v3o189t/a>tspa  class="comment">         */t/spa v3o190t/a>        if (ta href="+code=likely" class="sref">likelyt/a>((ta href="+code=s64" class="sref">s64t/a>)(ta href="+code=remote_clock" class="sref">remote_clockt/a> - ta href="+code=this_clock" class="sref">this_clockt/a>) < 0)) {3o191t/a>                ta href="+code=ptr" class="sref">ptrt/a> = &ta href="+code=scd" class="sref">scdt/a>->ta href="+code=clock" class="sref">clockt/a>;3o192t/a>                ta href="+code=old_val" class="sref">old_valt/a> = ta href="+code=remote_clock" class="sref">remote_clockt/a>;3o193t/a>                ta href="+code=val" class="sref">valt/a> = ta href="+code=this_clock" class="sref">this_clockt/a>;3o194t/a>        } else {3o195t/a>                tspa  class="comment">/*t/spa v3o196t/a>tspa  class="comment">                 * Should be rare, but possible:t/spa v3o197t/a>tspa  class="comment">                 */t/spa v3o198t/a>                ta href="+code=ptr" class="sref">ptrt/a> = &ta href="+code=my_scd" class="sref">my_scdt/a>->ta href="+code=clock" class="sref">clockt/a>;3o199t/a>                ta href="+code=old_val" class="sref">old_valt/a> = ta href="+code=this_clock" class="sref">this_clockt/a>;3o200t/a>                ta href="+code=val" class="sref">valt/a> = ta href="+code=remote_clock" class="sref">remote_clockt/a>;3o201t/a>        }3o202t/a>3o203t/a>        if (ta href="+code=cmpxchg64" class="sref">cmpxchg64t/a>(ta href="+code=ptr" class="sref">ptrt/a>, ta href="+code=old_val" class="sref">old_valt/a>, ta href="+code=val" class="sref">valt/a>) != ta href="+code=old_val" class="sref">old_valt/a>)3o204t/a>                goto ta href="+code=again" class="sref">againt/a>;3o205t/a>3o206t/a>        return ta href="+code=val" class="sref">valt/a>;3o207t/a>}3o208t/a>3o209t/a>tspa  class="comment">/*t/spa v3o2"
	 a>tspa  class="comment"> * Similar to cpu_clock(), but requires local IRQs to be disabled.t/spa v3o211t/a>tspa  class="comment"> *t/spa v3o212t/a>tspa  class="comment"> * See cpu_clock().t/spa v3o213t/a>tspa  class="comment"> */t/spa v3o214t/a>ta href="+code=u64" class="sref">u64t/a> ta href="+code=sched_clock_cpu" class="sref">sched_clock_cput/a>(int ta href="+code=cpu" class="sref">cput/a>)3o215t/a>{3o216t/a>        struct ta href="+code=sched_clock_data" class="sref">sched_clock_datat/a> *ta href="+code=scd" class="sref">scdt/a>;3o217t/a>        ta href="+code=u64" class="sref">u64t/a> ta href="+code=clock" class="sref">clockt/a>;3o218t/a>3o219t/a>        ta href="+code=WARN_ON_ONCE" class="sref">WARN_ON_ONCEt/a>(!ta href="+code=irqs_disabled" class="sref">irqs_disabledt/a>());3o22
	 a>3o221t/a>        if (ta href="+code=sched_clock_stable" class="sref">sched_clock_stablet/a>)3o222t/a>                return ta href="+code=sched_clock" class="sref">sched_clockt/a>();3o223t/a>3o224t/a>        if (ta href="+code=unlikely" class="sref">unlikelyt/a>(!ta href="+code=sched_clock_running" class="sref">sched_clock_runningt/a>))3o225t/a>                return 0ull;3o226t/a>3o227t/a>        ta href="+code=scd" class="sref">scdt/a> = ta href="+code=cpu_sdc" class="sref">cpu_sdct/a>(ta href="+code=cpu" class="sref">cput/a>);3o228t/a>3o229t/a>        if (ta href="+code=cpu" class="sref">cput/a> != ta href="+code=smp_processor_id" class="sref">smp_processor_idt/a>())3o230t/a>                ta href="+code=clock" class="sref">clockt/a> = ta href="+code=sched_clock_remote" class="sref">sched_clock_remotet/a>(ta href="+code=scd" class="sref">scdt/a>);3o231t/a>        else3o232t/a>                ta href="+code=clock" class="sref">clockt/a> = ta href="+code=sched_clock_local" class="sref">sched_clock_localt/a>(ta href="+code=scd" class="sref">scdt/a>);3o233t/a>3o234t/a>        return ta href="+code=clock" class="sref">clockt/a>;3o235t/a>}3o236t/a>3o237t/a>void ta href="+code=sched_clock_tick" class="sref">sched_clock_tickt/a>(void)3o238t/a>{3o239t/a>        struct ta href="+code=sched_clock_data" class="sref">sched_clock_datat/a> *ta href="+code=scd" class="sref">scdt/a>;3o240t/a>        ta href="+code=u64" class="sref">u64t/a> ta href="+code=now" class="sref">nowt/a>, ta href="+code=now_gtod" class="sref">now_gtodt/a>;3o241t/a>3o242t/a>        if (ta href="+code=sched_clock_stable" class="sref">sched_clock_stablet/a>)3o243t/a>                return;3o244t/a>3o245t/a>        if (ta href="+code=unlikely" class="sref">unlikelyt/a>(!ta href="+code=sched_clock_running" class="sref">sched_clock_runningt/a>))3o246t/a>                return;3o247t/a>3o248t/a>        ta href="+code=WARN_ON_ONCE" class="sref">WARN_ON_ONCEt/a>(!ta href="+code=irqs_disabled" class="sref">irqs_disabledt/a>());3o249t/a>3o250t/a>        ta href="+code=scd" class="sref">scdt/a> = ta href="+code=this_scd" class="sref">this_scdt/a>();3o251t/a>        ta href="+code=now_gtod" class="sref">now_gtodt/a> = ta href="+code=ktime_to_ns" class="sref">ktime_to_nst/a>(ta href="+code=ktime_get" class="sref">ktime_gett/a>());3o252t/a>        ta href="+code=now" class="sref">nowt/a> = ta href="+code=sched_clock" class="sref">sched_clockt/a>();3o253t/a>3o254t/a>        ta href="+code=scd" class="sref">scdt/a>->ta href="+code=tick_raw" class="sref">tick_rawt/a> = ta href="+code=now" class="sref">nowt/a>;3o255t/a>        ta href="+code=scd" class="sref">scdt/a>->ta href="+code=tick_gtod" class="sref">tick_gtodt/a> = ta href="+code=now_gtod" class="sref">now_gtodt/a>;3o256t/a>        ta href="+code=sched_clock_local" class="sref">sched_clock_localt/a>(ta href="+code=scd" class="sref">scdt/a>);3o257t/a>}3o258t/a>3o259t/a>tspa  class="comment">/*t/spa v3o26
	 a>tspa  class="comment"> * We are going deep-idle (irqs are disabled):t/spa v3o261t/a>tspa  class="comment"> */t/spa v3o262t/a>void ta href="+code=sched_clock_idle_sleep_event" class="sref">sched_clock_idle_sleep_eventt/a>(void)3o263t/a>{3o264t/a>        ta href="+code=sched_clock_cpu" class="sref">sched_clock_cput/a>(ta href="+code=smp_processor_id" class="sref">smp_processor_idt/a>());3o265t/a>}3o266t/a>ta href="+code=EXPORT_SYMBOL_GPL" class="sref">EXPORT_SYMBOL_GPLt/a>(ta href="+code=sched_clock_idle_sleep_event" class="sref">sched_clock_idle_sleep_eventt/a>);3o267t/a>3o268t/a>tspa  class="comment">/*t/spa v3o269t/a>tspa  class="comment"> * We just idled delta nanoseconds (called with irqs disabled):t/spa v3o27
	 a>tspa  class="comment"> */t/spa v3o271t/a>void ta href="+code=sched_clock_idle_wakeup_event" class="sref">sched_clock_idle_wakeup_eventt/a>(ta href="+code=u64" class="sref">u64t/a> ta href="+code=delta_ns" class="sref">delta_nst/a>)3o272t/a>{3o273t/a>        if (ta href="+code=timekeeping_suspended" class="sref">timekeeping_suspendedt/a>)3o274t/a>                return;3o275t/a>3o276t/a>        ta href="+code=sched_clock_tick" class="sref">sched_clock_tickt/a>();3o277t/a>        ta href="+code=touch_softlockup_watchdog" class="sref">touch_softlockup_watchdogt/a>();3o278t/a>}3o279t/a>ta href="+code=EXPORT_SYMBOL_GPL" class="sref">EXPORT_SYMBOL_GPLt/a>(ta href="+code=sched_clock_idle_wakeup_event" class="sref">sched_clock_idle_wakeup_eventt/a>);3o28
	 a>3o281t/a>tspa  class="comment">/*t/spa v3o282t/a>tspa  class="comment"> * As outlined at the top, provides a fast, high resolu2.ip, nanosecondt/spa v3o283t/a>tspa  class="comment"> * time source that is monotipic per cpu argument and has bounded driftt/spa v3o284t/a>tspa  class="comment"> * between cpus.t/spa v3o285t/a>tspa  class="comment"> *t/spa v3o286t/a>tspa  class="comment"> * ######################### BIG FAT WARNING ##########################t/spa v3o287t/a>tspa  class="comment"> * # when comparing cpu_clock(i) to cpu_clock(j) for i != j, time ca  #t/spa v3o288t/a>tspa  class="comment"> * # go backwards !!                                                  #t/spa v3o289t/a>tspa  class="comment"> * ####################################################################t/spa v3o29
	 a>tspa  class="comment"> */t/spa v3o291t/a>ta href="+code=u64" class="sref">u64t/a> ta href="+code=cpu_clock" class="sref">cpu_clockt/a>(int ta href="+code=cpu" class="sref">cput/a>)3o292t/a>{3o293t/a>        ta href="+code=u64" class="sref">u64t/a> ta href="+code=clock" class="sref">clockt/a>;3o294t/a>        unsigned long ta href="+code=flags" class="sref">flagst/a>;3o295t/a>3o296t/a>        ta href="+code=local_irq_save" class="sref">local_irq_savet/a>(ta href="+code=flags" class="sref">flagst/a>);3o297t/a>        ta href="+code=clock" class="sref">clockt/a> = ta href="+code=sched_clock_cpu" class="sref">sched_clock_cput/a>(ta href="+code=cpu" class="sref">cput/a>);3o298t/a>        ta href="+code=local_irq_restore" class="sref">local_irq_restoret/a>(ta href="+code=flags" class="sref">flagst/a>);3o299t/a>3o300t/a>        return ta href="+code=clock" class="sref">clockt/a>;3o301t/a>}3o302t/a>3o303t/a>tspa  class="comment">/*t/spa v3o304t/a>tspa  class="comment"> * Similar to cpu_clock() for the current cpu. Time will only be observedt/spa v3o305t/a>tspa  class="comment"> * to be monotipic if care is taken to only compare timestampt taken on thet/spa v3o306t/a>tspa  class="comment"> * sam  CPU.t/spa v3o307t/a>tspa  class="comment"> *t/spa v3o308t/a>tspa  class="comment"> * See cpu_clock().t/spa v3o309t/a>tspa  class="comment"> */t/spa v3o31
	 a>ta href="+code=u64" class="sref">u64t/a> ta href="+code=local_clock" class="sref">local_clockt/a>(void)3o311t/a>{3o312t/a>        ta href="+code=u64" class="sref">u64t/a> ta href="+code=clock" class="sref">clockt/a>;3o313t/a>        unsigned long ta href="+code=flags" class="sref">flagst/a>;3o314t/a>3o315t/a>        ta href="+code=local_irq_save" class="sref">local_irq_savet/a>(ta href="+code=flags" class="sref">flagst/a>);3o316t/a>        ta href="+code=clock" class="sref">clockt/a> = ta href="+code=sched_clock_cpu" class="sref">sched_clock_cput/a>(ta href="+code=smp_processor_id" class="sref">smp_processor_idt/a>());3o317t/a>        ta href="+code=local_irq_restore" class="sref">local_irq_restoret/a>(ta href="+code=flags" class="sref">flagst/a>);3o318t/a>3o319t/a>        return ta href="+code=clock" class="sref">clockt/a>;3o32
	 a>}3o321t/a>3o322t/a>#else tspa  class="comment">/* CONFIG_HAVE_UNSTABLE_SCHED_CLOCK */t/spa v3o323t/a>3o324t/a>void ta href="+code=sched_clock_init" class="sref">sched_clock_initt/a>(void)3o325t/a>{3o326t/a>        ta href="+code=sched_clock_running" class="sref">sched_clock_runningt/a> = 1;3o327t/a>}3o328t/a>3o329t/a>ta href="+code=u64" class="sref">u64t/a> ta href="+code=sched_clock_cpu" class="sref">sched_clock_cput/a>(int ta href="+code=cpu" class="sref">cput/a>)3o330t/a>{3o331t/a>        if (ta href="+code=unlikely" class="sref">unlikelyt/a>(!ta href="+code=sched_clock_running" class="sref">sched_clock_runningt/a>))3o332t/a>                return 0;3o333t/a>3o334t/a>        return ta href="+code=sched_clock" class="sref">sched_clockt/a>();3o335t/a>}3o336t/a>3o337t/a>ta href="+code=u64" class="sref">u64t/a> ta href="+code=cpu_clock" class="sref">cpu_clockt/a>(int ta href="+code=cpu" class="sref">cput/a>)3o338t/a>{3o339t/a>        return ta href="+code=sched_clock_cpu" class="sref">sched_clock_cput/a>(ta href="+code=cpu" class="sref">cput/a>);3o34
	 a>}3o341t/a>3o342t/a>ta href="+code=u64" class="sref">u64t/a> ta href="+code=local_clock" class="sref">local_clockt/a>(void)3o343t/a>{3o344t/a>        return ta href="+code=sched_clock_cpu" class="sref">sched_clock_cput/a>(0);3o345t/a>}3o346t/a>3o347t/a>#endif tspa  class="comment">/* CONFIG_HAVE_UNSTABLE_SCHED_CLOCK */t/spa v3o348t/a>3o349t/a>ta href="+code=EXPORT_SYMBOL_GPL" class="sref">EXPORT_SYMBOL_GPLt/a>(ta href="+code=cpu_clock" class="sref">cpu_clockt/a>L249" id 
L249" class=" clas
L338" class="line" nam  
L338">o338t/a>{3 ta href="+cod1BLE_SCHED_CLOCK */t/spa v3o26
	 a>ts_41line" nam  
L26class=


footer"> T="coriginal LXR tchdw carby Simi nf#L260"http://a claforge.net/projects/lxr">LXR 347t"> ass=, nt"> experit/a>al versionrby nf#L260"mailto:lxr@ts_ux.no Lxr@ts_ux.noass=.
subfooter"> Lxr.ts_ux.no kindly hostedrby nf#L260"http://www.redpill-ts_pro.no Redpill Ls_pro ASass=, * As or of Ls_ux 34nsult="comonooperarovis t cpices tince" cl5.