linux/kernel/async.c
<<
>>
Prefs
   1/*
   2 * async.c: Asynchronous function calls for boot performance
   3 *
   4 * (C) Copyright 2009 Intel Corporation
   5 * Author: Arjan van de Ven <arjan@linux.intel.com>
   6 *
   7 * This program is free software; you can redistribute it and/or
   8 * modify it under the terms of the GNU General Public License
   9 * as published by the Free Software Foundation; version 2
  10 * of the License.
  11 */
  12
  13
  14/*
  15
  16Goals and Theory of Operation
  17
  18The primary goal of this feature is to reduce the kernel boot time,
  19by doing various independent hardware delays and discovery operations
  20decoupled and not strictly serialized.
  21
  22More specifically, the asynchronous function call concept allows
  23certain operations (primarily during system boot) to happen
  24asynchronously, out of order, while these operations still
  25have their externally visible parts happen sequentially and in-order.
  26(not unlike how out-of-order CPUs retire their instructions in order)
  27
  28Key to the asynchronous function call implementation is the concept of
  29a "sequence cookie" (which, although it has an abstracted type, can be
  30thought of as a monotonically incrementing number).
  31
  32The async core will assign each scheduled event such a sequence cookie and
  33pass this to the called functions.
  34
  35The asynchronously called function should before doing a globally visible
  36operation, such as registering device numbers, call the
  37async_synchronize_cookie() function and pass in its own cookie. The
  38async_synchronize_cookie() function will make sure that all asynchronous
  39operations that were scheduled prior to the operation corresponding with the
  40cookie have completed.
  41
  42Subsystem/driver initialization code that scheduled asynchronous probe
  43functions, but which shares global resources with other drivers/subsystems
  44that do not use the asynchronous call feature, need to do a full
  45synchronization with the async_synchronize_full() function, before returning
  46from their init function. This is to maintain strict ordering between the
  47asynchronous and synchronous parts of the kernel.
  48
  49*/
  50
  51#include <linux/async.h>
  52#include <linux/atomic.h>
  53#include <linux/ktime.h>
  54#include <linux/export.h>
  55#include <linux/wait.h>
  56#include <linux/sched.h>
  57#include <linux/slab.h>
  58#include <linux/workqueue.h>
  59
  60#include "workqueue_internal.h"
  61
  62static async_cookie_t next_cookie = 1;
  63
  64#define MAX_WORK                32768
  65#define ASYNC_COOKIE_MAX        ULLONG_MAX      /* infinity cookie */
  66
  67static LIST_HEAD(async_global_pending); /* pending from all registered doms */
  68static ASYNC_DOMAIN(async_dfl_domain);
  69static DEFINE_SPINLOCK(async_lock);
  70
  71struct async_entry {
  72        struct list_head        domain_list;
  73        struct list_head        global_list;
  74        struct work_struct      work;
  75        async_cookie_t          cookie;
  76        async_func_ptr          *func;
  77        void                    *data;
  78        struct async_domain     *domain;
  79};
  80
  81static DECLARE_WAIT_QUEUE_HEAD(async_done);
  82
  83static atomic_t entry_count;
  84
  85static async_cookie_t lowest_in_progress(struct async_domain *domain)
  86{
  87        struct async_entry *first = NULL;
  88        async_cookie_t ret = ASYNC_COOKIE_MAX;
  89        unsigned long flags;
  90
  91        spin_lock_irqsave(&async_lock, flags);
  92
  93        if (domain) {
  94                if (!list_empty(&domain->pending))
  95                        first = list_first_entry(&domain->pending,
  96                                        struct async_entry, domain_list);
  97        } else {
  98                if (!list_empty(&async_global_pending))
  99                        first = list_first_entry(&async_global_pending,
 100                                        struct async_entry, global_list);
 101        }
 102
 103        if (first)
 104                ret = first->cookie;
 105
 106        spin_unlock_irqrestore(&async_lock, flags);
 107        return ret;
 108}
 109
 110/*
 111 * pick the first pending entry and run it
 112 */
 113static void async_run_entry_fn(struct work_struct *work)
 114{
 115        struct async_entry *entry =
 116                container_of(work, struct async_entry, work);
 117        unsigned long flags;
 118        ktime_t uninitialized_var(calltime), delta, rettime;
 119
 120        /* 1) run (and print duration) */
 121        if (initcall_debug && system_state == SYSTEM_BOOTING) {
 122                printk(KERN_DEBUG "calling  %lli_%pF @ %i\n",
 123                        (long long)entry->cookie,
 124                        entry->func, task_pid_nr(current));
 125                calltime = ktime_get();
 126        }
 127        entry->func(entry->data, entry->cookie);
 128        if (initcall_debug && system_state == SYSTEM_BOOTING) {
 129                rettime = ktime_get();
 130                delta = ktime_sub(rettime, calltime);
 131                printk(KERN_DEBUG "initcall %lli_%pF returned 0 after %lld usecs\n",
 132                        (long long)entry->cookie,
 133                        entry->func,
 134                        (long long)ktime_to_ns(delta) >> 10);
 135        }
 136
 137        /* 2) remove self from the pending queues */
 138        spin_lock_irqsave(&async_lock, flags);
 139        list_del_init(&entry->domain_list);
 140        list_del_init(&entry->global_list);
 141
 142        /* 3) free the entry */
 143        kfree(entry);
 144        atomic_dec(&entry_count);
 145
 146        spin_unlock_irqrestore(&async_lock, flags);
 147
 148        /* 4) wake up any waiters */
 149        wake_up(&async_done);
 150}
 151
 152static async_cookie_t __async_schedule(async_func_ptr *ptr, void *data, struct async_domain *domain)
 153{
 154        struct async_entry *entry;
 155        unsigned long flags;
 156        async_cookie_t newcookie;
 157
 158        /* allow irq-off callers */
 159        entry = kzalloc(sizeof(struct async_entry), GFP_ATOMIC);
 160
 161        /*
 162         * If we're out of memory or if there's too much work
 163         * pending already, we execute synchronously.
 164         */
 165        if (!entry || atomic_read(&entry_count) > MAX_WORK) {
 166                kfree(entry);
 167                spin_lock_irqsave(&async_lock, flags);
 168                newcookie = next_cookie++;
 169                spin_unlock_irqrestore(&async_lock, flags);
 170
 171                /* low on memory.. run synchronously */
 172                ptr(data, newcookie);
 173                return newcookie;
 174        }
 175        INIT_LIST_HEAD(&entry->domain_list);
 176        INIT_LIST_HEAD(&entry->global_list);
 177        INIT_WORK(&entry->work, async_run_entry_fn);
 178        entry->func = ptr;
 179        entry->data = data;
 180        entry->domain = domain;
entry->  82
spin_lock_irqsave(&async_lock, flags);
  83sta18ref="kernel/async.c#L14" id="L144" class=1"line" name="L84">  84
/* allow irq-off catnd
  85sta18   newcookie = entry->cookie,
next_cookie++;
  86{
<18ref="kernel/async.c#L137" id="L17" class=1"line" name="L87">  87   18   list_del_initentry->domain_list);
domain->pending))
  18>    8   if (domain->domain doms */  89   18           list_del_initentry->global_list);
, ding" class="sref">async_global_pending);   90
  91   1    func =d(&i"sentry_count) >   92
spin_unlock_irqrestore(&async_lock, flags);
  93   19ref="kernel/async.c#L14" id="L144" class=1"line" name="L94">  94   19   /* allow irq-markduled e callaskstrac/spandcted " id job,uot;ree Smoa hrtion.   95   19   current));
flags);
flags)PF_USED_IE_MAref="kernel/async.c#L1stati16" class=1"line" name="L96">  96   19ref="kernel/async.c#L137" id="L17" class=1"line" name="L97">  97   19   /* 2) remove a>(  98   19   work, rnal.hs="sef="+code=data" claste unb ver_wqref">spin_unlocke unb ver_wqhref, ef="+code=domain" clss="sref">entry->work,   19>   19ref="kernel/async.c#L120" id="L200" class2"line" name="L100"> 100<2a>   2              href="+code=newcookie" class="sref">newcookie;
 101<2a>   2    }
"kernel/async.c#L175" id="L202" class2"line" name="L102"> 102<2a>
 103<2a>   20 class="comment">         */*a href="kernel/async.c#L162" id="L204" class2"line" name="L104"> 104<2a>   20 class="comment">         */*chronize>((  95
synchroniz/*c@ptr:will make toynchronouasly */ 106<2a>   20 class="comment">from their/*c@& toyto theation ill make href="kernel/async.c#L162" id="L20" class=""line" name="L107"> 107<2a>   20 class="comment">asynchrono a href="kernel/async.c#L162" id="L20" class=""line" name="L108"> 108<2a>}
<20 class="comment">async_sync/*cRref="scted " id/a> ( 109<2a>
*/
/*cNote:o mainill make may beuction shgiste(re&#non-e(rof 110<2a>/*
a 111<2a>async_cookie_t __async_scdule(async_func_ptr *ptr, void *data, struct <"kernel/async.c#L153" id="L212" class2"line" name="L112"> 112<2a> 113<2a>sta213            href="+code=newcookie"hedule" class="sref">__async_schedule(ptr, void *data, async_dfl_domain);
 114<2a>{
<2 href="kernel/async.c#L175" id="L215" class2"line" name="L115"> 115<2a>   2    stcode=async_doneEXPORT_SYMBOL_GPsref">NULL;
EXPORT_SYMBOL_GPsef="+code=async_func_ptr" class="sref">__async_scdule( 116<2a>   21ref="kernel/async.c#L137" id="L217" class2"line" name="L117"> 117<2a>   21 class="comment">asynchrono/*a href="kernel/async.c#L162" id="L218" class2"line" name="L118"> 118<2a>   21 class="comment">async_sync/*cdule();
( 119<2a>
*/
/*c@ptr:will make toynchronouasly */ 120<2a>   22 class="comment">/*
a& toyto theation ill make href="kernel/async.c#L162" id="L221" class2"line" name="L121"> 121<2a>   22 class="comment"> * pick the f@);
 122<2a>   22 class="comment"> */
 123<2a>   22 class="comment">         * *cRref="scted " id/a> ( 124<2a>   22 class="comment">         */*c@);
);
 125<2a>   22 class="comment">synchroniz/*c 126<2a>   22 class="comment">from their/*ce_full() f the a);
 127<2a>   22 class="comment">asynchrono a may beuction shgiste(re&#non-e(rof 128<2a>   22 class="comment">async_sync/*a href="kernel/async.c#L172" id="L229" class2"line" name="L129"> 129<2a>   2      sode=async_lock" clasclass="sref">async_cookie_t async_dfl_domainclass="slass="sef="+code=async_func_ptr" class="sref">async_func_ptr *ptr, void *data, struct <"kernel/async.c#L134" id="L230" class2"line" name="L130"> 130<2a>   2            async_domain *domain)
 131<2a>   2      "kernel/async.c#L166" id="L232" class2"line" name="L132"> 132<2a>   2              href="+code=newcookie"hedule" class="sref">__async_schedule(ptr, void *data, domain)
 133<2a>   2      "kernel/async.c#L175" id="L234" class2"line" name="L134"> 134<2a>   2      code=async_doneEXPORT_SYMBOL_GPsref">NULL;
EXPORT_SYMBOL_GPsef="+code=async_func_ptr" class="slass="sref">async_dfl_domainclass="slass="sef="""kernel/async.c#L69" id="L6235" class2"line" name="L135"> 135<2a>   23ref="kernel/async.c#L146" id="L236" class2"line" name="L136"> 136<2a>
from their/*a href="kernel/async.c#L162" id="L237" class2"line" name="L137"> 137<2a>   23 class="comment">asynchrono a hronize_full() funfullp- a_full() futeredasly */ 138<2a>   23 class="comment">async_sync/* href="kernel/async.c#L162" id="L239" class2"line" name="L139"> 139<2a>   23 class="comment">*/
/*c mainill make  140<2a>   24 class="comment">/*
a 141<2a>
async_dfl_domainc_full() funfullef="+ef=""kernel/async.c#L153" id="L242" class2"line" name="L142"> 142<2a>   24 clas"kernel/async.c#L166" id="L243" class2"line" name="L143"> 143<2a>   2    async_dfl_domainc_full() funfulllass="sef="+code=async_funcs="sref">NULL;
 144<2a>   24href="kernel/async.c#L175" id="L245" class2"line" name="L145"> 145<2a>
NULL;
EXPORT_SYMBOL_GPsef="+code=async_func_ptr" c_full() funfullref">async_dfl_domainc_full() funfullef="""kernel/async.c#L69" id="L6246" class2"line" name="L146"> 146<2a>   24ref="kernel/async.c#L137" id="L247" class2"line" name="L147"> 147<2a>
asynchrono/*a href="kernel/async.c#L162" id="L248" class2"line" name="L148"> 148<2a>   24 class="comment">async_sync/*cdule);


ke tmain);
 149<2a>   24 class="comment">*/
/*c@);
 150<2a>}
<25 class="comment">/*
a< href="kernel/async.c#L162" id="L251" class2"line" name="L151"> 151<2a>
 * pick the fhronize_full() fun{a> );
 href="kernel/async.c#L162" id="L252" class2"line" name="L152"> 152<2a>sta25 class="comment"> */
< or these rory"L1inshould know_synclife or @);
 153<2a>{
<25 class="comment">         * *hrref="kernel/async.c#L162" id="L254" class2"line" name="L154"> 154<2a>   25 class="comment">         */*cPl/aer IN(
kver flush 155<2a>   25 class="comment">synchroniz/*a href="kernel/async.c#L172" id="L256" class2"line" name="L156"> 156<2a>   2    );
async_dfl_domainun doms */>);
async_domain *domain)
 157<2a>
 158<2a>   25   spin_unlock_irqve" clasasync_lock,  159<2a>   2    async_lockWARN_ONdomain->domain doms */list_empty(&domain->pending))
 160<2a>
domain->domain doms */ 161<2a>   26   spin_unlock_irqrestoreasync_lock,  162<2a> 163<2a>NULL;
EXPORT_SYMBOL_GPsef="+code=async_func_ptr" un doms */>);
async_dfl_domainun doms */>);
 164<2a> 165<2a>   26 class="comment">synchroniz/*a href="kernel/async.c#L162" id="L266" class2"line" name="L166"> 166<2a>   26 class="comment">from their/*cdomainc_full() funfulllass="sp- a_full() futeredasly */ 167<2a>   26 class="comment">asynchrono a @);
 168<2a>   26 class="comment">async_sync/* href="kernel/async.c#L162" id="L269" class2"line" name="L169"> 169<2a>   26 class="comment">*/
/*c mainill make  170<2a>
/*
a 171<2a>   27 class="comment"> * pick the a href="kernel/async.c#L172" id="L272" class2"line" name="L172"> 172<2a>   2      ef="+code=async_run_entry_e_full() funfulllass="sref">async_dfl_domainc_full() funfulllass="sef="+href="+code=async_domain" class="sref">async_domain *domain)
 173<2a>   27href="kernel/async.c#L154" id="L274" class2"line" name="L174"> 174<2a>   27    async_dfl_domainc_full() funa> ASYNC_COOKIE_MAX;
domain)
 175<2a>   2     176<2a>   2    NULL;
EXPORT_SYMBOL_GPsef="+code=async_func_ptr" c_full() funfulllass="sref">async_dfl_domainc_full() funfulllass="sef="""kernel/async.c#L69" id="L6277" class2"line" name="L177"> 177<2a>   27ref="kernel/async.c#L158" id="L278" class2"line" name="L178"> 178<2a>   27 class="comment">async_sync/*a href="kernel/async.c#L162" id="L279" class2"line" name="L179"> 179<2a>   27 class="comment">*/
/*cdomainc_full() funa>  180<2a>   28 class="comment">/*
a >( * pick the f@);
  82
 */
  83sta28 class="comment">         * *c mainill make   84
         */*ce_full() f the a);
  85sta28 class="comment">synchroniz/*chave bee a);nea href="kernel/async.c#L34" id="L326" class=2"line" name="L86">  86{
<28 class="comment">from their/*a href="kernel/async.c#L172" id="L27" class=2"line" name="L87">  87   28    async_dfl_domainc_full() funa> async_cookie_t cookie,
async_domain *domain)
  18>   28   if"kernel/async.c#L154" id="L29" class=2"line" name="L89">  89   28   ktime_t uninitialized_var(rettimedelta, rettime  90
  91   2    initcall_debug && system_state == SYSTEM_BOOTING) {
  92
printk(KERN_DEBUG "calling  &in,
task_pid_nr(current));
  93   29           return rettimektime_get();
  94   29   }
  95   29ref="kernel/async.c#L146" id="L26" class=2"line" name="L96">  96   29   currentasync_done);
flags)lowest_in_progressef="+code=async_funcass="sref">domain)
cookie);
  97   29ref="kernel/async.c#L158" id="L25" class=2"line" name="L98">  98   29   if (initcall_debug && system_state == SYSTEM_BOOTING) {
  19>   29           rettimektime_get();
 100<3a>   30           delta = ktime_sub(rettimerettime  91   30try" class="sref">entry->302" class3"line" name="L102"> 102<3a>
printk(KERN_DEBUG "calling  &inof,
 103<3a>   30                   task_pid_nr(current));
  94   30                   (long long)ktime_to_ns(delta) >> 10);
  95
 106<3a>   30 clas"kernel/async.c#L136" id="L307 class=3"line" name="L97">  97   30 classode=async_doneEXPORT_SYMBOL_GPsref">NULL;
EXPORT_SYMBOL_GPsef="+code=async_func_ptr" c_full() funa> async_dfl_domainc_full() funa>   98}
<30 claskernel/async.c#L135" id="L309 class=3"line" name="L19">  19
*/
/*a href="kernel/async.c#L162" id="L310" class3"line" name="L110"> 110<3a>/*
a >( 111<3a> * pick the f@ >( 112<3a> */
 113<3a>sta31 class="comment">         * *c mainill make  114<3a>{
<31 class="comment">         */*chave bee a);nea href="kernel/async.c#L34" id="L3315" class3"line" name="L115"> 115<3a>   31 class="comment">synchroniz/*a href="kernel/async.c#L172" id="L31" class=3"line" name="L116"> 116<3a>   31    async_dfl_domainc_full() funa> async_cookie_t cookie,
 117<3a>   31ref=""kernel/async.c#L166" id="L318" class3"line" name="L118"> 118<3a>   31   async_dfl_domainc_full() funa> cookie,
async_dfl_domain);
 119<3a>
 120<3a>   32 classode=async_doneEXPORT_SYMBOL_GPsref">NULL;
EXPORT_SYMBOL_GPsef="+code=async_func_ptr" c_full() funa> async_dfl_domainc_full() funa>  121<3a>   32try" class="sref">entry->322" class3"line" name="L122"> 122<3a>   32 class="comment"> *//*a href="kernel/async.c#L162" id="L323" class3"line" name="L123"> 123<3a>   32 class="comment">         * *c>));
));

 124<3a>   32 class="comment">         */* href="kernel/async.c#L162" id="L325" class3"line" name="L125"> 125<3a>   32 class="comment">synchroniz/*cRref="scsecs\trueef="secs\>));

 126<3a>   32 class="comment">from their/*a href="kernel/async.c#L172" id="L327" class3"line" name="L127"> 127<3a>   32 classode=async_doneboolref">async_dfl_boolef="+code=newcookie"a));
current));
 128<3a>   32   if"kernel/async.c#L154" id="L329" class3"line" name="L129"> 129<3a>   32           
current
current
current
 130<3a>   33ref="kernel/async.c#L91" id="L9331" class3"line" name="L131"> 131<3a>   33   
current
currentfunc =a));
async_run_entry_fn);
 132<3a>   33 clas"kernel/async.c#L175" id="L333" class3"line" name="L133"> 133<3a>   3      
The original LXR software#bytion ernel/asyhttp://sourceforge.net/projects/lxsr>LXR rom unip;lxs@="Lux.no lxs.="Lux.no kindly host;rebycernel/asyhttp://www.redpill-="Lpro.no">Redpill L"Lpro AS servic1insince/1995.