linux/block/blk-ioc.c
<<
" " " o/spa> " ospa> class="lxr_search">" ="+search" method="post" onsubmit="return do_search(this);">" " " Search " ospa> class="lxr_prefs" " " o/spa> < ="ajax+*" method="post" onsubmit="return false;">" oinput typ="vhidden" nam="vajax_lookup" id"vajax_lookup" alue="v">" <
odiv id"vfile_contents"
< <1o/a>ospa> class="comment">/*o/spa>
 < <2o/a>ospa> class="comment"> * Funcion>s related to io context handlingo/spa>
 < <3o/a>ospa> class="comment"> */o/spa>
 < <4o/a>#include <linux/kernel.ho/a>> < <5o/a>#include <linux/module.ho/a>> < <6o/a>#include <linux/init.ho/a>> < <7o/a>#include <linux/bio.ho/a>> < <8o/a>#include <linux/blkdev.ho/a>> < <9o/a>#include <linux/bootmem.ho/a>>	  <  class="comment">/* for max_pfn/max_low_pfn */o/spa>
 < 1#include <linux/slab.ho/a>> < 11o/a> < 12/oa>#include "blk.ho/a>" < 13o/a> < 14o/a>ospa> class="comment">/*o/spa>
 < 15o/a>ospa> class="comment"> * For io context allocaion>so/spa>
 < 16o/a>ospa> class="comment"> */o/spa>
 < 17o/a>static structkmem_cacheo/a> *oa href="+code=iocontext_cachep" class="sref">iocontext_cachepo/a>; < 18o/a> < 19o/a>ospa> class="comment">/**o/spa>
 < 20o/a>ospa> class="comment"> * get_io_context - increment reference count to io_contexto/spa>
 < 21o/a>ospa> class="comment"> * @ioc: io_context to geto/spa>
 < 22o/a>ospa> class="comment"> *o/spa>
 < 23o/a>ospa> class="comment"> * Increment reference count to @ioc.o/spa>
 < 24o/a>ospa> class="comment"> */o/spa>
 < 25o/a>voidget_io_contexto/a>(structio_contexto/a> *oa href="+code=ioc" class="sref">ioco/a>) < 26o/a>{ < 27o/a>        oa href="+code=BUG_ON" class="sref">BUG_ONo/a>(oa href="+code=atnmic_long_read" class="sref">atnmic_long_reado/a>(&oa href="+code=ioc" class="sref">ioco/a>->oa href="+code=refcount" class="sref">refcounto/a>) <= 0); < 28o/a>        oa href="+code=atnmic_long_inc" class="sref">atnmic_long_inco/a>(&oa href="+code=ioc" class="sref">ioco/a>->oa href="+code=refcount" class="sref">refcounto/a>); < 29o/a>} < 30o/a>oa href="+code=EXPORT_SYMBOL" class="sref">EXPORT_SYMBOLo/a>(oa href="+code=get_io_context" class="sref">get_io_contexto/a>); < 31o/a> < 32o/a>static voidicq_free_icq_rcuo/a>(structrcu_heado/a> *oa href="+code=head" class="sref">heado/a>) < 33o/a>{ < 34o/a>        structio_cqo/a> *oa href="+code=icq" class="sref">icqo/a> =container_ofo/a>(oa href="+code=head" class="sref">heado/a>, structio_cqo/a>,__rcu_heado/a>); < 35o/a> < 36o/a>        oa href="+code=kmem_cache_free" class="sref">kmem_cache_freeo/a>(oa href="+code=icq" class="sref">icqo/a>->oa href="+code=__rcu_icq_cache" class="sref">__rcu_icq_cacheo/a>,icqo/a>); < 37o/a>} < 38o/a> < 39o/a>ospa> class="comment">/* Exit an icq. Called with both ioc and q locked. */o/spa>
 < 40o/a>static voidioc_exit_icqo/a>(structio_cqo/a> *oa href="+code=icq" class="sref">icqo/a>) < 41o/a>{ < 42o/a>        structelevatnr_typ=o/a> *oa href="+code=et" class="sref">eto/a> =icqo/a>->oa href="+code=q" class="sref">qo/a>->oa href="+code=elevatnr" class="sref">elevatnro/a>->oa href="+code=typ=" class="sref">typ=o/a>; < 43o/a> < 44o/a>        if (oa href="+code=icq" class="sref">icqo/a>->oa href="+code=flags" class="sref">flagso/a> &ICQ_EXITEDo/a>) < 45o/a>                return; < 46o/a> < 47o/a>        if (oa href="+code=et" class="sref">eto/a>->oa href="+code=ops" class="sref">opso/a>.oa href="+code=elevatnr_exit_icq_fn" class="sref">elevatnr_exit_icq_fno/a>) < 48o/a>                oa href="+code=et" class="sref">eto/a>->oa href="+code=ops" class="sref">opso/a>.oa href="+code=elevatnr_exit_icq_fn" class="sref">elevatnr_exit_icq_fno/a>(oa href="+code=icq" class="sref">icqo/a>); < 49o/a> < 50o/a>        oa href="+code=icq" class="sref">icqo/a>->oa href="+code=flags" class="sref">flagso/a> |=ICQ_EXITEDo/a>; < 51o/a>} < 52o/a> < 53o/a>ospa> class="comment">/* Release an icq.  Called with both ioc and q locked. */o/spa>
 < 54o/a>static voidioc_destroy_icqo/a>(structio_cqo/a> *oa href="+code=icq" class="sref">icqo/a>) < 55o/a>{ < 56o/a>        structio_contexto/a> *oa href="+code=ioc" class="sref">ioco/a> =icqo/a>->oa href="+code=ioc" class="sref">ioco/a>; < 57o/a>        structrequest_queu=o/a> *oa href="+code=q" class="sref">qo/a> =icqo/a>->oa href="+code=q" class="sref">qo/a>; < 58o/a>        structelevatnr_typ=o/a> *oa href="+code=et" class="sref">eto/a> =qo/a>->oa href="+code=elevatnr" class="sref">elevatnro/a>->oa href="+code=typ=" class="sref">typ=o/a>; < 59o/a> < 60o/a>        oa href="+code=lockdep_assert_held" class="sref">lockdep_assert_heldo/a>(&oa href="+code=ioc" class="sref">ioco/a>->oa href="+code=lock" class="sref">locko/a>); < 61o/a>        oa href="+code=lockdep_assert_held" class="sref">lockdep_assert_heldo/a>(oa href="+code=q" class="sref">qo/a>->oa href="+code=queu=_lock" class="sref">queu=_locko/a>); < 62o/a> < 63o/a>        oa href="+code=radix_tree_delet=" class="sref">radix_tree_delet=o/a>(&oa href="+code=ioc" class="sref">ioco/a>->oa href="+code=icq_tree" class="sref">icq_treeo/a>,icqo/a>->oa href="+code=q" class="sref">qo/a>->oa href="+code=id" class="sref">ido/a>); < 64o/a>        oa href="+code=hlist_del_init" class="sref">hlist_del_inito/a>(&oa href="+code=icq" class="sref">icqo/a>->oa href="+code=ioc_node" class="sref">ioc_nodeo/a>); < 65o/a>        oa href="+code=list_del_init" class="sref">list_del_inito/a>(&oa href="+code=icq" class="sref">icqo/a>->oa href="+code=q_node" class="sref">q_nodeo/a>); < 66o/a> < 67o/a>        ospa> class="comment">/*o/spa>
 < 68o/a>ospa> class="comment">         * Both setting lookup hint to and clearing it from @icq are doneo/spa>
 < 69o/a>ospa> class="comment">         * under queu=_lock.  If it's not pointing to @icq now, it nevero/spa>
 < 70o/a>ospa> class="comment">         * will.  Hint assignment itself ca> race safely.o/spa>
 < 71o/a>ospa> class="comment">         */o/spa>
 < 72o/a>        if (oa href="+code=rcu_dereference_raw" class="sref">rcu_dereference_rawo/a>(oa href="+code=ioc" class="sref">ioco/a>->oa href="+code=icq_hint" class="sref">icq_hinto/a>) ==icqo/a>) < 73o/a>                oa href="+code=rcu_assign_pointer" class="sref">rcu_assign_pointero/a>(oa href="+code=ioc" class="sref">ioco/a>->oa href="+code=icq_hint" class="sref">icq_hinto/a>,NULLo/a>); < 74o/a> < 75o/a>        oa href="+code=ioc_exit_icq" class="sref">ioc_exit_icqo/a>(oa href="+code=icq" class="sref">icqo/a>); < 76o/a> < 77o/a>        ospa> class="comment">/*o/spa>
 < 78o/a>ospa> class="comment">         * @icq->q might have gone away by the time RCU callback ru>so/spa>
 < 79o/a>ospa> class="comment">         * making it impossible to det=rmine icq_cache.  Record it in @icq.o/spa>
 < 80o/a>ospa> class="comment">         */o/spa>
 < 81o/a>        oa href="+code=icq" class="sref">icqo/a>->oa href="+code=__rcu_icq_cache" class="sref">__rcu_icq_cacheo/a> =eto/a>->oa href="+code=icq_cache" class="sref">icq_cacheo/a>; < 82o/a>        oa href="+code=call_rcu" class="sref">call_rcuo/a>(&oa href="+code=icq" class="sref">icqo/a>->oa href="+code=__rcu_head" class="sref">__rcu_heado/a>,icq_free_icq_rcuo/a>); < 83o/a>} < 84o/a> < 85o/a>ospa> class="comment">/*o/spa>
 < 86o/a>ospa> class="comment"> * Slow path for ioc release in put_io_context().  Performs double-locko/spa>
 < 87o/a>ospa> class="comment"> * dancing to unlink all icq's and then frees ioc.o/spa>
 < 88o/a>ospa> class="comment"> */o/spa>
 < 89o/a>static voidioc_release_fno/a>(structwork_structo/a> *oa href="+code=work" class="sref">worko/a>) < 90o/a>{ < 91o/a>        structio_contexto/a> *oa href="+code=ioc" class="sref">ioco/a> =container_ofo/a>(oa href="+code=work" class="sref">worko/a>, structio_contexto/a>, < 92o/a>                                              oa href="+code=release_work" class="sref">release_worko/a>); < 93o/a>        u>signed long oa href="+code=flags" class="sref">flagso/a>; < 94o/a> < 95o/a>        ospa> class="comment">/*o/spa>
 < 96o/a>ospa> class="comment">         * Exiting icq may call into put_io_context() through elevatnro/spa>
 < 97o/a>ospa> class="comment">         * which will trigger lockdep warning.  The ioc's are guaranteed too/spa>
 < 98o/a>ospa> class="comment">         * be different, use a different locking subclass here.  Useo/spa>
 < 99o/a>ospa> class="comment">         * irqsave variant as there's no spin_lock_irq_nested().o/spa>
 <100o/a>ospa> class="comment">         */o/spa>
 <101o/a>        oa href="+code=spin_lock_irqsave_nested" class="sref">spin_lock_irqsave_nestedo/a>(&oa href="+code=ioc" class="sref">ioco/a>->oa href="+code=lock" class="sref">locko/a>,flagso/a>, 1); <102o/a> <103o/a>        while (!oa href="+code=hlist_empty" class="sref">hlist_emptyo/a>(&oa href="+code=ioc" class="sref">ioco/a>->oa href="+code=icq_list" class="sref">icq_listo/a>)) { <104o/a>                structio_cqo/a> *oa href="+code=icq" class="sref">icqo/a> =hlist_entryo/a>(oa href="+code=ioc" class="sref">ioco/a>->oa href="+code=icq_list" class="sref">icq_listo/a>.oa href="+code=first" class="sref">firsto/a>, <105o/a>                                                structio_cqo/a>,ioc_nodeo/a>); <106o/a>                structrequest_queu=o/a> *oa href="+code=q" class="sref">qo/a> =icqo/a>->oa href="+code=q" class="sref">qo/a>; <107o/a> <108o/a>                if (oa href="+code=spin_trylock" class="sref">spin_trylocko/a>(oa href="+code=q" class="sref">qo/a>->oa href="+code=queu=_lock" class="sref">queu=_locko/a>)) { <109o/a>                        oa href="+code=ioc_destroy_icq" class="sref">ioc_destroy_icqo/a>(oa href="+code=icq" class="sref">icqo/a>); <110o/a>                        oa href="+code=spin_unlock" class="sref">spin_unlocko/a>(oa href="+code=q" class="sref">qo/a>->oa href="+code=queu=_lock" class="sref">queu=_locko/a>); <111o/a>                } else { <112o/a>                        oa href="+code=spin_unlock_irqrestor=" class="sref">spin_unlock_irqrestor=o/a>(&oa href="+code=ioc" class="sref">ioco/a>->oa href="+code=lock" class="sref">locko/a>,flagso/a>); <113o/a>                        oa href="+code=cpu_relax" class="sref">cpu_relaxo/a>(); <114o/a>                        oa href="+code=spin_lock_irqsave_nested" class="sref">spin_lock_irqsave_nestedo/a>(&oa href="+code=ioc" class="sref">ioco/a>->oa href="+code=lock" class="sref">locko/a>,flagso/a>, 1); <115o/a>                } <116o/a>        } <117o/a> <118o/a>        oa href="+code=spin_unlock_irqrestor=" class="sref">spin_unlock_irqrestor=o/a>(&oa href="+code=ioc" class="sref">ioco/a>->oa href="+code=lock" class="sref">locko/a>,flagso/a>); <119o/a> <120o/a>        oa href="+code=kmem_cache_free" class="sref">kmem_cache_freeo/a>(oa href="+code=iocontext_cachep" class="sref">iocontext_cachepo/a>,ioco/a>); <121o/a>} <122o/a> <123o/a>ospa> class="comment">/**o/spa>
 <124o/a>ospa> class="comment"> * put_io_context - put a reference of io_contexto/spa>
 <125o/a>ospa> class="comment"> * @ioc: io_context to puto/spa>
 <126o/a>ospa> class="comment"> *o/spa>
 <127o/a>ospa> class="comment"> * Decrement reference count of @ioc and release it if the count reacheso/spa>
 <128o/a>ospa> class="comment"> * zero.o/spa>
 <129o/a>ospa> class="comment"> */o/spa>
 <130o/a>voidput_io_contexto/a>(structio_contexto/a> *oa href="+code=ioc" class="sref">ioco/a>) <131o/a>{ <132o/a>        u>signed long oa href="+code=flags" class="sref">flagso/a>; <133o/a>        oa href="+code=bool" class="sref">boolo/a> oa href="+code=free_ioc" class="sref">free_ioco/a> =falseo/a>; <134o/a> <135o/a>        if (oa href="+code=ioc" class="sref">ioco/a> ==NULLo/a>) <136o/a>                return; <137o/a> <138o/a>        oa href="+code=BUG_ON" class="sref">BUG_ONo/a>(oa href="+code=atnmic_long_read" class="sref">atnmic_long_reado/a>(&oa href="+code=ioc" class="sref">ioco/a>->oa href="+code=refcount" class="sref">refcounto/a>) <= 0); <139o/a> <140o/a>        ospa> class="comment">/*o/spa>
 <141o/a>ospa> class="comment">         * Releasing ioc requires reverse order double locking and we mayo/spa>
 <142o/a>ospa> class="comment">         * already be holding a queu=_lock.  Do it asynchronously from wq.o/spa>
 <143o/a>ospa> class="comment">         */o/spa>
 <144o/a>        if (oa href="+code=atnmic_long_dec_and_test" class="sref">atnmic_long_dec_and_testo/a>(&oa href="+code=ioc" class="sref">ioco/a>->oa href="+code=refcount" class="sref">refcounto/a>)) { <145o/a>                oa href="+code=spin_lock_irqsave" class="sref">spin_lock_irqsaveo/a>(&oa href="+code=ioc" class="sref">ioco/a>->oa href="+code=lock" class="sref">locko/a>,flagso/a>); <146o/a>                if (!oa href="+code=hlist_empty" class="sref">hlist_emptyo/a>(&oa href="+code=ioc" class="sref">ioco/a>->oa href="+code=icq_list" class="sref">icq_listo/a>)) <147o/a>                        oa href="+code=schedule_work" class="sref">schedule_worko/a>(&oa href="+code=ioc" class="sref">ioco/a>->oa href="+code=release_work" class="sref">release_worko/a>); <148o/a>                else <149o/a>                        oa href="+code=free_ioc" class="sref">free_ioco/a> =trueo/a>; <150o/a>                oa href="+code=spin_unlock_irqrestor=" class="sref">spin_unlock_irqrestor=o/a>(&oa href="+code=ioc" class="sref">ioco/a>->oa href="+code=lock" class="sref">locko/a>,flagso/a>); <151o/a>        } <152o/a> <153o/a>        if (oa href="+code=free_ioc" class="sref">free_ioco/a>) <154o/a>                oa href="+code=kmem_cache_free" class="sref">kmem_cache_freeo/a>(oa href="+code=iocontext_cachep" class="sref">iocontext_cachepo/a>,ioco/a>); <155o/a>} <156o/a>oa href="+code=EXPORT_SYMBOL" class="sref">EXPORT_SYMBOLo/a>(oa href="+code=put_io_context" class="sref">put_io_contexto/a>); <157o/a> <158o/a>ospa> class="comment">/**o/spa>
 <159o/a>ospa> class="comment"> * put_io_context_aciove - put aciove reference on ioco/spa>
 <160o/a>ospa> class="comment"> * @ioc: ioc of interesto/spa>
 <161o/a>ospa> class="comment"> *o/spa>
 <162o/a>ospa> class="comment"> * Undo get_io_context_aciove().  If aciove reference reaches zero aftero/spa>
 <163o/a>ospa> class="comment"> * put, @ioc ca> never issue further IOs and ioscheds are notified.o/spa>
 <164o/a>ospa> class="comment"> */o/spa>
 <165o/a>voidput_io_context_acioveo/a>(structio_contexto/a> *oa href="+code=ioc" class="sref">ioco/a>) <166o/a>{ <167o/a>        structhlist_nodeo/a> *oa href="+code=n" class="sref">no/a>; <168o/a>        u>signed long oa href="+code=flags" class="sref">flagso/a>; <169o/a>        structio_cqo/a> *oa href="+code=icq" class="sref">icqo/a>; <170o/a> <171o/a>        if (!oa href="+code=atnmic_dec_and_test" class="sref">atnmic_dec_and_testo/a>(&oa href="+code=ioc" class="sref">ioco/a>->oa href="+code=aciove_ref" class="sref">aciove_refo/a>)) { <172o/a>                oa href="+code=put_io_context" class="sref">put_io_contexto/a>(oa href="+code=ioc" class="sref">ioco/a>); <173o/a>                return; <174o/a>        } <175o/a> <176o/a>        ospa> class="comment">/*o/spa>
 <177o/a>ospa> class="comment">         * Need ioc lock to walk icq_list and q lock to exit icq.  Performo/spa>
 <178o/a>ospa> class="comment">         * reverse double locking.  Read comment in ioc_release_fn() foro/spa>
 <179o/a>ospa> class="comment">         * explanaion> on the nested locking annotaion>.o/spa>
 <180o/a>ospa> class="comment">         */o/spa>
 <181o/a>oa href="+code=retry" class="sref">retryo/a>: <182o/a>        oa href="+code=spin_lock_irqsave_nested" class="sref">spin_lock_irqsave_nestedo/a>(&oa href="+code=ioc" class="sref">ioco/a>->oa href="+code=lock" class="sref">locko/a>,flagso/a>, 1); <183o/a>        oa href="+code=hlist_fnr_each_entry" class="sref">hlist_fnr_each_entryo/a>(oa href="+code=icq" class="sref">icqo/a>,no/a>, &oa href="+code=ioc" class="sref">ioco/a>->oa href="+code=icq_list" class="sref">icq_listo/a>,ioc_nodeo/a>) { <184o/a>                if (oa href="+code=icq" class="sref">icqo/a>->oa href="+code=flags" class="sref">flagso/a> &ICQ_EXITEDo/a>) <185o/a>                        continue; <186o/a>                if (oa href="+code=spin_trylock" class="sref">spin_trylocko/a>(oa href="+code=icq" class="sref">icqo/a>->oa href="+code=q" class="sref">qo/a>->oa href="+code=queu=_lock" class="sref">queu=_locko/a>)) { <187o/a>                        oa href="+code=ioc_exit_icq" class="sref">ioc_exit_icqo/a>(oa href="+code=icq" class="sref">icqo/a>); <188o/a>                        oa href="+code=spin_unlock" class="sref">spin_unlocko/a>(oa href="+code=icq" class="sref">icqo/a>->oa href="+code=q" class="sref">qo/a>->oa href="+code=queu=_lock" class="sref">queu=_locko/a>); <189o/a>                } else { <190o/a>                        oa href="+code=spin_unlock_irqrestor=" class="sref">spin_unlock_irqrestor=o/a>(&oa href="+code=ioc" class="sref">ioco/a>->oa href="+code=lock" class="sref">locko/a>,flagso/a>); <191o/a>                        oa href="+code=cpu_relax" class="sref">cpu_relaxo/a>(); <192o/a>                        goto oa href="+code=retry" class="sref">retryo/a>; <193o/a>                } <194o/a>        } <195o/a>        oa href="+code=spin_unlock_irqrestor=" class="sref">spin_unlock_irqrestor=o/a>(&oa href="+code=ioc" class="sref">ioco/a>->oa href="+code=lock" class="sref">locko/a>,flagso/a>); <196o/a> <197o/a>        oa href="+code=put_io_context" class="sref">put_io_contexto/a>(oa href="+code=ioc" class="sref">ioco/a>); <198o/a>} <199o/a> <200o/a>ospa> class="comment">/* Called by the exiting task */o/spa>
 <201o/a>voidexit_io_contexto/a>(structtask_structo/a> *oa href="+code=task" class="sref">tasko/a>) <202o/a>{ <203o/a>        structio_contexto/a> *oa href="+code=ioc" class="sref">ioco/a>; <204o/a> <205o/a>        oa href="+code=task_lock" class="sref">task_locko/a>(oa href="+code=task" class="sref">tasko/a>); <206o/a>        oa href="+code=ioc" class="sref">ioco/a> = oa href="+code=task" class="sref">tasko/a>->oa href="+code=io_context" class="sref">io_contexto/a>; <207o/a>        oa href="+code=task" class="sref">tasko/a>->oa href="+code=io_context" class="sref">io_contexto/a> = oa href="+code=NULL" class="sref">NULLo/a>; <208o/a>        oa href="+code=task_unlock" class="sref">task_unlocko/a>(oa href="+code=task" class="sref">tasko/a>); <209o/a> <210o/a>        oa href="+code=atnmic_dec" class="sref">atnmic_deco/a>(&oa href="+code=ioc" class="sref">ioco/a>->oa href="+code=nr_tasks" class="sref">nr_taskso/a>); <211o/a>        oa href="+code=put_io_context_aciove" class="sref">put_io_context_acioveo/a>(oa href="+code=ioc" class="sref">ioco/a>); <212o/a>} <213o/a> <214o/a>ospa> class="comment">/**o/spa>
 <215o/a>ospa> class="comment"> * ioc_clear_queu= - break any ioc associaion> with the specified queu=o/spa>
 <216o/a>ospa> class="comment"> * @q: request_queu= being clearedo/spa>
 <217o/a>ospa> class="comment"> *o/spa>
 <218o/a>ospa> class="comment"> * Walk @q->icq_list and exit all io_cq's.  Must be called with @q locked.o/spa>
 <219o/a>ospa> class="comment"> */o/spa>
 <220o/a>voidioc_clear_queu=o/a>(structrequest_queu=o/a> *oa href="+code=q" class="sref">qo/a>) <221o/a>{ <222o/a>        oa href="+code=lockdep_assert_held" class="sref">lockdep_assert_heldo/a>(oa href="+code=q" class="sref">qo/a>->oa href="+code=queu=_lock" class="sref">queu=_locko/a>); <223o/a> <224o/a>        while (!oa href="+code=list_empty" class="sref">list_emptyo/a>(&oa href="+code=q" class="sref">qo/a>->oa href="+code=icq_list" class="sref">icq_listo/a>)) { <225o/a>                structio_cqo/a> *oa href="+code=icq" class="sref">icqo/a> =list_entryo/a>(oa href="+code=q" class="sref">qo/a>->oa href="+code=icq_list" class="sref">icq_listo/a>.oa href="+code=next" class="sref">nexto/a>, <226o/a>                                               structio_cqo/a>,q_nodeo/a>); <227o/a>                structio_contexto/a> *oa href="+code=ioc" class="sref">ioco/a> =icqo/a>->oa href="+code=ioc" class="sref">ioco/a>; <228o/a> <229o/a>                oa href="+code=spin_lock" class="sref">spin_locko/a>(&oa href="+code=ioc" class="sref">ioco/a>->oa href="+code=lock" class="sref">locko/a>); <230o/a>                oa href="+code=ioc_destroy_icq" class="sref">ioc_destroy_icqo/a>(oa href="+code=icq" class="sref">icqo/a>); <231o/a>                oa href="+code=spin_unlock" class="sref">spin_unlocko/a>(&oa href="+code=ioc" class="sref">ioco/a>->oa href="+code=lock" class="sref">locko/a>); <232o/a>        } <233o/a>} <234o/a> <235o/a>int oa href="+code=create_task_io_context" class="sref">create_task_io_contexto/a>(structtask_structo/a> *oa href="+code=task" class="sref">tasko/a>,gfp_to/a> oa href="+code=gfp_flags" class="sref">gfp_flagso/a>,nodeo/a>) <236o/a>{ <237o/a>        structio_contexto/a> *oa href="+code=ioc" class="sref">ioco/a>; <238o/a>        int oa href="+code=ret" class="sref">reto/a>; <239o/a> <240o/a>        oa href="+code=ioc" class="sref">ioco/a> =kmem_cache_alloc_nodeo/a>(oa href="+code=iocontext_cachep" class="sref">iocontext_cachepo/a>,gfp_flagso/a> |__GFP_ZEROo/a>, <241o/a>                                    oa href="+code=node" class="sref">nodeo/a>); <242o/a>        if (oa href="+code=unlikely" class="sref">unlikelyo/a>(!oa href="+code=ioc" class="sref">ioco/a>)) <243o/a>                return -oa href="+code=ENOMEM" class="sref">ENOMEMo/a>; <244o/a> <245o/a>        ospa> class="comment">/* initialize */o/spa>
 <246o/a>        oa href="+code=atnmic_long_set" class="sref">atnmic_long_seto/a>(&oa href="+code=ioc" class="sref">ioco/a>->oa href="+code=refcount" class="sref">refcounto/a>, 1); <247o/a>        oa href="+code=atnmic_set" class="sref">atnmic_seto/a>(&oa href="+code=ioc" class="sref">ioco/a>->oa href="+code=nr_tasks" class="sref">nr_taskso/a>, 1); <248o/a>        oa href="+code=atnmic_set" class="sref">atnmic_seto/a>(&oa href="+code=ioc" class="sref">ioco/a>->oa href="+code=aciove_ref" class="sref">aciove_refo/a>, 1); <249o/a>        oa href="+code=spin_lock_init" class="sref">spin_lock_inito/a>(&oa href="+code=ioc" class="sref">ioco/a>->oa href="+code=lock" class="sref">locko/a>); <250o/a>        oa href="+code=INIT_RADIX_TREE" class="sref">INIT_RADIX_TREEo/a>(&oa href="+code=ioc" class="sref">ioco/a>->oa href="+code=icq_tree" class="sref">icq_treeo/a>,GFP_ATOMICo/a> |__GFP_HIGHo/a>); <251o/a>        oa href="+code=INIT_HLIST_HEAD" class="sref">INIT_HLIST_HEADo/a>(&oa href="+code=ioc" class="sref">ioco/a>->oa href="+code=icq_list" class="sref">icq_listo/a>); <252o/a>        oa href="+code=INIT_WORK" class="sref">INIT_WORKo/a>(&oa href="+code=ioc" class="sref">ioco/a>->oa href="+code=release_work" class="sref">release_worko/a>,ioc_release_fno/a>); <253o/a> <254o/a>        ospa> class="comment">/*o/spa>
 <255o/a>ospa> class="comment">         * Try to install.  ioc shouldn't be installed if someone elseo/spa>
 <256o/a>ospa> class="comment">         * already did or @task, which isn't %current, is exiting.  Noteo/spa>
 <257o/a>ospa> class="comment">         * that we need to allow ioc creaton> on exiting %current as exito/spa>
 <258o/a>ospa> class="comment">         * path may issue IOs from e.g. exit_files().  The exit path iso/spa>
 <259o/a>ospa> class="comment">         * responsible for not issuing IO after exit_io_context().o/spa>
 <260o/a>ospa> class="comment">         */o/spa>
 <261o/a>        oa href="+code=task_lock" class="sref">task_locko/a>(oa href="+code=task" class="sref">tasko/a>); <262o/a>        if (!oa href="+code=task" class="sref">tasko/a>->oa href="+code=io_context" class="sref">io_contexto/a> && <263o/a>            (oa href="+code=task" class="sref">tasko/a> ==currento/a> || !(oa href="+code=task" class="sref">tasko/a>->oa href="+code=flags" class="sref">flagso/a> &PF_EXITINGo/a>))) <264o/a>                oa href="+code=task" class="sref">tasko/a>->oa href="+code=io_context" class="sref">io_contexto/a> = oa href="+code=ioc" class="sref">ioco/a>; <265o/a>        else <266o/a>                oa href="+code=kmem_cache_free" class="sref">kmem_cache_freeo/a>(oa href="+code=iocontext_cachep" class="sref">iocontext_cachepo/a>,ioco/a>); <267o/a> <268o/a>        oa href="+code=ret" class="sref">reto/a> = oa href="+code=task" class="sref">tasko/a>->oa href="+code=io_context" class="sref">io_contexto/a> ? 0 : -oa href="+code=EBUSY" class="sref">EBUSYo/a>; <269o/a> <270o/a>        oa href="+code=task_unlock" class="sref">task_unlocko/a>(oa href="+code=task" class="sref">tasko/a>); <271o/a> <272o/a>        return oa href="+code=ret" class="sref">reto/a>; <273o/a>} <274o/a> <275o/a>ospa> class="comment">/**o/spa>
 <276o/a>ospa> class="comment"> * get_task_io_context - get io_context of a tasko/spa>
 <277o/a>ospa> class="comment"> * @task: task of interesto/spa>
 <278o/a>ospa> class="comment"> * @gfp_flags: allocaton> flags, used if allocaton> is necessaryo/spa>
 <279o/a>ospa> class="comment"> * @node: allocaton> node, used if allocaton> is necessaryo/spa>
 <280o/a>ospa> class="comment"> *o/spa>
 <281o/a>ospa> class="comment"> * Return io_context of @task.  If it doesn't exist, it is created witho/spa>
 <282o/a>ospa> class="comment"> * @gfp_flags and @node.  The returned io_context has its reference counto/spa>
 <283o/a>ospa> class="comment"> * incremented.o/spa>
 <284o/a>ospa> class="comment"> *o/spa>
 <285o/a>ospa> class="comment"> * This functon> always goes through task_lock() and it's better to useo/spa>
 <286o/a>ospa> class="comment"> * %current->io_context + get_io_context() for %current.o/spa>
 <287o/a>ospa> class="comment"> */o/spa>
 <288o/a>structio_contexto/a> *oa href="+code=get_task_io_context" class="sref">get_task_io_contexto/a>(structtask_structo/a> *oa href="+code=task" class="sref">tasko/a>, <289o/a>                                       oa href="+code=gfp_t" class="sref">gfp_to/a> oa href="+code=gfp_flags" class="sref">gfp_flagso/a>,nodeo/a>) <290o/a>{ <291o/a>        structio_contexto/a> *oa href="+code=ioc" class="sref">ioco/a>; <292o/a> <293o/a>        oa href="+code=might_sleep_if" class="sref">might_sleep_ifo/a>(oa href="+code=gfp_flags" class="sref">gfp_flagso/a> &__GFP_WAITo/a>); <294o/a> <295o/a>        do { <296o/a>                oa href="+code=task_lock" class="sref">task_locko/a>(oa href="+code=task" class="sref">tasko/a>); <297o/a>                oa href="+code=ioc" class="sref">ioco/a> =tasko/a>->oa href="+code=io_context" class="sref">io_contexto/a>; <298o/a>                if (oa href="+code=likely" class="sref">likelyo/a>(oa href="+code=ioc" class="sref">ioco/a>)) { <299o/a>                        oa href="+code=get_io_context" class="sref">get_io_contexto/a>(oa href="+code=ioc" class="sref">ioco/a>); <300o/a>                        oa href="+code=task_unlock" class="sref">task_unlocko/a>(oa href="+code=task" class="sref">tasko/a>); <301o/a>                        return oa href="+code=ioc" class="sref">ioco/a>; <302o/a>                } <303o/a>                oa href="+code=task_unlock" class="sref">task_unlocko/a>(oa href="+code=task" class="sref">tasko/a>); <304o/a>        } while (!oa href="+code=create_task_io_context" class="sref">create_task_io_contexto/a>(oa href="+code=task" class="sref">tasko/a>, oa href="+code=gfp_flags" class="sref">gfp_flagso/a>,nodeo/a>)); <305o/a> <306o/a>        return oa href="+code=NULL" class="sref">NULLo/a>; <307o/a>} <308o/a>oa href="+code=EXPORT_SYMBOL" class="sref">EXPORT_SYMBOLo/a>(oa href="+code=get_task_io_context" class="sref">get_task_io_contexto/a>); <309o/a> <310o/a>ospa> class="comment">/**o/spa>
 <311o/a>ospa> class="comment"> * ioc_lookup_icq - lookup io_cq from ioco/spa>
 <312o/a>ospa> class="comment"> * @ioc: the associaied io_contexto/spa>
 <313o/a>ospa> class="comment"> * @q: the associaied request_queu=o/spa>
 <314o/a>ospa> class="comment"> *o/spa>
 <315o/a>ospa> class="comment"> * Look up io_cq associaied with @ioc - @q pair from @ioc.  Must be calledo/spa>
 <316o/a>ospa> class="comment"> * with @q->queu=_lock held.o/spa>
 <317o/a>ospa> class="comment"> */o/spa>
 <318o/a>structio_cqo/a> *oa href="+code=ioc_lookup_icq" class="sref">ioc_lookup_icqo/a>(structio_contexto/a> *oa href="+code=ioc" class="sref">ioco/a>, structrequest_queu=o/a> *oa href="+code=q" class="sref">qo/a>) <319o/a>{ <320o/a>        structio_cqo/a> *oa href="+code=icq" class="sref">icqo/a>; <321o/a> <322o/a>        oa href="+code=lockdep_assert_held" class="sref">lockdep_assert_heldo/a>(oa href="+code=q" class="sref">qo/a>->oa href="+code=queu=_lock" class="sref">queu=_locko/a>); <323o/a> <324o/a>        ospa> class="comment">/*o/spa>
 <325o/a>ospa> class="comment">         * icq's are indexed from @ioc using radix tree and hint pointer,o/spa>
 <326o/a>ospa> class="comment">         * both of which are protecied with RCU.  All removals are don=o/spa>
 <327o/a>ospa> class="comment">         * holding both q and ioc locks, and we're holding q lock - if w=o/spa>
 <328o/a>ospa> class="comment">         * find a icq which points to us, it's guaranteed to be valid.o/spa>
 <329o/a>ospa> class="comment">         */o/spa>
 <330o/a>        oa href="+code=rcu_read_lock" class="sref">rcu_read_locko/a>(); <331o/a>        oa href="+code=icq" class="sref">icqo/a> =rcu_dereferenceo/a>(oa href="+code=ioc" class="sref">ioco/a>->oa href="+code=icq_hint" class="sref">icq_hinto/a>); <332o/a>        if (oa href="+code=icq" class="sref">icqo/a> &&icqo/a>->oa href="+code=q" class="sref">qo/a> ==qo/a>) <333o/a>                goto oa href="+code=out" class="sref">outo/a>; <334o/a> <335o/a>        oa href="+code=icq" class="sref">icqo/a> =radix_tree_lookupo/a>(&oa href="+code=ioc" class="sref">ioco/a>->oa href="+code=icq_tree" class="sref">icq_treeo/a>,qo/a>->oa href="+code=id" class="sref">ido/a>); <336o/a>        if (oa href="+code=icq" class="sref">icqo/a> &&icqo/a>->oa href="+code=q" class="sref">qo/a> ==qo/a>) <337o/a>                oa href="+code=rcu_assign_pointer" class="sref">rcu_assign_pointero/a>(oa href="+code=ioc" class="sref">ioco/a>->oa href="+code=icq_hint" class="sref">icq_hinto/a>,icqo/a>); ospa> class="comment">/* allowed to race */o/spa>
 <338o/a>        else <339o/a>                oa href="+code=icq" class="sref">icqo/a> =NULLo/a>; <340o/a>oa href="+code=out" class="sref">outo/a>: <341o/a>        oa href="+code=rcu_read_unlock" class="sref">rcu_read_unlocko/a>(); <342o/a>        return oa href="+code=icq" class="sref">icqo/a>; <343o/a>} <344o/a>oa href="+code=EXPORT_SYMBOL" class="sref">EXPORT_SYMBOLo/a>(oa href="+code=ioc_lookup_icq" class="sref">ioc_lookup_icqo/a>); <345o/a> <346o/a>ospa> class="comment">/**o/spa>
 <347o/a>ospa> class="comment"> * ioc_create_icq - create and link io_cqo/spa>
 <348o/a>ospa> class="comment"> * @ioc: io_context of interesto/spa>
 <349o/a>ospa> class="comment"> * @q: request_queu= of interesto/spa>
 <350o/a>ospa> class="comment"> * @gfp_mask: allocaton> masko/spa>
 <351o/a>ospa> class="comment"> *o/spa>
 <352o/a>ospa> class="comment"> * Make sure io_cq linking @ioc and @q exists.  If icq doesn't exist, theyo/spa>
 <353o/a>ospa> class="comment"> * will be created using @gfp_mask.o/spa>
 <354o/a>ospa> class="comment"> *o/spa>
 <355o/a>ospa> class="comment"> * The caller is responsible for ensuring @ioc won't go away and @q iso/spa>
 <356o/a>ospa> class="comment"> * alove and will stay alove until this functon> returns.o/spa>
 <357o/a>ospa> class="comment"> */o/spa>
 <358o/a>structio_cqo/a> *oa href="+code=ioc_create_icq" class="sref">ioc_create_icqo/a>(structio_contexto/a> *oa href="+code=ioc" class="sref">ioco/a>, structrequest_queu=o/a> *oa href="+code=q" class="sref">qo/a>, <359o/a>                             oa href="+code=gfp_t" class="sref">gfp_to/a> oa href="+code=gfp_mask" class="sref">gfp_masko/a>) <360o/a>{ <361o/a>        structelevator_typ=o/a> *oa href="+code=et" class="sref">eto/a> = oa href="+code=q" class="sref">qo/a>->oa href="+code=elevator" class="sref">elevatoro/a>->oa href="+code=typ=" class="sref">typ=o/a>; <362o/a>        structio_cqo/a> *oa href="+code=icq" class="sref">icqo/a>; <363o/a> <364o/a>        ospa> class="comment">/* allocate stuff */o/spa>
 <365o/a>        oa href="+code=icq" class="sref">icqo/a> =kmem_cache_alloc_nodeo/a>(oa href="+code=et" class="sref">eto/a>->oa href="+code=icq_cache" class="sref">icq_cacheo/a>, oa href="+code=gfp_mask" class="sref">gfp_masko/a> |__GFP_ZEROo/a>, <366o/a>                                    oa href="+code=q" class="sref">qo/a>->oa href="+code=node" class="sref">nodeo/a>); <367o/a>        if (!oa href="+code=icq" class="sref">icqo/a>) <368o/a>                return oa href="+code=NULL" class="sref">NULLo/a>; <369o/a> <370o/a>        if (oa href="+code=radix_tree_preload" class="sref">radix_tree_preloado/a>(oa href="+code=gfp_mask" class="sref">gfp_masko/a>) < 0) { <371o/a>                oa href="+code=kmem_cache_free" class="sref">kmem_cache_freeo/a>(oa href="+code=et" class="sref">eto/a>->oa href="+code=icq_cache" class="sref">icq_cacheo/a>, oa href="+code=icq" class="sref">icqo/a>); <372o/a>                return oa href="+code=NULL" class="sref">NULLo/a>; <373o/a>        } <374o/a> <375o/a>        oa href="+code=icq" class="sref">icqo/a>->oa href="+code=ioc" class="sref">ioco/a> = oa href="+code=ioc" class="sref">ioco/a>; <376o/a>        oa href="+code=icq" class="sref">icqo/a>->oa href="+code=q" class="sref">qo/a> = oa href="+code=q" class="sref">qo/a>; <377o/a>        oa href="+code=INIT_LIST_HEAD" class="sref">INIT_LIST_HEADo/a>(&oa href="+code=icq" class="sref">icqo/a>->oa href="+code=q_node" class="sref">q_nodeo/a>); <378o/a>        oa href="+code=INIT_HLIST_NODE" class="sref">INIT_HLIST_NODEo/a>(&oa href="+code=icq" class="sref">icqo/a>->oa href="+code=ioc_node" class="sref">ioc_nodeo/a>); <379o/a> <380o/a>        ospa> class="comment">/* lock both q and ioc and try to link @icq */o/spa>
 <381o/a>        oa href="+code=spin_lock_irq" class="sref">spin_lock_irqo/a>(oa href="+code=q" class="sref">qo/a>->oa href="+code=queu=_lock" class="sref">queu=_locko/a>); <382o/a>        oa href="+code=spin_lock" class="sref">spin_locko/a>(&oa href="+code=ioc" class="sref">ioco/a>->oa href="+code=lock" class="sref">locko/a>); <383o/a> <384o/a>        if (oa href="+code=likely" class="sref">likelyo/a>(!oa href="+code=radix_tree_insert" class="sref">radix_tree_inserto/a>(&oa href="+code=ioc" class="sref">ioco/a>->oa href="+code=icq_tree" class="sref">icq_treeo/a>,qo/a>->oa href="+code=id" class="sref">ido/a>, oa href="+code=icq" class="sref">icqo/a>))) { <385o/a>                oa href="+code=hlist_add_head" class="sref">hlist_add_heado/a>(&oa href="+code=icq" class="sref">icqo/a>->oa href="+code=ioc_node" class="sref">ioc_nodeo/a>, &oa href="+code=ioc" class="sref">ioco/a>->oa href="+code=icq_list" class="sref">icq_listo/a>); <386o/a>                oa href="+code=list_add" class="sref">list_addo/a>(&oa href="+code=icq" class="sref">icqo/a>->oa href="+code=q_node" class="sref">q_nodeo/a>, &oa href="+code=q" class="sref">qo/a>->oa href="+code=icq_list" class="sref">icq_listo/a>); <387o/a>                if (oa href="+code=et" class="sref">eto/a>->oa href="+code=ops" class="sref">opso/a>.oa href="+code=elevator_init_icq_fn" class="sref">elevator_init_icq_fno/a>) <388o/a>                        oa href="+code=et" class="sref">eto/a>->oa href="+code=ops" class="sref">opso/a>.oa href="+code=elevator_init_icq_fn" class="sref">elevator_init_icq_fno/a>(oa href="+code=icq" class="sref">icqo/a>); <389o/a>        } else { <390o/a>                oa href="+code=kmem_cache_free" class="sref">kmem_cache_freeo/a>(oa href="+code=et" class="sref">eto/a>->oa href="+code=icq_cache" class="sref">icq_cacheo/a>, oa href="+code=icq" class="sref">icqo/a>); <391o/a>                oa href="+code=icq" class="sref">icqo/a> =ioc_lookup_icqo/a>(oa href="+code=ioc" class="sref">ioco/a>, oa href="+code=q" class="sref">qo/a>); <392o/a>                if (!oa href="+code=icq" class="sref">icqo/a>) <393o/a>                        oa href="+code=printk" class="sref">printko/a>(oa href="+code=KERN_ERR" class="sref">KERN_ERRo/a> ospa> class="string">"cfq: icq link failed!\n"o/spa>
); <394o/a>        } <395o/a> <396o/a>        oa href="+code=spin_unlock" class="sref">spin_unlocko/a>(&oa href="+code=ioc" class="sref">ioco/a>->oa href="+code=lock" class="sref">locko/a>); <397o/a>        oa href="+code=spin_unlock_irq" class="sref">spin_unlock_irqo/a>(oa href="+code=q" class="sref">qo/a>->oa href="+code=queu=_lock" class="sref">queu=_locko/a>); <398o/a>        oa href="+code=radix_tree_preload_end" class="sref">radix_tree_preload_endo/a>(); <399o/a>        return oa href="+code=icq" class="sref">icqo/a>; <400o/a>} <401o/a> <402o/a>static__inito/a> oa href="+code=blk_ioc_init" class="sref">blk_ioc_inito/a>(void) <403o/a>{ <404o/a>        oa href="+code=iocontext_cachep" class="sref">iocontext_cachepo/a> =kmem_cache_createo/a>(ospa> class="string">"blkdev_ioc"o/spa>
, <405o/a>                        sizeof(structio_contexto/a>), 0, oa href="+code=SLAB_PANIC" class="sref">SLAB_PANICo/a>, oa href="+code=NULL" class="sref">NULLo/a>); <406o/a>        return 0; <407o/a>} <408o/a>oa href="+code=subsys_initcall" class="sref">subsys_initcallo/a>(oa href="+code=blk_ioc_init" class="sref">blk_ioc_inito/a>); <409o/a>
lxr.linux.no kindly hosted by Redpill Linpro ASo/a>, provider of Linux consulting and operaton>s services since 1995.