linux/mm/bounce.c
<<
>>
Prefs
   1/* bounce buffer handling for block devices
   2 *
   3 * - Split from highmem.c
   4 */
   5
   6#include <linux/mm.h>
   7#include <linux/export.h>
   8#include <linux/swap.h>
   9#include <linux/gfp.h>
  10#include <linux/bio.h>
  11#include <linux/pagemap.h>
  12#include <linux/mempool.h>
  13#include <linux/blkdev.h>
  14#include <linux/init.h>
  15#include <linux/hash.h>
  16#include <linux/highmem.h>
  17#include <linux/bootmem.h>
  18#include <asm/tlbflush.h>
  19
  20#include <trace/events/block.h>
  21
  22#define POOL_SIZE       64
  23#define ISA_POOL_SIZE   16
  24
  25static mempool_t *page_pool, *isa_page_pool;
  26
  27#if defined(CONFIG_HIGHMEM) || defined(CONFIG_NEED_BOUNCE_POOL)
  28static __init int init_emergency_pool(void)
  29{
  30#if defined(CONFIG_HIGHMEM) && !defined(CONFIG_MEMORY_HOTPLUG)
  31        if (max_pfn <= max_low_pfn)
  32                return 0;
  33#endif
  34
  35        page_pool = mempool_create_page_pool(POOL_SIZE, 0);
  36        BUG_ON(!page_pool);
  37        printk("bounce pool size: %d pages\n", POOL_SIZE);
  38
  39        return 0;
  40}
  41
  42__initcall(init_emergency_pool);
  43#endif
  44
  45#ifdef CONFIG_HIGHMEM
  46/*
  47 * highmem version, map in to vec
  48 */
  49static void bounce_copy_vec(struct bio_vec *to, unsigned char *vfrom)
  50{
  51        unsigned long flags;
  52        unsigned char *vto;
  53
  54        local_irq_save(flags);
  55        vto = kmap_atomic(to->bv_page);
  56        memcpy(vto + to->bv_offset, vfrom, to->bv_len);
  57        kunmap_atomic(vto);
  58        local_irq_restore(flags);
  59}
  60
  61#else /* CONFIG_HIGHMEM */
  62
  63#define bounce_copy_vec(to, vfrom)      \
  64        memcpy(page_address((to)->bv_page) + (to)->bv_offset, vfrom, (to)->bv_len)
  65
  66#endif /* CONFIG_HIGHMEM */
  67
  68/*
  69 * allocate pages in the DMA region for the ISA pool
  70 */
  71static void *mempool_alloc_pages_isa(gfp_t gfp_mask, void *data)
  72{
  73        return mempool_alloc_pages(gfp_mask | GFP_DMA, data);
  74}
  75
  76/*
  77 * gets called "every" time someone init's a queue with BLK_BOUNCE_ISA
  78 * as the max address, so check if the pool has already been created.
  79 */
  80int init_emergency_isa_pool(void)
  81{
  82        if (isa_page_pool)
  83                return 0;
  84
  85        isa_page_pool = mempool_create(ISA_POOL_SIZE, mempool_alloc_pages_isa,
  86                                       mempool_free_pages, (void *) 0);
  87        BUG_ON(!isa_page_pool);
  88
  89        printk("isa bounce pool size: %d pages\n", ISA_POOL_SIZE);
  90        return 0;
  91}
  92
  93/*
  94 * Simple bounce buffer support for highmem pages. Depending on the
  95 * queue gfp mask set, *to may or may not be a highmem page. kmap it
  96 * always, it will do the Right Thing
  97 */
  98static void copy_to_high_bio_irq(struct bio *to, struct bio *from)
  99{
 100        unsigned char *vfrom;
 101        struct bio_vec *tovec, *fromvec;
 102        int i;
 103
 104        __bio_for_each_segment(tovec, to, i, 0) {
 105                fromvec = from->bi_io_vec + i;
 106
 107                /*
 108                 * not bounced
 109                 */
 110                if (tovec->bv_page == fromvec->bv_page)
 111                        continue;
 112
 113                /*
 114                 * fromvec->bv_offset and fromvec->bv_len might have been
 115                 * modified by the block layer, so use the original copy,
 116                 * bounce_copy_vec already uses tovec->bv_len
 117                 */
 118                vfrom = page_address(fromvec->bv_page) + tovec->bv_offset;
 119
 120                bounce_copy_vec(tovec, vfrom);
 121                flush_dcache_page(tovec->bv_page);
 122        }
 123}
 124
 125static void bounce_end_io(struct bio *bio, mempool_t *pool, int err)
 126{
 127        struct bio *bio_orig = bio->bi_private;
 128        struct bio_vec *bvec, *org_vec;
 129        int i;
 130
 131        if (test_bit(BIO_EOPNOTSUPP, &bio->bi_flags))
 132                set_bit(BIO_EOPNOTSUPP, &bio_orig->bi_flags);
 133
 134        /*
 135         * free up bounce indirect pages used
 136         */
 137        __bio_for_each_segment(bvec, bio, i, 0) {
 138                org_vec = bio_orig->bi_io_vec + i;
 139                if (bvec->bv_page == org_vec->bv_page)
 140                        continue;
 141
 142                dec_zone_page_state(bvec->bv_page, NR_BOUNCE);
 143                mempool_free(bvec->bv_page, pool);
 144        }
 145
 146        bio_endio(bio_orig, err);
 147        bio_put(bio);
 148}
 149
 150static void bounce_end_io_write(struct bio *bio, int err)
 151{
 152        bounce_end_io(bio, page_pool, err);
 153}
 154
 155static void 1  57<1a>         151{
 152        bouncee=bio_put name="L58">  58    1    <1 href="+code=local_irq_restore1" class="1sref">local_irq_restore<1/a>(<1 href="+code=flags" class="sr1ef">flags1);
 136 124
 125static void bounce_end_io(struct bio *  60
gfp_t   61#els1e err)
 126{
 127        struct bio *ev_offset<1/a>, i;
 131        if (test_bit((s  6lass="sref">ISA_POOL_      if (  97 145
err);
  65
  66#endi1f          151{
 152        bouncunce.c#L617" id="L67" class="line"1 name16href="+code=local_irq_restore1an>
 153}
 136a154" class="lbloc4" class="line" name="L124"> 124
bio *(gfp_t gfp_mask,1 void1* 136 151{
 152        bounc(gfp_mask | GFP_DMA1,1data1);

 153}
 124
bio *t   177         136 151{
 152        bouncounce.c#L178" id="L78" class="line1" nam17href="+code=local_irq_restore1o check i1f the pool has already b1een c18href="+code=flags" class="sr1ef="mm/bo1unce.c#L80" id="L80" cla1ss="l18L10" cllass="line" name="L43">  43de=defined" class="sref">defined(mempool_au.c#snapshotlinebl class=4" class="line" name="L124"reque.c#ommen"sref">mempool_reque.c#ommen/bounce.c#L125" id="Lss="line" name="qref=52ass="line" name="L124"> 124
  81{
bvec-&gass="line" name="L124"" class="line" name href="mmclass="sref">boun class="line" name href="m">linux/highmem.h>
de=GFP_D1e.c#L83" id="L83" class=1"line1 name="L83">  ass="line" name="L124">acking_dev_inf5" class="line" nacking_dev_inf5/bounce.c#L125" id="L1da href="+code=or1daf="m">linux/highmem.h>
ef">data1Iass="line" name="L124"ef="mm/_spaclass="line" nameef="mm/_spacl/bounce.c#L125" id="Lmapp88" sref">mempool_aapp88"f="m">linux/highmem.h>

  86    1     18ref="+code=err" class="sref">err)
ef">bio->i, 0) {
linux/highmem.h>

bvec, *BUG_ON(!b>printk(1"1isa bounce pooref="+code=bi_io_vecref cla_di err !href="mm/bounce.cWRIT_zone_page_stateWRIT_e="L+code=bio" class="sr="sr1an>, (  90  1      ret1urn 0;
1da_caplinebl class=_requir"line" name="L30"1da_caplinebl class=_requir"lorig<131" id="L131" classss="line" name="qref=/a>        if (((bi_flags);

 134ss=att'efs(a haapp88", decide whes="r or;
/*to employ93" id="L93" can cto guarantee inebl 3">  93;
         * 2href="+co2e=bio" class="sref">bio<2a> *<2 href="+code=f"+code=bi_io_vecref" class="line" name="L136"> 136a>         */i, 0) {
 137        __b29">  99{
(i, 0) {
 14tatic                         continumapp88" sref">mempool_aapp88"f="mn class="comment">     mapp88" sref">mempool_     mapp88"nt">         */boun2mvec" cla2s="sref">fromvec;
mapp88" sref">mempool_aapp88"f="md="L28" class="line" nameass="line2 name="L102"> 102   2    i204ce_copy_vec(org_vec-><2o_for_eac2_segment(tovec, mempool_aapp88"f="m/a>        if (org_vec-><2oool_t" cl= ISA_POOL_e=data" class="sref">daapp88" sref">mempool_aapp88"f="m/a>        if ( 136hosnf="m/a>        if (        if (teL131" " class="sref">dMS_SNAP_STABL_zone_page_stateMS_SNAP_STABL_/bou"sref">org_vec-><2osref">pagec" class="sref">bi_io_v2c2+  106
2a hre20  /*
POOL_SIZE);
2         2  * not bounced
<2 href21="line" name="L38">  38
<2href="+co2e=bv_page" class="sref">2v_pag21" id="L59"" name="L38">  38
<2huct mempool_au.c#snapshotlinebl class=4" class="line" name="L124"reque.c#ommen"sref">mempool_reque.c#ommen/bounce.c#L125" id="Lss="line" name="qref=52ass="line" name="L124"> 124
 21="L82t  112
2a hre214ode=POOL_SIZE" class="sref">POOL_SIZE);
2 _for_eac2class="line" name="L114"2 114<2a>  38
<2hool_t" clL115" class="line" name=2L115"21href="+code=bv_len" class="sref">bv_len)a href="+code=CO"span class="comment">         * 2bounce.c#2116" id="L116" class="li2e" na21/bounce.c#L66" id="L66" clas2ady uses 2ovec->bv_len
<2 href21Thing
 136mempool_reque.c#ommen/bounce.c#L125" id="Lss="line" name="qref=52ass="line" name="L124"> 124
 126{
mempool_cr2href="+co2e=bv_page" class="sref">2v_pag21bio_orig->bounce_end_io(struct  136pan" e="L+code=bio" class="sr="sr2x86_64/tlbbflush.h|include/asm-xten"+cod22io_vec *b22="L100" class="line" name="L100"> 1" class="line" name href="mmclass="sref">boun class="line" name href="m">linux/highmem.h>2< 124
defineNULsf="m">linux/highmem.h>2&1" id="L221
bvec        mempool_rwe="L="L84" class="linecref cla_di  126{
boun2a>       664
bvec-&gass="line" name="L124"/
  49i, 0) {
linux/highmem.h>23">  23#define mempool_tlass=22s="line" name="L144"> 144   " class="line" name="L136"> 136a>         */i, 0) {
 126{
        __b2"sref">pagge_pool, *i, 0) {
 14tatic bio_22     28static ->bi_flags);
org_vec;
  69<<<<<<<<<<<<<<<<<* is de.cinspaon> 134sbelowe" name="fn?;
 129   2    i2t  116                 * bounce_copy_vec alr2BIO_EOPNO2SUPP" class="sref">BIO_E2PNOTS2PP, &)
     toEMORnt">         */Commen/a154" EMORY_HOTPLUG)
ommen/a154" EMORnt">         */pan" ame="L136"> 136pan" e="L+code=bio" class="sr="sr2SUPP" cla2s="sref">BIO_EOPNOTSUPP<2a>, &23->org_vec-><2ass="sref2>bio_orig->  32               2="L133" c2ass="line" name="L133"> 233tovecbi_flags);
 135<<<<<<<<<* irk,ine" nameding on the
 116                 * bounce_copy_vec alr2o_for_eac2_segment(bvec1ode=err" class="sref">err egment" class="sref">__b2> = bio_orig-> 136cment">mhr<26" id="L126" class="line" name="L126"> 126{
        if ( 136ef=vc n> 14tatic 2v_pag24href="+code=flags" class="sr2 href="+c2de=bv_page" class="sref"2bv_pa24/a> ==  126ode=mnt">         */    NOIO{
         136cment">end_io" class="sref">boun240" id="L240" class="line" name="L240"> 240                =fromvece.c#L125" id="Lounerbounce.c#L138" id="L138" class="line" name="L138"> 138         136cment"><* >  8ofspan class="comment"> */
boun24UPP" cla2code=bvec" class="sref">2vec->  38
<2_free2  38
<2_"L133" c2ef">bv_page, pool)2
ISA_POOL_      if (  49< href="mm/bounce.c#L127" id="L127" class="line" name="L127"> 127 138                org_vecbvec,   55        " name="ode=mnt">         */(struct  152     ss="line" name="qref=/a>        if (        end_io" class="sref">boun24 = bio_put(->,   to)->i, 0) {
boun2io);
2a href="mm/bounce.c#L1482 id="25bv_offset" class="sref">bv_offset  56        i, 0) {
fromvec->(struct          */  55        dec_zone_page_state2"+code=bi2" class="sref">bio, 2nt   90  2e_end_io<2a>(        mempool_rwe="L="href="mm/bounce.cWRIT_zone_page_stateWRIT_e="L+ egment" class="sref">__b2ge_pool" 2lass="sref">page_pool(from)
bv_len);
<7"26" id="L126" cl href="mm/bounce.c#L99" id="L99" class="line" name="L2rr);
2a href="mm/bounce.c#L1532 id="25pool" class="sref">pool)2437e=boun2e_end_ioce_end_io_write<2a>(s 25lass="sref">ISA_POOL_SIZEef="+code=i" cla" class="sref">tovec, vfromi, 0) {
 14">dec_zone_page_state2"   ef="+code=i" cl ss="sref">bv_len);
                 */
  55        vfrom =   56        2  57<25" class="sref">bvecef="+code=i" cl ne" name="L117"> 117 Depname="L117"> 11 Depclass="sref">vfromi, 0) {
 14"">vfrom = i, 0) {
fromvec->  58    2    <25bio_orig->" ncpyclass="sref">vfrombv_len);
<7"ef="+code=i" cl ne" name="L117"> 117 104ref="mm/bounce.c#L10543to)->dec_zone_page_state2" class="2sref">local_irq_restore<2/a>(<26/a> ==  unDepname="L117"> 11 unDepclass="sref">vfromi, 0) {
 14">dec_zone_page_state2ef">flags2);
  38
<2e+code=bi2 name="L60">  60
(  61#els2e bi_flags);
,  134/*26a>>
<2  href="mm/bounce.c#L15"2i>(s 26" class="line" name="L135"> 135         * 2s="line" 2name="L65">  65
1ode=err" class="sref">errs="comment">         * 2sc#L147" 2name="L66">  66#endi2f bvecdec_zone_page_state2e = 2an>
 136tra" Et and>         */ 126{
dec_zone_page_state2cate page2s in the DMA region for 2the I27)
bi_flags);
gfp_mask,2 void27" id="L91" class="line" namid="L134" cat least  hr> 134swaGFP_DMA2,27         * 2ref">data2);
 144 136         */i, 0) {
 126{
        __b2a
  49< href="mm/bounce.c#L1_ice.c_idxe" name="L126"> 126ice.c_idxorig, er /a>          277bvecnce.c#L55" id="L55" class="line" name="L55">  55        __b2a = ->  55        i, 0) {
 14tatic nce.c#L55" id="L55" class="line" name="L55">  to)->i, 0) {
boun2ef="mm/bo2unce.c#L80" id="L80" cla2ss="l28/a> ==   56        i, 0) {
fromvec->(                href="+code=tovec" class2">  81{
  32               2
de=GFP_D2e.c#L83" id="L83" class=2"line2 name="L83">  ef="mm/bounce.c#L127" id="L127" class="line" name="L127"> 127 126{
        if (fromvec->data2 144  27" id="L127" class="line" name="L127"> 127te |hr<1OTPLUTPLUe="L144"> 144  86    2     28>bio_endio( 127 126{
        if (dec_zone_page_state2
c#L147" 2""mm/bounce.c#L17" id="L277         127mempool_7 126{
        if (mempool_7dec_zone_page_state2
 = BUG_ON(!2>printk(2"29href="+code=f"+code=bi_io_vecre27" id="L127" class="line" name="L127"> 127 136ef=vc n> 14mhr<26" id="L126" class="line" name="L126"> 126{
        if ( 136ef=vc n> 14tatic  127 1_idxorigmhr<26" id="L126" class="line" name="L126"> 126{
        if ( 1_idxorigtatic =b27" id="L127" class="line" name="L127"> 127 126{
        if (__b2ade=GFP_D2#L94" id="L94" class="li2ne" n29c" class="sref">tovec,  127)
data2L95" id="L95" class="lin2e" na29lass="sref">ISA_POOL_ref=a>        mempool_rwe="L="href="mm/bounce.cREADe=test_bit" claREAD">errs="comment">         * 2ge. kmap 2it
ef="+code=i" cl1=b27" id="L127" class="line" name="L127"> 127)
 136a154" class="lbloc4" ctatic         __b2a = ->,  127)
bio<3a> *<30/a> == mempool_rwe="L="href="mm/bounce.cREADe=test_bit" claREAD">errs="comment">         * 39">  99{
(,  127)
(fromvec;
 102   3    i304ce_copy_vec<=>,  127        struct  126{
 126{
30/bounce.c#L66" id="L66" clas3class="li3e" name="L106"> 106
3a hre30   153lk#ommen/a154" ame="L136"> 136alk#ommen/a154" 4" class="line" name="L124"reque.c#ommen"sref">mempool_reque.c#ommen/bounce.c#L125" id="Lss="line" name="qref=52ass="line" name="L124"> 124
 126{
         * 399ass="li3esm-v850/tlbflush.h|incl3ounce30code=egment" class="sref">__b3         3  * not bounced
<3 href31sa bounce poor"+code=bio" class=mu.c#a154" ame="L136"> 136mu.c#a154" s="ltatic 3v_pag31nce_copy_vec<"+code=bi_io_vemc void bounce_end_io(struct   90  311" id="L311" class="line" name="L311"> 31bi_flags);
 112
3a hre31"mm/bounce.c#L113" id="L113" class="* Data-less  12,e="Lhan cto a154" s=         * 3hool_t" c3L115" class="line" name=3L115"31lass="sref">Iref=s" class="sref">1od_hasf clae" name="L126"> 126hasf cla4" cl26" id="L126" class="line" name="L126"> 126{
         * 3bounce.c#3116" id="L116" class="li3e" na31 dec_zone_page_state3ady uses 3ovec->bv_len
<3 href31  3v_pag31bio_orig-ode=bio" class=mu.c#a154" ame="L136"> 136mu.c#a154" s="l="L84" class="line"u.c#snapshotlinebl class="sref">mempool_au.c#snapshotlinebl class=4" cle.c#L125" id="Lss="line" name="qref=5226" id="L126" class="line" name="L126"> 126{
dec_zone_page_state3x86_64/tl3bflush.h|include/asm-xte3"+cod32href="+code=flags" class="sr3"mm/bounc3e.c#L20" id="L20" class=3ef">b32="L100" classuss="sref">bi_flags);
 fn iass=" system -- iass=atmcase,;
  23#define          * 3pool_t" c3lass="sref">mempool_tIref=sle.c#L125" id="Lss="line" name="qref=/a>        if (        L131" " class="sref">d    DMAa>        __b3"sref">pa3ge_pool, *        )
ommen/a154" EMORnt">         */Calk#maxEMORY_HOTPLUG)
alk#maxEMORnt">L131" 131" "s" class="sref">mu.c#a154" ame="L136"> 136mu.c#a154" s="lrs="comment">         * 3ONFIG_HIG3HMEM) || bvecE" cla>dec_zone_page_state3e="L28"> 3 28static ->(struct org_vec;
__b3_mm/bounc3 name="L129"> 129   3    i33nce_copy_vec( s" class="sref">mempool_free_pages, (void *) 0);
BIO_E3PNOTS3PP, &BIO_EOPNOTSUPP<3a>, &33->  32               3="L133" c3ass="line" name="L133"> 333bi_flags);
 135<* slowepath;
pa3nce indirect pages used<3span>33="L116"> 116         * 3o_for_eac3_segment( 136 126{
(struct  152     mu.c#a154" ame="L136"> 136mu.c#a154" s="lr>dec_zone_page_state3B="L28"> 3f="+code=bio_orig" class3"sref33href="+code=local_irq_restore3href="+co3e=bv_page" class="sref">3v_pag34href="+code=flags" class="sr3 href="+c3de=bv_page" class="sref"3bv_pa34/a> ="> 152     EXPORT_SYMBass="sref">defineEXPORT_SYMBas4" cle.c#L125" id="L3lk#ommen/a154" ame="L136"> 136alk#ommen/a154" 4" cr>dec_zone_page_state340" id="L340" class="line" name="L340"> 340
T=" inal LXR softwar" "yss=" ec_zone_phttp://sourcelorge.net/projects/lxr">LXR pan unitya hrefthis experi claal versaon>"ysec_zone_paailto:lxr@linux.no">lxr@linux.noa hr.
lxr.linux.no kindly hosned>"ysec_zone_phttp://www.redpill-linpro.no">Redpill Linpro ASa hrefprovid"r of Linuxb" csultan cand operata cs ="rvices =iame=1995.