linux/fs/bio-integrity.c
<<
>>
Prefs
   1/*
   2 * bio-integrity.c - bio data integrity extensions
   3 *
   4 * Copyright (C) 2007, 2008, 2009 Oracle Corporation
   5 * Written by: Martin K. Petersen <martin.petersen@oracle.com>
   6 *
   7 * This program is free software; you can redistribute it and/or
   8 * modify it under the terms of the GNU General Public License version
   9 * 2 as published by the Free Software Foundation.
  10 *
  11 * This program is distributed in the hope that it will be useful, but
  12 * WITHOUT ANY WARRANTY; without even the implied warranty of
  13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14 * General Public License for more details.
  15 *
  16 * You should have received a copy of the GNU General Public License
  17 * along with this program; see the file COPYING.  If not, write to
  18 * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139,
  19 * USA.
  20 *
  21 */
  22
  23#include <linux/blkdev.h>
  24#include <linux/mempool.h>
  25#include <linux/export.h>
  26#include <linux/bio.h>
  27#include <linux/workqueue.h>
  28#include <linux/slab.h>
  29
  30#define BIP_INLINE_VECS 4
  31
  32static struct kmem_cache *bip_slab;
  33static struct workqueue_struct *kintegrityd_wq;
  34
  35/**
  36 * bio_integrity_alloc - Allocate integrity payload and attach it to bio
  37 * @bio:        bio to attach integrity metadata to
  38 * @gfp_mask:   Memory allocation mask
  39 * @nr_vecs:    Number of integrity metadata scatter-gather elements
  40 *
  41 * Description: This function prepares a bio for attaching integrity
  42 * metadata.  nr_vecs specifies the maximum number of pages containing
  43 * integrity metadata that can be attached.
  44 */
  45struct bio_integrity_payload *bio_integrity_alloc(struct bio *bio,
  46                                                  gfp_t gfp_mask,
  47                                                  unsigned int nr_vecs)
  48{
  49        struct bio_integrity_payload *bip;
  50        struct bio_set *bs = bio->bi_pool;
  51        unsigned long idx = BIO_POOL_NONE;
  52        unsigned inline_vecs;
  53
  54        if (!bs) {
  55                bip = kmalloc(sizeof(struct bio_integrity_payload) +
  56                              sizeof(struct bio_vec) * nr_vecs, gfp_mask);
  57                inline_vecs = nr_vecs;
  58        } else {
  59                bip = mempool_alloc(bs->bio_integrity_pool, gfp_mask);
  60                inline_vecs = BIP_INLINE_VECS;
  61        }
  62
  63        if (unlikely(!bip))
  64                return NULL;
  65
  66        memset(bip, 0, sizeof(*bip));
  67
  68        if (nr_vecs > inline_vecs) {
  69                bip->bip_vec = bvec_alloc(gfp_mask, nr_vecs, &idx,
  70                                          bs->bvec_integrity_pool);
  71                if (!bip->bip_vec)
  72                        goto err;
  73        } else {
  74                bip->bip_vec = bip->bip_inline_vecs;
  75        }
  76
  77        bip->bip_slab = idx;
  78        bip->bip_bio = bio;
  79        bio->bi_integrity = bip;
  80
  81        return bip;
  82err:
  83        mempool_free(bip, bs->bio_integrity_pool);
  84        return NULL;
  85}
  86EXPORT_SYMBOL(bio_integrity_alloc);
  87
  88/**
  89 * bio_integrity_free - Free bio integrity payload
  90 * @bio:        bio containing bip to be freed
  91 *
  92 * Description: Used to free the integrity portion of a bio. Usually
  93 * called from bio_free().
  94 */
  95void bio_integrity_free(struct bio *bio)
  96{
  97        struct bio_integrity_payload *bip = bio->bi_integrity;
  98        struct bio_set *bs = bio->bi_pool;
  99
 100        if (bip->bip_owns_buf)
 101                kfree(bip->bip_buf);
 102
 103        if (bs) {
 104                if (bip->bip_slab != BIO_POOL_NONE)
 105                        bvec_free(bs->bvec_integrity_pool, bip->bip_vec,
 106                                  bip->bip_slab);
 107
 108                mempool_free(bip, bs->bio_integrity_pool);
 109        } else {
 110                kfree(bip);
 111        }
 112
 113        bio->bi_integrity = NULL;
 114}
 115EXPORT_SYMBOL(bio_integrity_free);
 116
 117/**
 118 * bio_integrity_add_page - Attach integrity metadata
 119 * @bio:        bio to update
 120 * @page:       page containing integrity metadata
 121 * @len:        number of bytes of integrity metadata in page
 122 * @offset:     start offset within page
 123 *
 124 * Description: Attach a page containing integrity metadata to bio.
 125 */
 126int bio_integrity_add_page(struct bio *bio, struct page *page,
 127                           unsigned int len, unsigned int offset)
 128{
 129        struct bio_integrity_payload *bip = bio->bi_integrity;
 130        struct bio_vec *iv;
 131
 132        if (bip->bip_vcnt >= bvec_nr_vecs(bip->bip_slab)) {
 133                printk(KERN_ERR "%s: bip_vec full\n", __func__);
 134                return 0;
 135        }
 136
 137        iv = bip_vec_idx(bip, bip->bip_vcnt);
 138        BUG_ON(iv == NULL);
 139
 140        iv->bv_page = page;
 141        iv->bv_len = len;
 142        iv->bv_offset = offset;
 143        bip->bip_vcnt++;
 144
 145        return len;
 146}
 147EXPORT_SYMBOL(bio_integrity_add_page);
 148
 149static int bdev_integrity_enabled(struct block_device *bdev, int rw)
 150{
 151        struct blk_integrity *bi = bdev_get_integrity(bdev);
 152
 153        if (bi == NULL)
 154                return 0;
 155
 156        if (rw == READ && bi->verify_fn != NULL &&
 157            (bi->flags & INTEGRITY_FLAG_READ))
 158                return 1;
 159
 160        if (rw == WRITE && bi->generate_fn != NULL &&
 161            (bi->flags & INTEGRITY_FLAG_WRITE))
 162                return 1;
 163
 164        return 0;
 165}
 166
 167/**
 168 * bio_integrity_enabled - Check whether integrity can be passed
 169 * @bio:        bio to check
 170 *
 171 * Description: Determines whether bio_integrity_prep() can be called
 172 * on this bio or not.  bio data direction and target device must be
 173 * set prior to calling.  The functions honors the write_generate and
 174 * read_verify flags in sysfs.
 175 */
 176int bio_integrity_enabled(struct bio *bio)
 177{
 178        /* Already protected? */
 179        if (bio_integrity(bio))
 180                return 0;
 181
 182        return bdev_integrity_enabled(bio->bi_bdev, bio_data_dir(bio));
 183}
 184EXPORT_SYMBOL(bio_integrity_enabled);
 185
 186/**
 187 * bio_integrity_hw_sectors - Convert 512b sectors to hardware ditto
 188 * @bi:         blk_integrity profile for device
 189 * @sectors:    Number of 512 sectors to convert
 190 *
 191 * Description: The block layer calculates everything in 512 byte
 192 * sectors but integrity metadata is done in terms of the hardware
 193 * sector size of the storage device.  Convert the block layer sectors
 194 * to physical sectors.
 195 */
 196static inline unsigned int bio_integrity_hw_sectors(struct blk_integrity *bi,
 197                                                    unsigned int sectors)
 198{
 199        /* At this point there are only 512b or 4096b DIF/EPP devices */
 200        if (bi->sector_size == 4096)
 201                return sectors >>= 3;
 202
 203        return sectors;
 204}
 205
 206/**
 207 * bio_integrity_tag_size - Retrieve integrity tag space
 208 * @bio:        bio to inspect
 209 *
 210 * Description: Returns the maximum number of tag bytes that can be
 211 * attached to this bio. Filesystems can use this to determine how
 212 * much metadata to attach to an I/O.
 213 */
 214unsigned int bio_integrity_tag_size(struct bio *bio)
 215{
 216        struct blk_integrity *bi = bdev_get_integrity(bio->bi_bdev);
 217
 218        BUG_ON(bio->bi_size == 0);
 219
 220        return bi->tag_size * (bio->bi_size / bi->sector_size);
 221}
 222EXPORT_SYMBOL(bio_integrity_tag_size);
 223
 224int bio_integrity_tag(struct bio *bio, void *tag_buf, unsigned int len, int set)
 225{
 226        struct bio_integrity_payload *bip = bio->bi_integrity;
 227        struct blk_integrity *bi = bdev_get_integrity(bio->bi_bdev);
 228        unsigned int nr_sectors;
 229
 230        BUG_ON(bip->bip_buf == NULL);
 231
 232        if (bi->tag_size == 0)
 233                return -1;
 234
 235        nr_sectors = bio_integrity_hw_sectors(bi,
 236                                        DIV_ROUND_UP(len, bi->tag_size));
 237
 238        if (nr_sectors * bi->tuple_size > bip->bip_size) {
 239                printk(KERN_ERR "%s: tag too big for bio: %u > %u\n",
 240                       __func__, nr_sectors * bi->tuple_size, bip->bip_size);
 241                return -1;
 242        }
 243
 244        if (set)
 245                bi->set_tag_fn(bip->bip_buf, tag_buf, nr_sectors);
 246        else
 247                bi->get_tag_fn(bip->bip_buf, tag_buf, nr_sectors);
 248
 249        return 0;
 250}
 251
 252/**
 253 * bio_integrity_set_tag - Attach a tag buffer to a bio
 254 * @bio:        bio to attach buffer to
 255 * @tag_buf:    Pointer to a buffer containing tag data
 256 * @len:        Length of the included buffer
 257 *
 258 * Description: Use this function to tag a bio by leveraging the extra
 259 * space provided by devices formatted with integrity protection.  The
 260 * size of the integrity buffer must be <= to the size reported by
 261 * bio_integrity_tag_size().
 262 */
 263int bio_integrity_set_tag(struct bio *bio, void *tag_buf, unsigned int len)
 264{
 265        BUG_ON(bio_data_dir(bio) != WRITE);
 266
 267        return bio_integrity_tag(bio, tag_buf, len, 1);
 268}
 269EXPORT_SYMBOL(bio_integrity_set_tag);
 270
 271/**
 272 * bio_integrity_get_tag - Retrieve a tag buffer from a bio
 273 * @bio:        bio to retrieve buffer from
 274 * @tag_buf:    Pointer to a buffer for the tag data
 275 * @len:        Length of the target buffer
 276 *
 277 * Description: Use this function to retrieve the tag buffer from a
 278 * completed I/O. The size of the integrity buffer must be <= to the
 279 * size reported by bio_integrity_tag_size().
 280 */
 281int bio_integrity_get_tag(struct bio *bio, void *tag_buf, unsigned int len)
 282{
 283        BUG_ON(bio_data_dir(bio) != READ);
 284
 285        return bio_integrity_tag(bio, tag_buf, len, 0);
 286}
 287EXPORT_SYMBOL(bio_integrity_get_tag);
 288
 289/**
 290 * bio_integrity_generate - Generate integrity metadata for a bio
 291 * @bio:        bio to generate integrity metadata for
 292 *
 293 * Description: Generates integrity metadata for a bio by calling the
 294 * block device's generation callback function.  The bio must have a
 295 * bip attached with enough room to accommodate the generated
 296 * integrity metadata.
 297 */
 298static void bio_integrity_generate(struct bio *bio)
 299{
 300        struct blk_integrity *bi = bdev_get_integrity(bio->bi_bdev);
 301        struct blk_integrity_exchg bix;
 302        struct bio_vec *bv;
 303        sector_t sector = bio->bi_sector;
 304        unsigned int i, sectors, total;
 305        void *prot_buf = bio->bi_integrity->bip_buf;
 306
 307        total = 0;
 308        bix.disk_name = bio->bi_bdev->bd_disk->disk_name;
 309        bix.sector_size = bi->sector_size;
 310
 311        bio_for_each_segment(bv, bio, i) {
 312                void *kaddr = kmap_atomic(bv->bv_page);
 313                bix.data_buf = kaddr + bv->bv_offset;
 314                bix.data_size = bv->bv_len;
 315                bix.prot_buf = prot_buf;
 316                bix.sector = sector;
 317
 318                bi->generate_fn(&bix);
 319
 320                sectors = bv->bv_len / bi->sector_size;
 321                sector += sectors;
 322                prot_buf += sectors * bi->tuple_size;
 323                total += sectors * bi->tuple_size;
 324                BUG_ON(total > bio->bi_integrity->bip_size);
 325
 326                kunmap_atomic(kaddr);
 327        }
 328}
 329
 330static inline unsigned short blk_integrity_tuple_size(struct blk_integrity *bi)
 331{
 332        if (bi)
 333                return bi->tuple_size;
 334
 335        return 0;
 336}
 337
 338/**
 339 * bio_integrity_prep - Prepare bio for integrity I/O
 340 * @bio:        bio to prepare
 341 *
 342 * Description: Allocates a buffer for integrity metadata, maps the
 343 * pages and attaches them to a bio.  The bio must have data
 344 * direction, target device and start sector set priot to calling.  In
 345 * the WRITE case, integrity metadata will be generated using the
 346 * block device's integrity function.  In the READ case, the buffer
 347 * will be prepared for DMA and a suitable end_io handler set up.
 348 */
 349int bio_integrity_prep(struct bio *bio)
 350{
 351        struct bio_integrity_payload *bip;
 352        struct blk_integrity *bi;
 353        struct request_queue *q;
 354        void *buf;
 355        unsigned long start, end;
 356        unsigned int len, nr_pages;
 357        unsigned int bytes, offset, i;
 358        unsigned int sectors;
 359
 360        bi = bdev_get_integrity(bio->bi_bdev);
 361        q = bdev_get_queue(bio->bi_bdev);
 362        BUG_ON(bi == NULL);
 363        BUG_ON(bio_integrity(bio));
 364
 365        sectors = bio_integrity_hw_sectors(bi, bio_sectors(bio));
 366
 367        /* Allocate kernel buffer for protection data */
 368        len = sectors * blk_integrity_tuple_size(bi);
 369        buf = kmalloc(len, GFP_NOIO | q->bounce_gfp);
 370        if (unlikely(buf == NULL)) {
 371                printk(KERN_ERR "could not allocate integrity buffer\n");
 372                return -ENOMEM;
 373        }
 374
 375        end = (((unsigned long) buf) + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
 376        start = ((unsigned long) buf) >> PAGE_SHIFT;
 377        nr_pages = end - start;
 378
 379        /* Allocate bio integrity payload and integrity vectors */
 380        bip = bio_integrity_alloc(bio, GFP_NOIO, nr_pages);
 381        if (unlikely(bip == NULL)) {
 382                printk(KERN_ERR "could not allocate data integrity bioset\n");
 383                kfree(buf);
 384                return -EIO;
 385        }
 386
 387        bip->bip_owns_buf = 1;
 388        bip->bip_buf = buf;
 389        bip->bip_size = len;
 390        bip->bip_sector = bio->bi_sector;
 391
 392        /* Map it */
 393        offset = offset_in_page(buf);
 394        for (i = 0 ; i < nr_pages ; i++) {
 395                int ret;
 396                bytes = PAGE_SIZE - offset;
 397
 398                if (len <= 0)
 399                        break;
 400
 401                if (bytes > len)
 402                        bytes = len;
 403
 404                ret = bio_integrity_add_page(bio, virt_to_page(buf),
 405                                             bytes, offset);
 406
 407                if (ret == 0)
 408                        return 0;
 409
 410                if (ret < bytes)
 411                        break;
 412
 413                buf += bytes;
 414                len -= bytes;
 415                offset = 0;
 416        }
 417
 418        /* Install custom I/O completion handler if read verify is enabled */
 419        if (bio_data_dir(bio) == READ) {
 420                bip->bip_end_io = bio->bi_end_io;
 421                bio->bi_end_io = bio_integrity_endio;
 422        }
 423
 424        /* Auto-generate integrity metadata if this is a write */
 425        if (bio_data_dir(bio) == WRITE)
 426                bio_integrity_generate(bio);
 427
 428        return 0;
 429}
 430EXPORT_SYMBOL(bio_integrity_prep);
 431
 432/**
 433 * bio_integrity_verify - Verify integrity metadata for a bio
 434 * @bio:        bio to verify
 435 *
 436 * Description: This function is called to verify the integrity of a
 437 * bio.  The data in the bio io_vec is compared to the integrity
 438 * metadata returned by the HBA.
 439 */
 440static int bio_integrity_verify(struct bio *bio)
 441{
 442        struct blk_integrity *bi = bdev_get_integrity(bio->bi_bdev);
 443        struct blk_integrity_exchg bix;
 444        struct bio_vec *bv;
 445        sector_t sector = bio->bi_integrity->bip_sector;
 446        unsigned int i, sectors, total, ret;
 447        void *prot_buf = bio->bi_integrity->bip_buf;
 448
 449        ret = total = 0;
 450        bix.disk_name = bio->bi_bdev->bd_disk->disk_name;
 451        bix.sector_size = bi->sector_size;
 452
 453        bio_for_each_segment(bv, bio, i) {
 454                void *kaddr = kmap_atomic(bv->bv_page);
 455                bix.data_buf = kaddr + bv->bv_offset;
 456                bix.data_size = bv->bv_len;
 457                bix.prot_buf = prot_buf;
 458                bix.sector = sector;
 459
 460                ret = bi->verify_fn(&bix);
 461
 462                if (ret) {
 463                        kunmap_atomic(kaddr);
 464                        return ret;
 465                }
 466
 467                sectors = bv->bv_len / bi->sector_size;
 468                sector += sectors;
 469                prot_buf += sectors * bi->tuple_size;
 470                total += sectors * bi->tuple_size;
 471                BUG_ON(total > bio->bi_integrity->bip_size);
 472
 473                kunmap_atomic(kaddr);
 474        }
 475
 476        return ret;
 477}
 478
 479/**
 480 * bio_integrity_verify_fn - Integrity I/O completion worker
 481 * @work:       Work struct stored in bio to be verified
 482 *
 483 * Description: This workqueue function is called to complete a READ
 484 * request.  The function verifies the transferred integrity metadata
 485 * and then calls the original bio end_io function.
 486 */
 487static void bio_integrity_verify_fn(struct work_struct *work)
 488{
 489        struct bio_integrity_payload *bip =
 490                container_of(work, struct bio_integrity_payload, bip_work);
 491        struct bio *bio = bip->bip_bio;
 492        int error;
 493
 494        error = bio_integrity_verify(bio);
 495
 496        /* Restore original bio completion handler */
 497        bio->bi_end_io = bip->bip_end_io;
 498        bio_endio(bio, error);
 499}
 500
 501/**
 502 * bio_integrity_endio - Integrity I/O completion function
 503 * @bio:        Protected bio
 504 * @error:      Pointer to errno
 505 *
 506 * Description: Completion for integrity I/O
 507 *
 508 * Normally I/O completion is done in interrupt context.  However,
 509 * verifying I/O integrity is a time-consuming task which must be run
 510 * in process context.  This function postpones completion
 511 * accordingly.
 512 */
 513void bio_integrity_endio(struct bio *bio, int error)
 514{
 515        struct bio_integrity_payload *bip = bio->bi_integrity;
 516
 517        BUG_ON(bip->bip_bio != bio);
 518
 519        /* In case of an I/O error there is no point in verifying the
 520         * integrity metadata.  Restore original bio end_io handler
 521         * and run it.
 522         */
 523        if (error) {
 524                bio->bi_end_io = bip->bip_end_io;
 525                bio_endio(bio, error);
 526
 527                return;
 528        }
 529
 530        INIT_WORK(&bip->bip_work, bio_integrity_verify_fn);
 531        queue_work(kintegrityd_wq, &bip->bip_work);
 532}
 533EXPORT_SYMBOL(bio_integrity_endio);
 534
 535/**
 536 * bio_integrity_mark_head - Advance bip_vec skip bytes
 537 * @bip:        Integrity vector to advance
 538 * @skip:       Number of bytes to advance it
 539 */
 540void bio_integrity_mark_head(struct bio_integrity_payload *bip,
 541                             unsigned int skip)
 542{
 543        struct bio_vec *iv;
 544        unsigned int i;
 545
 546        bip_for_each_vec(iv, bip, i) {
 547                if (skip == 0) {
 548                        bip->bip_idx = i;
 549                        return;
 550                } else if (skip >= iv->bv_len) {
 551                        skip -= iv->bv_len;
 552                } else { /* skip < iv->bv_len) */
 553                        iv->bv_offset += skip;
 554                        iv->bv_len -= skip;
 555                        bip->bip_idx = i;
 556                        return;
 557                }
 558        }
 559}
 560
 561/**
 562 * bio_integrity_mark_tail - Truncate bip_vec to be len bytes long
 563 * @bip:        Integrity vector to truncate
 564 * @len:        New length of integrity vector
 565 */
 566void bio_integrity_mark_tail(struct bio_integrity_payload *bip,
 567                             unsigned int len)
 568{
 569        struct bio_vec *iv;
 570        unsigned int i;
 571
 572        bip_for_each_vec(iv, bip, i) {
 573                if (len == 0) {
 574                        bip->bip_vcnt = i;
 575                        return;
 576                } else if (len >= iv->bv_len) {
 577                        len -= iv->bv_len;
 578                } else { /* len < iv->bv_len) */
 579                        iv->bv_len = len;
 580                        len = 0;
 581                }
 582        }
 583}
 584
 585/**
 586 * bio_integrity_advance - Advance integrity vector
 587 * @bio:        bio whose integrity vector to update
 588 * @bytes_done: number of data bytes that have been completed
 589 *
 590 * Description: This function calculates how many integrity bytes the
 591 * number of completed data bytes correspond to and advances the
 592 * integrity vector accordingly.
 593 */
 594void bio_integrity_advance(struct bio *bio, unsigned int bytes_done)
 595{
 596        struct bio_integrity_payload *bip = bio->bi_integrity;
 597        struct blk_integrity *bi = bdev_get_integrity(bio->bi_bdev);
 598        unsigned int nr_sectors;
 599
 600        BUG_ON(bip == NULL);
 601        BUG_ON(bi == NULL);
 602
 603        nr_sectors = bio_integrity_hw_sectors(bi, bytes_done >> 9);
 604        bio_integrity_mark_head(bip, nr_sectors * bi->tuple_size);
 605}
 606EXPORT_SYMBOL(bio_integrity_advance);
 607
 608/**
 609 * bio_integrity_trim - Trim integrity vector
 610 * @bio:        bio whose integrity vector to update
 611 * @offset:     offset to first data sector
 612 * @sectors:    number of data sectors
 613 *
 614 * Description: Used to trim the integrity vector in a cloned bio.
 615 * The ivec will be advanced corresponding to 'offset' data sectors
 616 * and the length will be truncated corresponding to 'len' data
 617 * sectors.
 618 */
 619void bio_integrity_trim(struct bio *bio, unsigned int offset,
 620                        unsigned int sectors)
 621{
 622        struct bio_integrity_payload *bip = bio->bi_integrity;
 623        struct blk_integrity *bi = bdev_get_integrity(bio->bi_bdev);
 624        unsigned int nr_sectors;
 625
 626        BUG_ON(bip == NULL);
 627        BUG_ON(bi == NULL);
 628        BUG_ON(!bio_flagged(bio, BIO_CLONED));
 629
 630        nr_sectors = bio_integrity_hw_sectors(bi, sectors);
 631        bip->bip_sector = bip->bip_sector + offset;
 632        bio_integrity_mark_head(bip, offset * bi->tuple_size);
 633        bio_integrity_mark_tail(bip, sectors * bi->tuple_size);
 634}
 635EXPORT_SYMBOL(bio_integrity_trim);
 636
 637/**
 638 * bio_integrity_split - Split integrity metadata
 639 * @bio:        Protected bio
 640 * @bp:         Resulting bio_pair
 641 * @sectors:    Offset
 642 *
 643 * Description: Splits an integrity page into a bio_pair.
 644 */
 645void bio_integrity_split(struct bio *bio, struct bio_pair *bp, int sectors)
 646{
 647        struct blk_integrity *bi;
 648        struct bio_integrity_payload *bip = bio->bi_integrity;
 649        unsigned int nr_sectors;
 650
 651        if (bio_integrity(bio) == 0)
 652                return;
 653
 654        bi = bdev_get_integrity(bio->bi_bdev);
 655        BUG_ON(bi == NULL);
 656        BUG_ON(bip->bip_vcnt != 1);
 657
 658        nr_sectors = bio_integrity_hw_sectors(bi, sectors);
 659
 660        bp->bio1.bi_integrity = &bp->bip1;
 661        bp->bio2.bi_integrity = &bp->bip2;
 662
 663        bp->iv1 = bip->bip_vec[bip->bip_idx];
 664        bp->iv2 = bip->bip_vec[bip->bip_idx];
 665
 666        bp->bip1.bip_vec = &bp->iv1;
 667        bp->bip2.bip_vec = &bp->iv2;
 668
 669        bp->iv1.bv_len = sectors * bi->tuple_size;
 670        bp->iv2.bv_offset += sectors * bi->tuple_size;
 671        bp->iv2.bv_len -= sectors * bi->tuple_size;
 672
 673        bp->bip1.bip_sector = bio->bi_integrity->bip_sector;
 674        bp->bip2.bip_sector = bio->bi_integrity->bip_sector + nr_sectors;
 675
 676        bp->bip1.bip_vcnt = bp->bip2.bip_vcnt = 1;
 677        bp->bip1.bip_idx = bp->bip2.bip_idx = 0;
 678}
 679EXPORT_SYMBOL(bio_integrity_split);
 680
 681/**
 682 * bio_integrity_clone - Callback for cloning bios with integrity metadata
 683 * @bio:        New bio
 684 * @bio_src:    Original bio
 685 * @gfp_mask:   Memory allocation mask
 686 *
 687 * Description: Called to allocate a bip when cloning a bio
 688 */
 689int bio_integrity_clone(struct bio *bio, struct bio *bio_src,
 690                        gfp_t gfp_mask)
 691{
 692        struct bio_integrity_payload *bip_src = bio_src->bi_integrity;
 693        struct bio_integrity_payload *bip;
 694
 695        BUG_ON(bip_src == NULL);
 696
 697        bip = bio_integrity_alloc(bio, gfp_mask, bip_src->bip_vcnt);
 698
 699        if (bip == NULL)
 700                return -EIO;
 701
 702        memcpy(bip->bip_vec, bip_src->bip_vec,
 703               bip_src->bip_vcnt * sizeof(struct bio_vec));
 704
 705        bip->bip_sector = bip_src->bip_sector;
 706        bip->bip_vcnt = bip_src->bip_vcnt;
 707        bip->bip_idx = bip_src->bip_idx;
 708
 709        return 0;
 710}
 711EXPORT_SYMBOL(bio_integrity_clone);
 712
 713int bioset_integrity_create(struct bio_set *bs, int pool_size)
 714{
 715        if (bs->bio_integrity_pool)
 716                return 0;
 717
 718        bs->bio_integrity_pool = mempool_create_slab_pool(pool_size, bip_slab);
 719
 720        bs->bvec_integrity_pool = biovec_create_pool(bs, pool_size);
 721        if (!bs->bvec_integrity_pool)
 722                return -1;
 723
 724        if (!bs->bio_integrity_pool)
 725                return -1;
 726
 727        return 0;
 728}
 729EXPORT_SYMBOL(bioset_integrity_create);
 730
 731void bioset_integrity_free(struct bio_set *bs)
 732{
 733        if (bs->bio_integrity_pool)
 734                mempool_destroy(bs->bio_integrity_pool);
 735
 736        if (bs->bvec_integrity_pool)
 737                mempool_destroy(bs->bio_integrity_pool);
 738}
 739EXPORT_SYMBOL(bioset_integrity_free);
 740
 741void __init bio_integrity_init(void)
 742{
 743        /*
 744         * kintegrityd won't block much but may burn a lot of CPU cycles.
 745         * Make it highpri CPU intensive wq with max concurrency of 1.
 746         */
 747        kintegrityd_wq = alloc_workqueue("kintegrityd", WQ_MEM_RECLAIM |
 748                                         WQ_HIGHPRI | WQ_CPU_INTENSIVE, 1);
 749        if (!kintegrityd_wq)
 750                panic("Failed to create kintegrityd\n");
 751
 752        bip_slab = kmem_cache_create("bio_integrity_payload",
 753                                     sizeof(struct bio_integrity_payload) +
 754                                     sizeof(struct bio_vec) * BIP_INLINE_VECS,
 755                                     0, SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
 756        if (!bip_slab)
 757                panic("Failed to create slab\n");
 758}
 759
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.