linux/drivers/infiniband/hw/mthca/mthca_mr.c
<<
>>
Prefs
   1/*
   2 * Copyright (c) 2004 Topspin Communications.  All rights reserved.
   3 * Copyright (c) 2005 Mellanox Technologies. All rights reserved.
   4 *
   5 * This software is available to you under a choice of one of two
   6 * licenses.  You may choose to be licensed under the terms of the GNU
   7 * General Public License (GPL) Version 2, available from the file
   8 * COPYING in the main directory of this source tree, or the
   9 * OpenIB.org BSD license below:
  10 *
  11 *     Redistribution and use in source and binary forms, with or
  12 *     without modification, are permitted provided that the following
  13 *     conditions are met:
  14 *
  15 *      - Redistributions of source code must retain the above
  16 *        copyright notice, this list of conditions and the following
  17 *        disclaimer.
  18 *
  19 *      - Redistributions in binary form must reproduce the above
  20 *        copyright notice, this list of conditions and the following
  21 *        disclaimer in the documentation and/or other materials
  22 *        provided with the distribution.
  23 *
  24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  31 * SOFTWARE.
  32 *
  33 * $Id: mthca_mr.c 1349 2004-12-16 21:09:43Z roland $
  34 */
  35
  36#include <linux/slab.h>
  37#include <linux/errno.h>
  38
  39#include "mthca_dev.h"
  40#include "mthca_cmd.h"
  41#include "mthca_memfree.h"
  42
  43struct mthca_mtt {
  44        struct mthca_buddy *buddy;
  45        int                 order;
  46        u32                 first_seg;
  47};
  48
  49/*
  50 * Must be packed because mtt_seg is 64 bits but only aligned to 32 bits.
  51 */
  52struct mthca_mpt_entry {
  53        __be32 flags;
  54        __be32 page_size;
  55        __be32 key;
  56        __be32 pd;
  57        __be64 start;
  58        __be64 length;
  59        __be32 lkey;
  60        __be32 window_count;
  61        __be32 window_count_limit;
  62        __be64 mtt_seg;
  63        __be32 mtt_sz;          /* Arbel only */
  64        u32    reserved[2];
  65} __attribute__((packed));
  66
  67#define MTHCA_MPT_FLAG_SW_OWNS       (0xfUL << 28)
  68#define MTHCA_MPT_FLAG_MIO           (1 << 17)
  69#define MTHCA_MPT_FLAG_BIND_ENABLE   (1 << 15)
  70#define MTHCA_MPT_FLAG_PHYSICAL      (1 <<  9)
  71#define MTHCA_MPT_FLAG_REGION        (1 <<  8)
  72
  73#define MTHCA_MTT_FLAG_PRESENT       1
  74
  75#define MTHCA_MPT_STATUS_SW 0xF0
  76#define MTHCA_MPT_STATUS_HW 0x00
  77
  78#define SINAI_FMR_KEY_INC 0x1000000
  79
  80/*
  81 * Buddy allocator for MTT segments (currently not very efficient
  82 * since it doesn't keep a free list and just searches linearly
  83 * through the bitmaps)
  84 */
  85
  86static u32 mthca_buddy_alloc(struct mthca_buddy *buddy, int order)
  87{
  88        int o;
  89        int m;
  90        u32 seg;
  91
  92        spin_lock(&buddy->lock);
  93
  94        for (o = order; o <= buddy->max_order; ++o) {
  95                m = 1 << (buddy->max_order - o);
  96                seg = find_first_bit(buddy->bits[o], m);
  97                if (seg < m)
  98                        goto found;
  99        }
 100
 101        spin_unlock(&buddy->lock);
 102        return -1;
 103
 104 found:
 105        clear_bit(seg, buddy->bits[o]);
 106
 107        while (o > order) {
 108                --o;
 109                seg <<= 1;
 110                set_bit(seg ^ 1, buddy->bits[o]);
 111        }
 112
 113        spin_unlock(&buddy->lock);
 114
 115        seg <<= order;
 116
 117        return seg;
 118}
 119
 120static void mthca_buddy_free(struct mthca_buddy *buddy, u32 seg, int order)
 121{
 122        seg >>= order;
 123
 124        spin_lock(&buddy->lock);
 125
 126        while (test_bit(seg ^ 1, buddy->bits[order])) {
 127                clear_bit(seg ^ 1, buddy->bits[order]);
 128                seg >>= 1;
 129                ++order;
 130        }
 131
 132        set_bit(seg, buddy->bits[order]);
 133
 134        spin_unlock(&buddy->lock);
 135}
 136
 137static int mthca_buddy_init(struct mthca_buddy *buddy, int max_order)
 138{
 139        int i, s;
 140
 141        buddy->max_order = max_order;
 142        spin_lock_init(&buddy->lock);
 143
 144        buddy->bits = kzalloc((buddy->max_order + 1) * sizeof (long *),
 145                              GFP_KERNEL);
 146        if (!buddy->bits)
 147                goto err_out;
 148
 149        for (i = 0; i <= buddy->max_order; ++i) {
 150                s = BITS_TO_LONGS(1 << (buddy->max_order - i));
 151                buddy->bits[i] = kmalloc(s * sizeof (long), GFP_KERNEL);
 152                if (!buddy->bits[i])
 153                        goto err_out_free;
 154                bitmap_zero(buddy->bits[i],
 155                            1 << (buddy->max_order - i));
 156        }
 157
 158        set_bit(0, buddy->bits[buddy->max_order]);
 159
 160        return 0;
 161
 162err_out_free:
 163        for (i = 0; i <= buddy->max_order; ++i)
 164                kfree(buddy->bits[i]);
 165
 166        kfree(buddy->bits);
 167
 168err_out:
 169        return -ENOMEM;
 170}
 171
 172static void mthca_buddy_cleanup(struct mthca_buddy *buddy)
 173{
 174        int i;
 175
 176        for (i = 0; i <= buddy->max_order; ++i)
 177                kfree(buddy->bits[i]);
 178
 179        kfree(buddy->bits);
 180}
 181
 182static u32 mthca_alloc_mtt_range(struct mthca_dev *dev, int order,
 183                                 struct mthca_buddy *buddy)
 184{
 185        u32 seg = mthca_buddy_alloc(buddy, order);
 186
 187        if (seg == -1)
 188                return -1;
 189
 190        if (mthca_is_memfree(dev))
 191                if (mthca_table_get_range(dev, dev->mr_table.mtt_table, seg,
 192                                          seg + (1 << order) - 1)) {
 193                        mthca_buddy_free(buddy, seg, order);
 194                        seg = -1;
 195                }
 196
 197        return seg;
 198}
 199
 200static struct mthca_mtt *__mthca_alloc_mtt(struct mthca_dev *dev, int size,
 201                                           struct mthca_buddy *buddy)
 202{
 203        struct mthca_mtt *mtt;
 204        int i;
 205
 206        if (size <= 0)
 207                return ERR_PTR(-EINVAL);
 208
 209        mtt = kmalloc(sizeof *mtt, GFP_KERNEL);
 210        if (!mtt)
 211                return ERR_PTR(-ENOMEM);
 212
 213        mtt->buddy = buddy;
 214        mtt->order = 0;
 215        for (i = MTHCA_MTT_SEG_SIZE / 8; i < size; i <<= 1)
 216                ++mtt->order;
 217
 218        mtt->first_seg = mthca_alloc_mtt_range(dev, mtt->order, buddy);
 219        if (mtt->first_seg == -1) {
 220                kfree(mtt);
 221                return ERR_PTR(-ENOMEM);
 222        }
 223
 224        return mtt;
 225}
 226
 227struct mthca_mtt *mthca_alloc_mtt(struct mthca_dev *dev, int size)
 228{
 229        return __mthca_alloc_mtt(dev, size, &dev->mr_table.mtt_buddy);
 230}
 231
 232void mthca_free_mtt(struct mthca_dev *dev, struct mthca_mtt *mtt)
 233{
 234        if (!mtt)
 235                return;
 236
 237        mthca_buddy_free(mtt->buddy, mtt->first_seg, mtt->order);
 238
 239        mthca_table_put_range(dev, dev->mr_table.mtt_table,
 240                              mtt->first_seg,
 241                              mtt->first_seg + (1 << mtt->order) - 1);
 242
 243        kfree(mtt);
 244}
 245
 246static int __mthca_write_mtt(struct mthca_dev *dev, struct mthca_mtt *mtt,
 247                             int start_index, u64 *buffer_list, int list_len)
 248{
 249        struct mthca_mailbox *mailbox;
 250        __be64 *mtt_entry;
 251        int err = 0;
 252        u8 status;
 253        int i;
 254
 255        mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL);
 256        if (IS_ERR(mailbox))
 257                return PTR_ERR(mailbox);
 258        mtt_entry = mailbox->buf;
 259
 260        while (list_len > 0) {
 261                mtt_entry[0] = cpu_to_be64(dev->mr_table.mtt_base +
 262                                           mtt->first_seg * MTHCA_MTT_SEG_SIZE +
 263                                           start_index * 8);
 264                mtt_entry[1] = 0;
 265                for (i = 0; i < list_len && i < MTHCA_MAILBOX_SIZE / 8 - 2; ++i)
 266                        mtt_entry[i + 2] = cpu_to_be64(buffer_list[i] |
 267                                                       MTHCA_MTT_FLAG_PRESENT);
 268
 269                /*
 270                 * If we have an odd number of entries to write, add
 271                 * one more dummy entry for firmware efficiency.
 272                 */
 273                if (i & 1)
 274                        mtt_entry[i + 2] = 0;
 275
 276                err = mthca_WRITE_MTT(dev, mailbox, (i + 1) & ~1, &status);
 277                if (err) {
 278                        mthca_warn(dev, "WRITE_MTT failed (%d)\n", err);
 279                        goto out;
 280                }
 281                if (status) {
 282                        mthca_warn(dev, "WRITE_MTT returned status 0x%02x\n",
 283                                   status);
 284                        err = -EINVAL;
 285                        goto out;
 286                }
 287
 288                list_len    -= i;
 289                start_index += i;
 290                buffer_list += i;
 291        }
 292
 293out:
 294        mthca_free_mailbox(dev, mailbox);
 295        return err;
 296}
 297
 298int mthca_write_mtt_size(struct mthca_dev *dev)
 299{
 300        if (dev->mr_table.fmr_mtt_buddy != &dev->mr_table.mtt_buddy ||
 301            !(dev->mthca_flags & MTHCA_FLAG_FMR))
 302                /*
 303                 * Be friendly to WRITE_MTT command
 304                 * and leave two empty slots for the
 305                 * index and reserved fields of the
 306                 * mailbox.
 307                 */
 308                return PAGE_SIZE / sizeof (u64) - 2;
 309
 310        /* For Arbel, all MTTs must fit in the same page. */
 311        return mthca_is_memfree(dev) ? (PAGE_SIZE / sizeof (u64)) : 0x7ffffff;
 312}
 313
 314static void mthca_tavor_write_mtt_seg(struct mthca_dev *dev,
 315                                      struct mthca_mtt *mtt, int start_index,
 316                                      u64 *buffer_list, int list_len)
 317{
 318        u64 __iomem *mtts;
 319        int i;
 320
 321        mtts = dev->mr_table.tavor_fmr.mtt_base + mtt->first_seg * MTHCA_MTT_SEG_SIZE +
 322                start_index * sizeof (u64);
 323        for (i = 0; i < list_len; ++i)
 324                mthca_write64_raw(cpu_to_be64(buffer_list[i] | MTHCA_MTT_FLAG_PRESENT),
 325                                  mtts + i);
 326}
 327
 328static void mthca_arbel_write_mtt_seg(struct mthca_dev *dev,
 329                                      struct mthca_mtt *mtt, int start_index,
 330                                      u64 *buffer_list, int list_len)
 331{
 332        __be64 *mtts;
 333        dma_addr_t dma_handle;
 334        int i;
 335        int s = start_index * sizeof (u64);
 336
 337        /* For Arbel, all MTTs must fit in the same page. */
 338        BUG_ON(s / PAGE_SIZE != (s + list_len * sizeof(u64) - 1) / PAGE_SIZE);
 339        /* Require full segments */
 340        BUG_ON(s % MTHCA_MTT_SEG_SIZE);
 341
 342        mtts = mthca_table_find(dev->mr_table.mtt_table, mtt->first_seg +
 343                                s / MTHCA_MTT_SEG_SIZE, &dma_handle);
 344
 345        BUG_ON(!mtts);
 346
 347        for (i = 0; i < list_len; ++i)
 348                mtts[i] = cpu_to_be64(buffer_list[i] | MTHCA_MTT_FLAG_PRESENT);
 349
 350        dma_sync_single(&dev->pdev->dev, dma_handle, list_len * sizeof (u64), DMA_TO_DEVICE);
 351}
 352
 353int mthca_write_mtt(struct mthca_dev *dev, struct mthca_mtt *mtt,
 354                    int start_index, u64 *buffer_list, int list_len)
 355{
 356        int size = mthca_write_mtt_size(dev);
 357        int chunk;
 358
 359        if (dev->mr_table.fmr_mtt_buddy != &dev->mr_table.mtt_buddy ||
 360            !(dev->mthca_flags & MTHCA_FLAG_FMR))
 361                return __mthca_write_mtt(dev, mtt, start_index, buffer_list, list_len);
 362
 363        while (list_len > 0) {
 364                chunk = min(size, list_len);
 365                if (mthca_is_memfree(dev))
 366                        mthca_arbel_write_mtt_seg(dev, mtt, start_index,
 367                                                  buffer_list, chunk);
 368                else
 369                        mthca_tavor_write_mtt_seg(dev, mtt, start_index,
 370                                                  buffer_list, chunk);
 371
 372                list_len    -= chunk;
 373                start_index += chunk;
 374                buffer_list += chunk;
 375        }
 376
 377        return 0;
 378}
 379
 380static inline u32 tavor_hw_index_to_key(u32 ind)
 381{
 382        return ind;
 383}
 384
 385static inline u32 tavor_key_to_hw_index(u32 key)
 386{
 387        return key;
 388}
 389
 390static inline u32 arbel_hw_index_to_key(u32 ind)
 391{
 392        return (ind >> 24) | (ind << 8);
 393}
 394
 395static inline u32 arbel_key_to_hw_index(u32 key)
 396{
 397        return (key << 24) | (key >> 8);
 398}
 399
 400static inline u32 hw_index_to_key(struct mthca_dev *dev, u32 ind)
 401{
 402        if (mthca_is_memfree(dev))
 403                return arbel_hw_index_to_key(ind);
 404        else
 405                return tavor_hw_index_to_key(ind);
 406}
 407
 408static inline u32 key_to_hw_index(struct mthca_dev *dev, u32 key)
 409{
 410        if (mthca_is_memfree(dev))
 411                return arbel_key_to_hw_index(key);
 412        else
 413                return tavor_key_to_hw_index(key);
 414}
 415
 416static inline u32 adjust_key(struct mthca_dev *dev, u32 key)
 417{
 418        if (dev->mthca_flags & MTHCA_FLAG_SINAI_OPT)
 419                return ((key << 20) & 0x800000) | (key & 0x7fffff);
 420        else
 421                return key;
 422}
 423
 424int mthca_mr_alloc(struct mthca_dev *dev, u32 pd, int buffer_size_shift,
 425                   u64 iova, u64 total_size, u32 access, struct mthca_mr *mr)
 426{
 427        struct mthca_mailbox *mailbox;
 428        struct mthca_mpt_entry *mpt_entry;
 429        u32 key;
 430        int i;
 431        int err;
 432        u8 status;
 433
 434        WARN_ON(buffer_size_shift >= 32);
 435
 436        key = mthca_alloc(&dev->mr_table.mpt_alloc);
 437        if (key == -1)
 438                return -ENOMEM;
 439        key = adjust_key(dev, key);
 440        mr->ibmr.rkey = mr->ibmr.lkey = hw_index_to_key(dev, key);
 441
 442        if (mthca_is_memfree(dev)) {
 443                err = mthca_table_get(dev, dev->mr_table.mpt_table, key);
 444                if (err)
 445                        goto err_out_mpt_free;
 446        }
 447
 448        mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL);
 449        if (IS_ERR(mailbox)) {
 450                err = PTR_ERR(mailbox);
 451                goto err_out_table;
 452        }
 453        mpt_entry = mailbox->buf;
 454
 455        mpt_entry->flags = cpu_to_be32(MTHCA_MPT_FLAG_SW_OWNS     |
 456                                       MTHCA_MPT_FLAG_MIO         |
 457                                       MTHCA_MPT_FLAG_REGION      |
 458                                       access);
 459        if (!mr->mtt)
 460                mpt_entry->flags |= cpu_to_be32(MTHCA_MPT_FLAG_PHYSICAL);
 461
 462        mpt_entry->page_size = cpu_to_be32(buffer_size_shift - 12);
 463        mpt_entry->key       = cpu_to_be32(key);
 464        mpt_entry->pd        = cpu_to_be32(pd);
 465        mpt_entry->start     = cpu_to_be64(iova);
 466        mpt_entry->length    = cpu_to_be64(total_size);
 467
 468        memset(&mpt_entry->lkey, 0,
 469               sizeof *mpt_entry - offsetof(struct mthca_mpt_entry, lkey));
 470
 471        if (mr->mtt)
 472                mpt_entry->mtt_seg =
 473                        cpu_to_be64(dev->mr_table.mtt_base +
 474                                    mr->mtt->first_seg * MTHCA_MTT_SEG_SIZE);
 475
 476        if (0) {
 477                mthca_dbg(dev, "Dumping MPT entry %08x:\n", mr->ibmr.lkey);
 478                for (i = 0; i < sizeof (struct mthca_mpt_entry) / 4; ++i) {
 479                        if (i % 4 == 0)
 480                                printk("[%02x] ", i * 4);
 481                        printk(" %08x", be32_to_cpu(((__be32 *) mpt_entry)[i]));
 482                        if ((i + 1) % 4 == 0)
 483                                printk("\n");
 484                }
 485        }
 486
 487        err = mthca_SW2HW_MPT(dev, mailbox,
 488                              key & (dev->limits.num_mpts - 1),
 489                              &status);
 490        if (err) {
 491                mthca_warn(dev, "SW2HW_MPT failed (%d)\n", err);
 492                goto err_out_mailbox;
 493        } else if (status) {
 494                mthca_warn(dev, "SW2HW_MPT returned status 0x%02x\n",
 495                           status);
 496                err = -EINVAL;
 497                goto err_out_mailbox;
 498        }
 499
 500        mthca_free_mailbox(dev, mailbox);
 501        return err;
 502
 503err_out_mailbox:
 504        mthca_free_mailbox(dev, mailbox);
 505
 506err_out_table:
 507        mthca_table_put(dev, dev->mr_table.mpt_table, key);
 508
 509err_out_mpt_free:
 510        mthca_free(&dev->mr_table.mpt_alloc, key);
 511        return err;
 512}
 513
 514int mthca_mr_alloc_notrans(struct mthca_dev *dev, u32 pd,
 515                           u32 access, struct mthca_mr *mr)
 516{
 517        mr->mtt = NULL;
 518        return mthca_mr_alloc(dev, pd, 12, 0, ~0ULL, access, mr);
 519}
 520
 521int mthca_mr_alloc_phys(struct mthca_dev *dev, u32 pd,
 522                        u64 *buffer_list, int buffer_size_shift,
 523                        int list_len, u64 iova, u64 total_size,
 524                        u32 access, struct mthca_mr *mr)
 525{
 526        int err;
 527
 528        mr->mtt = mthca_alloc_mtt(dev, list_len);
 529        if (IS_ERR(mr->mtt))
 530                return PTR_ERR(mr->mtt);
 531
 532        err = mthca_write_mtt(dev, mr->mtt, 0, buffer_list, list_len);
 533        if (err) {
 534                mthca_free_mtt(dev, mr->mtt);
 535                return err;
 536        }
 537
 538        err = mthca_mr_alloc(dev, pd, buffer_size_shift, iova,
 539                             total_size, access, mr);
 540        if (err)
 541                mthca_free_mtt(dev, mr->mtt);
 542
 543        return err;
 544}
 545
 546/* Free mr or fmr */
 547static void mthca_free_region(struct mthca_dev *dev, u32 lkey)
 548{
 549        mthca_table_put(dev, dev->mr_table.mpt_table,
 550                        key_to_hw_index(dev, lkey));
 551
 552        mthca_free(&dev->mr_table.mpt_alloc, key_to_hw_index(dev, lkey));
 553}
 554
 555void mthca_free_mr(struct mthca_dev *dev, struct mthca_mr *mr)
 556{
 557        int err;
 558        u8 status;
 559
 560        err = mthca_HW2SW_MPT(dev, NULL,
 561                              key_to_hw_index(dev, mr->ibmr.lkey) &
 562                              (dev->limits.num_mpts - 1),
 563                              &status);
 564        if (err)
 565                mthca_warn(dev, "HW2SW_MPT failed (%d)\n", err);
 566        else if (status)
 567                mthca_warn(dev, "HW2SW_MPT returned status 0x%02x\n",
 568                           status);
 569
 570        mthca_free_region(dev, mr->ibmr.lkey);
 571        mthca_free_mtt(dev, mr->mtt);
 572}
 573
 574int mthca_fmr_alloc(struct mthca_dev *dev, u32 pd,
 575                    u32 access, struct mthca_fmr *mr)
 576{
 577        struct mthca_mpt_entry *mpt_entry;
 578        struct mthca_mailbox *mailbox;
 579        u64 mtt_seg;
 580        u32 key, idx;
 581        u8 status;
 582        int list_len = mr->attr.max_pages;
 583        int err = -ENOMEM;
 584        int i;
 585
 586        if (mr->attr.page_shift < 12 || mr->attr.page_shift >= 32)
 587                return -EINVAL;
 588
 589        /* For Arbel, all MTTs must fit in the same page. */
 590        if (mthca_is_memfree(dev) &&
 591            mr->attr.max_pages * sizeof *mr->mem.arbel.mtts > PAGE_SIZE)
 592                return -EINVAL;
 593
 594        mr->maps = 0;
 595
 596        key = mthca_alloc(&dev->mr_table.mpt_alloc);
 597        if (key == -1)
 598                return -ENOMEM;
 599        key = adjust_key(dev, key);
 600
 601        idx = key & (dev->limits.num_mpts - 1);
 602        mr->ibmr.rkey = mr->ibmr.lkey = hw_index_to_key(dev, key);
 603
 604        if (mthca_is_memfree(dev)) {
 605                err = mthca_table_get(dev, dev->mr_table.mpt_table, key);
 606                if (err)
 607                        goto err_out_mpt_free;
 608
 609                mr->mem.arbel.mpt = mthca_table_find(dev->mr_table.mpt_table, key, NULL);
 610                BUG_ON(!mr->mem.arbel.mpt);
 611        } else
 612                mr->mem.tavor.mpt = dev->mr_table.tavor_fmr.mpt_base +
 613                        sizeof *(mr->mem.tavor.mpt) * idx;
 614
 615        mr->mtt = __mthca_alloc_mtt(dev, list_len, dev->mr_table.fmr_mtt_buddy);
 616        if (IS_ERR(mr->mtt)) {
 617                err = PTR_ERR(mr->mtt);
 618                goto err_out_table;
 619        }
 620
 621        mtt_seg = mr->mtt->first_seg * MTHCA_MTT_SEG_SIZE;
 622
 623        if (mthca_is_memfree(dev)) {
 624                mr->mem.arbel.mtts = mthca_table_find(dev->mr_table.mtt_table,
 625                                                      mr->mtt->first_seg,
 626                                                      &mr->mem.arbel.dma_handle);
 627                BUG_ON(!mr->mem.arbel.mtts);
 628        } else
 629                mr->mem.tavor.mtts = dev->mr_table.tavor_fmr.mtt_base + mtt_seg;
 630
 631        mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL);
 632        if (IS_ERR(mailbox)) {
 633                err = PTR_ERR(mailbox);
 634                goto err_out_free_mtt;
 635        }
 636
 637        mpt_entry = mailbox->buf;
 638
 639        mpt_entry->flags = cpu_to_be32(MTHCA_MPT_FLAG_SW_OWNS     |
 640                                       MTHCA_MPT_FLAG_MIO         |
 641                                       MTHCA_MPT_FLAG_REGION      |
 642                                       access);
 643
 644        mpt_entry->page_size = cpu_to_be32(mr->attr.page_shift - 12);
 645        mpt_entry->key       = cpu_to_be32(key);
 646        mpt_entry->pd        = cpu_to_be32(pd);
 647        memset(&mpt_entry->start, 0,
 648               sizeof *mpt_entry - offsetof(struct mthca_mpt_entry, start));
 649        mpt_entry->mtt_seg   = cpu_to_be64(dev->mr_table.mtt_base + mtt_seg);
 650
 651        if (0) {
 652                mthca_dbg(dev, "Dumping MPT entry %08x:\n", mr->ibmr.lkey);
 653                for (i = 0; i < sizeof (struct mthca_mpt_entry) / 4; ++i) {
 654                        if (i % 4 == 0)
 655                                printk("[%02x] ", i * 4);
 656                        printk(" %08x", be32_to_cpu(((__be32 *) mpt_entry)[i]));
 657                        if ((i + 1) % 4 == 0)
 658                                printk("\n");
 659                }
 660        }
 661
 662        err = mthca_SW2HW_MPT(dev, mailbox,
 663                              key & (dev->limits.num_mpts - 1),
 664                              &status);
 665        if (err) {
 666                mthca_warn(dev, "SW2HW_MPT failed (%d)\n", err);
 667                goto err_out_mailbox_free;
 668        }
 669        if (status) {
 670                mthca_warn(dev, "SW2HW_MPT returned status 0x%02x\n",
 671                           status);
 672                err = -EINVAL;
 673                goto err_out_mailbox_free;
 674        }
 675
 676        mthca_free_mailbox(dev, mailbox);
 677        return 0;
 678
 679err_out_mailbox_free:
 680        mthca_free_mailbox(dev, mailbox);
 681
 682err_out_free_mtt:
 683        mthca_free_mtt(dev, mr->mtt);
 684
 685err_out_table:
 686        mthca_table_put(dev, dev->mr_table.mpt_table, key);
 687
 688err_out_mpt_free:
 689        mthca_free(&dev->mr_table.mpt_alloc, key);
 690        return err;
 691}
 692
 693int mthca_free_fmr(struct mthca_dev *dev, struct mthca_fmr *fmr)
 694{
 695        if (fmr->maps)
 696                return -EBUSY;
 697
 698        mthca_free_region(dev, fmr->ibmr.lkey);
 699        mthca_free_mtt(dev, fmr->mtt);
 700
 701        return 0;
 702}
 703
 704static inline int mthca_check_fmr(struct mthca_fmr *fmr, u64 *page_list,
 705                                  int list_len, u64 iova)
 706{
 707        int i, page_mask;
 708
 709        if (list_len > fmr->attr.max_pages)
 710                return -EINVAL;
 711
 712        page_mask = (1 << fmr->attr.page_shift) - 1;
 713
 714        /* We are getting page lists, so va must be page aligned. */
 715        if (iova & page_mask)
 716                return -EINVAL;
 717
 718        /* Trust the user not to pass misaligned data in page_list */
 719        if (0)
 720                for (i = 0; i < list_len; ++i) {
 721                        if (page_list[i] & ~page_mask)
 722                                return -EINVAL;
 723                }
 724
 725        if (fmr->maps >= fmr->attr.max_maps)
 726                return -EINVAL;
 727
 728        return 0;
 729}
 730
 731
 732int mthca_tavor_map_phys_fmr(struct ib_fmr *ibfmr, u64 *page_list,
 733                             int list_len, u64 iova)
 734{
 735        struct mthca_fmr *fmr = to_mfmr(ibfmr);
 736        struct mthca_dev *dev = to_mdev(ibfmr->device);
 737        struct mthca_mpt_entry mpt_entry;
 738        u32 key;
 739        int i, err;
 740
 741        err = mthca_check_fmr(fmr, page_list, list_len, iova);
 742        if (err)
 743                return err;
 744
 745        ++fmr->maps;
 746
 747        key = tavor_key_to_hw_index(fmr->ibmr.lkey);
 748        key += dev->limits.num_mpts;
 749        fmr->ibmr.lkey = fmr->ibmr.rkey = tavor_hw_index_to_key(key);
 750
 751        writeb(MTHCA_MPT_STATUS_SW, fmr->mem.tavor.mpt);
 752
 753        for (i = 0; i < list_len; ++i) {
 754                __be64 mtt_entry = cpu_to_be64(page_list[i] |
 755                                               MTHCA_MTT_FLAG_PRESENT);
 756                mthca_write64_raw(mtt_entry, fmr->mem.tavor.mtts + i);
 757        }
 758
 759        mpt_entry.lkey   = cpu_to_be32(key);
 760        mpt_entry.length = cpu_to_be64(list_len * (1ull << fmr->attr.page_shift));
 761        mpt_entry.start  = cpu_to_be64(iova);
 762
 763        __raw_writel((__force u32) mpt_entry.lkey, &fmr->mem.tavor.mpt->key);
 764        memcpy_toio(&fmr->mem.tavor.mpt->start, &mpt_entry.start,
 765                    offsetof(struct mthca_mpt_entry, window_count) -
 766                    offsetof(struct mthca_mpt_entry, start));
 767
 768        writeb(MTHCA_MPT_STATUS_HW, fmr->mem.tavor.mpt);
 769
 770        return 0;
 771}
 772
 773int mthca_arbel_map_phys_fmr(struct ib_fmr *ibfmr, u64 *page_list,
 774                             int list_len, u64 iova)
 775{
 776        struct mthca_fmr *fmr = to_mfmr(ibfmr);
 777        struct mthca_dev *dev = to_mdev(ibfmr->device);
 778        u32 key;
 779        int i, err;
 780
 781        err = mthca_check_fmr(fmr, page_list, list_len, iova);
 782        if (err)
 783                return err;
 784
 785        ++fmr->maps;
 786
 787        key = arbel_key_to_hw_index(fmr->ibmr.lkey);
 788        if (dev->mthca_flags & MTHCA_FLAG_SINAI_OPT)
 789                key += SINAI_FMR_KEY_INC;
 790        else
 791                key += dev->limits.num_mpts;
 792        fmr->ibmr.lkey = fmr->ibmr.rkey = arbel_hw_index_to_key(key);
 793
 794        *(u8 *) fmr->mem.arbel.mpt = MTHCA_MPT_STATUS_SW;
 795
 796        wmb();
 797
 798        for (i = 0; i < list_len; ++i)
 799                fmr->mem.arbel.mtts[i] = cpu_to_be64(page_list[i] |
 800                                                     MTHCA_MTT_FLAG_PRESENT);
 801
 802        dma_sync_single(&dev->pdev->dev, fmr->mem.arbel.dma_handle,
 803                        list_len * sizeof(u64), DMA_TO_DEVICE);
 804
 805        fmr->mem.arbel.mpt->key    = cpu_to_be32(key);
 806        fmr->mem.arbel.mpt->lkey   = cpu_to_be32(key);
 807        fmr->mem.arbel.mpt->length = cpu_to_be64(list_len * (1ull << fmr->attr.page_shift));
 808        fmr->mem.arbel.mpt->start  = cpu_to_be64(iova);
 809
 810        wmb();
 811
 812        *(u8 *) fmr->mem.arbel.mpt = MTHCA_MPT_STATUS_HW;
 813
 814        wmb();
 815
 816        return 0;
 817}
 818
 819void mthca_tavor_fmr_unmap(struct mthca_dev *dev, struct mthca_fmr *fmr)
 820{
 821        u32 key;
 822
 823        if (!fmr->maps)
 824                return;
 825
 826        key = tavor_key_to_hw_index(fmr->ibmr.lkey);
 827        key &= dev->limits.num_mpts - 1;
 828        fmr->ibmr.lkey = fmr->ibmr.rkey = tavor_hw_index_to_key(key);
 829
 830        fmr->maps = 0;
 831
 832        writeb(MTHCA_MPT_STATUS_SW, fmr->mem.tavor.mpt);
 833}
 834
 835void mthca_arbel_fmr_unmap(struct mthca_dev *dev, struct mthca_fmr *fmr)
 836{
 837        u32 key;
 838
 839        if (!fmr->maps)
 840                return;
 841
 842        key = arbel_key_to_hw_index(fmr->ibmr.lkey);
 843        key &= dev->limits.num_mpts - 1;
 844        key = adjust_key(dev, key);
 845        fmr->ibmr.lkey = fmr->ibmr.rkey = arbel_hw_index_to_key(key);
 846
 847        fmr->maps = 0;
 848
 849        *(u8 *) fmr->mem.arbel.mpt = MTHCA_MPT_STATUS_SW;
 850}
 851
 852int mthca_init_mr_table(struct mthca_dev *dev)
 853{
 854        unsigned long addr;
 855        int mpts, mtts, err, i;
 856
 857        err = mthca_alloc_init(&dev->mr_table.mpt_alloc,
 858                               dev->limits.num_mpts,
 859                               ~0, dev->limits.reserved_mrws);
 860        if (err)
 861                return err;
 862
 863        if (!mthca_is_memfree(dev) &&
 864            (dev->mthca_flags & MTHCA_FLAG_DDR_HIDDEN))
 865                dev->limits.fmr_reserved_mtts = 0;
 866        else
 867                dev->mthca_flags |= MTHCA_FLAG_FMR;
 868
 869        if (dev->mthca_flags & MTHCA_FLAG_SINAI_OPT)
 870                mthca_dbg(dev, "Memory key throughput optimization activated.\n");
 871
 872        err = mthca_buddy_init(&dev->mr_table.mtt_buddy,
 873                               fls(dev->limits.num_mtt_segs - 1));
 874
 875        if (err)
 876                goto err_mtt_buddy;
 877
 878        dev->mr_table.tavor_fmr.mpt_base = NULL;
 879        dev->mr_table.tavor_fmr.mtt_base = NULL;
 880
 881        if (dev->limits.fmr_reserved_mtts) {
 882                i = fls(dev->limits.fmr_reserved_mtts - 1);
 883
 884                if (i >= 31) {
 885                        mthca_warn(dev, "Unable to reserve 2^31 FMR MTTs.\n");
 886                        err = -EINVAL;
 887                        goto err_fmr_mpt;
 888                }
 889                mpts = mtts = 1 << i;
 890        } else {
 891                mtts = dev->limits.num_mtt_segs;
 892                mpts = dev->limits.num_mpts;
 893        }
 894
 895        if (!mthca_is_memfree(dev) &&
 896            (dev->mthca_flags & MTHCA_FLAG_FMR)) {
 897
 898                addr = pci_resource_start(dev->pdev, 4) +
 899                        ((pci_resource_len(dev->pdev, 4) - 1) &
 900                         dev->mr_table.mpt_base);
 901
 902                dev->mr_table.tavor_fmr.mpt_base =
 903                        ioremap(addr, mpts * sizeof(struct mthca_mpt_entry));
 904
 905                if (!dev->mr_table.tavor_fmr.mpt_base) {
 906                        mthca_warn(dev, "MPT ioremap for FMR failed.\n");
 907                        err = -ENOMEM;
 908                        goto err_fmr_mpt;
 909                }
 910
 911                addr = pci_resource_start(dev->pdev, 4) +
 912                        ((pci_resource_len(dev->pdev, 4) - 1) &
 913                         dev->mr_table.mtt_base);
 914
 915                dev->mr_table.tavor_fmr.mtt_base =
 916                        ioremap(addr, mtts * MTHCA_MTT_SEG_SIZE);
 917                if (!dev->mr_table.tavor_fmr.mtt_base) {
 918                        mthca_warn(dev, "MTT ioremap for FMR failed.\n");
 919                        err = -ENOMEM;
 920                        goto err_fmr_mtt;
 921                }
 922        }
 923
 924        if (dev->limits.fmr_reserved_mtts) {
 925                err = mthca_buddy_init(&dev->mr_table.tavor_fmr.mtt_buddy, fls(mtts - 1));
 926                if (err)
 927                        goto err_fmr_mtt_buddy;
 928
 929                /* Prevent regular MRs from using FMR keys */
 930                err = mthca_buddy_alloc(&dev->mr_table.mtt_buddy, fls(mtts - 1));
 931                if (err)
 932                        goto err_reserve_fmr;
 933
 934                dev->mr_table.fmr_mtt_buddy =
 935                        &dev->mr_table.tavor_fmr.mtt_buddy;
 936        } else
 937                dev->mr_table.fmr_mtt_buddy = &dev->mr_table.mtt_buddy;
 938
 939        /* FMR table is always the first, take reserved MTTs out of there */
 940        if (dev->limits.reserved_mtts) {
 941                i = fls(dev->limits.reserved_mtts - 1);
 942
 943                if (mthca_alloc_mtt_range(dev, i,
 944                                          dev->mr_table.fmr_mtt_buddy) == -1) {
 945                        mthca_warn(dev, "MTT table of order %d is too small.\n",
 946                                  dev->mr_table.fmr_mtt_buddy->max_order);
 947                        err = -ENOMEM;
 948                        goto err_reserve_mtts;
 949                }
 950        }
 951
 952        return 0;
 953
 954err_reserve_mtts:
 955err_reserve_fmr:
 956        if (dev->limits.fmr_reserved_mtts)
 957                mthca_buddy_cleanup(&dev->mr_table.tavor_fmr.mtt_buddy);
 958
 959err_fmr_mtt_buddy:
 960        if (dev->mr_table.tavor_fmr.mtt_base)
 961                iounmap(dev->mr_table.tavor_fmr.mtt_base);
 962
 963err_fmr_mtt:
 964        if (dev->mr_table.tavor_fmr.mpt_base)
 965                iounmap(dev->mr_table.tavor_fmr.mpt_base);
 966
 967err_fmr_mpt:
 968        mthca_buddy_cleanup(&dev->mr_table.mtt_buddy);
 969
 970err_mtt_buddy:
 971        mthca_alloc_cleanup(&dev->mr_table.mpt_alloc);
 972
 973        return err;
 974}
 975
 976void mthca_cleanup_mr_table(struct mthca_dev *dev)
 977{
 978        /* XXX check if any MRs are still allocated? */
 979        if (dev->limits.fmr_reserved_mtts)
 980                mthca_buddy_cleanup(&dev->mr_table.tavor_fmr.mtt_buddy);
 981
 982        mthca_buddy_cleanup(&dev->mr_table.mtt_buddy);
 983
 984        if (dev->mr_table.tavor_fmr.mtt_base)
 985                iounmap(dev->mr_table.tavor_fmr.mtt_base);
 986        if (dev->mr_table.tavor_fmr.mpt_base)
 987                iounmap(dev->mr_table.tavor_fmr.mpt_base);
 988
 989        mthca_alloc_cleanup(&dev->mr_table.mpt_alloc);
 990}
 991
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.