linux/drivers/infiniband/hw/hns/hns_roce_hem.c
<<
>>
Prefs
   1/*
   2 * Copyright (c) 2016 Hisilicon Limited.
   3 * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved.
   4 *
   5 * This software is available to you under a choice of one of two
   6 * licenses.  You may choose to be licensed under the terms of the GNU
   7 * General Public License (GPL) Version 2, available from the file
   8 * COPYING in the main directory of this source tree, or the
   9 * OpenIB.org BSD license below:
  10 *
  11 *     Redistribution and use in source and binary forms, with or
  12 *     without modification, are permitted provided that the following
  13 *     conditions are met:
  14 *
  15 *      - Redistributions of source code must retain the above
  16 *        copyright notice, this list of conditions and the following
  17 *        disclaimer.
  18 *
  19 *      - Redistributions in binary form must reproduce the above
  20 *        copyright notice, this list of conditions and the following
  21 *        disclaimer in the documentation and/or other materials
  22 *        provided with the distribution.
  23 *
  24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  31 * SOFTWARE.
  32 */
  33
  34#include <linux/platform_device.h>
  35#include "hns_roce_device.h"
  36#include "hns_roce_hem.h"
  37#include "hns_roce_common.h"
  38
  39#define HEM_INDEX_BUF                   BIT(0)
  40#define HEM_INDEX_L0                    BIT(1)
  41#define HEM_INDEX_L1                    BIT(2)
  42struct hns_roce_hem_index {
  43        u64 buf;
  44        u64 l0;
  45        u64 l1;
  46        u32 inited; /* indicate which index is available */
  47};
  48
  49bool hns_roce_check_whether_mhop(struct hns_roce_dev *hr_dev, u32 type)
  50{
  51        int hop_num = 0;
  52
  53        switch (type) {
  54        case HEM_TYPE_QPC:
  55                hop_num = hr_dev->caps.qpc_hop_num;
  56                break;
  57        case HEM_TYPE_MTPT:
  58                hop_num = hr_dev->caps.mpt_hop_num;
  59                break;
  60        case HEM_TYPE_CQC:
  61                hop_num = hr_dev->caps.cqc_hop_num;
  62                break;
  63        case HEM_TYPE_SRQC:
  64                hop_num = hr_dev->caps.srqc_hop_num;
  65                break;
  66        case HEM_TYPE_SCCC:
  67                hop_num = hr_dev->caps.sccc_hop_num;
  68                break;
  69        case HEM_TYPE_QPC_TIMER:
  70                hop_num = hr_dev->caps.qpc_timer_hop_num;
  71                break;
  72        case HEM_TYPE_CQC_TIMER:
  73                hop_num = hr_dev->caps.cqc_timer_hop_num;
  74                break;
  75        case HEM_TYPE_GMV:
  76                hop_num = hr_dev->caps.gmv_hop_num;
  77                break;
  78        default:
  79                return false;
  80        }
  81
  82        return hop_num ? true : false;
  83}
  84
  85static bool hns_roce_check_hem_null(struct hns_roce_hem **hem, u64 hem_idx,
  86                                    u32 bt_chunk_num, u64 hem_max_num)
  87{
  88        u64 start_idx = round_down(hem_idx, bt_chunk_num);
  89        u64 check_max_num = start_idx + bt_chunk_num;
  90        u64 i;
  91
  92        for (i = start_idx; (i < check_max_num) && (i < hem_max_num); i++)
  93                if (i != hem_idx && hem[i])
  94                        return false;
  95
  96        return true;
  97}
  98
  99static bool hns_roce_check_bt_null(u64 **bt, u64 ba_idx, u32 bt_chunk_num)
 100{
 101        u64 start_idx = round_down(ba_idx, bt_chunk_num);
 102        int i;
 103
 104        for (i = 0; i < bt_chunk_num; i++)
 105                if (i != ba_idx && bt[start_idx + i])
 106                        return false;
 107
 108        return true;
 109}
 110
 111static int hns_roce_get_bt_num(u32 table_type, u32 hop_num)
 112{
 113        if (check_whether_bt_num_3(table_type, hop_num))
 114                return 3;
 115        else if (check_whether_bt_num_2(table_type, hop_num))
 116                return 2;
 117        else if (check_whether_bt_num_1(table_type, hop_num))
 118                return 1;
 119        else
 120                return 0;
 121}
 122
 123static int get_hem_table_config(struct hns_roce_dev *hr_dev,
 124                                struct hns_roce_hem_mhop *mhop,
 125                                u32 type)
 126{
 127        struct device *dev = hr_dev->dev;
 128
 129        switch (type) {
 130        case HEM_TYPE_QPC:
 131                mhop->buf_chunk_size = 1 << (hr_dev->caps.qpc_buf_pg_sz
 132                                             + PAGE_SHIFT);
 133                mhop->bt_chunk_size = 1 << (hr_dev->caps.qpc_ba_pg_sz
 134                                             + PAGE_SHIFT);
 135                mhop->ba_l0_num = hr_dev->caps.qpc_bt_num;
 136                mhop->hop_num = hr_dev->caps.qpc_hop_num;
 137                break;
 138        case HEM_TYPE_MTPT:
 139                mhop->buf_chunk_size = 1 << (hr_dev->caps.mpt_buf_pg_sz
 140                                             + PAGE_SHIFT);
 141                mhop->bt_chunk_size = 1 << (hr_dev->caps.mpt_ba_pg_sz
 142                                             + PAGE_SHIFT);
 143                mhop->ba_l0_num = hr_dev->caps.mpt_bt_num;
 144                mhop->hop_num = hr_dev->caps.mpt_hop_num;
 145                break;
 146        case HEM_TYPE_CQC:
 147                mhop->buf_chunk_size = 1 << (hr_dev->caps.cqc_buf_pg_sz
 148                                             + PAGE_SHIFT);
 149                mhop->bt_chunk_size = 1 << (hr_dev->caps.cqc_ba_pg_sz
 150                                            + PAGE_SHIFT);
 151                mhop->ba_l0_num = hr_dev->caps.cqc_bt_num;
 152                mhop->hop_num = hr_dev->caps.cqc_hop_num;
 153                break;
 154        case HEM_TYPE_SCCC:
 155                mhop->buf_chunk_size = 1 << (hr_dev->caps.sccc_buf_pg_sz
 156                                             + PAGE_SHIFT);
 157                mhop->bt_chunk_size = 1 << (hr_dev->caps.sccc_ba_pg_sz
 158                                            + PAGE_SHIFT);
 159                mhop->ba_l0_num = hr_dev->caps.sccc_bt_num;
 160                mhop->hop_num = hr_dev->caps.sccc_hop_num;
 161                break;
 162        case HEM_TYPE_QPC_TIMER:
 163                mhop->buf_chunk_size = 1 << (hr_dev->caps.qpc_timer_buf_pg_sz
 164                                             + PAGE_SHIFT);
 165                mhop->bt_chunk_size = 1 << (hr_dev->caps.qpc_timer_ba_pg_sz
 166                                            + PAGE_SHIFT);
 167                mhop->ba_l0_num = hr_dev->caps.qpc_timer_bt_num;
 168                mhop->hop_num = hr_dev->caps.qpc_timer_hop_num;
 169                break;
 170        case HEM_TYPE_CQC_TIMER:
 171                mhop->buf_chunk_size = 1 << (hr_dev->caps.cqc_timer_buf_pg_sz
 172                                             + PAGE_SHIFT);
 173                mhop->bt_chunk_size = 1 << (hr_dev->caps.cqc_timer_ba_pg_sz
 174                                            + PAGE_SHIFT);
 175                mhop->ba_l0_num = hr_dev->caps.cqc_timer_bt_num;
 176                mhop->hop_num = hr_dev->caps.cqc_timer_hop_num;
 177                break;
 178        case HEM_TYPE_SRQC:
 179                mhop->buf_chunk_size = 1 << (hr_dev->caps.srqc_buf_pg_sz
 180                                             + PAGE_SHIFT);
 181                mhop->bt_chunk_size = 1 << (hr_dev->caps.srqc_ba_pg_sz
 182                                             + PAGE_SHIFT);
 183                mhop->ba_l0_num = hr_dev->caps.srqc_bt_num;
 184                mhop->hop_num = hr_dev->caps.srqc_hop_num;
 185                break;
 186        case HEM_TYPE_GMV:
 187                mhop->buf_chunk_size = 1 << (hr_dev->caps.gmv_buf_pg_sz +
 188                                             PAGE_SHIFT);
 189                mhop->bt_chunk_size = 1 << (hr_dev->caps.gmv_ba_pg_sz +
 190                                            PAGE_SHIFT);
 191                mhop->ba_l0_num = hr_dev->caps.gmv_bt_num;
 192                mhop->hop_num = hr_dev->caps.gmv_hop_num;
 193                break;
 194        default:
 195                dev_err(dev, "table %u not support multi-hop addressing!\n",
 196                        type);
 197                return -EINVAL;
 198        }
 199
 200        return 0;
 201}
 202
 203int hns_roce_calc_hem_mhop(struct hns_roce_dev *hr_dev,
 204                           struct hns_roce_hem_table *table, unsigned long *obj,
 205                           struct hns_roce_hem_mhop *mhop)
 206{
 207        struct device *dev = hr_dev->dev;
 208        u32 chunk_ba_num;
 209        u32 chunk_size;
 210        u32 table_idx;
 211        u32 bt_num;
 212
 213        if (get_hem_table_config(hr_dev, mhop, table->type))
 214                return -EINVAL;
 215
 216        if (!obj)
 217                return 0;
 218
 219        /*
 220         * QPC/MTPT/CQC/SRQC/SCCC alloc hem for buffer pages.
 221         * MTT/CQE alloc hem for bt pages.
 222         */
 223        bt_num = hns_roce_get_bt_num(table->type, mhop->hop_num);
 224        chunk_ba_num = mhop->bt_chunk_size / BA_BYTE_LEN;
 225        chunk_size = table->type < HEM_TYPE_MTT ? mhop->buf_chunk_size :
 226                              mhop->bt_chunk_size;
 227        table_idx = *obj / (chunk_size / table->obj_size);
 228        switch (bt_num) {
 229        case 3:
 230                mhop->l2_idx = table_idx & (chunk_ba_num - 1);
 231                mhop->l1_idx = table_idx / chunk_ba_num & (chunk_ba_num - 1);
 232                mhop->l0_idx = (table_idx / chunk_ba_num) / chunk_ba_num;
 233                break;
 234        case 2:
 235                mhop->l1_idx = table_idx & (chunk_ba_num - 1);
 236                mhop->l0_idx = table_idx / chunk_ba_num;
 237                break;
 238        case 1:
 239                mhop->l0_idx = table_idx;
 240                break;
 241        default:
 242                dev_err(dev, "table %u not support hop_num = %u!\n",
 243                        table->type, mhop->hop_num);
 244                return -EINVAL;
 245        }
 246        if (mhop->l0_idx >= mhop->ba_l0_num)
 247                mhop->l0_idx %= mhop->ba_l0_num;
 248
 249        return 0;
 250}
 251
 252static struct hns_roce_hem *hns_roce_alloc_hem(struct hns_roce_dev *hr_dev,
 253                                               int npages,
 254                                               unsigned long hem_alloc_size,
 255                                               gfp_t gfp_mask)
 256{
 257        struct hns_roce_hem_chunk *chunk = NULL;
 258        struct hns_roce_hem *hem;
 259        struct scatterlist *mem;
 260        int order;
 261        void *buf;
 262
 263        WARN_ON(gfp_mask & __GFP_HIGHMEM);
 264
 265        hem = kmalloc(sizeof(*hem),
 266                      gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN));
 267        if (!hem)
 268                return NULL;
 269
 270        INIT_LIST_HEAD(&hem->chunk_list);
 271
 272        order = get_order(hem_alloc_size);
 273
 274        while (npages > 0) {
 275                if (!chunk) {
 276                        chunk = kmalloc(sizeof(*chunk),
 277                                gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN));
 278                        if (!chunk)
 279                                goto fail;
 280
 281                        sg_init_table(chunk->mem, HNS_ROCE_HEM_CHUNK_LEN);
 282                        chunk->npages = 0;
 283                        chunk->nsg = 0;
 284                        memset(chunk->buf, 0, sizeof(chunk->buf));
 285                        list_add_tail(&chunk->list, &hem->chunk_list);
 286                }
 287
 288                while (1 << order > npages)
 289                        --order;
 290
 291                /*
 292                 * Alloc memory one time. If failed, don't alloc small block
 293                 * memory, directly return fail.
 294                 */
 295                mem = &chunk->mem[chunk->npages];
 296                buf = dma_alloc_coherent(hr_dev->dev, PAGE_SIZE << order,
 297                                &sg_dma_address(mem), gfp_mask);
 298                if (!buf)
 299                        goto fail;
 300
 301                chunk->buf[chunk->npages] = buf;
 302                sg_dma_len(mem) = PAGE_SIZE << order;
 303
 304                ++chunk->npages;
 305                ++chunk->nsg;
 306                npages -= 1 << order;
 307        }
 308
 309        return hem;
 310
 311fail:
 312        hns_roce_free_hem(hr_dev, hem);
 313        return NULL;
 314}
 315
 316void hns_roce_free_hem(struct hns_roce_dev *hr_dev, struct hns_roce_hem *hem)
 317{
 318        struct hns_roce_hem_chunk *chunk, *tmp;
 319        int i;
 320
 321        if (!hem)
 322                return;
 323
 324        list_for_each_entry_safe(chunk, tmp, &hem->chunk_list, list) {
 325                for (i = 0; i < chunk->npages; ++i)
 326                        dma_free_coherent(hr_dev->dev,
 327                                   sg_dma_len(&chunk->mem[i]),
 328                                   chunk->buf[i],
 329                                   sg_dma_address(&chunk->mem[i]));
 330                kfree(chunk);
 331        }
 332
 333        kfree(hem);
 334}
 335
 336static int calc_hem_config(struct hns_roce_dev *hr_dev,
 337                           struct hns_roce_hem_table *table, unsigned long obj,
 338                           struct hns_roce_hem_mhop *mhop,
 339                           struct hns_roce_hem_index *index)
 340{
 341        struct ib_device *ibdev = &hr_dev->ib_dev;
 342        unsigned long mhop_obj = obj;
 343        u32 l0_idx, l1_idx, l2_idx;
 344        u32 chunk_ba_num;
 345        u32 bt_num;
 346        int ret;
 347
 348        ret = hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, mhop);
 349        if (ret)
 350                return ret;
 351
 352        l0_idx = mhop->l0_idx;
 353        l1_idx = mhop->l1_idx;
 354        l2_idx = mhop->l2_idx;
 355        chunk_ba_num = mhop->bt_chunk_size / BA_BYTE_LEN;
 356        bt_num = hns_roce_get_bt_num(table->type, mhop->hop_num);
 357        switch (bt_num) {
 358        case 3:
 359                index->l1 = l0_idx * chunk_ba_num + l1_idx;
 360                index->l0 = l0_idx;
 361                index->buf = l0_idx * chunk_ba_num * chunk_ba_num +
 362                             l1_idx * chunk_ba_num + l2_idx;
 363                break;
 364        case 2:
 365                index->l0 = l0_idx;
 366                index->buf = l0_idx * chunk_ba_num + l1_idx;
 367                break;
 368        case 1:
 369                index->buf = l0_idx;
 370                break;
 371        default:
 372                ibdev_err(ibdev, "table %u not support mhop.hop_num = %u!\n",
 373                          table->type, mhop->hop_num);
 374                return -EINVAL;
 375        }
 376
 377        if (unlikely(index->buf >= table->num_hem)) {
 378                ibdev_err(ibdev, "table %u exceed hem limt idx %llu, max %lu!\n",
 379                          table->type, index->buf, table->num_hem);
 380                return -EINVAL;
 381        }
 382
 383        return 0;
 384}
 385
 386static void free_mhop_hem(struct hns_roce_dev *hr_dev,
 387                          struct hns_roce_hem_table *table,
 388                          struct hns_roce_hem_mhop *mhop,
 389                          struct hns_roce_hem_index *index)
 390{
 391        u32 bt_size = mhop->bt_chunk_size;
 392        struct device *dev = hr_dev->dev;
 393
 394        if (index->inited & HEM_INDEX_BUF) {
 395                hns_roce_free_hem(hr_dev, table->hem[index->buf]);
 396                table->hem[index->buf] = NULL;
 397        }
 398
 399        if (index->inited & HEM_INDEX_L1) {
 400                dma_free_coherent(dev, bt_size, table->bt_l1[index->l1],
 401                                  table->bt_l1_dma_addr[index->l1]);
 402                table->bt_l1[index->l1] = NULL;
 403        }
 404
 405        if (index->inited & HEM_INDEX_L0) {
 406                dma_free_coherent(dev, bt_size, table->bt_l0[index->l0],
 407                                  table->bt_l0_dma_addr[index->l0]);
 408                table->bt_l0[index->l0] = NULL;
 409        }
 410}
 411
 412static int alloc_mhop_hem(struct hns_roce_dev *hr_dev,
 413                          struct hns_roce_hem_table *table,
 414                          struct hns_roce_hem_mhop *mhop,
 415                          struct hns_roce_hem_index *index)
 416{
 417        u32 bt_size = mhop->bt_chunk_size;
 418        struct device *dev = hr_dev->dev;
 419        struct hns_roce_hem_iter iter;
 420        gfp_t flag;
 421        u64 bt_ba;
 422        u32 size;
 423        int ret;
 424
 425        /* alloc L1 BA's chunk */
 426        if ((check_whether_bt_num_3(table->type, mhop->hop_num) ||
 427             check_whether_bt_num_2(table->type, mhop->hop_num)) &&
 428             !table->bt_l0[index->l0]) {
 429                table->bt_l0[index->l0] = dma_alloc_coherent(dev, bt_size,
 430                                            &table->bt_l0_dma_addr[index->l0],
 431                                            GFP_KERNEL);
 432                if (!table->bt_l0[index->l0]) {
 433                        ret = -ENOMEM;
 434                        goto out;
 435                }
 436                index->inited |= HEM_INDEX_L0;
 437        }
 438
 439        /* alloc L2 BA's chunk */
 440        if (check_whether_bt_num_3(table->type, mhop->hop_num) &&
 441            !table->bt_l1[index->l1])  {
 442                table->bt_l1[index->l1] = dma_alloc_coherent(dev, bt_size,
 443                                            &table->bt_l1_dma_addr[index->l1],
 444                                            GFP_KERNEL);
 445                if (!table->bt_l1[index->l1]) {
 446                        ret = -ENOMEM;
 447                        goto err_alloc_hem;
 448                }
 449                index->inited |= HEM_INDEX_L1;
 450                *(table->bt_l0[index->l0] + mhop->l1_idx) =
 451                                               table->bt_l1_dma_addr[index->l1];
 452        }
 453
 454        /*
 455         * alloc buffer space chunk for QPC/MTPT/CQC/SRQC/SCCC.
 456         * alloc bt space chunk for MTT/CQE.
 457         */
 458        size = table->type < HEM_TYPE_MTT ? mhop->buf_chunk_size : bt_size;
 459        flag = (table->lowmem ? GFP_KERNEL : GFP_HIGHUSER) | __GFP_NOWARN;
 460        table->hem[index->buf] = hns_roce_alloc_hem(hr_dev, size >> PAGE_SHIFT,
 461                                                    size, flag);
 462        if (!table->hem[index->buf]) {
 463                ret = -ENOMEM;
 464                goto err_alloc_hem;
 465        }
 466
 467        index->inited |= HEM_INDEX_BUF;
 468        hns_roce_hem_first(table->hem[index->buf], &iter);
 469        bt_ba = hns_roce_hem_addr(&iter);
 470        if (table->type < HEM_TYPE_MTT) {
 471                if (mhop->hop_num == 2)
 472                        *(table->bt_l1[index->l1] + mhop->l2_idx) = bt_ba;
 473                else if (mhop->hop_num == 1)
 474                        *(table->bt_l0[index->l0] + mhop->l1_idx) = bt_ba;
 475        } else if (mhop->hop_num == 2) {
 476                *(table->bt_l0[index->l0] + mhop->l1_idx) = bt_ba;
 477        }
 478
 479        return 0;
 480err_alloc_hem:
 481        free_mhop_hem(hr_dev, table, mhop, index);
 482out:
 483        return ret;
 484}
 485
 486static int set_mhop_hem(struct hns_roce_dev *hr_dev,
 487                        struct hns_roce_hem_table *table, unsigned long obj,
 488                        struct hns_roce_hem_mhop *mhop,
 489                        struct hns_roce_hem_index *index)
 490{
 491        struct ib_device *ibdev = &hr_dev->ib_dev;
 492        int step_idx;
 493        int ret = 0;
 494
 495        if (index->inited & HEM_INDEX_L0) {
 496                ret = hr_dev->hw->set_hem(hr_dev, table, obj, 0);
 497                if (ret) {
 498                        ibdev_err(ibdev, "set HEM step 0 failed!\n");
 499                        goto out;
 500                }
 501        }
 502
 503        if (index->inited & HEM_INDEX_L1) {
 504                ret = hr_dev->hw->set_hem(hr_dev, table, obj, 1);
 505                if (ret) {
 506                        ibdev_err(ibdev, "set HEM step 1 failed!\n");
 507                        goto out;
 508                }
 509        }
 510
 511        if (index->inited & HEM_INDEX_BUF) {
 512                if (mhop->hop_num == HNS_ROCE_HOP_NUM_0)
 513                        step_idx = 0;
 514                else
 515                        step_idx = mhop->hop_num;
 516                ret = hr_dev->hw->set_hem(hr_dev, table, obj, step_idx);
 517                if (ret)
 518                        ibdev_err(ibdev, "set HEM step last failed!\n");
 519        }
 520out:
 521        return ret;
 522}
 523
 524static int hns_roce_table_mhop_get(struct hns_roce_dev *hr_dev,
 525                                   struct hns_roce_hem_table *table,
 526                                   unsigned long obj)
 527{
 528        struct ib_device *ibdev = &hr_dev->ib_dev;
 529        struct hns_roce_hem_index index = {};
 530        struct hns_roce_hem_mhop mhop = {};
 531        int ret;
 532
 533        ret = calc_hem_config(hr_dev, table, obj, &mhop, &index);
 534        if (ret) {
 535                ibdev_err(ibdev, "calc hem config failed!\n");
 536                return ret;
 537        }
 538
 539        mutex_lock(&table->mutex);
 540        if (table->hem[index.buf]) {
 541                refcount_inc(&table->hem[index.buf]->refcount);
 542                goto out;
 543        }
 544
 545        ret = alloc_mhop_hem(hr_dev, table, &mhop, &index);
 546        if (ret) {
 547                ibdev_err(ibdev, "alloc mhop hem failed!\n");
 548                goto out;
 549        }
 550
 551        /* set HEM base address to hardware */
 552        if (table->type < HEM_TYPE_MTT) {
 553                ret = set_mhop_hem(hr_dev, table, obj, &mhop, &index);
 554                if (ret) {
 555                        ibdev_err(ibdev, "set HEM address to HW failed!\n");
 556                        goto err_alloc;
 557                }
 558        }
 559
 560        refcount_set(&table->hem[index.buf]->refcount, 1);
 561        goto out;
 562
 563err_alloc:
 564        free_mhop_hem(hr_dev, table, &mhop, &index);
 565out:
 566        mutex_unlock(&table->mutex);
 567        return ret;
 568}
 569
 570int hns_roce_table_get(struct hns_roce_dev *hr_dev,
 571                       struct hns_roce_hem_table *table, unsigned long obj)
 572{
 573        struct device *dev = hr_dev->dev;
 574        unsigned long i;
 575        int ret = 0;
 576
 577        if (hns_roce_check_whether_mhop(hr_dev, table->type))
 578                return hns_roce_table_mhop_get(hr_dev, table, obj);
 579
 580        i = obj / (table->table_chunk_size / table->obj_size);
 581
 582        mutex_lock(&table->mutex);
 583
 584        if (table->hem[i]) {
 585                refcount_inc(&table->hem[i]->refcount);
 586                goto out;
 587        }
 588
 589        table->hem[i] = hns_roce_alloc_hem(hr_dev,
 590                                       table->table_chunk_size >> PAGE_SHIFT,
 591                                       table->table_chunk_size,
 592                                       (table->lowmem ? GFP_KERNEL :
 593                                        GFP_HIGHUSER) | __GFP_NOWARN);
 594        if (!table->hem[i]) {
 595                ret = -ENOMEM;
 596                goto out;
 597        }
 598
 599        /* Set HEM base address(128K/page, pa) to Hardware */
 600        if (hr_dev->hw->set_hem(hr_dev, table, obj, HEM_HOP_STEP_DIRECT)) {
 601                hns_roce_free_hem(hr_dev, table->hem[i]);
 602                table->hem[i] = NULL;
 603                ret = -ENODEV;
 604                dev_err(dev, "set HEM base address to HW failed.\n");
 605                goto out;
 606        }
 607
 608        refcount_set(&table->hem[i]->refcount, 1);
 609out:
 610        mutex_unlock(&table->mutex);
 611        return ret;
 612}
 613
 614static void clear_mhop_hem(struct hns_roce_dev *hr_dev,
 615                           struct hns_roce_hem_table *table, unsigned long obj,
 616                           struct hns_roce_hem_mhop *mhop,
 617                           struct hns_roce_hem_index *index)
 618{
 619        struct ib_device *ibdev = &hr_dev->ib_dev;
 620        u32 hop_num = mhop->hop_num;
 621        u32 chunk_ba_num;
 622        int step_idx;
 623
 624        index->inited = HEM_INDEX_BUF;
 625        chunk_ba_num = mhop->bt_chunk_size / BA_BYTE_LEN;
 626        if (check_whether_bt_num_2(table->type, hop_num)) {
 627                if (hns_roce_check_hem_null(table->hem, index->buf,
 628                                            chunk_ba_num, table->num_hem))
 629                        index->inited |= HEM_INDEX_L0;
 630        } else if (check_whether_bt_num_3(table->type, hop_num)) {
 631                if (hns_roce_check_hem_null(table->hem, index->buf,
 632                                            chunk_ba_num, table->num_hem)) {
 633                        index->inited |= HEM_INDEX_L1;
 634                        if (hns_roce_check_bt_null(table->bt_l1, index->l1,
 635                                                   chunk_ba_num))
 636                                index->inited |= HEM_INDEX_L0;
 637                }
 638        }
 639
 640        if (table->type < HEM_TYPE_MTT) {
 641                if (hop_num == HNS_ROCE_HOP_NUM_0)
 642                        step_idx = 0;
 643                else
 644                        step_idx = hop_num;
 645
 646                if (hr_dev->hw->clear_hem(hr_dev, table, obj, step_idx))
 647                        ibdev_warn(ibdev, "failed to clear hop%u HEM.\n", hop_num);
 648
 649                if (index->inited & HEM_INDEX_L1)
 650                        if (hr_dev->hw->clear_hem(hr_dev, table, obj, 1))
 651                                ibdev_warn(ibdev, "failed to clear HEM step 1.\n");
 652
 653                if (index->inited & HEM_INDEX_L0)
 654                        if (hr_dev->hw->clear_hem(hr_dev, table, obj, 0))
 655                                ibdev_warn(ibdev, "failed to clear HEM step 0.\n");
 656        }
 657}
 658
 659static void hns_roce_table_mhop_put(struct hns_roce_dev *hr_dev,
 660                                    struct hns_roce_hem_table *table,
 661                                    unsigned long obj,
 662                                    int check_refcount)
 663{
 664        struct ib_device *ibdev = &hr_dev->ib_dev;
 665        struct hns_roce_hem_index index = {};
 666        struct hns_roce_hem_mhop mhop = {};
 667        int ret;
 668
 669        ret = calc_hem_config(hr_dev, table, obj, &mhop, &index);
 670        if (ret) {
 671                ibdev_err(ibdev, "calc hem config failed!\n");
 672                return;
 673        }
 674
 675        if (!check_refcount)
 676                mutex_lock(&table->mutex);
 677        else if (!refcount_dec_and_mutex_lock(&table->hem[index.buf]->refcount,
 678                                              &table->mutex))
 679                return;
 680
 681        clear_mhop_hem(hr_dev, table, obj, &mhop, &index);
 682        free_mhop_hem(hr_dev, table, &mhop, &index);
 683
 684        mutex_unlock(&table->mutex);
 685}
 686
 687void hns_roce_table_put(struct hns_roce_dev *hr_dev,
 688                        struct hns_roce_hem_table *table, unsigned long obj)
 689{
 690        struct device *dev = hr_dev->dev;
 691        unsigned long i;
 692
 693        if (hns_roce_check_whether_mhop(hr_dev, table->type)) {
 694                hns_roce_table_mhop_put(hr_dev, table, obj, 1);
 695                return;
 696        }
 697
 698        i = obj / (table->table_chunk_size / table->obj_size);
 699
 700        if (!refcount_dec_and_mutex_lock(&table->hem[i]->refcount,
 701                                         &table->mutex))
 702                return;
 703
 704        if (hr_dev->hw->clear_hem(hr_dev, table, obj, HEM_HOP_STEP_DIRECT))
 705                dev_warn(dev, "failed to clear HEM base address.\n");
 706
 707        hns_roce_free_hem(hr_dev, table->hem[i]);
 708        table->hem[i] = NULL;
 709
 710        mutex_unlock(&table->mutex);
 711}
 712
 713void *hns_roce_table_find(struct hns_roce_dev *hr_dev,
 714                          struct hns_roce_hem_table *table,
 715                          unsigned long obj, dma_addr_t *dma_handle)
 716{
 717        struct hns_roce_hem_chunk *chunk;
 718        struct hns_roce_hem_mhop mhop;
 719        struct hns_roce_hem *hem;
 720        unsigned long mhop_obj = obj;
 721        unsigned long obj_per_chunk;
 722        unsigned long idx_offset;
 723        int offset, dma_offset;
 724        void *addr = NULL;
 725        u32 hem_idx = 0;
 726        int length;
 727        int i, j;
 728
 729        if (!table->lowmem)
 730                return NULL;
 731
 732        mutex_lock(&table->mutex);
 733
 734        if (!hns_roce_check_whether_mhop(hr_dev, table->type)) {
 735                obj_per_chunk = table->table_chunk_size / table->obj_size;
 736                hem = table->hem[obj / obj_per_chunk];
 737                idx_offset = obj % obj_per_chunk;
 738                dma_offset = offset = idx_offset * table->obj_size;
 739        } else {
 740                u32 seg_size = 64; /* 8 bytes per BA and 8 BA per segment */
 741
 742                if (hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop))
 743                        goto out;
 744                /* mtt mhop */
 745                i = mhop.l0_idx;
 746                j = mhop.l1_idx;
 747                if (mhop.hop_num == 2)
 748                        hem_idx = i * (mhop.bt_chunk_size / BA_BYTE_LEN) + j;
 749                else if (mhop.hop_num == 1 ||
 750                         mhop.hop_num == HNS_ROCE_HOP_NUM_0)
 751                        hem_idx = i;
 752
 753                hem = table->hem[hem_idx];
 754                dma_offset = offset = obj * seg_size % mhop.bt_chunk_size;
 755                if (mhop.hop_num == 2)
 756                        dma_offset = offset = 0;
 757        }
 758
 759        if (!hem)
 760                goto out;
 761
 762        list_for_each_entry(chunk, &hem->chunk_list, list) {
 763                for (i = 0; i < chunk->npages; ++i) {
 764                        length = sg_dma_len(&chunk->mem[i]);
 765                        if (dma_handle && dma_offset >= 0) {
 766                                if (length > (u32)dma_offset)
 767                                        *dma_handle = sg_dma_address(
 768                                                &chunk->mem[i]) + dma_offset;
 769                                dma_offset -= length;
 770                        }
 771
 772                        if (length > (u32)offset) {
 773                                addr = chunk->buf[i] + offset;
 774                                goto out;
 775                        }
 776                        offset -= length;
 777                }
 778        }
 779
 780out:
 781        mutex_unlock(&table->mutex);
 782        return addr;
 783}
 784
 785int hns_roce_init_hem_table(struct hns_roce_dev *hr_dev,
 786                            struct hns_roce_hem_table *table, u32 type,
 787                            unsigned long obj_size, unsigned long nobj,
 788                            int use_lowmem)
 789{
 790        unsigned long obj_per_chunk;
 791        unsigned long num_hem;
 792
 793        if (!hns_roce_check_whether_mhop(hr_dev, type)) {
 794                table->table_chunk_size = hr_dev->caps.chunk_sz;
 795                obj_per_chunk = table->table_chunk_size / obj_size;
 796                num_hem = DIV_ROUND_UP(nobj, obj_per_chunk);
 797
 798                table->hem = kcalloc(num_hem, sizeof(*table->hem), GFP_KERNEL);
 799                if (!table->hem)
 800                        return -ENOMEM;
 801        } else {
 802                struct hns_roce_hem_mhop mhop = {};
 803                unsigned long buf_chunk_size;
 804                unsigned long bt_chunk_size;
 805                unsigned long bt_chunk_num;
 806                unsigned long num_bt_l0;
 807                u32 hop_num;
 808
 809                if (get_hem_table_config(hr_dev, &mhop, type))
 810                        return -EINVAL;
 811
 812                buf_chunk_size = mhop.buf_chunk_size;
 813                bt_chunk_size = mhop.bt_chunk_size;
 814                num_bt_l0 = mhop.ba_l0_num;
 815                hop_num = mhop.hop_num;
 816
 817                obj_per_chunk = buf_chunk_size / obj_size;
 818                num_hem = DIV_ROUND_UP(nobj, obj_per_chunk);
 819                bt_chunk_num = bt_chunk_size / BA_BYTE_LEN;
 820
 821                if (type >= HEM_TYPE_MTT)
 822                        num_bt_l0 = bt_chunk_num;
 823
 824                table->hem = kcalloc(num_hem, sizeof(*table->hem),
 825                                         GFP_KERNEL);
 826                if (!table->hem)
 827                        goto err_kcalloc_hem_buf;
 828
 829                if (check_whether_bt_num_3(type, hop_num)) {
 830                        unsigned long num_bt_l1;
 831
 832                        num_bt_l1 = DIV_ROUND_UP(num_hem, bt_chunk_num);
 833                        table->bt_l1 = kcalloc(num_bt_l1,
 834                                               sizeof(*table->bt_l1),
 835                                               GFP_KERNEL);
 836                        if (!table->bt_l1)
 837                                goto err_kcalloc_bt_l1;
 838
 839                        table->bt_l1_dma_addr = kcalloc(num_bt_l1,
 840                                                 sizeof(*table->bt_l1_dma_addr),
 841                                                 GFP_KERNEL);
 842
 843                        if (!table->bt_l1_dma_addr)
 844                                goto err_kcalloc_l1_dma;
 845                }
 846
 847                if (check_whether_bt_num_2(type, hop_num) ||
 848                        check_whether_bt_num_3(type, hop_num)) {
 849                        table->bt_l0 = kcalloc(num_bt_l0, sizeof(*table->bt_l0),
 850                                               GFP_KERNEL);
 851                        if (!table->bt_l0)
 852                                goto err_kcalloc_bt_l0;
 853
 854                        table->bt_l0_dma_addr = kcalloc(num_bt_l0,
 855                                                 sizeof(*table->bt_l0_dma_addr),
 856                                                 GFP_KERNEL);
 857                        if (!table->bt_l0_dma_addr)
 858                                goto err_kcalloc_l0_dma;
 859                }
 860        }
 861
 862        table->type = type;
 863        table->num_hem = num_hem;
 864        table->obj_size = obj_size;
 865        table->lowmem = use_lowmem;
 866        mutex_init(&table->mutex);
 867
 868        return 0;
 869
 870err_kcalloc_l0_dma:
 871        kfree(table->bt_l0);
 872        table->bt_l0 = NULL;
 873
 874err_kcalloc_bt_l0:
 875        kfree(table->bt_l1_dma_addr);
 876        table->bt_l1_dma_addr = NULL;
 877
 878err_kcalloc_l1_dma:
 879        kfree(table->bt_l1);
 880        table->bt_l1 = NULL;
 881
 882err_kcalloc_bt_l1:
 883        kfree(table->hem);
 884        table->hem = NULL;
 885
 886err_kcalloc_hem_buf:
 887        return -ENOMEM;
 888}
 889
 890static void hns_roce_cleanup_mhop_hem_table(struct hns_roce_dev *hr_dev,
 891                                            struct hns_roce_hem_table *table)
 892{
 893        struct hns_roce_hem_mhop mhop;
 894        u32 buf_chunk_size;
 895        u64 obj;
 896        int i;
 897
 898        if (hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop))
 899                return;
 900        buf_chunk_size = table->type < HEM_TYPE_MTT ? mhop.buf_chunk_size :
 901                                        mhop.bt_chunk_size;
 902
 903        for (i = 0; i < table->num_hem; ++i) {
 904                obj = i * buf_chunk_size / table->obj_size;
 905                if (table->hem[i])
 906                        hns_roce_table_mhop_put(hr_dev, table, obj, 0);
 907        }
 908
 909        kfree(table->hem);
 910        table->hem = NULL;
 911        kfree(table->bt_l1);
 912        table->bt_l1 = NULL;
 913        kfree(table->bt_l1_dma_addr);
 914        table->bt_l1_dma_addr = NULL;
 915        kfree(table->bt_l0);
 916        table->bt_l0 = NULL;
 917        kfree(table->bt_l0_dma_addr);
 918        table->bt_l0_dma_addr = NULL;
 919}
 920
 921void hns_roce_cleanup_hem_table(struct hns_roce_dev *hr_dev,
 922                                struct hns_roce_hem_table *table)
 923{
 924        struct device *dev = hr_dev->dev;
 925        unsigned long i;
 926
 927        if (hns_roce_check_whether_mhop(hr_dev, table->type)) {
 928                hns_roce_cleanup_mhop_hem_table(hr_dev, table);
 929                return;
 930        }
 931
 932        for (i = 0; i < table->num_hem; ++i)
 933                if (table->hem[i]) {
 934                        if (hr_dev->hw->clear_hem(hr_dev, table,
 935                            i * table->table_chunk_size / table->obj_size, 0))
 936                                dev_err(dev, "Clear HEM base address failed.\n");
 937
 938                        hns_roce_free_hem(hr_dev, table->hem[i]);
 939                }
 940
 941        kfree(table->hem);
 942}
 943
 944void hns_roce_cleanup_hem(struct hns_roce_dev *hr_dev)
 945{
 946        if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SRQ)
 947                hns_roce_cleanup_hem_table(hr_dev,
 948                                           &hr_dev->srq_table.table);
 949        hns_roce_cleanup_hem_table(hr_dev, &hr_dev->cq_table.table);
 950        if (hr_dev->caps.qpc_timer_entry_sz)
 951                hns_roce_cleanup_hem_table(hr_dev,
 952                                           &hr_dev->qpc_timer_table);
 953        if (hr_dev->caps.cqc_timer_entry_sz)
 954                hns_roce_cleanup_hem_table(hr_dev,
 955                                           &hr_dev->cqc_timer_table);
 956        if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL)
 957                hns_roce_cleanup_hem_table(hr_dev,
 958                                           &hr_dev->qp_table.sccc_table);
 959        if (hr_dev->caps.trrl_entry_sz)
 960                hns_roce_cleanup_hem_table(hr_dev,
 961                                           &hr_dev->qp_table.trrl_table);
 962
 963        if (hr_dev->caps.gmv_entry_sz)
 964                hns_roce_cleanup_hem_table(hr_dev, &hr_dev->gmv_table);
 965
 966        hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.irrl_table);
 967        hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.qp_table);
 968        hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtpt_table);
 969}
 970
 971struct hns_roce_hem_item {
 972        struct list_head list; /* link all hems in the same bt level */
 973        struct list_head sibling; /* link all hems in last hop for mtt */
 974        void *addr;
 975        dma_addr_t dma_addr;
 976        size_t count; /* max ba numbers */
 977        int start; /* start buf offset in this hem */
 978        int end; /* end buf offset in this hem */
 979};
 980
 981/* All HEM items are linked in a tree structure */
 982struct hns_roce_hem_head {
 983        struct list_head branch[HNS_ROCE_MAX_BT_REGION];
 984        struct list_head root;
 985        struct list_head leaf;
 986};
 987
 988static struct hns_roce_hem_item *
 989hem_list_alloc_item(struct hns_roce_dev *hr_dev, int start, int end, int count,
 990                    bool exist_bt, int bt_level)
 991{
 992        struct hns_roce_hem_item *hem;
 993
 994        hem = kzalloc(sizeof(*hem), GFP_KERNEL);
 995        if (!hem)
 996                return NULL;
 997
 998        if (exist_bt) {
 999                hem->addr = dma_alloc_coherent(hr_dev->dev, count * BA_BYTE_LEN,
1000                                               &hem->dma_addr, GFP_KERNEL);
1001                if (!hem->addr) {
1002                        kfree(hem);
1003                        return NULL;
1004                }
1005        }
1006
1007        hem->count = count;
1008        hem->start = start;
1009        hem->end = end;
1010        INIT_LIST_HEAD(&hem->list);
1011        INIT_LIST_HEAD(&hem->sibling);
1012
1013        return hem;
1014}
1015
1016static void hem_list_free_item(struct hns_roce_dev *hr_dev,
1017                               struct hns_roce_hem_item *hem, bool exist_bt)
1018{
1019        if (exist_bt)
1020                dma_free_coherent(hr_dev->dev, hem->count * BA_BYTE_LEN,
1021                                  hem->addr, hem->dma_addr);
1022        kfree(hem);
1023}
1024
1025static void hem_list_free_all(struct hns_roce_dev *hr_dev,
1026                              struct list_head *head, bool exist_bt)
1027{
1028        struct hns_roce_hem_item *hem, *temp_hem;
1029
1030        list_for_each_entry_safe(hem, temp_hem, head, list) {
1031                list_del(&hem->list);
1032                hem_list_free_item(hr_dev, hem, exist_bt);
1033        }
1034}
1035
1036static void hem_list_link_bt(struct hns_roce_dev *hr_dev, void *base_addr,
1037                             u64 table_addr)
1038{
1039        *(u64 *)(base_addr) = table_addr;
1040}
1041
1042/* assign L0 table address to hem from root bt */
1043static void hem_list_assign_bt(struct hns_roce_dev *hr_dev,
1044                               struct hns_roce_hem_item *hem, void *cpu_addr,
1045                               u64 phy_addr)
1046{
1047        hem->addr = cpu_addr;
1048        hem->dma_addr = (dma_addr_t)phy_addr;
1049}
1050
1051static inline bool hem_list_page_is_in_range(struct hns_roce_hem_item *hem,
1052                                             int offset)
1053{
1054        return (hem->start <= offset && offset <= hem->end);
1055}
1056
1057static struct hns_roce_hem_item *hem_list_search_item(struct list_head *ba_list,
1058                                                      int page_offset)
1059{
1060        struct hns_roce_hem_item *hem, *temp_hem;
1061        struct hns_roce_hem_item *found = NULL;
1062
1063        list_for_each_entry_safe(hem, temp_hem, ba_list, list) {
1064                if (hem_list_page_is_in_range(hem, page_offset)) {
1065                        found = hem;
1066                        break;
1067                }
1068        }
1069
1070        return found;
1071}
1072
1073static bool hem_list_is_bottom_bt(int hopnum, int bt_level)
1074{
1075        /*
1076         * hopnum    base address table levels
1077         * 0            L0(buf)
1078         * 1            L0 -> buf
1079         * 2            L0 -> L1 -> buf
1080         * 3            L0 -> L1 -> L2 -> buf
1081         */
1082        return bt_level >= (hopnum ? hopnum - 1 : hopnum);
1083}
1084
1085/*
1086 * calc base address entries num
1087 * @hopnum: num of mutihop addressing
1088 * @bt_level: base address table level
1089 * @unit: ba entries per bt page
1090 */
1091static u32 hem_list_calc_ba_range(int hopnum, int bt_level, int unit)
1092{
1093        u32 step;
1094        int max;
1095        int i;
1096
1097        if (hopnum <= bt_level)
1098                return 0;
1099        /*
1100         * hopnum  bt_level   range
1101         * 1          0       unit
1102         * ------------
1103         * 2          0       unit * unit
1104         * 2          1       unit
1105         * ------------
1106         * 3          0       unit * unit * unit
1107         * 3          1       unit * unit
1108         * 3          2       unit
1109         */
1110        step = 1;
1111        max = hopnum - bt_level;
1112        for (i = 0; i < max; i++)
1113                step = step * unit;
1114
1115        return step;
1116}
1117
1118/*
1119 * calc the root ba entries which could cover all regions
1120 * @regions: buf region array
1121 * @region_cnt: array size of @regions
1122 * @unit: ba entries per bt page
1123 */
1124int hns_roce_hem_list_calc_root_ba(const struct hns_roce_buf_region *regions,
1125                                   int region_cnt, int unit)
1126{
1127        struct hns_roce_buf_region *r;
1128        int total = 0;
1129        int step;
1130        int i;
1131
1132        for (i = 0; i < region_cnt; i++) {
1133                r = (struct hns_roce_buf_region *)&regions[i];
1134                if (r->hopnum > 1) {
1135                        step = hem_list_calc_ba_range(r->hopnum, 1, unit);
1136                        if (step > 0)
1137                                total += (r->count + step - 1) / step;
1138                } else {
1139                        total += r->count;
1140                }
1141        }
1142
1143        return total;
1144}
1145
1146static int hem_list_alloc_mid_bt(struct hns_roce_dev *hr_dev,
1147                                 const struct hns_roce_buf_region *r, int unit,
1148                                 int offset, struct list_head *mid_bt,
1149                                 struct list_head *btm_bt)
1150{
1151        struct hns_roce_hem_item *hem_ptrs[HNS_ROCE_MAX_BT_LEVEL] = { NULL };
1152        struct list_head temp_list[HNS_ROCE_MAX_BT_LEVEL];
1153        struct hns_roce_hem_item *cur, *pre;
1154        const int hopnum = r->hopnum;
1155        int start_aligned;
1156        int distance;
1157        int ret = 0;
1158        int max_ofs;
1159        int level;
1160        u32 step;
1161        int end;
1162
1163        if (hopnum <= 1)
1164                return 0;
1165
1166        if (hopnum > HNS_ROCE_MAX_BT_LEVEL) {
1167                dev_err(hr_dev->dev, "invalid hopnum %d!\n", hopnum);
1168                return -EINVAL;
1169        }
1170
1171        if (offset < r->offset) {
1172                dev_err(hr_dev->dev, "invalid offset %d, min %u!\n",
1173                        offset, r->offset);
1174                return -EINVAL;
1175        }
1176
1177        distance = offset - r->offset;
1178        max_ofs = r->offset + r->count - 1;
1179        for (level = 0; level < hopnum; level++)
1180                INIT_LIST_HEAD(&temp_list[level]);
1181
1182        /* config L1 bt to last bt and link them to corresponding parent */
1183        for (level = 1; level < hopnum; level++) {
1184                cur = hem_list_search_item(&mid_bt[level], offset);
1185                if (cur) {
1186                        hem_ptrs[level] = cur;
1187                        continue;
1188                }
1189
1190                step = hem_list_calc_ba_range(hopnum, level, unit);
1191                if (step < 1) {
1192                        ret = -EINVAL;
1193                        goto err_exit;
1194                }
1195
1196                start_aligned = (distance / step) * step + r->offset;
1197                end = min_t(int, start_aligned + step - 1, max_ofs);
1198                cur = hem_list_alloc_item(hr_dev, start_aligned, end, unit,
1199                                          true, level);
1200                if (!cur) {
1201                        ret = -ENOMEM;
1202                        goto err_exit;
1203                }
1204                hem_ptrs[level] = cur;
1205                list_add(&cur->list, &temp_list[level]);
1206                if (hem_list_is_bottom_bt(hopnum, level))
1207                        list_add(&cur->sibling, &temp_list[0]);
1208
1209                /* link bt to parent bt */
1210                if (level > 1) {
1211                        pre = hem_ptrs[level - 1];
1212                        step = (cur->start - pre->start) / step * BA_BYTE_LEN;
1213                        hem_list_link_bt(hr_dev, pre->addr + step,
1214                                         cur->dma_addr);
1215                }
1216        }
1217
1218        list_splice(&temp_list[0], btm_bt);
1219        for (level = 1; level < hopnum; level++)
1220                list_splice(&temp_list[level], &mid_bt[level]);
1221
1222        return 0;
1223
1224err_exit:
1225        for (level = 1; level < hopnum; level++)
1226                hem_list_free_all(hr_dev, &temp_list[level], true);
1227
1228        return ret;
1229}
1230
1231static struct hns_roce_hem_item *
1232alloc_root_hem(struct hns_roce_dev *hr_dev, int unit, int *max_ba_num,
1233               const struct hns_roce_buf_region *regions, int region_cnt)
1234{
1235        const struct hns_roce_buf_region *r;
1236        struct hns_roce_hem_item *hem;
1237        int ba_num;
1238        int offset;
1239
1240        ba_num = hns_roce_hem_list_calc_root_ba(regions, region_cnt, unit);
1241        if (ba_num < 1)
1242                return ERR_PTR(-ENOMEM);
1243
1244        if (ba_num > unit)
1245                return ERR_PTR(-ENOBUFS);
1246
1247        offset = regions[0].offset;
1248        /* indicate to last region */
1249        r = &regions[region_cnt - 1];
1250        hem = hem_list_alloc_item(hr_dev, offset, r->offset + r->count - 1,
1251                                  ba_num, true, 0);
1252        if (!hem)
1253                return ERR_PTR(-ENOMEM);
1254
1255        *max_ba_num = ba_num;
1256
1257        return hem;
1258}
1259
1260static int alloc_fake_root_bt(struct hns_roce_dev *hr_dev, void *cpu_base,
1261                              u64 phy_base, const struct hns_roce_buf_region *r,
1262                              struct list_head *branch_head,
1263                              struct list_head *leaf_head)
1264{
1265        struct hns_roce_hem_item *hem;
1266
1267        hem = hem_list_alloc_item(hr_dev, r->offset, r->offset + r->count - 1,
1268                                  r->count, false, 0);
1269        if (!hem)
1270                return -ENOMEM;
1271
1272        hem_list_assign_bt(hr_dev, hem, cpu_base, phy_base);
1273        list_add(&hem->list, branch_head);
1274        list_add(&hem->sibling, leaf_head);
1275
1276        return r->count;
1277}
1278
1279static int setup_middle_bt(struct hns_roce_dev *hr_dev, void *cpu_base,
1280                           int unit, const struct hns_roce_buf_region *r,
1281                           const struct list_head *branch_head)
1282{
1283        struct hns_roce_hem_item *hem, *temp_hem;
1284        int total = 0;
1285        int offset;
1286        int step;
1287
1288        step = hem_list_calc_ba_range(r->hopnum, 1, unit);
1289        if (step < 1)
1290                return -EINVAL;
1291
1292        /* if exist mid bt, link L1 to L0 */
1293        list_for_each_entry_safe(hem, temp_hem, branch_head, list) {
1294                offset = (hem->start - r->offset) / step * BA_BYTE_LEN;
1295                hem_list_link_bt(hr_dev, cpu_base + offset, hem->dma_addr);
1296                total++;
1297        }
1298
1299        return total;
1300}
1301
1302static int
1303setup_root_hem(struct hns_roce_dev *hr_dev, struct hns_roce_hem_list *hem_list,
1304               int unit, int max_ba_num, struct hns_roce_hem_head *head,
1305               const struct hns_roce_buf_region *regions, int region_cnt)
1306{
1307        const struct hns_roce_buf_region *r;
1308        struct hns_roce_hem_item *root_hem;
1309        void *cpu_base;
1310        u64 phy_base;
1311        int i, total;
1312        int ret;
1313
1314        root_hem = list_first_entry(&head->root,
1315                                    struct hns_roce_hem_item, list);
1316        if (!root_hem)
1317                return -ENOMEM;
1318
1319        total = 0;
1320        for (i = 0; i < region_cnt && total < max_ba_num; i++) {
1321                r = &regions[i];
1322                if (!r->count)
1323                        continue;
1324
1325                /* all regions's mid[x][0] shared the root_bt's trunk */
1326                cpu_base = root_hem->addr + total * BA_BYTE_LEN;
1327                phy_base = root_hem->dma_addr + total * BA_BYTE_LEN;
1328
1329                /* if hopnum is 0 or 1, cut a new fake hem from the root bt
1330                 * which's address share to all regions.
1331                 */
1332                if (hem_list_is_bottom_bt(r->hopnum, 0))
1333                        ret = alloc_fake_root_bt(hr_dev, cpu_base, phy_base, r,
1334                                                 &head->branch[i], &head->leaf);
1335                else
1336                        ret = setup_middle_bt(hr_dev, cpu_base, unit, r,
1337                                              &hem_list->mid_bt[i][1]);
1338
1339                if (ret < 0)
1340                        return ret;
1341
1342                total += ret;
1343        }
1344
1345        list_splice(&head->leaf, &hem_list->btm_bt);
1346        list_splice(&head->root, &hem_list->root_bt);
1347        for (i = 0; i < region_cnt; i++)
1348                list_splice(&head->branch[i], &hem_list->mid_bt[i][0]);
1349
1350        return 0;
1351}
1352
1353static int hem_list_alloc_root_bt(struct hns_roce_dev *hr_dev,
1354                                  struct hns_roce_hem_list *hem_list, int unit,
1355                                  const struct hns_roce_buf_region *regions,
1356                                  int region_cnt)
1357{
1358        struct hns_roce_hem_item *root_hem;
1359        struct hns_roce_hem_head head;
1360        int max_ba_num;
1361        int ret;
1362        int i;
1363
1364        root_hem = hem_list_search_item(&hem_list->root_bt, regions[0].offset);
1365        if (root_hem)
1366                return 0;
1367
1368        max_ba_num = 0;
1369        root_hem = alloc_root_hem(hr_dev, unit, &max_ba_num, regions,
1370                                  region_cnt);
1371        if (IS_ERR(root_hem))
1372                return PTR_ERR(root_hem);
1373
1374        /* List head for storing all allocated HEM items */
1375        INIT_LIST_HEAD(&head.root);
1376        INIT_LIST_HEAD(&head.leaf);
1377        for (i = 0; i < region_cnt; i++)
1378                INIT_LIST_HEAD(&head.branch[i]);
1379
1380        hem_list->root_ba = root_hem->dma_addr;
1381        list_add(&root_hem->list, &head.root);
1382        ret = setup_root_hem(hr_dev, hem_list, unit, max_ba_num, &head, regions,
1383                             region_cnt);
1384        if (ret) {
1385                for (i = 0; i < region_cnt; i++)
1386                        hem_list_free_all(hr_dev, &head.branch[i], false);
1387
1388                hem_list_free_all(hr_dev, &head.root, true);
1389        }
1390
1391        return ret;
1392}
1393
1394/* construct the base address table and link them by address hop config */
1395int hns_roce_hem_list_request(struct hns_roce_dev *hr_dev,
1396                              struct hns_roce_hem_list *hem_list,
1397                              const struct hns_roce_buf_region *regions,
1398                              int region_cnt, unsigned int bt_pg_shift)
1399{
1400        const struct hns_roce_buf_region *r;
1401        int ofs, end;
1402        int unit;
1403        int ret;
1404        int i;
1405
1406        if (region_cnt > HNS_ROCE_MAX_BT_REGION) {
1407                dev_err(hr_dev->dev, "invalid region region_cnt %d!\n",
1408                        region_cnt);
1409                return -EINVAL;
1410        }
1411
1412        unit = (1 << bt_pg_shift) / BA_BYTE_LEN;
1413        for (i = 0; i < region_cnt; i++) {
1414                r = &regions[i];
1415                if (!r->count)
1416                        continue;
1417
1418                end = r->offset + r->count;
1419                for (ofs = r->offset; ofs < end; ofs += unit) {
1420                        ret = hem_list_alloc_mid_bt(hr_dev, r, unit, ofs,
1421                                                    hem_list->mid_bt[i],
1422                                                    &hem_list->btm_bt);
1423                        if (ret) {
1424                                dev_err(hr_dev->dev,
1425                                        "alloc hem trunk fail ret=%d!\n", ret);
1426                                goto err_alloc;
1427                        }
1428                }
1429        }
1430
1431        ret = hem_list_alloc_root_bt(hr_dev, hem_list, unit, regions,
1432                                     region_cnt);
1433        if (ret)
1434                dev_err(hr_dev->dev, "alloc hem root fail ret=%d!\n", ret);
1435        else
1436                return 0;
1437
1438err_alloc:
1439        hns_roce_hem_list_release(hr_dev, hem_list);
1440
1441        return ret;
1442}
1443
1444void hns_roce_hem_list_release(struct hns_roce_dev *hr_dev,
1445                               struct hns_roce_hem_list *hem_list)
1446{
1447        int i, j;
1448
1449        for (i = 0; i < HNS_ROCE_MAX_BT_REGION; i++)
1450                for (j = 0; j < HNS_ROCE_MAX_BT_LEVEL; j++)
1451                        hem_list_free_all(hr_dev, &hem_list->mid_bt[i][j],
1452                                          j != 0);
1453
1454        hem_list_free_all(hr_dev, &hem_list->root_bt, true);
1455        INIT_LIST_HEAD(&hem_list->btm_bt);
1456        hem_list->root_ba = 0;
1457}
1458
1459void hns_roce_hem_list_init(struct hns_roce_hem_list *hem_list)
1460{
1461        int i, j;
1462
1463        INIT_LIST_HEAD(&hem_list->root_bt);
1464        INIT_LIST_HEAD(&hem_list->btm_bt);
1465        for (i = 0; i < HNS_ROCE_MAX_BT_REGION; i++)
1466                for (j = 0; j < HNS_ROCE_MAX_BT_LEVEL; j++)
1467                        INIT_LIST_HEAD(&hem_list->mid_bt[i][j]);
1468}
1469
1470void *hns_roce_hem_list_find_mtt(struct hns_roce_dev *hr_dev,
1471                                 struct hns_roce_hem_list *hem_list,
1472                                 int offset, int *mtt_cnt, u64 *phy_addr)
1473{
1474        struct list_head *head = &hem_list->btm_bt;
1475        struct hns_roce_hem_item *hem, *temp_hem;
1476        void *cpu_base = NULL;
1477        u64 phy_base = 0;
1478        int nr = 0;
1479
1480        list_for_each_entry_safe(hem, temp_hem, head, sibling) {
1481                if (hem_list_page_is_in_range(hem, offset)) {
1482                        nr = offset - hem->start;
1483                        cpu_base = hem->addr + nr * BA_BYTE_LEN;
1484                        phy_base = hem->dma_addr + nr * BA_BYTE_LEN;
1485                        nr = hem->end + 1 - offset;
1486                        break;
1487                }
1488        }
1489
1490        if (mtt_cnt)
1491                *mtt_cnt = nr;
1492
1493        if (phy_addr)
1494                *phy_addr = phy_base;
1495
1496        return cpu_base;
1497}
1498