drivers/infiniband/hw/hns/hns_roce_hem.c

   1 /*
   2  * Copyright (c) 2016 Hisilicon Limited.
   3  * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved.
   4  *
   5  * This software is available to you under a choice of one of two
   6  * licenses.  You may choose to be licensed under the terms of the GNU
   7  * General Public License (GPL) Version 2, available from the file
   8  * COPYING in the main directory of this source tree, or the
   9  * OpenIB.org BSD license below:
  10  *
  11  *     Redistribution and use in source and binary forms, with or
  12  *     without modification, are permitted provided that the following
  13  *     conditions are met:
  14  *
  15  *      - Redistributions of source code must retain the above
  16  *        copyright notice, this list of conditions and the following
  17  *        disclaimer.
  18  *
  19  *      - Redistributions in binary form must reproduce the above
  20  *        copyright notice, this list of conditions and the following
  21  *        disclaimer in the documentation and/or other materials
  22  *        provided with the distribution.
  23  *
  24  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  25  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  26  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  27  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  28  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  29  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  30  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  31  * SOFTWARE.
  32  */
  33
  34 #include <linux/platform_device.h>
  35 #include "hns_roce_device.h"
  36 #include "hns_roce_hem.h"
  37 #include "hns_roce_common.h"
  38
  39 #define HEM_INDEX_BUF                   BIT(0)
  40 #define HEM_INDEX_L0                    BIT(1)
  41 #define HEM_INDEX_L1                    BIT(2)
  42 struct hns_roce_hem_index {
  43         u64 buf;
  44         u64 l0;
  45         u64 l1;
  46         u32 inited; /* indicate which index is available */
  47 };
  48
  49 bool hns_roce_check_whether_mhop(struct hns_roce_dev *hr_dev, u32 type)
  50 {
  51         int hop_num = 0;
  52
  53         switch (type) {
  54         case HEM_TYPE_QPC:
  55                 hop_num = hr_dev->caps.qpc_hop_num;
  56                 break;
  57         case HEM_TYPE_MTPT:
  58                 hop_num = hr_dev->caps.mpt_hop_num;
  59                 break;
  60         case HEM_TYPE_CQC:
  61                 hop_num = hr_dev->caps.cqc_hop_num;
  62                 break;
  63         case HEM_TYPE_SRQC:
  64                 hop_num = hr_dev->caps.srqc_hop_num;
  65                 break;
  66         case HEM_TYPE_SCCC:
  67                 hop_num = hr_dev->caps.sccc_hop_num;
  68                 break;
  69         case HEM_TYPE_QPC_TIMER:
  70                 hop_num = hr_dev->caps.qpc_timer_hop_num;
  71                 break;
  72         case HEM_TYPE_CQC_TIMER:
  73                 hop_num = hr_dev->caps.cqc_timer_hop_num;
  74                 break;
  75         case HEM_TYPE_GMV:
  76                 hop_num = hr_dev->caps.gmv_hop_num;
  77                 break;
  78         default:
  79                 return false;
  80         }
  81
  82         return hop_num ? true : false;
  83 }
  84
  85 static bool hns_roce_check_hem_null(struct hns_roce_hem **hem, u64 hem_idx,
  86                                     u32 bt_chunk_num, u64 hem_max_num)
  87 {
  88         u64 start_idx = round_down(hem_idx, bt_chunk_num);
  89         u64 check_max_num = start_idx + bt_chunk_num;
  90         u64 i;
  91
  92         for (i = start_idx; (i < check_max_num) && (i < hem_max_num); i++)
  93                 if (i != hem_idx && hem[i])
  94                         return false;
  95
  96         return true;
  97 }
  98
  99 static bool hns_roce_check_bt_null(u64 **bt, u64 ba_idx, u32 bt_chunk_num)
 100 {
 101         u64 start_idx = round_down(ba_idx, bt_chunk_num);
 102         int i;
 103
 104         for (i = 0; i < bt_chunk_num; i++)
 105                 if (i != ba_idx && bt[start_idx + i])
 106                         return false;
 107
 108         return true;
 109 }
 110
 111 static int hns_roce_get_bt_num(u32 table_type, u32 hop_num)
 112 {
 113         if (check_whether_bt_num_3(table_type, hop_num))
 114                 return 3;
 115         else if (check_whether_bt_num_2(table_type, hop_num))
 116                 return 2;
 117         else if (check_whether_bt_num_1(table_type, hop_num))
 118                 return 1;
 119         else
 120                 return 0;
 121 }
 122
 123 static int get_hem_table_config(struct hns_roce_dev *hr_dev,
 124                                 struct hns_roce_hem_mhop *mhop,
 125                                 u32 type)
 126 {
 127         struct device *dev = hr_dev->dev;
 128
 129         switch (type) {
 130         case HEM_TYPE_QPC:
 131                 mhop->buf_chunk_size = 1 << (hr_dev->caps.qpc_buf_pg_sz
 132                                              + PAGE_SHIFT);
 133                 mhop->bt_chunk_size = 1 << (hr_dev->caps.qpc_ba_pg_sz
 134                                              + PAGE_SHIFT);
 135                 mhop->ba_l0_num = hr_dev->caps.qpc_bt_num;
 136                 mhop->hop_num = hr_dev->caps.qpc_hop_num;
 137                 break;
 138         case HEM_TYPE_MTPT:
 139                 mhop->buf_chunk_size = 1 << (hr_dev->caps.mpt_buf_pg_sz
 140                                              + PAGE_SHIFT);
 141                 mhop->bt_chunk_size = 1 << (hr_dev->caps.mpt_ba_pg_sz
 142                                              + PAGE_SHIFT);
 143                 mhop->ba_l0_num = hr_dev->caps.mpt_bt_num;
 144                 mhop->hop_num = hr_dev->caps.mpt_hop_num;
 145                 break;
 146         case HEM_TYPE_CQC:
 147                 mhop->buf_chunk_size = 1 << (hr_dev->caps.cqc_buf_pg_sz
 148                                              + PAGE_SHIFT);
 149                 mhop->bt_chunk_size = 1 << (hr_dev->caps.cqc_ba_pg_sz
 150                                             + PAGE_SHIFT);
 151                 mhop->ba_l0_num = hr_dev->caps.cqc_bt_num;
 152                 mhop->hop_num = hr_dev->caps.cqc_hop_num;
 153                 break;
 154         case HEM_TYPE_SCCC:
 155                 mhop->buf_chunk_size = 1 << (hr_dev->caps.sccc_buf_pg_sz
 156                                              + PAGE_SHIFT);
 157                 mhop->bt_chunk_size = 1 << (hr_dev->caps.sccc_ba_pg_sz
 158                                             + PAGE_SHIFT);
 159                 mhop->ba_l0_num = hr_dev->caps.sccc_bt_num;
 160                 mhop->hop_num = hr_dev->caps.sccc_hop_num;
 161                 break;
 162         case HEM_TYPE_QPC_TIMER:
 163                 mhop->buf_chunk_size = 1 << (hr_dev->caps.qpc_timer_buf_pg_sz
 164                                              + PAGE_SHIFT);
 165                 mhop->bt_chunk_size = 1 << (hr_dev->caps.qpc_timer_ba_pg_sz
 166                                             + PAGE_SHIFT);
 167                 mhop->ba_l0_num = hr_dev->caps.qpc_timer_bt_num;
 168                 mhop->hop_num = hr_dev->caps.qpc_timer_hop_num;
 169                 break;
 170         case HEM_TYPE_CQC_TIMER:
 171                 mhop->buf_chunk_size = 1 << (hr_dev->caps.cqc_timer_buf_pg_sz
 172                                              + PAGE_SHIFT);
 173                 mhop->bt_chunk_size = 1 << (hr_dev->caps.cqc_timer_ba_pg_sz
 174                                             + PAGE_SHIFT);
 175                 mhop->ba_l0_num = hr_dev->caps.cqc_timer_bt_num;
 176                 mhop->hop_num = hr_dev->caps.cqc_timer_hop_num;
 177                 break;
 178         case HEM_TYPE_SRQC:
 179                 mhop->buf_chunk_size = 1 << (hr_dev->caps.srqc_buf_pg_sz
 180                                              + PAGE_SHIFT);
 181                 mhop->bt_chunk_size = 1 << (hr_dev->caps.srqc_ba_pg_sz
 182                                              + PAGE_SHIFT);
 183                 mhop->ba_l0_num = hr_dev->caps.srqc_bt_num;
 184                 mhop->hop_num = hr_dev->caps.srqc_hop_num;
 185                 break;
 186         case HEM_TYPE_GMV:
 187                 mhop->buf_chunk_size = 1 << (hr_dev->caps.gmv_buf_pg_sz +
 188                                              PAGE_SHIFT);
 189                 mhop->bt_chunk_size = 1 << (hr_dev->caps.gmv_ba_pg_sz +
 190                                             PAGE_SHIFT);
 191                 mhop->ba_l0_num = hr_dev->caps.gmv_bt_num;
 192                 mhop->hop_num = hr_dev->caps.gmv_hop_num;
 193                 break;
 194         default:
 195                 dev_err(dev, "table %u not support multi-hop addressing!\n",
 196                         type);
 197                 return -EINVAL;
 198         }
 199
 200         return 0;
 201 }
 202
 203 int hns_roce_calc_hem_mhop(struct hns_roce_dev *hr_dev,
 204                            struct hns_roce_hem_table *table, unsigned long *obj,
 205                            struct hns_roce_hem_mhop *mhop)
 206 {
 207         struct device *dev = hr_dev->dev;
 208         u32 chunk_ba_num;
 209         u32 chunk_size;
 210         u32 table_idx;
 211         u32 bt_num;
 212
 213         if (get_hem_table_config(hr_dev, mhop, table->type))
 214                 return -EINVAL;
 215
 216         if (!obj)
 217                 return 0;
 218
 219         /*
 220          * QPC/MTPT/CQC/SRQC/SCCC alloc hem for buffer pages.
 221          * MTT/CQE alloc hem for bt pages.
 222          */
 223         bt_num = hns_roce_get_bt_num(table->type, mhop->hop_num);
 224         chunk_ba_num = mhop->bt_chunk_size / BA_BYTE_LEN;
 225         chunk_size = table->type < HEM_TYPE_MTT ? mhop->buf_chunk_size :
 226                               mhop->bt_chunk_size;
 227         table_idx = *obj / (chunk_size / table->obj_size);
 228         switch (bt_num) {
 229         case 3:
 230                 mhop->l2_idx = table_idx & (chunk_ba_num - 1);
 231                 mhop->l1_idx = table_idx / chunk_ba_num & (chunk_ba_num - 1);
 232                 mhop->l0_idx = (table_idx / chunk_ba_num) / chunk_ba_num;
 233                 break;
 234         case 2:
 235                 mhop->l1_idx = table_idx & (chunk_ba_num - 1);
 236                 mhop->l0_idx = table_idx / chunk_ba_num;
 237                 break;
 238         case 1:
 239                 mhop->l0_idx = table_idx;
 240                 break;
 241         default:
 242                 dev_err(dev, "table %u not support hop_num = %u!\n",
 243                         table->type, mhop->hop_num);
 244                 return -EINVAL;
 245         }
 246         if (mhop->l0_idx >= mhop->ba_l0_num)
 247                 mhop->l0_idx %= mhop->ba_l0_num;
 248
 249         return 0;
 250 }
 251
 252 static struct hns_roce_hem *hns_roce_alloc_hem(struct hns_roce_dev *hr_dev,
 253                                                int npages,
 254                                                unsigned long hem_alloc_size,
 255                                                gfp_t gfp_mask)
 256 {
 257         struct hns_roce_hem_chunk *chunk = NULL;
 258         struct hns_roce_hem *hem;
 259         struct scatterlist *mem;
 260         int order;
 261         void *buf;
 262
 263         WARN_ON(gfp_mask & __GFP_HIGHMEM);
 264
 265         hem = kmalloc(sizeof(*hem),
 266                       gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN));
 267         if (!hem)
 268                 return NULL;
 269
 270         INIT_LIST_HEAD(&hem->chunk_list);
 271
 272         order = get_order(hem_alloc_size);
 273
 274         while (npages > 0) {
 275                 if (!chunk) {
 276                         chunk = kmalloc(sizeof(*chunk),
 277                                 gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN));
 278                         if (!chunk)
 279                                 goto fail;
 280
 281                         sg_init_table(chunk->mem, HNS_ROCE_HEM_CHUNK_LEN);
 282                         chunk->npages = 0;
 283                         chunk->nsg = 0;
 284                         memset(chunk->buf, 0, sizeof(chunk->buf));
 285                         list_add_tail(&chunk->list, &hem->chunk_list);
 286                 }
 287
 288                 while (1 << order > npages)
 289                         --order;
 290
 291                 /*
 292                  * Alloc memory one time. If failed, don't alloc small block
 293                  * memory, directly return fail.
 294                  */
 295                 mem = &chunk->mem[chunk->npages];
 296                 buf = dma_alloc_coherent(hr_dev->dev, PAGE_SIZE << order,
 297                                 &sg_dma_address(mem), gfp_mask);
 298                 if (!buf)
 299                         goto fail;
 300
 301                 chunk->buf[chunk->npages] = buf;
 302                 sg_dma_len(mem) = PAGE_SIZE << order;
 303
 304                 ++chunk->npages;
 305                 ++chunk->nsg;
 306                 npages -= 1 << order;
 307         }
 308
 309         return hem;
 310
 311 fail:
 312         hns_roce_free_hem(hr_dev, hem);
 313         return NULL;
 314 }
 315
 316 void hns_roce_free_hem(struct hns_roce_dev *hr_dev, struct hns_roce_hem *hem)
 317 {
 318         struct hns_roce_hem_chunk *chunk, *tmp;
 319         int i;
 320
 321         if (!hem)
 322                 return;
 323
 324         list_for_each_entry_safe(chunk, tmp, &hem->chunk_list, list) {
 325                 for (i = 0; i < chunk->npages; ++i)
 326                         dma_free_coherent(hr_dev->dev,
 327                                    sg_dma_len(&chunk->mem[i]),
 328                                    chunk->buf[i],
 329                                    sg_dma_address(&chunk->mem[i]));
 330                 kfree(chunk);
 331         }
 332
 333         kfree(hem);
 334 }
 335
 336 static int calc_hem_config(struct hns_roce_dev *hr_dev,
 337                            struct hns_roce_hem_table *table, unsigned long obj,
 338                            struct hns_roce_hem_mhop *mhop,
 339                            struct hns_roce_hem_index *index)
 340 {
 341         struct ib_device *ibdev = &hr_dev->ib_dev;
 342         unsigned long mhop_obj = obj;
 343         u32 l0_idx, l1_idx, l2_idx;
 344         u32 chunk_ba_num;
 345         u32 bt_num;
 346         int ret;
 347
 348         ret = hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, mhop);
 349         if (ret)
 350                 return ret;
 351
 352         l0_idx = mhop->l0_idx;
 353         l1_idx = mhop->l1_idx;
 354         l2_idx = mhop->l2_idx;
 355         chunk_ba_num = mhop->bt_chunk_size / BA_BYTE_LEN;
 356         bt_num = hns_roce_get_bt_num(table->type, mhop->hop_num);
 357         switch (bt_num) {
 358         case 3:
 359                 index->l1 = l0_idx * chunk_ba_num + l1_idx;
 360                 index->l0 = l0_idx;
 361                 index->buf = l0_idx * chunk_ba_num * chunk_ba_num +
 362                              l1_idx * chunk_ba_num + l2_idx;
 363                 break;
 364         case 2:
 365                 index->l0 = l0_idx;
 366                 index->buf = l0_idx * chunk_ba_num + l1_idx;
 367                 break;
 368         case 1:
 369                 index->buf = l0_idx;
 370                 break;
 371         default:
 372                 ibdev_err(ibdev, "table %u not support mhop.hop_num = %u!\n",
 373                           table->type, mhop->hop_num);
 374                 return -EINVAL;
 375         }
 376
 377         if (unlikely(index->buf >= table->num_hem)) {
 378                 ibdev_err(ibdev, "table %u exceed hem limt idx %llu, max %lu!\n",
 379                           table->type, index->buf, table->num_hem);
 380                 return -EINVAL;
 381         }
 382
 383         return 0;
 384 }
 385
 386 static void free_mhop_hem(struct hns_roce_dev *hr_dev,
 387                           struct hns_roce_hem_table *table,
 388                           struct hns_roce_hem_mhop *mhop,
 389                           struct hns_roce_hem_index *index)
 390 {
 391         u32 bt_size = mhop->bt_chunk_size;
 392         struct device *dev = hr_dev->dev;
 393
 394         if (index->inited & HEM_INDEX_BUF) {
 395                 hns_roce_free_hem(hr_dev, table->hem[index->buf]);
 396                 table->hem[index->buf] = NULL;
 397         }
 398
 399         if (index->inited & HEM_INDEX_L1) {
 400                 dma_free_coherent(dev, bt_size, table->bt_l1[index->l1],
 401                                   table->bt_l1_dma_addr[index->l1]);
 402                 table->bt_l1[index->l1] = NULL;
 403         }
 404
 405         if (index->inited & HEM_INDEX_L0) {
 406                 dma_free_coherent(dev, bt_size, table->bt_l0[index->l0],
 407                                   table->bt_l0_dma_addr[index->l0]);
 408                 table->bt_l0[index->l0] = NULL;
 409         }
 410 }
 411
 412 static int alloc_mhop_hem(struct hns_roce_dev *hr_dev,
 413                           struct hns_roce_hem_table *table,
 414                           struct hns_roce_hem_mhop *mhop,
 415                           struct hns_roce_hem_index *index)
 416 {
 417         u32 bt_size = mhop->bt_chunk_size;
 418         struct device *dev = hr_dev->dev;
 419         struct hns_roce_hem_iter iter;
 420         gfp_t flag;
 421         u64 bt_ba;
 422         u32 size;
 423         int ret;
 424
 425         /* alloc L1 BA's chunk */
 426         if ((check_whether_bt_num_3(table->type, mhop->hop_num) ||
 427              check_whether_bt_num_2(table->type, mhop->hop_num)) &&
 428              !table->bt_l0[index->l0]) {
 429                 table->bt_l0[index->l0] = dma_alloc_coherent(dev, bt_size,
 430                                             &table->bt_l0_dma_addr[index->l0],
 431                                             GFP_KERNEL);
 432                 if (!table->bt_l0[index->l0]) {
 433                         ret = -ENOMEM;
 434                         goto out;
 435                 }
 436                 index->inited |= HEM_INDEX_L0;
 437         }
 438
 439         /* alloc L2 BA's chunk */
 440         if (check_whether_bt_num_3(table->type, mhop->hop_num) &&
 441             !table->bt_l1[index->l1])  {
 442                 table->bt_l1[index->l1] = dma_alloc_coherent(dev, bt_size,
 443                                             &table->bt_l1_dma_addr[index->l1],
 444                                             GFP_KERNEL);
 445                 if (!table->bt_l1[index->l1]) {
 446                         ret = -ENOMEM;
 447                         goto err_alloc_hem;
 448                 }
 449                 index->inited |= HEM_INDEX_L1;
 450                 *(table->bt_l0[index->l0] + mhop->l1_idx) =
 451                                                table->bt_l1_dma_addr[index->l1];
 452         }
 453
 454         /*
 455          * alloc buffer space chunk for QPC/MTPT/CQC/SRQC/SCCC.
 456          * alloc bt space chunk for MTT/CQE.
 457          */
 458         size = table->type < HEM_TYPE_MTT ? mhop->buf_chunk_size : bt_size;
 459         flag = (table->lowmem ? GFP_KERNEL : GFP_HIGHUSER) | __GFP_NOWARN;
 460         table->hem[index->buf] = hns_roce_alloc_hem(hr_dev, size >> PAGE_SHIFT,
 461                                                     size, flag);
 462         if (!table->hem[index->buf]) {
 463                 ret = -ENOMEM;
 464                 goto err_alloc_hem;
 465         }
 466
 467         index->inited |= HEM_INDEX_BUF;
 468         hns_roce_hem_first(table->hem[index->buf], &iter);
 469         bt_ba = hns_roce_hem_addr(&iter);
 470         if (table->type < HEM_TYPE_MTT) {
 471                 if (mhop->hop_num == 2)
 472                         *(table->bt_l1[index->l1] + mhop->l2_idx) = bt_ba;
 473                 else if (mhop->hop_num == 1)
 474                         *(table->bt_l0[index->l0] + mhop->l1_idx) = bt_ba;
 475         } else if (mhop->hop_num == 2) {
 476                 *(table->bt_l0[index->l0] + mhop->l1_idx) = bt_ba;
 477         }
 478
 479         return 0;
 480 err_alloc_hem:
 481         free_mhop_hem(hr_dev, table, mhop, index);
 482 out:
 483         return ret;
 484 }
 485
 486 static int set_mhop_hem(struct hns_roce_dev *hr_dev,
 487                         struct hns_roce_hem_table *table, unsigned long obj,
 488                         struct hns_roce_hem_mhop *mhop,
 489                         struct hns_roce_hem_index *index)
 490 {
 491         struct ib_device *ibdev = &hr_dev->ib_dev;
 492         int step_idx;
 493         int ret = 0;
 494
 495         if (index->inited & HEM_INDEX_L0) {
 496                 ret = hr_dev->hw->set_hem(hr_dev, table, obj, 0);
 497                 if (ret) {
 498                         ibdev_err(ibdev, "set HEM step 0 failed!\n");
 499                         goto out;
 500                 }
 501         }
 502
 503         if (index->inited & HEM_INDEX_L1) {
 504                 ret = hr_dev->hw->set_hem(hr_dev, table, obj, 1);
 505                 if (ret) {
 506                         ibdev_err(ibdev, "set HEM step 1 failed!\n");
 507                         goto out;
 508                 }
 509         }
 510
 511         if (index->inited & HEM_INDEX_BUF) {
 512                 if (mhop->hop_num == HNS_ROCE_HOP_NUM_0)
 513                         step_idx = 0;
 514                 else
 515                         step_idx = mhop->hop_num;
 516                 ret = hr_dev->hw->set_hem(hr_dev, table, obj, step_idx);
 517                 if (ret)
 518                         ibdev_err(ibdev, "set HEM step last failed!\n");
 519         }
 520 out:
 521         return ret;
 522 }
 523
 524 static int hns_roce_table_mhop_get(struct hns_roce_dev *hr_dev,
 525                                    struct hns_roce_hem_table *table,
 526                                    unsigned long obj)
 527 {
 528         struct ib_device *ibdev = &hr_dev->ib_dev;
 529         struct hns_roce_hem_index index = {};
 530         struct hns_roce_hem_mhop mhop = {};
 531         int ret;
 532
 533         ret = calc_hem_config(hr_dev, table, obj, &mhop, &index);
 534         if (ret) {
 535                 ibdev_err(ibdev, "calc hem config failed!\n");
 536                 return ret;
 537         }
 538
 539         mutex_lock(&table->mutex);
 540         if (table->hem[index.buf]) {
 541                 refcount_inc(&table->hem[index.buf]->refcount);
 542                 goto out;
 543         }
 544
 545         ret = alloc_mhop_hem(hr_dev, table, &mhop, &index);
 546         if (ret) {
 547                 ibdev_err(ibdev, "alloc mhop hem failed!\n");
 548                 goto out;
 549         }
 550
 551         /* set HEM base address to hardware */
 552         if (table->type < HEM_TYPE_MTT) {
 553                 ret = set_mhop_hem(hr_dev, table, obj, &mhop, &index);
 554                 if (ret) {
 555                         ibdev_err(ibdev, "set HEM address to HW failed!\n");
 556                         goto err_alloc;
 557                 }
 558         }
 559
 560         refcount_set(&table->hem[index.buf]->refcount, 1);
 561         goto out;
 562
 563 err_alloc:
 564         free_mhop_hem(hr_dev, table, &mhop, &index);
 565 out:
 566         mutex_unlock(&table->mutex);
 567         return ret;
 568 }
 569
 570 int hns_roce_table_get(struct hns_roce_dev *hr_dev,
 571                        struct hns_roce_hem_table *table, unsigned long obj)
 572 {
 573         struct device *dev = hr_dev->dev;
 574         unsigned long i;
 575         int ret = 0;
 576
 577         if (hns_roce_check_whether_mhop(hr_dev, table->type))
 578                 return hns_roce_table_mhop_get(hr_dev, table, obj);
 579
 580         i = obj / (table->table_chunk_size / table->obj_size);
 581
 582         mutex_lock(&table->mutex);
 583
 584         if (table->hem[i]) {
 585                 refcount_inc(&table->hem[i]->refcount);
 586                 goto out;
 587         }
 588
 589         table->hem[i] = hns_roce_alloc_hem(hr_dev,
 590                                        table->table_chunk_size >> PAGE_SHIFT,
 591                                        table->table_chunk_size,
 592                                        (table->lowmem ? GFP_KERNEL :
 593                                         GFP_HIGHUSER) | __GFP_NOWARN);
 594         if (!table->hem[i]) {
 595                 ret = -ENOMEM;
 596                 goto out;
 597         }
 598
 599         /* Set HEM base address(128K/page, pa) to Hardware */
 600         if (hr_dev->hw->set_hem(hr_dev, table, obj, HEM_HOP_STEP_DIRECT)) {
 601                 hns_roce_free_hem(hr_dev, table->hem[i]);
 602                 table->hem[i] = NULL;
 603                 ret = -ENODEV;
 604                 dev_err(dev, "set HEM base address to HW failed.\n");
 605                 goto out;
 606         }
 607
 608         refcount_set(&table->hem[i]->refcount, 1);
 609 out:
 610         mutex_unlock(&table->mutex);
 611         return ret;
 612 }
 613
 614 static void clear_mhop_hem(struct hns_roce_dev *hr_dev,
 615                            struct hns_roce_hem_table *table, unsigned long obj,
 616                            struct hns_roce_hem_mhop *mhop,
 617                            struct hns_roce_hem_index *index)
 618 {
 619         struct ib_device *ibdev = &hr_dev->ib_dev;
 620         u32 hop_num = mhop->hop_num;
 621         u32 chunk_ba_num;
 622         int step_idx;
 623
 624         index->inited = HEM_INDEX_BUF;
 625         chunk_ba_num = mhop->bt_chunk_size / BA_BYTE_LEN;
 626         if (check_whether_bt_num_2(table->type, hop_num)) {
 627                 if (hns_roce_check_hem_null(table->hem, index->buf,
 628                                             chunk_ba_num, table->num_hem))
 629                         index->inited |= HEM_INDEX_L0;
 630         } else if (check_whether_bt_num_3(table->type, hop_num)) {
 631                 if (hns_roce_check_hem_null(table->hem, index->buf,
 632                                             chunk_ba_num, table->num_hem)) {
 633                         index->inited |= HEM_INDEX_L1;
 634                         if (hns_roce_check_bt_null(table->bt_l1, index->l1,
 635                                                    chunk_ba_num))
 636                                 index->inited |= HEM_INDEX_L0;
 637                 }
 638         }
 639
 640         if (table->type < HEM_TYPE_MTT) {
 641                 if (hop_num == HNS_ROCE_HOP_NUM_0)
 642                         step_idx = 0;
 643                 else
 644                         step_idx = hop_num;
 645
 646                 if (hr_dev->hw->clear_hem(hr_dev, table, obj, step_idx))
 647                         ibdev_warn(ibdev, "failed to clear hop%u HEM.\n", hop_num);
 648
 649                 if (index->inited & HEM_INDEX_L1)
 650                         if (hr_dev->hw->clear_hem(hr_dev, table, obj, 1))
 651                                 ibdev_warn(ibdev, "failed to clear HEM step 1.\n");
 652
 653                 if (index->inited & HEM_INDEX_L0)
 654                         if (hr_dev->hw->clear_hem(hr_dev, table, obj, 0))
 655                                 ibdev_warn(ibdev, "failed to clear HEM step 0.\n");
 656         }
 657 }
 658
 659 static void hns_roce_table_mhop_put(struct hns_roce_dev *hr_dev,
 660                                     struct hns_roce_hem_table *table,
 661                                     unsigned long obj,
 662                                     int check_refcount)
 663 {
 664         struct ib_device *ibdev = &hr_dev->ib_dev;
 665         struct hns_roce_hem_index index = {};
 666         struct hns_roce_hem_mhop mhop = {};
 667         int ret;
 668
 669         ret = calc_hem_config(hr_dev, table, obj, &mhop, &index);
 670         if (ret) {
 671                 ibdev_err(ibdev, "calc hem config failed!\n");
 672                 return;
 673         }
 674
 675         if (!check_refcount)
 676                 mutex_lock(&table->mutex);
 677         else if (!refcount_dec_and_mutex_lock(&table->hem[index.buf]->refcount,
 678                                               &table->mutex))
 679                 return;
 680
 681         clear_mhop_hem(hr_dev, table, obj, &mhop, &index);
 682         free_mhop_hem(hr_dev, table, &mhop, &index);
 683
 684         mutex_unlock(&table->mutex);
 685 }
 686
 687 void hns_roce_table_put(struct hns_roce_dev *hr_dev,
 688                         struct hns_roce_hem_table *table, unsigned long obj)
 689 {
 690         struct device *dev = hr_dev->dev;
 691         unsigned long i;
 692
 693         if (hns_roce_check_whether_mhop(hr_dev, table->type)) {
 694                 hns_roce_table_mhop_put(hr_dev, table, obj, 1);
 695                 return;
 696         }
 697
 698         i = obj / (table->table_chunk_size / table->obj_size);
 699
 700         if (!refcount_dec_and_mutex_lock(&table->hem[i]->refcount,
 701                                          &table->mutex))
 702                 return;
 703
 704         if (hr_dev->hw->clear_hem(hr_dev, table, obj, HEM_HOP_STEP_DIRECT))
 705                 dev_warn(dev, "failed to clear HEM base address.\n");
 706
 707         hns_roce_free_hem(hr_dev, table->hem[i]);
 708         table->hem[i] = NULL;
 709
 710         mutex_unlock(&table->mutex);
 711 }
 712
 713 void *hns_roce_table_find(struct hns_roce_dev *hr_dev,
 714                           struct hns_roce_hem_table *table,
 715                           unsigned long obj, dma_addr_t *dma_handle)
 716 {
 717         struct hns_roce_hem_chunk *chunk;
 718         struct hns_roce_hem_mhop mhop;
 719         struct hns_roce_hem *hem;
 720         unsigned long mhop_obj = obj;
 721         unsigned long obj_per_chunk;
 722         unsigned long idx_offset;
 723         int offset, dma_offset;
 724         void *addr = NULL;
 725         u32 hem_idx = 0;
 726         int length;
 727         int i, j;
 728
 729         if (!table->lowmem)
 730                 return NULL;
 731
 732         mutex_lock(&table->mutex);
 733
 734         if (!hns_roce_check_whether_mhop(hr_dev, table->type)) {
 735                 obj_per_chunk = table->table_chunk_size / table->obj_size;
 736                 hem = table->hem[obj / obj_per_chunk];
 737                 idx_offset = obj % obj_per_chunk;
 738                 dma_offset = offset = idx_offset * table->obj_size;
 739         } else {
 740                 u32 seg_size = 64; /* 8 bytes per BA and 8 BA per segment */
 741
 742                 if (hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop))
 743                         goto out;
 744                 /* mtt mhop */
 745                 i = mhop.l0_idx;
 746                 j = mhop.l1_idx;
 747                 if (mhop.hop_num == 2)
 748                         hem_idx = i * (mhop.bt_chunk_size / BA_BYTE_LEN) + j;
 749                 else if (mhop.hop_num == 1 ||
 750                          mhop.hop_num == HNS_ROCE_HOP_NUM_0)
 751                         hem_idx = i;
 752
 753                 hem = table->hem[hem_idx];
 754                 dma_offset = offset = obj * seg_size % mhop.bt_chunk_size;
 755                 if (mhop.hop_num == 2)
 756                         dma_offset = offset = 0;
 757         }
 758
 759         if (!hem)
 760                 goto out;
 761
 762         list_for_each_entry(chunk, &hem->chunk_list, list) {
 763                 for (i = 0; i < chunk->npages; ++i) {
 764                         length = sg_dma_len(&chunk->mem[i]);
 765                         if (dma_handle && dma_offset >= 0) {
 766                                 if (length > (u32)dma_offset)
 767                                         *dma_handle = sg_dma_address(
 768                                                 &chunk->mem[i]) + dma_offset;
 769                                 dma_offset -= length;
 770                         }
 771
 772                         if (length > (u32)offset) {
 773                                 addr = chunk->buf[i] + offset;
 774                                 goto out;
 775                         }
 776                         offset -= length;
 777                 }
 778         }
 779
 780 out:
 781         mutex_unlock(&table->mutex);
 782         return addr;
 783 }
 784
 785 int hns_roce_init_hem_table(struct hns_roce_dev *hr_dev,
 786                             struct hns_roce_hem_table *table, u32 type,
 787                             unsigned long obj_size, unsigned long nobj,
 788                             int use_lowmem)
 789 {
 790         unsigned long obj_per_chunk;
 791         unsigned long num_hem;
 792
 793         if (!hns_roce_check_whether_mhop(hr_dev, type)) {
 794                 table->table_chunk_size = hr_dev->caps.chunk_sz;
 795                 obj_per_chunk = table->table_chunk_size / obj_size;
 796                 num_hem = DIV_ROUND_UP(nobj, obj_per_chunk);
 797
 798                 table->hem = kcalloc(num_hem, sizeof(*table->hem), GFP_KERNEL);
 799                 if (!table->hem)
 800                         return -ENOMEM;
 801         } else {
 802                 struct hns_roce_hem_mhop mhop = {};
 803                 unsigned long buf_chunk_size;
 804                 unsigned long bt_chunk_size;
 805                 unsigned long bt_chunk_num;
 806                 unsigned long num_bt_l0;
 807                 u32 hop_num;
 808
 809                 if (get_hem_table_config(hr_dev, &mhop, type))
 810                         return -EINVAL;
 811
 812                 buf_chunk_size = mhop.buf_chunk_size;
 813                 bt_chunk_size = mhop.bt_chunk_size;
 814                 num_bt_l0 = mhop.ba_l0_num;
 815                 hop_num = mhop.hop_num;
 816
 817                 obj_per_chunk = buf_chunk_size / obj_size;
 818                 num_hem = DIV_ROUND_UP(nobj, obj_per_chunk);
 819                 bt_chunk_num = bt_chunk_size / BA_BYTE_LEN;
 820
 821                 if (type >= HEM_TYPE_MTT)
 822                         num_bt_l0 = bt_chunk_num;
 823
 824                 table->hem = kcalloc(num_hem, sizeof(*table->hem),
 825                                          GFP_KERNEL);
 826                 if (!table->hem)
 827                         goto err_kcalloc_hem_buf;
 828
 829                 if (check_whether_bt_num_3(type, hop_num)) {
 830                         unsigned long num_bt_l1;
 831
 832                         num_bt_l1 = DIV_ROUND_UP(num_hem, bt_chunk_num);
 833                         table->bt_l1 = kcalloc(num_bt_l1,
 834                                                sizeof(*table->bt_l1),
 835                                                GFP_KERNEL);
 836                         if (!table->bt_l1)
 837                                 goto err_kcalloc_bt_l1;
 838
 839                         table->bt_l1_dma_addr = kcalloc(num_bt_l1,
 840                                                  sizeof(*table->bt_l1_dma_addr),
 841                                                  GFP_KERNEL);
 842
 843                         if (!table->bt_l1_dma_addr)
 844                                 goto err_kcalloc_l1_dma;
 845                 }
 846
 847                 if (check_whether_bt_num_2(type, hop_num) ||
 848                         check_whether_bt_num_3(type, hop_num)) {
 849                         table->bt_l0 = kcalloc(num_bt_l0, sizeof(*table->bt_l0),
 850                                                GFP_KERNEL);
 851                         if (!table->bt_l0)
 852                                 goto err_kcalloc_bt_l0;
 853
 854                         table->bt_l0_dma_addr = kcalloc(num_bt_l0,
 855                                                  sizeof(*table->bt_l0_dma_addr),
 856                                                  GFP_KERNEL);
 857                         if (!table->bt_l0_dma_addr)
 858                                 goto err_kcalloc_l0_dma;
 859                 }
 860         }
 861
 862         table->type = type;
 863         table->num_hem = num_hem;
 864         table->obj_size = obj_size;
 865         table->lowmem = use_lowmem;
 866         mutex_init(&table->mutex);
 867
 868         return 0;
 869
 870 err_kcalloc_l0_dma:
 871         kfree(table->bt_l0);
 872         table->bt_l0 = NULL;
 873
 874 err_kcalloc_bt_l0:
 875         kfree(table->bt_l1_dma_addr);
 876         table->bt_l1_dma_addr = NULL;
 877
 878 err_kcalloc_l1_dma:
 879         kfree(table->bt_l1);
 880         table->bt_l1 = NULL;
 881
 882 err_kcalloc_bt_l1:
 883         kfree(table->hem);
 884         table->hem = NULL;
 885
 886 err_kcalloc_hem_buf:
 887         return -ENOMEM;
 888 }
 889
 890 static void hns_roce_cleanup_mhop_hem_table(struct hns_roce_dev *hr_dev,
 891                                             struct hns_roce_hem_table *table)
 892 {
 893         struct hns_roce_hem_mhop mhop;
 894         u32 buf_chunk_size;
 895         u64 obj;
 896         int i;
 897
 898         if (hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop))
 899                 return;
 900         buf_chunk_size = table->type < HEM_TYPE_MTT ? mhop.buf_chunk_size :
 901                                         mhop.bt_chunk_size;
 902
 903         for (i = 0; i < table->num_hem; ++i) {
 904                 obj = i * buf_chunk_size / table->obj_size;
 905                 if (table->hem[i])
 906                         hns_roce_table_mhop_put(hr_dev, table, obj, 0);
 907         }
 908
 909         kfree(table->hem);
 910         table->hem = NULL;
 911         kfree(table->bt_l1);
 912         table->bt_l1 = NULL;
 913         kfree(table->bt_l1_dma_addr);
 914         table->bt_l1_dma_addr = NULL;
 915         kfree(table->bt_l0);
 916         table->bt_l0 = NULL;
 917         kfree(table->bt_l0_dma_addr);
 918         table->bt_l0_dma_addr = NULL;
 919 }
 920
 921 void hns_roce_cleanup_hem_table(struct hns_roce_dev *hr_dev,
 922                                 struct hns_roce_hem_table *table)
 923 {
 924         struct device *dev = hr_dev->dev;
 925         unsigned long i;
 926
 927         if (hns_roce_check_whether_mhop(hr_dev, table->type)) {
 928                 hns_roce_cleanup_mhop_hem_table(hr_dev, table);
 929                 return;
 930         }
 931
 932         for (i = 0; i < table->num_hem; ++i)
 933                 if (table->hem[i]) {
 934                         if (hr_dev->hw->clear_hem(hr_dev, table,
 935                             i * table->table_chunk_size / table->obj_size, 0))
 936                                 dev_err(dev, "Clear HEM base address failed.\n");
 937
 938                         hns_roce_free_hem(hr_dev, table->hem[i]);
 939                 }
 940
 941         kfree(table->hem);
 942 }
 943
 944 void hns_roce_cleanup_hem(struct hns_roce_dev *hr_dev)
 945 {
 946         if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SRQ)
 947                 hns_roce_cleanup_hem_table(hr_dev,
 948                                            &hr_dev->srq_table.table);
 949         hns_roce_cleanup_hem_table(hr_dev, &hr_dev->cq_table.table);
 950         if (hr_dev->caps.qpc_timer_entry_sz)
 951                 hns_roce_cleanup_hem_table(hr_dev,
 952                                            &hr_dev->qpc_timer_table);
 953         if (hr_dev->caps.cqc_timer_entry_sz)
 954                 hns_roce_cleanup_hem_table(hr_dev,
 955                                            &hr_dev->cqc_timer_table);
 956         if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL)
 957                 hns_roce_cleanup_hem_table(hr_dev,
 958                                            &hr_dev->qp_table.sccc_table);
 959         if (hr_dev->caps.trrl_entry_sz)
 960                 hns_roce_cleanup_hem_table(hr_dev,
 961                                            &hr_dev->qp_table.trrl_table);
 962
 963         if (hr_dev->caps.gmv_entry_sz)
 964                 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->gmv_table);
 965
 966         hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.irrl_table);
 967         hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.qp_table);
 968         hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtpt_table);
 969 }
 970
 971 struct hns_roce_hem_item {
 972         struct list_head list; /* link all hems in the same bt level */
 973         struct list_head sibling; /* link all hems in last hop for mtt */
 974         void *addr;
 975         dma_addr_t dma_addr;
 976         size_t count; /* max ba numbers */
 977         int start; /* start buf offset in this hem */
 978         int end; /* end buf offset in this hem */
 979 };
 980
 981 /* All HEM items are linked in a tree structure */
 982 struct hns_roce_hem_head {
 983         struct list_head branch[HNS_ROCE_MAX_BT_REGION];
 984         struct list_head root;
 985         struct list_head leaf;
 986 };
 987
 988 static struct hns_roce_hem_item *
 989 hem_list_alloc_item(struct hns_roce_dev *hr_dev, int start, int end, int count,
 990                     bool exist_bt, int bt_level)
 991 {
 992         struct hns_roce_hem_item *hem;
 993
 994         hem = kzalloc(sizeof(*hem), GFP_KERNEL);
 995         if (!hem)
 996                 return NULL;
 997
 998         if (exist_bt) {
 999                 hem->addr = dma_alloc_coherent(hr_dev->dev, count * BA_BYTE_LEN,
1000                                                &hem->dma_addr, GFP_KERNEL);
1001                 if (!hem->addr) {
1002                         kfree(hem);
1003                         return NULL;
1004                 }
1005         }
1006
1007         hem->count = count;
1008         hem->start = start;
1009         hem->end = end;
1010         INIT_LIST_HEAD(&hem->list);
1011         INIT_LIST_HEAD(&hem->sibling);
1012
1013         return hem;
1014 }
1015
1016 static void hem_list_free_item(struct hns_roce_dev *hr_dev,
1017                                struct hns_roce_hem_item *hem, bool exist_bt)
1018 {
1019         if (exist_bt)
1020                 dma_free_coherent(hr_dev->dev, hem->count * BA_BYTE_LEN,
1021                                   hem->addr, hem->dma_addr);
1022         kfree(hem);
1023 }
1024
1025 static void hem_list_free_all(struct hns_roce_dev *hr_dev,
1026                               struct list_head *head, bool exist_bt)
1027 {
1028         struct hns_roce_hem_item *hem, *temp_hem;
1029
1030         list_for_each_entry_safe(hem, temp_hem, head, list) {
1031                 list_del(&hem->list);
1032                 hem_list_free_item(hr_dev, hem, exist_bt);
1033         }
1034 }
1035
1036 static void hem_list_link_bt(struct hns_roce_dev *hr_dev, void *base_addr,
1037                              u64 table_addr)
1038 {
1039         *(u64 *)(base_addr) = table_addr;
1040 }
1041
1042 /* assign L0 table address to hem from root bt */
1043 static void hem_list_assign_bt(struct hns_roce_dev *hr_dev,
1044                                struct hns_roce_hem_item *hem, void *cpu_addr,
1045                                u64 phy_addr)
1046 {
1047         hem->addr = cpu_addr;
1048         hem->dma_addr = (dma_addr_t)phy_addr;
1049 }
1050
1051 static inline bool hem_list_page_is_in_range(struct hns_roce_hem_item *hem,
1052                                              int offset)
1053 {
1054         return (hem->start <= offset && offset <= hem->end);
1055 }
1056
1057 static struct hns_roce_hem_item *hem_list_search_item(struct list_head *ba_list,
1058                                                       int page_offset)
1059 {
1060         struct hns_roce_hem_item *hem, *temp_hem;
1061         struct hns_roce_hem_item *found = NULL;
1062
1063         list_for_each_entry_safe(hem, temp_hem, ba_list, list) {
1064                 if (hem_list_page_is_in_range(hem, page_offset)) {
1065                         found = hem;
1066                         break;
1067                 }
1068         }
1069
1070         return found;
1071 }
1072
1073 static bool hem_list_is_bottom_bt(int hopnum, int bt_level)
1074 {
1075         /*
1076          * hopnum    base address table levels
1077          * 0            L0(buf)
1078          * 1            L0 -> buf
1079          * 2            L0 -> L1 -> buf
1080          * 3            L0 -> L1 -> L2 -> buf
1081          */
1082         return bt_level >= (hopnum ? hopnum - 1 : hopnum);
1083 }
1084
1085 /*
1086  * calc base address entries num
1087  * @hopnum: num of mutihop addressing
1088  * @bt_level: base address table level
1089  * @unit: ba entries per bt page
1090  */
1091 static u32 hem_list_calc_ba_range(int hopnum, int bt_level, int unit)
1092 {
1093         u32 step;
1094         int max;
1095         int i;
1096
1097         if (hopnum <= bt_level)
1098                 return 0;
1099         /*
1100          * hopnum  bt_level   range
1101          * 1          0       unit
1102          * ------------
1103          * 2          0       unit * unit
1104          * 2          1       unit
1105          * ------------
1106          * 3          0       unit * unit * unit
1107          * 3          1       unit * unit
1108          * 3          2       unit
1109          */
1110         step = 1;
1111         max = hopnum - bt_level;
1112         for (i = 0; i < max; i++)
1113                 step = step * unit;
1114
1115         return step;
1116 }
1117
1118 /*
1119  * calc the root ba entries which could cover all regions
1120  * @regions: buf region array
1121  * @region_cnt: array size of @regions
1122  * @unit: ba entries per bt page
1123  */
1124 int hns_roce_hem_list_calc_root_ba(const struct hns_roce_buf_region *regions,
1125                                    int region_cnt, int unit)
1126 {
1127         struct hns_roce_buf_region *r;
1128         int total = 0;
1129         int step;
1130         int i;
1131
1132         for (i = 0; i < region_cnt; i++) {
1133                 r = (struct hns_roce_buf_region *)&regions[i];
1134                 if (r->hopnum > 1) {
1135                         step = hem_list_calc_ba_range(r->hopnum, 1, unit);
1136                         if (step > 0)
1137                                 total += (r->count + step - 1) / step;
1138                 } else {
1139                         total += r->count;
1140                 }
1141         }
1142
1143         return total;
1144 }
1145
1146 static int hem_list_alloc_mid_bt(struct hns_roce_dev *hr_dev,
1147                                  const struct hns_roce_buf_region *r, int unit,
1148                                  int offset, struct list_head *mid_bt,
1149                                  struct list_head *btm_bt)
1150 {
1151         struct hns_roce_hem_item *hem_ptrs[HNS_ROCE_MAX_BT_LEVEL] = { NULL };
1152         struct list_head temp_list[HNS_ROCE_MAX_BT_LEVEL];
1153         struct hns_roce_hem_item *cur, *pre;
1154         const int hopnum = r->hopnum;
1155         int start_aligned;
1156         int distance;
1157         int ret = 0;
1158         int max_ofs;
1159         int level;
1160         u32 step;
1161         int end;
1162
1163         if (hopnum <= 1)
1164                 return 0;
1165
1166         if (hopnum > HNS_ROCE_MAX_BT_LEVEL) {
1167                 dev_err(hr_dev->dev, "invalid hopnum %d!\n", hopnum);
1168                 return -EINVAL;
1169         }
1170
1171         if (offset < r->offset) {
1172                 dev_err(hr_dev->dev, "invalid offset %d, min %u!\n",
1173                         offset, r->offset);
1174                 return -EINVAL;
1175         }
1176
1177         distance = offset - r->offset;
1178         max_ofs = r->offset + r->count - 1;
1179         for (level = 0; level < hopnum; level++)
1180                 INIT_LIST_HEAD(&temp_list[level]);
1181
1182         /* config L1 bt to last bt and link them to corresponding parent */
1183         for (level = 1; level < hopnum; level++) {
1184                 cur = hem_list_search_item(&mid_bt[level], offset);
1185                 if (cur) {
1186                         hem_ptrs[level] = cur;
1187                         continue;
1188                 }
1189
1190                 step = hem_list_calc_ba_range(hopnum, level, unit);
1191                 if (step < 1) {
1192                         ret = -EINVAL;
1193                         goto err_exit;
1194                 }
1195
1196                 start_aligned = (distance / step) * step + r->offset;
1197                 end = min_t(int, start_aligned + step - 1, max_ofs);
1198                 cur = hem_list_alloc_item(hr_dev, start_aligned, end, unit,
1199                                           true, level);
1200                 if (!cur) {
1201                         ret = -ENOMEM;
1202                         goto err_exit;
1203                 }
1204                 hem_ptrs[level] = cur;
1205                 list_add(&cur->list, &temp_list[level]);
1206                 if (hem_list_is_bottom_bt(hopnum, level))
1207                         list_add(&cur->sibling, &temp_list[0]);
1208
1209                 /* link bt to parent bt */
1210                 if (level > 1) {
1211                         pre = hem_ptrs[level - 1];
1212                         step = (cur->start - pre->start) / step * BA_BYTE_LEN;
1213                         hem_list_link_bt(hr_dev, pre->addr + step,
1214                                          cur->dma_addr);
1215                 }
1216         }
1217
1218         list_splice(&temp_list[0], btm_bt);
1219         for (level = 1; level < hopnum; level++)
1220                 list_splice(&temp_list[level], &mid_bt[level]);
1221
1222         return 0;
1223
1224 err_exit:
1225         for (level = 1; level < hopnum; level++)
1226                 hem_list_free_all(hr_dev, &temp_list[level], true);
1227
1228         return ret;
1229 }
1230
1231 static struct hns_roce_hem_item *
1232 alloc_root_hem(struct hns_roce_dev *hr_dev, int unit, int *max_ba_num,
1233                const struct hns_roce_buf_region *regions, int region_cnt)
1234 {
1235         const struct hns_roce_buf_region *r;
1236         struct hns_roce_hem_item *hem;
1237         int ba_num;
1238         int offset;
1239
1240         ba_num = hns_roce_hem_list_calc_root_ba(regions, region_cnt, unit);
1241         if (ba_num < 1)
1242                 return ERR_PTR(-ENOMEM);
1243
1244         if (ba_num > unit)
1245                 return ERR_PTR(-ENOBUFS);
1246
1247         offset = regions[0].offset;
1248         /* indicate to last region */
1249         r = &regions[region_cnt - 1];
1250         hem = hem_list_alloc_item(hr_dev, offset, r->offset + r->count - 1,
1251                                   ba_num, true, 0);
1252         if (!hem)
1253                 return ERR_PTR(-ENOMEM);
1254
1255         *max_ba_num = ba_num;
1256
1257         return hem;
1258 }
1259
1260 static int alloc_fake_root_bt(struct hns_roce_dev *hr_dev, void *cpu_base,
1261                               u64 phy_base, const struct hns_roce_buf_region *r,
1262                               struct list_head *branch_head,
1263                               struct list_head *leaf_head)
1264 {
1265         struct hns_roce_hem_item *hem;
1266
1267         hem = hem_list_alloc_item(hr_dev, r->offset, r->offset + r->count - 1,
1268                                   r->count, false, 0);
1269         if (!hem)
1270                 return -ENOMEM;
1271
1272         hem_list_assign_bt(hr_dev, hem, cpu_base, phy_base);
1273         list_add(&hem->list, branch_head);
1274         list_add(&hem->sibling, leaf_head);
1275
1276         return r->count;
1277 }
1278
1279 static int setup_middle_bt(struct hns_roce_dev *hr_dev, void *cpu_base,
1280                            int unit, const struct hns_roce_buf_region *r,
1281                            const struct list_head *branch_head)
1282 {
1283         struct hns_roce_hem_item *hem, *temp_hem;
1284         int total = 0;
1285         int offset;
1286         int step;
1287
1288         step = hem_list_calc_ba_range(r->hopnum, 1, unit);
1289         if (step < 1)
1290                 return -EINVAL;
1291
1292         /* if exist mid bt, link L1 to L0 */
1293         list_for_each_entry_safe(hem, temp_hem, branch_head, list) {
1294                 offset = (hem->start - r->offset) / step * BA_BYTE_LEN;
1295                 hem_list_link_bt(hr_dev, cpu_base + offset, hem->dma_addr);
1296                 total++;
1297         }
1298
1299         return total;
1300 }
1301
1302 static int
1303 setup_root_hem(struct hns_roce_dev *hr_dev, struct hns_roce_hem_list *hem_list,
1304                int unit, int max_ba_num, struct hns_roce_hem_head *head,
1305                const struct hns_roce_buf_region *regions, int region_cnt)
1306 {
1307         const struct hns_roce_buf_region *r;
1308         struct hns_roce_hem_item *root_hem;
1309         void *cpu_base;
1310         u64 phy_base;
1311         int i, total;
1312         int ret;
1313
1314         root_hem = list_first_entry(&head->root,
1315                                     struct hns_roce_hem_item, list);
1316         if (!root_hem)
1317                 return -ENOMEM;
1318
1319         total = 0;
1320         for (i = 0; i < region_cnt && total < max_ba_num; i++) {
1321                 r = &regions[i];
1322                 if (!r->count)
1323                         continue;
1324
1325                 /* all regions's mid[x][0] shared the root_bt's trunk */
1326                 cpu_base = root_hem->addr + total * BA_BYTE_LEN;
1327                 phy_base = root_hem->dma_addr + total * BA_BYTE_LEN;
1328
1329                 /* if hopnum is 0 or 1, cut a new fake hem from the root bt
1330                  * which's address share to all regions.
1331                  */
1332                 if (hem_list_is_bottom_bt(r->hopnum, 0))
1333                         ret = alloc_fake_root_bt(hr_dev, cpu_base, phy_base, r,
1334                                                  &head->branch[i], &head->leaf);
1335                 else
1336                         ret = setup_middle_bt(hr_dev, cpu_base, unit, r,
1337                                               &hem_list->mid_bt[i][1]);
1338
1339                 if (ret < 0)
1340                         return ret;
1341
1342                 total += ret;
1343         }
1344
1345         list_splice(&head->leaf, &hem_list->btm_bt);
1346         list_splice(&head->root, &hem_list->root_bt);
1347         for (i = 0; i < region_cnt; i++)
1348                 list_splice(&head->branch[i], &hem_list->mid_bt[i][0]);
1349
1350         return 0;
1351 }
1352
1353 static int hem_list_alloc_root_bt(struct hns_roce_dev *hr_dev,
1354                                   struct hns_roce_hem_list *hem_list, int unit,
1355                                   const struct hns_roce_buf_region *regions,
1356                                   int region_cnt)
1357 {
1358         struct hns_roce_hem_item *root_hem;
1359         struct hns_roce_hem_head head;
1360         int max_ba_num;
1361         int ret;
1362         int i;
1363
1364         root_hem = hem_list_search_item(&hem_list->root_bt, regions[0].offset);
1365         if (root_hem)
1366                 return 0;
1367
1368         max_ba_num = 0;
1369         root_hem = alloc_root_hem(hr_dev, unit, &max_ba_num, regions,
1370                                   region_cnt);
1371         if (IS_ERR(root_hem))
1372                 return PTR_ERR(root_hem);
1373
1374         /* List head for storing all allocated HEM items */
1375         INIT_LIST_HEAD(&head.root);
1376         INIT_LIST_HEAD(&head.leaf);
1377         for (i = 0; i < region_cnt; i++)
1378                 INIT_LIST_HEAD(&head.branch[i]);
1379
1380         hem_list->root_ba = root_hem->dma_addr;
1381         list_add(&root_hem->list, &head.root);
1382         ret = setup_root_hem(hr_dev, hem_list, unit, max_ba_num, &head, regions,
1383                              region_cnt);
1384         if (ret) {
1385                 for (i = 0; i < region_cnt; i++)
1386                         hem_list_free_all(hr_dev, &head.branch[i], false);
1387
1388                 hem_list_free_all(hr_dev, &head.root, true);
1389         }
1390
1391         return ret;
1392 }
1393
1394 /* construct the base address table and link them by address hop config */
1395 int hns_roce_hem_list_request(struct hns_roce_dev *hr_dev,
1396                               struct hns_roce_hem_list *hem_list,
1397                               const struct hns_roce_buf_region *regions,
1398                               int region_cnt, unsigned int bt_pg_shift)
1399 {
1400         const struct hns_roce_buf_region *r;
1401         int ofs, end;
1402         int unit;
1403         int ret;
1404         int i;
1405
1406         if (region_cnt > HNS_ROCE_MAX_BT_REGION) {
1407                 dev_err(hr_dev->dev, "invalid region region_cnt %d!\n",
1408                         region_cnt);
1409                 return -EINVAL;
1410         }
1411
1412         unit = (1 << bt_pg_shift) / BA_BYTE_LEN;
1413         for (i = 0; i < region_cnt; i++) {
1414                 r = &regions[i];
1415                 if (!r->count)
1416                         continue;
1417
1418                 end = r->offset + r->count;
1419                 for (ofs = r->offset; ofs < end; ofs += unit) {
1420                         ret = hem_list_alloc_mid_bt(hr_dev, r, unit, ofs,
1421                                                     hem_list->mid_bt[i],
1422                                                     &hem_list->btm_bt);
1423                         if (ret) {
1424                                 dev_err(hr_dev->dev,
1425                                         "alloc hem trunk fail ret=%d!\n", ret);
1426                                 goto err_alloc;
1427                         }
1428                 }
1429         }
1430
1431         ret = hem_list_alloc_root_bt(hr_dev, hem_list, unit, regions,
1432                                      region_cnt);
1433         if (ret)
1434                 dev_err(hr_dev->dev, "alloc hem root fail ret=%d!\n", ret);
1435         else
1436                 return 0;
1437
1438 err_alloc:
1439         hns_roce_hem_list_release(hr_dev, hem_list);
1440
1441         return ret;
1442 }
1443
1444 void hns_roce_hem_list_release(struct hns_roce_dev *hr_dev,
1445                                struct hns_roce_hem_list *hem_list)
1446 {
1447         int i, j;
1448
1449         for (i = 0; i < HNS_ROCE_MAX_BT_REGION; i++)
1450                 for (j = 0; j < HNS_ROCE_MAX_BT_LEVEL; j++)
1451                         hem_list_free_all(hr_dev, &hem_list->mid_bt[i][j],
1452                                           j != 0);
1453
1454         hem_list_free_all(hr_dev, &hem_list->root_bt, true);
1455         INIT_LIST_HEAD(&hem_list->btm_bt);
1456         hem_list->root_ba = 0;
1457 }
1458
1459 void hns_roce_hem_list_init(struct hns_roce_hem_list *hem_list)
1460 {
1461         int i, j;
1462
1463         INIT_LIST_HEAD(&hem_list->root_bt);
1464         INIT_LIST_HEAD(&hem_list->btm_bt);
1465         for (i = 0; i < HNS_ROCE_MAX_BT_REGION; i++)
1466                 for (j = 0; j < HNS_ROCE_MAX_BT_LEVEL; j++)
1467                         INIT_LIST_HEAD(&hem_list->mid_bt[i][j]);
1468 }
1469
1470 void *hns_roce_hem_list_find_mtt(struct hns_roce_dev *hr_dev,
1471                                  struct hns_roce_hem_list *hem_list,
1472                                  int offset, int *mtt_cnt, u64 *phy_addr)
1473 {
1474         struct list_head *head = &hem_list->btm_bt;
1475         struct hns_roce_hem_item *hem, *temp_hem;
1476         void *cpu_base = NULL;
1477         u64 phy_base = 0;
1478         int nr = 0;
1479
1480         list_for_each_entry_safe(hem, temp_hem, head, sibling) {
1481                 if (hem_list_page_is_in_range(hem, offset)) {
1482                         nr = offset - hem->start;
1483                         cpu_base = hem->addr + nr * BA_BYTE_LEN;
1484                         phy_base = hem->dma_addr + nr * BA_BYTE_LEN;
1485                         nr = hem->end + 1 - offset;
1486                         break;
1487                 }
1488         }
1489
1490         if (mtt_cnt)
1491                 *mtt_cnt = nr;
1492
1493         if (phy_addr)
1494                 *phy_addr = phy_base;
1495
1496         return cpu_base;
1497 }