fs/ksmbd/transport_rdma.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  *   Copyright (C) 2017, Microsoft Corporation.
   4  *   Copyright (C) 2018, LG Electronics.
   5  *
   6  *   Author(s): Long Li <longli@microsoft.com>,
   7  *              Hyunchul Lee <hyc.lee@gmail.com>
   8  *
   9  *   This program is free software;  you can redistribute it and/or modify
  10  *   it under the terms of the GNU General Public License as published by
  11  *   the Free Software Foundation; either version 2 of the License, or
  12  *   (at your option) any later version.
  13  *
  14  *   This program is distributed in the hope that it will be useful,
  15  *   but WITHOUT ANY WARRANTY;  without even the implied warranty of
  16  *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See
  17  *   the GNU General Public License for more details.
  18  */
  19
  20 #define SUBMOD_NAME     "smb_direct"
  21
  22 #include <linux/kthread.h>
  23 #include <linux/list.h>
  24 #include <linux/mempool.h>
  25 #include <linux/highmem.h>
  26 #include <linux/scatterlist.h>
  27 #include <rdma/ib_verbs.h>
  28 #include <rdma/rdma_cm.h>
  29 #include <rdma/rw.h>
  30
  31 #include "glob.h"
  32 #include "connection.h"
  33 #include "smb_common.h"
  34 #include "smbstatus.h"
  35 #include "transport_rdma.h"
  36
  37 #define SMB_DIRECT_PORT 5445
  38
  39 #define SMB_DIRECT_VERSION_LE           cpu_to_le16(0x0100)
  40
  41 /* SMB_DIRECT negotiation timeout in seconds */
  42 #define SMB_DIRECT_NEGOTIATE_TIMEOUT            120
  43
  44 #define SMB_DIRECT_MAX_SEND_SGES                8
  45 #define SMB_DIRECT_MAX_RECV_SGES                1
  46
  47 /*
  48  * Default maximum number of RDMA read/write outstanding on this connection
  49  * This value is possibly decreased during QP creation on hardware limit
  50  */
  51 #define SMB_DIRECT_CM_INITIATOR_DEPTH           8
  52
  53 /* Maximum number of retries on data transfer operations */
  54 #define SMB_DIRECT_CM_RETRY                     6
  55 /* No need to retry on Receiver Not Ready since SMB_DIRECT manages credits */
  56 #define SMB_DIRECT_CM_RNR_RETRY         0
  57
  58 /*
  59  * User configurable initial values per SMB_DIRECT transport connection
  60  * as defined in [MS-SMBD] 3.1.1.1
  61  * Those may change after a SMB_DIRECT negotiation
  62  */
  63 /* The local peer's maximum number of credits to grant to the peer */
  64 static int smb_direct_receive_credit_max = 255;
  65
  66 /* The remote peer's credit request of local peer */
  67 static int smb_direct_send_credit_target = 255;
  68
  69 /* The maximum single message size can be sent to remote peer */
  70 static int smb_direct_max_send_size = 8192;
  71
  72 /*  The maximum fragmented upper-layer payload receive size supported */
  73 static int smb_direct_max_fragmented_recv_size = 1024 * 1024;
  74
  75 /*  The maximum single-message size which can be received */
  76 static int smb_direct_max_receive_size = 8192;
  77
  78 static int smb_direct_max_read_write_size = 1024 * 1024;
  79
  80 static int smb_direct_max_outstanding_rw_ops = 8;
  81
  82 static struct smb_direct_listener {
  83         struct rdma_cm_id       *cm_id;
  84 } smb_direct_listener;
  85
  86 static struct workqueue_struct *smb_direct_wq;
  87
  88 enum smb_direct_status {
  89         SMB_DIRECT_CS_NEW = 0,
  90         SMB_DIRECT_CS_CONNECTED,
  91         SMB_DIRECT_CS_DISCONNECTING,
  92         SMB_DIRECT_CS_DISCONNECTED,
  93 };
  94
  95 struct smb_direct_transport {
  96         struct ksmbd_transport  transport;
  97
  98         enum smb_direct_status  status;
  99         bool                    full_packet_received;
 100         wait_queue_head_t       wait_status;
 101
 102         struct rdma_cm_id       *cm_id;
 103         struct ib_cq            *send_cq;
 104         struct ib_cq            *recv_cq;
 105         struct ib_pd            *pd;
 106         struct ib_qp            *qp;
 107
 108         int                     max_send_size;
 109         int                     max_recv_size;
 110         int                     max_fragmented_send_size;
 111         int                     max_fragmented_recv_size;
 112         int                     max_rdma_rw_size;
 113
 114         spinlock_t              reassembly_queue_lock;
 115         struct list_head        reassembly_queue;
 116         int                     reassembly_data_length;
 117         int                     reassembly_queue_length;
 118         int                     first_entry_offset;
 119         wait_queue_head_t       wait_reassembly_queue;
 120
 121         spinlock_t              receive_credit_lock;
 122         int                     recv_credits;
 123         int                     count_avail_recvmsg;
 124         int                     recv_credit_max;
 125         int                     recv_credit_target;
 126
 127         spinlock_t              recvmsg_queue_lock;
 128         struct list_head        recvmsg_queue;
 129
 130         spinlock_t              empty_recvmsg_queue_lock;
 131         struct list_head        empty_recvmsg_queue;
 132
 133         int                     send_credit_target;
 134         atomic_t                send_credits;
 135         spinlock_t              lock_new_recv_credits;
 136         int                     new_recv_credits;
 137         atomic_t                rw_avail_ops;
 138
 139         wait_queue_head_t       wait_send_credits;
 140         wait_queue_head_t       wait_rw_avail_ops;
 141
 142         mempool_t               *sendmsg_mempool;
 143         struct kmem_cache       *sendmsg_cache;
 144         mempool_t               *recvmsg_mempool;
 145         struct kmem_cache       *recvmsg_cache;
 146
 147         wait_queue_head_t       wait_send_payload_pending;
 148         atomic_t                send_payload_pending;
 149         wait_queue_head_t       wait_send_pending;
 150         atomic_t                send_pending;
 151
 152         struct delayed_work     post_recv_credits_work;
 153         struct work_struct      send_immediate_work;
 154         struct work_struct      disconnect_work;
 155
 156         bool                    negotiation_requested;
 157 };
 158
 159 #define KSMBD_TRANS(t) ((struct ksmbd_transport *)&((t)->transport))
 160
 161 enum {
 162         SMB_DIRECT_MSG_NEGOTIATE_REQ = 0,
 163         SMB_DIRECT_MSG_DATA_TRANSFER
 164 };
 165
 166 static struct ksmbd_transport_ops ksmbd_smb_direct_transport_ops;
 167
 168 struct smb_direct_send_ctx {
 169         struct list_head        msg_list;
 170         int                     wr_cnt;
 171         bool                    need_invalidate_rkey;
 172         unsigned int            remote_key;
 173 };
 174
 175 struct smb_direct_sendmsg {
 176         struct smb_direct_transport     *transport;
 177         struct ib_send_wr       wr;
 178         struct list_head        list;
 179         int                     num_sge;
 180         struct ib_sge           sge[SMB_DIRECT_MAX_SEND_SGES];
 181         struct ib_cqe           cqe;
 182         u8                      packet[];
 183 };
 184
 185 struct smb_direct_recvmsg {
 186         struct smb_direct_transport     *transport;
 187         struct list_head        list;
 188         int                     type;
 189         struct ib_sge           sge;
 190         struct ib_cqe           cqe;
 191         bool                    first_segment;
 192         u8                      packet[];
 193 };
 194
 195 struct smb_direct_rdma_rw_msg {
 196         struct smb_direct_transport     *t;
 197         struct ib_cqe           cqe;
 198         struct completion       *completion;
 199         struct rdma_rw_ctx      rw_ctx;
 200         struct sg_table         sgt;
 201         struct scatterlist      sg_list[0];
 202 };
 203
 204 static inline int get_buf_page_count(void *buf, int size)
 205 {
 206         return DIV_ROUND_UP((uintptr_t)buf + size, PAGE_SIZE) -
 207                 (uintptr_t)buf / PAGE_SIZE;
 208 }
 209
 210 static void smb_direct_destroy_pools(struct smb_direct_transport *transport);
 211 static void smb_direct_post_recv_credits(struct work_struct *work);
 212 static int smb_direct_post_send_data(struct smb_direct_transport *t,
 213                                      struct smb_direct_send_ctx *send_ctx,
 214                                      struct kvec *iov, int niov,
 215                                      int remaining_data_length);
 216
 217 static inline struct smb_direct_transport *
 218 smb_trans_direct_transfort(struct ksmbd_transport *t)
 219 {
 220         return container_of(t, struct smb_direct_transport, transport);
 221 }
 222
 223 static inline void
 224 *smb_direct_recvmsg_payload(struct smb_direct_recvmsg *recvmsg)
 225 {
 226         return (void *)recvmsg->packet;
 227 }
 228
 229 static inline bool is_receive_credit_post_required(int receive_credits,
 230                                                    int avail_recvmsg_count)
 231 {
 232         return receive_credits <= (smb_direct_receive_credit_max >> 3) &&
 233                 avail_recvmsg_count >= (receive_credits >> 2);
 234 }
 235
 236 static struct
 237 smb_direct_recvmsg *get_free_recvmsg(struct smb_direct_transport *t)
 238 {
 239         struct smb_direct_recvmsg *recvmsg = NULL;
 240
 241         spin_lock(&t->recvmsg_queue_lock);
 242         if (!list_empty(&t->recvmsg_queue)) {
 243                 recvmsg = list_first_entry(&t->recvmsg_queue,
 244                                            struct smb_direct_recvmsg,
 245                                            list);
 246                 list_del(&recvmsg->list);
 247         }
 248         spin_unlock(&t->recvmsg_queue_lock);
 249         return recvmsg;
 250 }
 251
 252 static void put_recvmsg(struct smb_direct_transport *t,
 253                         struct smb_direct_recvmsg *recvmsg)
 254 {
 255         ib_dma_unmap_single(t->cm_id->device, recvmsg->sge.addr,
 256                             recvmsg->sge.length, DMA_FROM_DEVICE);
 257
 258         spin_lock(&t->recvmsg_queue_lock);
 259         list_add(&recvmsg->list, &t->recvmsg_queue);
 260         spin_unlock(&t->recvmsg_queue_lock);
 261 }
 262
 263 static struct
 264 smb_direct_recvmsg *get_empty_recvmsg(struct smb_direct_transport *t)
 265 {
 266         struct smb_direct_recvmsg *recvmsg = NULL;
 267
 268         spin_lock(&t->empty_recvmsg_queue_lock);
 269         if (!list_empty(&t->empty_recvmsg_queue)) {
 270                 recvmsg = list_first_entry(&t->empty_recvmsg_queue,
 271                                            struct smb_direct_recvmsg, list);
 272                 list_del(&recvmsg->list);
 273         }
 274         spin_unlock(&t->empty_recvmsg_queue_lock);
 275         return recvmsg;
 276 }
 277
 278 static void put_empty_recvmsg(struct smb_direct_transport *t,
 279                               struct smb_direct_recvmsg *recvmsg)
 280 {
 281         ib_dma_unmap_single(t->cm_id->device, recvmsg->sge.addr,
 282                             recvmsg->sge.length, DMA_FROM_DEVICE);
 283
 284         spin_lock(&t->empty_recvmsg_queue_lock);
 285         list_add_tail(&recvmsg->list, &t->empty_recvmsg_queue);
 286         spin_unlock(&t->empty_recvmsg_queue_lock);
 287 }
 288
 289 static void enqueue_reassembly(struct smb_direct_transport *t,
 290                                struct smb_direct_recvmsg *recvmsg,
 291                                int data_length)
 292 {
 293         spin_lock(&t->reassembly_queue_lock);
 294         list_add_tail(&recvmsg->list, &t->reassembly_queue);
 295         t->reassembly_queue_length++;
 296         /*
 297          * Make sure reassembly_data_length is updated after list and
 298          * reassembly_queue_length are updated. On the dequeue side
 299          * reassembly_data_length is checked without a lock to determine
 300          * if reassembly_queue_length and list is up to date
 301          */
 302         virt_wmb();
 303         t->reassembly_data_length += data_length;
 304         spin_unlock(&t->reassembly_queue_lock);
 305 }
 306
 307 static struct smb_direct_recvmsg *get_first_reassembly(struct smb_direct_transport *t)
 308 {
 309         if (!list_empty(&t->reassembly_queue))
 310                 return list_first_entry(&t->reassembly_queue,
 311                                 struct smb_direct_recvmsg, list);
 312         else
 313                 return NULL;
 314 }
 315
 316 static void smb_direct_disconnect_rdma_work(struct work_struct *work)
 317 {
 318         struct smb_direct_transport *t =
 319                 container_of(work, struct smb_direct_transport,
 320                              disconnect_work);
 321
 322         if (t->status == SMB_DIRECT_CS_CONNECTED) {
 323                 t->status = SMB_DIRECT_CS_DISCONNECTING;
 324                 rdma_disconnect(t->cm_id);
 325         }
 326 }
 327
 328 static void
 329 smb_direct_disconnect_rdma_connection(struct smb_direct_transport *t)
 330 {
 331         if (t->status == SMB_DIRECT_CS_CONNECTED)
 332                 queue_work(smb_direct_wq, &t->disconnect_work);
 333 }
 334
 335 static void smb_direct_send_immediate_work(struct work_struct *work)
 336 {
 337         struct smb_direct_transport *t = container_of(work,
 338                         struct smb_direct_transport, send_immediate_work);
 339
 340         if (t->status != SMB_DIRECT_CS_CONNECTED)
 341                 return;
 342
 343         smb_direct_post_send_data(t, NULL, NULL, 0, 0);
 344 }
 345
 346 static struct smb_direct_transport *alloc_transport(struct rdma_cm_id *cm_id)
 347 {
 348         struct smb_direct_transport *t;
 349         struct ksmbd_conn *conn;
 350
 351         t = kzalloc(sizeof(*t), GFP_KERNEL);
 352         if (!t)
 353                 return NULL;
 354
 355         t->cm_id = cm_id;
 356         cm_id->context = t;
 357
 358         t->status = SMB_DIRECT_CS_NEW;
 359         init_waitqueue_head(&t->wait_status);
 360
 361         spin_lock_init(&t->reassembly_queue_lock);
 362         INIT_LIST_HEAD(&t->reassembly_queue);
 363         t->reassembly_data_length = 0;
 364         t->reassembly_queue_length = 0;
 365         init_waitqueue_head(&t->wait_reassembly_queue);
 366         init_waitqueue_head(&t->wait_send_credits);
 367         init_waitqueue_head(&t->wait_rw_avail_ops);
 368
 369         spin_lock_init(&t->receive_credit_lock);
 370         spin_lock_init(&t->recvmsg_queue_lock);
 371         INIT_LIST_HEAD(&t->recvmsg_queue);
 372
 373         spin_lock_init(&t->empty_recvmsg_queue_lock);
 374         INIT_LIST_HEAD(&t->empty_recvmsg_queue);
 375
 376         init_waitqueue_head(&t->wait_send_payload_pending);
 377         atomic_set(&t->send_payload_pending, 0);
 378         init_waitqueue_head(&t->wait_send_pending);
 379         atomic_set(&t->send_pending, 0);
 380
 381         spin_lock_init(&t->lock_new_recv_credits);
 382
 383         INIT_DELAYED_WORK(&t->post_recv_credits_work,
 384                           smb_direct_post_recv_credits);
 385         INIT_WORK(&t->send_immediate_work, smb_direct_send_immediate_work);
 386         INIT_WORK(&t->disconnect_work, smb_direct_disconnect_rdma_work);
 387
 388         conn = ksmbd_conn_alloc();
 389         if (!conn)
 390                 goto err;
 391         conn->transport = KSMBD_TRANS(t);
 392         KSMBD_TRANS(t)->conn = conn;
 393         KSMBD_TRANS(t)->ops = &ksmbd_smb_direct_transport_ops;
 394         return t;
 395 err:
 396         kfree(t);
 397         return NULL;
 398 }
 399
 400 static void free_transport(struct smb_direct_transport *t)
 401 {
 402         struct smb_direct_recvmsg *recvmsg;
 403
 404         wake_up_interruptible(&t->wait_send_credits);
 405
 406         ksmbd_debug(RDMA, "wait for all send posted to IB to finish\n");
 407         wait_event(t->wait_send_payload_pending,
 408                    atomic_read(&t->send_payload_pending) == 0);
 409         wait_event(t->wait_send_pending,
 410                    atomic_read(&t->send_pending) == 0);
 411
 412         cancel_work_sync(&t->disconnect_work);
 413         cancel_delayed_work_sync(&t->post_recv_credits_work);
 414         cancel_work_sync(&t->send_immediate_work);
 415
 416         if (t->qp) {
 417                 ib_drain_qp(t->qp);
 418                 ib_destroy_qp(t->qp);
 419         }
 420
 421         ksmbd_debug(RDMA, "drain the reassembly queue\n");
 422         do {
 423                 spin_lock(&t->reassembly_queue_lock);
 424                 recvmsg = get_first_reassembly(t);
 425                 if (recvmsg) {
 426                         list_del(&recvmsg->list);
 427                         spin_unlock(&t->reassembly_queue_lock);
 428                         put_recvmsg(t, recvmsg);
 429                 } else {
 430                         spin_unlock(&t->reassembly_queue_lock);
 431                 }
 432         } while (recvmsg);
 433         t->reassembly_data_length = 0;
 434
 435         if (t->send_cq)
 436                 ib_free_cq(t->send_cq);
 437         if (t->recv_cq)
 438                 ib_free_cq(t->recv_cq);
 439         if (t->pd)
 440                 ib_dealloc_pd(t->pd);
 441         if (t->cm_id)
 442                 rdma_destroy_id(t->cm_id);
 443
 444         smb_direct_destroy_pools(t);
 445         ksmbd_conn_free(KSMBD_TRANS(t)->conn);
 446         kfree(t);
 447 }
 448
 449 static struct smb_direct_sendmsg
 450 *smb_direct_alloc_sendmsg(struct smb_direct_transport *t)
 451 {
 452         struct smb_direct_sendmsg *msg;
 453
 454         msg = mempool_alloc(t->sendmsg_mempool, GFP_KERNEL);
 455         if (!msg)
 456                 return ERR_PTR(-ENOMEM);
 457         msg->transport = t;
 458         INIT_LIST_HEAD(&msg->list);
 459         msg->num_sge = 0;
 460         return msg;
 461 }
 462
 463 static void smb_direct_free_sendmsg(struct smb_direct_transport *t,
 464                                     struct smb_direct_sendmsg *msg)
 465 {
 466         int i;
 467
 468         if (msg->num_sge > 0) {
 469                 ib_dma_unmap_single(t->cm_id->device,
 470                                     msg->sge[0].addr, msg->sge[0].length,
 471                                     DMA_TO_DEVICE);
 472                 for (i = 1; i < msg->num_sge; i++)
 473                         ib_dma_unmap_page(t->cm_id->device,
 474                                           msg->sge[i].addr, msg->sge[i].length,
 475                                           DMA_TO_DEVICE);
 476         }
 477         mempool_free(msg, t->sendmsg_mempool);
 478 }
 479
 480 static int smb_direct_check_recvmsg(struct smb_direct_recvmsg *recvmsg)
 481 {
 482         switch (recvmsg->type) {
 483         case SMB_DIRECT_MSG_DATA_TRANSFER: {
 484                 struct smb_direct_data_transfer *req =
 485                         (struct smb_direct_data_transfer *)recvmsg->packet;
 486                 struct smb2_hdr *hdr = (struct smb2_hdr *)(recvmsg->packet
 487                                 + le32_to_cpu(req->data_offset) - 4);
 488                 ksmbd_debug(RDMA,
 489                             "CreditGranted: %u, CreditRequested: %u, DataLength: %u, RemainingDataLength: %u, SMB: %x, Command: %u\n",
 490                             le16_to_cpu(req->credits_granted),
 491                             le16_to_cpu(req->credits_requested),
 492                             req->data_length, req->remaining_data_length,
 493                             hdr->ProtocolId, hdr->Command);
 494                 break;
 495         }
 496         case SMB_DIRECT_MSG_NEGOTIATE_REQ: {
 497                 struct smb_direct_negotiate_req *req =
 498                         (struct smb_direct_negotiate_req *)recvmsg->packet;
 499                 ksmbd_debug(RDMA,
 500                             "MinVersion: %u, MaxVersion: %u, CreditRequested: %u, MaxSendSize: %u, MaxRecvSize: %u, MaxFragmentedSize: %u\n",
 501                             le16_to_cpu(req->min_version),
 502                             le16_to_cpu(req->max_version),
 503                             le16_to_cpu(req->credits_requested),
 504                             le32_to_cpu(req->preferred_send_size),
 505                             le32_to_cpu(req->max_receive_size),
 506                             le32_to_cpu(req->max_fragmented_size));
 507                 if (le16_to_cpu(req->min_version) > 0x0100 ||
 508                     le16_to_cpu(req->max_version) < 0x0100)
 509                         return -EOPNOTSUPP;
 510                 if (le16_to_cpu(req->credits_requested) <= 0 ||
 511                     le32_to_cpu(req->max_receive_size) <= 128 ||
 512                     le32_to_cpu(req->max_fragmented_size) <=
 513                                         128 * 1024)
 514                         return -ECONNABORTED;
 515
 516                 break;
 517         }
 518         default:
 519                 return -EINVAL;
 520         }
 521         return 0;
 522 }
 523
 524 static void recv_done(struct ib_cq *cq, struct ib_wc *wc)
 525 {
 526         struct smb_direct_recvmsg *recvmsg;
 527         struct smb_direct_transport *t;
 528
 529         recvmsg = container_of(wc->wr_cqe, struct smb_direct_recvmsg, cqe);
 530         t = recvmsg->transport;
 531
 532         if (wc->status != IB_WC_SUCCESS || wc->opcode != IB_WC_RECV) {
 533                 if (wc->status != IB_WC_WR_FLUSH_ERR) {
 534                         pr_err("Recv error. status='%s (%d)' opcode=%d\n",
 535                                ib_wc_status_msg(wc->status), wc->status,
 536                                wc->opcode);
 537                         smb_direct_disconnect_rdma_connection(t);
 538                 }
 539                 put_empty_recvmsg(t, recvmsg);
 540                 return;
 541         }
 542
 543         ksmbd_debug(RDMA, "Recv completed. status='%s (%d)', opcode=%d\n",
 544                     ib_wc_status_msg(wc->status), wc->status,
 545                     wc->opcode);
 546
 547         ib_dma_sync_single_for_cpu(wc->qp->device, recvmsg->sge.addr,
 548                                    recvmsg->sge.length, DMA_FROM_DEVICE);
 549
 550         switch (recvmsg->type) {
 551         case SMB_DIRECT_MSG_NEGOTIATE_REQ:
 552                 if (wc->byte_len < sizeof(struct smb_direct_negotiate_req)) {
 553                         put_empty_recvmsg(t, recvmsg);
 554                         return;
 555                 }
 556                 t->negotiation_requested = true;
 557                 t->full_packet_received = true;
 558                 wake_up_interruptible(&t->wait_status);
 559                 break;
 560         case SMB_DIRECT_MSG_DATA_TRANSFER: {
 561                 struct smb_direct_data_transfer *data_transfer =
 562                         (struct smb_direct_data_transfer *)recvmsg->packet;
 563                 unsigned int data_length;
 564                 int avail_recvmsg_count, receive_credits;
 565
 566                 if (wc->byte_len <
 567                     offsetof(struct smb_direct_data_transfer, padding)) {
 568                         put_empty_recvmsg(t, recvmsg);
 569                         return;
 570                 }
 571
 572                 data_length = le32_to_cpu(data_transfer->data_length);
 573                 if (data_length) {
 574                         if (wc->byte_len < sizeof(struct smb_direct_data_transfer) +
 575                             (u64)data_length) {
 576                                 put_empty_recvmsg(t, recvmsg);
 577                                 return;
 578                         }
 579
 580                         if (t->full_packet_received)
 581                                 recvmsg->first_segment = true;
 582
 583                         if (le32_to_cpu(data_transfer->remaining_data_length))
 584                                 t->full_packet_received = false;
 585                         else
 586                                 t->full_packet_received = true;
 587
 588                         enqueue_reassembly(t, recvmsg, (int)data_length);
 589                         wake_up_interruptible(&t->wait_reassembly_queue);
 590
 591                         spin_lock(&t->receive_credit_lock);
 592                         receive_credits = --(t->recv_credits);
 593                         avail_recvmsg_count = t->count_avail_recvmsg;
 594                         spin_unlock(&t->receive_credit_lock);
 595                 } else {
 596                         put_empty_recvmsg(t, recvmsg);
 597
 598                         spin_lock(&t->receive_credit_lock);
 599                         receive_credits = --(t->recv_credits);
 600                         avail_recvmsg_count = ++(t->count_avail_recvmsg);
 601                         spin_unlock(&t->receive_credit_lock);
 602                 }
 603
 604                 t->recv_credit_target =
 605                                 le16_to_cpu(data_transfer->credits_requested);
 606                 atomic_add(le16_to_cpu(data_transfer->credits_granted),
 607                            &t->send_credits);
 608
 609                 if (le16_to_cpu(data_transfer->flags) &
 610                     SMB_DIRECT_RESPONSE_REQUESTED)
 611                         queue_work(smb_direct_wq, &t->send_immediate_work);
 612
 613                 if (atomic_read(&t->send_credits) > 0)
 614                         wake_up_interruptible(&t->wait_send_credits);
 615
 616                 if (is_receive_credit_post_required(receive_credits, avail_recvmsg_count))
 617                         mod_delayed_work(smb_direct_wq,
 618                                          &t->post_recv_credits_work, 0);
 619                 break;
 620         }
 621         default:
 622                 break;
 623         }
 624 }
 625
 626 static int smb_direct_post_recv(struct smb_direct_transport *t,
 627                                 struct smb_direct_recvmsg *recvmsg)
 628 {
 629         struct ib_recv_wr wr;
 630         int ret;
 631
 632         recvmsg->sge.addr = ib_dma_map_single(t->cm_id->device,
 633                                               recvmsg->packet, t->max_recv_size,
 634                                               DMA_FROM_DEVICE);
 635         ret = ib_dma_mapping_error(t->cm_id->device, recvmsg->sge.addr);
 636         if (ret)
 637                 return ret;
 638         recvmsg->sge.length = t->max_recv_size;
 639         recvmsg->sge.lkey = t->pd->local_dma_lkey;
 640         recvmsg->cqe.done = recv_done;
 641
 642         wr.wr_cqe = &recvmsg->cqe;
 643         wr.next = NULL;
 644         wr.sg_list = &recvmsg->sge;
 645         wr.num_sge = 1;
 646
 647         ret = ib_post_recv(t->qp, &wr, NULL);
 648         if (ret) {
 649                 pr_err("Can't post recv: %d\n", ret);
 650                 ib_dma_unmap_single(t->cm_id->device,
 651                                     recvmsg->sge.addr, recvmsg->sge.length,
 652                                     DMA_FROM_DEVICE);
 653                 smb_direct_disconnect_rdma_connection(t);
 654                 return ret;
 655         }
 656         return ret;
 657 }
 658
 659 static int smb_direct_read(struct ksmbd_transport *t, char *buf,
 660                            unsigned int size)
 661 {
 662         struct smb_direct_recvmsg *recvmsg;
 663         struct smb_direct_data_transfer *data_transfer;
 664         int to_copy, to_read, data_read, offset;
 665         u32 data_length, remaining_data_length, data_offset;
 666         int rc;
 667         struct smb_direct_transport *st = smb_trans_direct_transfort(t);
 668
 669 again:
 670         if (st->status != SMB_DIRECT_CS_CONNECTED) {
 671                 pr_err("disconnected\n");
 672                 return -ENOTCONN;
 673         }
 674
 675         /*
 676          * No need to hold the reassembly queue lock all the time as we are
 677          * the only one reading from the front of the queue. The transport
 678          * may add more entries to the back of the queue at the same time
 679          */
 680         if (st->reassembly_data_length >= size) {
 681                 int queue_length;
 682                 int queue_removed = 0;
 683
 684                 /*
 685                  * Need to make sure reassembly_data_length is read before
 686                  * reading reassembly_queue_length and calling
 687                  * get_first_reassembly. This call is lock free
 688                  * as we never read at the end of the queue which are being
 689                  * updated in SOFTIRQ as more data is received
 690                  */
 691                 virt_rmb();
 692                 queue_length = st->reassembly_queue_length;
 693                 data_read = 0;
 694                 to_read = size;
 695                 offset = st->first_entry_offset;
 696                 while (data_read < size) {
 697                         recvmsg = get_first_reassembly(st);
 698                         data_transfer = smb_direct_recvmsg_payload(recvmsg);
 699                         data_length = le32_to_cpu(data_transfer->data_length);
 700                         remaining_data_length =
 701                                 le32_to_cpu(data_transfer->remaining_data_length);
 702                         data_offset = le32_to_cpu(data_transfer->data_offset);
 703
 704                         /*
 705                          * The upper layer expects RFC1002 length at the
 706                          * beginning of the payload. Return it to indicate
 707                          * the total length of the packet. This minimize the
 708                          * change to upper layer packet processing logic. This
 709                          * will be eventually remove when an intermediate
 710                          * transport layer is added
 711                          */
 712                         if (recvmsg->first_segment && size == 4) {
 713                                 unsigned int rfc1002_len =
 714                                         data_length + remaining_data_length;
 715                                 *((__be32 *)buf) = cpu_to_be32(rfc1002_len);
 716                                 data_read = 4;
 717                                 recvmsg->first_segment = false;
 718                                 ksmbd_debug(RDMA,
 719                                             "returning rfc1002 length %d\n",
 720                                             rfc1002_len);
 721                                 goto read_rfc1002_done;
 722                         }
 723
 724                         to_copy = min_t(int, data_length - offset, to_read);
 725                         memcpy(buf + data_read, (char *)data_transfer + data_offset + offset,
 726                                to_copy);
 727
 728                         /* move on to the next buffer? */
 729                         if (to_copy == data_length - offset) {
 730                                 queue_length--;
 731                                 /*
 732                                  * No need to lock if we are not at the
 733                                  * end of the queue
 734                                  */
 735                                 if (queue_length) {
 736                                         list_del(&recvmsg->list);
 737                                 } else {
 738                                         spin_lock_irq(&st->reassembly_queue_lock);
 739                                         list_del(&recvmsg->list);
 740                                         spin_unlock_irq(&st->reassembly_queue_lock);
 741                                 }
 742                                 queue_removed++;
 743                                 put_recvmsg(st, recvmsg);
 744                                 offset = 0;
 745                         } else {
 746                                 offset += to_copy;
 747                         }
 748
 749                         to_read -= to_copy;
 750                         data_read += to_copy;
 751                 }
 752
 753                 spin_lock_irq(&st->reassembly_queue_lock);
 754                 st->reassembly_data_length -= data_read;
 755                 st->reassembly_queue_length -= queue_removed;
 756                 spin_unlock_irq(&st->reassembly_queue_lock);
 757
 758                 spin_lock(&st->receive_credit_lock);
 759                 st->count_avail_recvmsg += queue_removed;
 760                 if (is_receive_credit_post_required(st->recv_credits, st->count_avail_recvmsg)) {
 761                         spin_unlock(&st->receive_credit_lock);
 762                         mod_delayed_work(smb_direct_wq,
 763                                          &st->post_recv_credits_work, 0);
 764                 } else {
 765                         spin_unlock(&st->receive_credit_lock);
 766                 }
 767
 768                 st->first_entry_offset = offset;
 769                 ksmbd_debug(RDMA,
 770                             "returning to thread data_read=%d reassembly_data_length=%d first_entry_offset=%d\n",
 771                             data_read, st->reassembly_data_length,
 772                             st->first_entry_offset);
 773 read_rfc1002_done:
 774                 return data_read;
 775         }
 776
 777         ksmbd_debug(RDMA, "wait_event on more data\n");
 778         rc = wait_event_interruptible(st->wait_reassembly_queue,
 779                                       st->reassembly_data_length >= size ||
 780                                        st->status != SMB_DIRECT_CS_CONNECTED);
 781         if (rc)
 782                 return -EINTR;
 783
 784         goto again;
 785 }
 786
 787 static void smb_direct_post_recv_credits(struct work_struct *work)
 788 {
 789         struct smb_direct_transport *t = container_of(work,
 790                 struct smb_direct_transport, post_recv_credits_work.work);
 791         struct smb_direct_recvmsg *recvmsg;
 792         int receive_credits, credits = 0;
 793         int ret;
 794         int use_free = 1;
 795
 796         spin_lock(&t->receive_credit_lock);
 797         receive_credits = t->recv_credits;
 798         spin_unlock(&t->receive_credit_lock);
 799
 800         if (receive_credits < t->recv_credit_target) {
 801                 while (true) {
 802                         if (use_free)
 803                                 recvmsg = get_free_recvmsg(t);
 804                         else
 805                                 recvmsg = get_empty_recvmsg(t);
 806                         if (!recvmsg) {
 807                                 if (use_free) {
 808                                         use_free = 0;
 809                                         continue;
 810                                 } else {
 811                                         break;
 812                                 }
 813                         }
 814
 815                         recvmsg->type = SMB_DIRECT_MSG_DATA_TRANSFER;
 816                         recvmsg->first_segment = false;
 817
 818                         ret = smb_direct_post_recv(t, recvmsg);
 819                         if (ret) {
 820                                 pr_err("Can't post recv: %d\n", ret);
 821                                 put_recvmsg(t, recvmsg);
 822                                 break;
 823                         }
 824                         credits++;
 825                 }
 826         }
 827
 828         spin_lock(&t->receive_credit_lock);
 829         t->recv_credits += credits;
 830         t->count_avail_recvmsg -= credits;
 831         spin_unlock(&t->receive_credit_lock);
 832
 833         spin_lock(&t->lock_new_recv_credits);
 834         t->new_recv_credits += credits;
 835         spin_unlock(&t->lock_new_recv_credits);
 836
 837         if (credits)
 838                 queue_work(smb_direct_wq, &t->send_immediate_work);
 839 }
 840
 841 static void send_done(struct ib_cq *cq, struct ib_wc *wc)
 842 {
 843         struct smb_direct_sendmsg *sendmsg, *sibling;
 844         struct smb_direct_transport *t;
 845         struct list_head *pos, *prev, *end;
 846
 847         sendmsg = container_of(wc->wr_cqe, struct smb_direct_sendmsg, cqe);
 848         t = sendmsg->transport;
 849
 850         ksmbd_debug(RDMA, "Send completed. status='%s (%d)', opcode=%d\n",
 851                     ib_wc_status_msg(wc->status), wc->status,
 852                     wc->opcode);
 853
 854         if (wc->status != IB_WC_SUCCESS || wc->opcode != IB_WC_SEND) {
 855                 pr_err("Send error. status='%s (%d)', opcode=%d\n",
 856                        ib_wc_status_msg(wc->status), wc->status,
 857                        wc->opcode);
 858                 smb_direct_disconnect_rdma_connection(t);
 859         }
 860
 861         if (sendmsg->num_sge > 1) {
 862                 if (atomic_dec_and_test(&t->send_payload_pending))
 863                         wake_up(&t->wait_send_payload_pending);
 864         } else {
 865                 if (atomic_dec_and_test(&t->send_pending))
 866                         wake_up(&t->wait_send_pending);
 867         }
 868
 869         /* iterate and free the list of messages in reverse. the list's head
 870          * is invalid.
 871          */
 872         for (pos = &sendmsg->list, prev = pos->prev, end = sendmsg->list.next;
 873              prev != end; pos = prev, prev = prev->prev) {
 874                 sibling = container_of(pos, struct smb_direct_sendmsg, list);
 875                 smb_direct_free_sendmsg(t, sibling);
 876         }
 877
 878         sibling = container_of(pos, struct smb_direct_sendmsg, list);
 879         smb_direct_free_sendmsg(t, sibling);
 880 }
 881
 882 static int manage_credits_prior_sending(struct smb_direct_transport *t)
 883 {
 884         int new_credits;
 885
 886         spin_lock(&t->lock_new_recv_credits);
 887         new_credits = t->new_recv_credits;
 888         t->new_recv_credits = 0;
 889         spin_unlock(&t->lock_new_recv_credits);
 890
 891         return new_credits;
 892 }
 893
 894 static int smb_direct_post_send(struct smb_direct_transport *t,
 895                                 struct ib_send_wr *wr)
 896 {
 897         int ret;
 898
 899         if (wr->num_sge > 1)
 900                 atomic_inc(&t->send_payload_pending);
 901         else
 902                 atomic_inc(&t->send_pending);
 903
 904         ret = ib_post_send(t->qp, wr, NULL);
 905         if (ret) {
 906                 pr_err("failed to post send: %d\n", ret);
 907                 if (wr->num_sge > 1) {
 908                         if (atomic_dec_and_test(&t->send_payload_pending))
 909                                 wake_up(&t->wait_send_payload_pending);
 910                 } else {
 911                         if (atomic_dec_and_test(&t->send_pending))
 912                                 wake_up(&t->wait_send_pending);
 913                 }
 914                 smb_direct_disconnect_rdma_connection(t);
 915         }
 916         return ret;
 917 }
 918
 919 static void smb_direct_send_ctx_init(struct smb_direct_transport *t,
 920                                      struct smb_direct_send_ctx *send_ctx,
 921                                      bool need_invalidate_rkey,
 922                                      unsigned int remote_key)
 923 {
 924         INIT_LIST_HEAD(&send_ctx->msg_list);
 925         send_ctx->wr_cnt = 0;
 926         send_ctx->need_invalidate_rkey = need_invalidate_rkey;
 927         send_ctx->remote_key = remote_key;
 928 }
 929
 930 static int smb_direct_flush_send_list(struct smb_direct_transport *t,
 931                                       struct smb_direct_send_ctx *send_ctx,
 932                                       bool is_last)
 933 {
 934         struct smb_direct_sendmsg *first, *last;
 935         int ret;
 936
 937         if (list_empty(&send_ctx->msg_list))
 938                 return 0;
 939
 940         first = list_first_entry(&send_ctx->msg_list,
 941                                  struct smb_direct_sendmsg,
 942                                  list);
 943         last = list_last_entry(&send_ctx->msg_list,
 944                                struct smb_direct_sendmsg,
 945                                list);
 946
 947         last->wr.send_flags = IB_SEND_SIGNALED;
 948         last->wr.wr_cqe = &last->cqe;
 949         if (is_last && send_ctx->need_invalidate_rkey) {
 950                 last->wr.opcode = IB_WR_SEND_WITH_INV;
 951                 last->wr.ex.invalidate_rkey = send_ctx->remote_key;
 952         }
 953
 954         ret = smb_direct_post_send(t, &first->wr);
 955         if (!ret) {
 956                 smb_direct_send_ctx_init(t, send_ctx,
 957                                          send_ctx->need_invalidate_rkey,
 958                                          send_ctx->remote_key);
 959         } else {
 960                 atomic_add(send_ctx->wr_cnt, &t->send_credits);
 961                 wake_up(&t->wait_send_credits);
 962                 list_for_each_entry_safe(first, last, &send_ctx->msg_list,
 963                                          list) {
 964                         smb_direct_free_sendmsg(t, first);
 965                 }
 966         }
 967         return ret;
 968 }
 969
 970 static int wait_for_credits(struct smb_direct_transport *t,
 971                             wait_queue_head_t *waitq, atomic_t *credits)
 972 {
 973         int ret;
 974
 975         do {
 976                 if (atomic_dec_return(credits) >= 0)
 977                         return 0;
 978
 979                 atomic_inc(credits);
 980                 ret = wait_event_interruptible(*waitq,
 981                                                atomic_read(credits) > 0 ||
 982                                                 t->status != SMB_DIRECT_CS_CONNECTED);
 983
 984                 if (t->status != SMB_DIRECT_CS_CONNECTED)
 985                         return -ENOTCONN;
 986                 else if (ret < 0)
 987                         return ret;
 988         } while (true);
 989 }
 990
 991 static int wait_for_send_credits(struct smb_direct_transport *t,
 992                                  struct smb_direct_send_ctx *send_ctx)
 993 {
 994         int ret;
 995
 996         if (send_ctx &&
 997             (send_ctx->wr_cnt >= 16 || atomic_read(&t->send_credits) <= 1)) {
 998                 ret = smb_direct_flush_send_list(t, send_ctx, false);
 999                 if (ret)
1000                         return ret;
1001         }
1002
1003         return wait_for_credits(t, &t->wait_send_credits, &t->send_credits);
1004 }
1005
1006 static int smb_direct_create_header(struct smb_direct_transport *t,
1007                                     int size, int remaining_data_length,
1008                                     struct smb_direct_sendmsg **sendmsg_out)
1009 {
1010         struct smb_direct_sendmsg *sendmsg;
1011         struct smb_direct_data_transfer *packet;
1012         int header_length;
1013         int ret;
1014
1015         sendmsg = smb_direct_alloc_sendmsg(t);
1016         if (IS_ERR(sendmsg))
1017                 return PTR_ERR(sendmsg);
1018
1019         /* Fill in the packet header */
1020         packet = (struct smb_direct_data_transfer *)sendmsg->packet;
1021         packet->credits_requested = cpu_to_le16(t->send_credit_target);
1022         packet->credits_granted = cpu_to_le16(manage_credits_prior_sending(t));
1023
1024         packet->flags = 0;
1025         packet->reserved = 0;
1026         if (!size)
1027                 packet->data_offset = 0;
1028         else
1029                 packet->data_offset = cpu_to_le32(24);
1030         packet->data_length = cpu_to_le32(size);
1031         packet->remaining_data_length = cpu_to_le32(remaining_data_length);
1032         packet->padding = 0;
1033
1034         ksmbd_debug(RDMA,
1035                     "credits_requested=%d credits_granted=%d data_offset=%d data_length=%d remaining_data_length=%d\n",
1036                     le16_to_cpu(packet->credits_requested),
1037                     le16_to_cpu(packet->credits_granted),
1038                     le32_to_cpu(packet->data_offset),
1039                     le32_to_cpu(packet->data_length),
1040                     le32_to_cpu(packet->remaining_data_length));
1041
1042         /* Map the packet to DMA */
1043         header_length = sizeof(struct smb_direct_data_transfer);
1044         /* If this is a packet without payload, don't send padding */
1045         if (!size)
1046                 header_length =
1047                         offsetof(struct smb_direct_data_transfer, padding);
1048
1049         sendmsg->sge[0].addr = ib_dma_map_single(t->cm_id->device,
1050                                                  (void *)packet,
1051                                                  header_length,
1052                                                  DMA_TO_DEVICE);
1053         ret = ib_dma_mapping_error(t->cm_id->device, sendmsg->sge[0].addr);
1054         if (ret) {
1055                 smb_direct_free_sendmsg(t, sendmsg);
1056                 return ret;
1057         }
1058
1059         sendmsg->num_sge = 1;
1060         sendmsg->sge[0].length = header_length;
1061         sendmsg->sge[0].lkey = t->pd->local_dma_lkey;
1062
1063         *sendmsg_out = sendmsg;
1064         return 0;
1065 }
1066
1067 static int get_sg_list(void *buf, int size, struct scatterlist *sg_list, int nentries)
1068 {
1069         bool high = is_vmalloc_addr(buf);
1070         struct page *page;
1071         int offset, len;
1072         int i = 0;
1073
1074         if (nentries < get_buf_page_count(buf, size))
1075                 return -EINVAL;
1076
1077         offset = offset_in_page(buf);
1078         buf -= offset;
1079         while (size > 0) {
1080                 len = min_t(int, PAGE_SIZE - offset, size);
1081                 if (high)
1082                         page = vmalloc_to_page(buf);
1083                 else
1084                         page = kmap_to_page(buf);
1085
1086                 if (!sg_list)
1087                         return -EINVAL;
1088                 sg_set_page(sg_list, page, len, offset);
1089                 sg_list = sg_next(sg_list);
1090
1091                 buf += PAGE_SIZE;
1092                 size -= len;
1093                 offset = 0;
1094                 i++;
1095         }
1096         return i;
1097 }
1098
1099 static int get_mapped_sg_list(struct ib_device *device, void *buf, int size,
1100                               struct scatterlist *sg_list, int nentries,
1101                               enum dma_data_direction dir)
1102 {
1103         int npages;
1104
1105         npages = get_sg_list(buf, size, sg_list, nentries);
1106         if (npages <= 0)
1107                 return -EINVAL;
1108         return ib_dma_map_sg(device, sg_list, npages, dir);
1109 }
1110
1111 static int post_sendmsg(struct smb_direct_transport *t,
1112                         struct smb_direct_send_ctx *send_ctx,
1113                         struct smb_direct_sendmsg *msg)
1114 {
1115         int i;
1116
1117         for (i = 0; i < msg->num_sge; i++)
1118                 ib_dma_sync_single_for_device(t->cm_id->device,
1119                                               msg->sge[i].addr, msg->sge[i].length,
1120                                               DMA_TO_DEVICE);
1121
1122         msg->cqe.done = send_done;
1123         msg->wr.opcode = IB_WR_SEND;
1124         msg->wr.sg_list = &msg->sge[0];
1125         msg->wr.num_sge = msg->num_sge;
1126         msg->wr.next = NULL;
1127
1128         if (send_ctx) {
1129                 msg->wr.wr_cqe = NULL;
1130                 msg->wr.send_flags = 0;
1131                 if (!list_empty(&send_ctx->msg_list)) {
1132                         struct smb_direct_sendmsg *last;
1133
1134                         last = list_last_entry(&send_ctx->msg_list,
1135                                                struct smb_direct_sendmsg,
1136                                                list);
1137                         last->wr.next = &msg->wr;
1138                 }
1139                 list_add_tail(&msg->list, &send_ctx->msg_list);
1140                 send_ctx->wr_cnt++;
1141                 return 0;
1142         }
1143
1144         msg->wr.wr_cqe = &msg->cqe;
1145         msg->wr.send_flags = IB_SEND_SIGNALED;
1146         return smb_direct_post_send(t, &msg->wr);
1147 }
1148
1149 static int smb_direct_post_send_data(struct smb_direct_transport *t,
1150                                      struct smb_direct_send_ctx *send_ctx,
1151                                      struct kvec *iov, int niov,
1152                                      int remaining_data_length)
1153 {
1154         int i, j, ret;
1155         struct smb_direct_sendmsg *msg;
1156         int data_length;
1157         struct scatterlist sg[SMB_DIRECT_MAX_SEND_SGES - 1];
1158
1159         ret = wait_for_send_credits(t, send_ctx);
1160         if (ret)
1161                 return ret;
1162
1163         data_length = 0;
1164         for (i = 0; i < niov; i++)
1165                 data_length += iov[i].iov_len;
1166
1167         ret = smb_direct_create_header(t, data_length, remaining_data_length,
1168                                        &msg);
1169         if (ret) {
1170                 atomic_inc(&t->send_credits);
1171                 return ret;
1172         }
1173
1174         for (i = 0; i < niov; i++) {
1175                 struct ib_sge *sge;
1176                 int sg_cnt;
1177
1178                 sg_init_table(sg, SMB_DIRECT_MAX_SEND_SGES - 1);
1179                 sg_cnt = get_mapped_sg_list(t->cm_id->device,
1180                                             iov[i].iov_base, iov[i].iov_len,
1181                                             sg, SMB_DIRECT_MAX_SEND_SGES - 1,
1182                                             DMA_TO_DEVICE);
1183                 if (sg_cnt <= 0) {
1184                         pr_err("failed to map buffer\n");
1185                         ret = -ENOMEM;
1186                         goto err;
1187                 } else if (sg_cnt + msg->num_sge > SMB_DIRECT_MAX_SEND_SGES) {
1188                         pr_err("buffer not fitted into sges\n");
1189                         ret = -E2BIG;
1190                         ib_dma_unmap_sg(t->cm_id->device, sg, sg_cnt,
1191                                         DMA_TO_DEVICE);
1192                         goto err;
1193                 }
1194
1195                 for (j = 0; j < sg_cnt; j++) {
1196                         sge = &msg->sge[msg->num_sge];
1197                         sge->addr = sg_dma_address(&sg[j]);
1198                         sge->length = sg_dma_len(&sg[j]);
1199                         sge->lkey  = t->pd->local_dma_lkey;
1200                         msg->num_sge++;
1201                 }
1202         }
1203
1204         ret = post_sendmsg(t, send_ctx, msg);
1205         if (ret)
1206                 goto err;
1207         return 0;
1208 err:
1209         smb_direct_free_sendmsg(t, msg);
1210         atomic_inc(&t->send_credits);
1211         return ret;
1212 }
1213
1214 static int smb_direct_writev(struct ksmbd_transport *t,
1215                              struct kvec *iov, int niovs, int buflen,
1216                              bool need_invalidate, unsigned int remote_key)
1217 {
1218         struct smb_direct_transport *st = smb_trans_direct_transfort(t);
1219         int remaining_data_length;
1220         int start, i, j;
1221         int max_iov_size = st->max_send_size -
1222                         sizeof(struct smb_direct_data_transfer);
1223         int ret;
1224         struct kvec vec;
1225         struct smb_direct_send_ctx send_ctx;
1226
1227         if (st->status != SMB_DIRECT_CS_CONNECTED)
1228                 return -ENOTCONN;
1229
1230         //FIXME: skip RFC1002 header..
1231         buflen -= 4;
1232         iov[0].iov_base += 4;
1233         iov[0].iov_len -= 4;
1234
1235         remaining_data_length = buflen;
1236         ksmbd_debug(RDMA, "Sending smb (RDMA): smb_len=%u\n", buflen);
1237
1238         smb_direct_send_ctx_init(st, &send_ctx, need_invalidate, remote_key);
1239         start = i = 0;
1240         buflen = 0;
1241         while (true) {
1242                 buflen += iov[i].iov_len;
1243                 if (buflen > max_iov_size) {
1244                         if (i > start) {
1245                                 remaining_data_length -=
1246                                         (buflen - iov[i].iov_len);
1247                                 ret = smb_direct_post_send_data(st, &send_ctx,
1248                                                                 &iov[start], i - start,
1249                                                                 remaining_data_length);
1250                                 if (ret)
1251                                         goto done;
1252                         } else {
1253                                 /* iov[start] is too big, break it */
1254                                 int nvec  = (buflen + max_iov_size - 1) /
1255                                                 max_iov_size;
1256
1257                                 for (j = 0; j < nvec; j++) {
1258                                         vec.iov_base =
1259                                                 (char *)iov[start].iov_base +
1260                                                 j * max_iov_size;
1261                                         vec.iov_len =
1262                                                 min_t(int, max_iov_size,
1263                                                       buflen - max_iov_size * j);
1264                                         remaining_data_length -= vec.iov_len;
1265                                         ret = smb_direct_post_send_data(st, &send_ctx, &vec, 1,
1266                                                                         remaining_data_length);
1267                                         if (ret)
1268                                                 goto done;
1269                                 }
1270                                 i++;
1271                                 if (i == niovs)
1272                                         break;
1273                         }
1274                         start = i;
1275                         buflen = 0;
1276                 } else {
1277                         i++;
1278                         if (i == niovs) {
1279                                 /* send out all remaining vecs */
1280                                 remaining_data_length -= buflen;
1281                                 ret = smb_direct_post_send_data(st, &send_ctx,
1282                                                                 &iov[start], i - start,
1283                                                                 remaining_data_length);
1284                                 if (ret)
1285                                         goto done;
1286                                 break;
1287                         }
1288                 }
1289         }
1290
1291 done:
1292         ret = smb_direct_flush_send_list(st, &send_ctx, true);
1293
1294         /*
1295          * As an optimization, we don't wait for individual I/O to finish
1296          * before sending the next one.
1297          * Send them all and wait for pending send count to get to 0
1298          * that means all the I/Os have been out and we are good to return
1299          */
1300
1301         wait_event(st->wait_send_payload_pending,
1302                    atomic_read(&st->send_payload_pending) == 0);
1303         return ret;
1304 }
1305
1306 static void read_write_done(struct ib_cq *cq, struct ib_wc *wc,
1307                             enum dma_data_direction dir)
1308 {
1309         struct smb_direct_rdma_rw_msg *msg = container_of(wc->wr_cqe,
1310                                                           struct smb_direct_rdma_rw_msg, cqe);
1311         struct smb_direct_transport *t = msg->t;
1312
1313         if (wc->status != IB_WC_SUCCESS) {
1314                 pr_err("read/write error. opcode = %d, status = %s(%d)\n",
1315                        wc->opcode, ib_wc_status_msg(wc->status), wc->status);
1316                 smb_direct_disconnect_rdma_connection(t);
1317         }
1318
1319         if (atomic_inc_return(&t->rw_avail_ops) > 0)
1320                 wake_up(&t->wait_rw_avail_ops);
1321
1322         rdma_rw_ctx_destroy(&msg->rw_ctx, t->qp, t->qp->port,
1323                             msg->sg_list, msg->sgt.nents, dir);
1324         sg_free_table_chained(&msg->sgt, SG_CHUNK_SIZE);
1325         complete(msg->completion);
1326         kfree(msg);
1327 }
1328
1329 static void read_done(struct ib_cq *cq, struct ib_wc *wc)
1330 {
1331         read_write_done(cq, wc, DMA_FROM_DEVICE);
1332 }
1333
1334 static void write_done(struct ib_cq *cq, struct ib_wc *wc)
1335 {
1336         read_write_done(cq, wc, DMA_TO_DEVICE);
1337 }
1338
1339 static int smb_direct_rdma_xmit(struct smb_direct_transport *t, void *buf,
1340                                 int buf_len, u32 remote_key, u64 remote_offset,
1341                                 u32 remote_len, bool is_read)
1342 {
1343         struct smb_direct_rdma_rw_msg *msg;
1344         int ret;
1345         DECLARE_COMPLETION_ONSTACK(completion);
1346         struct ib_send_wr *first_wr = NULL;
1347
1348         ret = wait_for_credits(t, &t->wait_rw_avail_ops, &t->rw_avail_ops);
1349         if (ret < 0)
1350                 return ret;
1351
1352         /* TODO: mempool */
1353         msg = kmalloc(offsetof(struct smb_direct_rdma_rw_msg, sg_list) +
1354                       sizeof(struct scatterlist) * SG_CHUNK_SIZE, GFP_KERNEL);
1355         if (!msg) {
1356                 atomic_inc(&t->rw_avail_ops);
1357                 return -ENOMEM;
1358         }
1359
1360         msg->sgt.sgl = &msg->sg_list[0];
1361         ret = sg_alloc_table_chained(&msg->sgt,
1362                                      get_buf_page_count(buf, buf_len),
1363                                      msg->sg_list, SG_CHUNK_SIZE);
1364         if (ret) {
1365                 atomic_inc(&t->rw_avail_ops);
1366                 kfree(msg);
1367                 return -ENOMEM;
1368         }
1369
1370         ret = get_sg_list(buf, buf_len, msg->sgt.sgl, msg->sgt.orig_nents);
1371         if (ret <= 0) {
1372                 pr_err("failed to get pages\n");
1373                 goto err;
1374         }
1375
1376         ret = rdma_rw_ctx_init(&msg->rw_ctx, t->qp, t->qp->port,
1377                                msg->sg_list, get_buf_page_count(buf, buf_len),
1378                                0, remote_offset, remote_key,
1379                                is_read ? DMA_FROM_DEVICE : DMA_TO_DEVICE);
1380         if (ret < 0) {
1381                 pr_err("failed to init rdma_rw_ctx: %d\n", ret);
1382                 goto err;
1383         }
1384
1385         msg->t = t;
1386         msg->cqe.done = is_read ? read_done : write_done;
1387         msg->completion = &completion;
1388         first_wr = rdma_rw_ctx_wrs(&msg->rw_ctx, t->qp, t->qp->port,
1389                                    &msg->cqe, NULL);
1390
1391         ret = ib_post_send(t->qp, first_wr, NULL);
1392         if (ret) {
1393                 pr_err("failed to post send wr: %d\n", ret);
1394                 goto err;
1395         }
1396
1397         wait_for_completion(&completion);
1398         return 0;
1399
1400 err:
1401         atomic_inc(&t->rw_avail_ops);
1402         if (first_wr)
1403                 rdma_rw_ctx_destroy(&msg->rw_ctx, t->qp, t->qp->port,
1404                                     msg->sg_list, msg->sgt.nents,
1405                                     is_read ? DMA_FROM_DEVICE : DMA_TO_DEVICE);
1406         sg_free_table_chained(&msg->sgt, SG_CHUNK_SIZE);
1407         kfree(msg);
1408         return ret;
1409 }
1410
1411 static int smb_direct_rdma_write(struct ksmbd_transport *t, void *buf,
1412                                  unsigned int buflen, u32 remote_key,
1413                                  u64 remote_offset, u32 remote_len)
1414 {
1415         return smb_direct_rdma_xmit(smb_trans_direct_transfort(t), buf, buflen,
1416                                     remote_key, remote_offset,
1417                                     remote_len, false);
1418 }
1419
1420 static int smb_direct_rdma_read(struct ksmbd_transport *t, void *buf,
1421                                 unsigned int buflen, u32 remote_key,
1422                                 u64 remote_offset, u32 remote_len)
1423 {
1424         return smb_direct_rdma_xmit(smb_trans_direct_transfort(t), buf, buflen,
1425                                     remote_key, remote_offset,
1426                                     remote_len, true);
1427 }
1428
1429 static void smb_direct_disconnect(struct ksmbd_transport *t)
1430 {
1431         struct smb_direct_transport *st = smb_trans_direct_transfort(t);
1432
1433         ksmbd_debug(RDMA, "Disconnecting cm_id=%p\n", st->cm_id);
1434
1435         smb_direct_disconnect_rdma_work(&st->disconnect_work);
1436         wait_event_interruptible(st->wait_status,
1437                                  st->status == SMB_DIRECT_CS_DISCONNECTED);
1438         free_transport(st);
1439 }
1440
1441 static int smb_direct_cm_handler(struct rdma_cm_id *cm_id,
1442                                  struct rdma_cm_event *event)
1443 {
1444         struct smb_direct_transport *t = cm_id->context;
1445
1446         ksmbd_debug(RDMA, "RDMA CM event. cm_id=%p event=%s (%d)\n",
1447                     cm_id, rdma_event_msg(event->event), event->event);
1448
1449         switch (event->event) {
1450         case RDMA_CM_EVENT_ESTABLISHED: {
1451                 t->status = SMB_DIRECT_CS_CONNECTED;
1452                 wake_up_interruptible(&t->wait_status);
1453                 break;
1454         }
1455         case RDMA_CM_EVENT_DEVICE_REMOVAL:
1456         case RDMA_CM_EVENT_DISCONNECTED: {
1457                 t->status = SMB_DIRECT_CS_DISCONNECTED;
1458                 wake_up_interruptible(&t->wait_status);
1459                 wake_up_interruptible(&t->wait_reassembly_queue);
1460                 wake_up(&t->wait_send_credits);
1461                 break;
1462         }
1463         case RDMA_CM_EVENT_CONNECT_ERROR: {
1464                 t->status = SMB_DIRECT_CS_DISCONNECTED;
1465                 wake_up_interruptible(&t->wait_status);
1466                 break;
1467         }
1468         default:
1469                 pr_err("Unexpected RDMA CM event. cm_id=%p, event=%s (%d)\n",
1470                        cm_id, rdma_event_msg(event->event),
1471                        event->event);
1472                 break;
1473         }
1474         return 0;
1475 }
1476
1477 static void smb_direct_qpair_handler(struct ib_event *event, void *context)
1478 {
1479         struct smb_direct_transport *t = context;
1480
1481         ksmbd_debug(RDMA, "Received QP event. cm_id=%p, event=%s (%d)\n",
1482                     t->cm_id, ib_event_msg(event->event), event->event);
1483
1484         switch (event->event) {
1485         case IB_EVENT_CQ_ERR:
1486         case IB_EVENT_QP_FATAL:
1487                 smb_direct_disconnect_rdma_connection(t);
1488                 break;
1489         default:
1490                 break;
1491         }
1492 }
1493
1494 static int smb_direct_send_negotiate_response(struct smb_direct_transport *t,
1495                                               int failed)
1496 {
1497         struct smb_direct_sendmsg *sendmsg;
1498         struct smb_direct_negotiate_resp *resp;
1499         int ret;
1500
1501         sendmsg = smb_direct_alloc_sendmsg(t);
1502         if (IS_ERR(sendmsg))
1503                 return -ENOMEM;
1504
1505         resp = (struct smb_direct_negotiate_resp *)sendmsg->packet;
1506         if (failed) {
1507                 memset(resp, 0, sizeof(*resp));
1508                 resp->min_version = cpu_to_le16(0x0100);
1509                 resp->max_version = cpu_to_le16(0x0100);
1510                 resp->status = STATUS_NOT_SUPPORTED;
1511         } else {
1512                 resp->status = STATUS_SUCCESS;
1513                 resp->min_version = SMB_DIRECT_VERSION_LE;
1514                 resp->max_version = SMB_DIRECT_VERSION_LE;
1515                 resp->negotiated_version = SMB_DIRECT_VERSION_LE;
1516                 resp->reserved = 0;
1517                 resp->credits_requested =
1518                                 cpu_to_le16(t->send_credit_target);
1519                 resp->credits_granted = cpu_to_le16(manage_credits_prior_sending(t));
1520                 resp->max_readwrite_size = cpu_to_le32(t->max_rdma_rw_size);
1521                 resp->preferred_send_size = cpu_to_le32(t->max_send_size);
1522                 resp->max_receive_size = cpu_to_le32(t->max_recv_size);
1523                 resp->max_fragmented_size =
1524                                 cpu_to_le32(t->max_fragmented_recv_size);
1525         }
1526
1527         sendmsg->sge[0].addr = ib_dma_map_single(t->cm_id->device,
1528                                                  (void *)resp, sizeof(*resp),
1529                                                  DMA_TO_DEVICE);
1530         ret = ib_dma_mapping_error(t->cm_id->device, sendmsg->sge[0].addr);
1531         if (ret) {
1532                 smb_direct_free_sendmsg(t, sendmsg);
1533                 return ret;
1534         }
1535
1536         sendmsg->num_sge = 1;
1537         sendmsg->sge[0].length = sizeof(*resp);
1538         sendmsg->sge[0].lkey = t->pd->local_dma_lkey;
1539
1540         ret = post_sendmsg(t, NULL, sendmsg);
1541         if (ret) {
1542                 smb_direct_free_sendmsg(t, sendmsg);
1543                 return ret;
1544         }
1545
1546         wait_event(t->wait_send_pending,
1547                    atomic_read(&t->send_pending) == 0);
1548         return 0;
1549 }
1550
1551 static int smb_direct_accept_client(struct smb_direct_transport *t)
1552 {
1553         struct rdma_conn_param conn_param;
1554         struct ib_port_immutable port_immutable;
1555         u32 ird_ord_hdr[2];
1556         int ret;
1557
1558         memset(&conn_param, 0, sizeof(conn_param));
1559         conn_param.initiator_depth = min_t(u8, t->cm_id->device->attrs.max_qp_rd_atom,
1560                                            SMB_DIRECT_CM_INITIATOR_DEPTH);
1561         conn_param.responder_resources = 0;
1562
1563         t->cm_id->device->ops.get_port_immutable(t->cm_id->device,
1564                                                  t->cm_id->port_num,
1565                                                  &port_immutable);
1566         if (port_immutable.core_cap_flags & RDMA_CORE_PORT_IWARP) {
1567                 ird_ord_hdr[0] = conn_param.responder_resources;
1568                 ird_ord_hdr[1] = 1;
1569                 conn_param.private_data = ird_ord_hdr;
1570                 conn_param.private_data_len = sizeof(ird_ord_hdr);
1571         } else {
1572                 conn_param.private_data = NULL;
1573                 conn_param.private_data_len = 0;
1574         }
1575         conn_param.retry_count = SMB_DIRECT_CM_RETRY;
1576         conn_param.rnr_retry_count = SMB_DIRECT_CM_RNR_RETRY;
1577         conn_param.flow_control = 0;
1578
1579         ret = rdma_accept(t->cm_id, &conn_param);
1580         if (ret) {
1581                 pr_err("error at rdma_accept: %d\n", ret);
1582                 return ret;
1583         }
1584
1585         wait_event_interruptible(t->wait_status,
1586                                  t->status != SMB_DIRECT_CS_NEW);
1587         if (t->status != SMB_DIRECT_CS_CONNECTED)
1588                 return -ENOTCONN;
1589         return 0;
1590 }
1591
1592 static int smb_direct_negotiate(struct smb_direct_transport *t)
1593 {
1594         int ret;
1595         struct smb_direct_recvmsg *recvmsg;
1596         struct smb_direct_negotiate_req *req;
1597
1598         recvmsg = get_free_recvmsg(t);
1599         if (!recvmsg)
1600                 return -ENOMEM;
1601         recvmsg->type = SMB_DIRECT_MSG_NEGOTIATE_REQ;
1602
1603         ret = smb_direct_post_recv(t, recvmsg);
1604         if (ret) {
1605                 pr_err("Can't post recv: %d\n", ret);
1606                 goto out;
1607         }
1608
1609         t->negotiation_requested = false;
1610         ret = smb_direct_accept_client(t);
1611         if (ret) {
1612                 pr_err("Can't accept client\n");
1613                 goto out;
1614         }
1615
1616         smb_direct_post_recv_credits(&t->post_recv_credits_work.work);
1617
1618         ksmbd_debug(RDMA, "Waiting for SMB_DIRECT negotiate request\n");
1619         ret = wait_event_interruptible_timeout(t->wait_status,
1620                                                t->negotiation_requested ||
1621                                                 t->status == SMB_DIRECT_CS_DISCONNECTED,
1622                                                SMB_DIRECT_NEGOTIATE_TIMEOUT * HZ);
1623         if (ret <= 0 || t->status == SMB_DIRECT_CS_DISCONNECTED) {
1624                 ret = ret < 0 ? ret : -ETIMEDOUT;
1625                 goto out;
1626         }
1627
1628         ret = smb_direct_check_recvmsg(recvmsg);
1629         if (ret == -ECONNABORTED)
1630                 goto out;
1631
1632         req = (struct smb_direct_negotiate_req *)recvmsg->packet;
1633         t->max_recv_size = min_t(int, t->max_recv_size,
1634                                  le32_to_cpu(req->preferred_send_size));
1635         t->max_send_size = min_t(int, t->max_send_size,
1636                                  le32_to_cpu(req->max_receive_size));
1637         t->max_fragmented_send_size =
1638                         le32_to_cpu(req->max_fragmented_size);
1639
1640         ret = smb_direct_send_negotiate_response(t, ret);
1641 out:
1642         if (recvmsg)
1643                 put_recvmsg(t, recvmsg);
1644         return ret;
1645 }
1646
1647 static int smb_direct_init_params(struct smb_direct_transport *t,
1648                                   struct ib_qp_cap *cap)
1649 {
1650         struct ib_device *device = t->cm_id->device;
1651         int max_send_sges, max_pages, max_rw_wrs, max_send_wrs;
1652
1653         /* need 2 more sge. because a SMB_DIRECT header will be mapped,
1654          * and maybe a send buffer could be not page aligned.
1655          */
1656         t->max_send_size = smb_direct_max_send_size;
1657         max_send_sges = DIV_ROUND_UP(t->max_send_size, PAGE_SIZE) + 2;
1658         if (max_send_sges > SMB_DIRECT_MAX_SEND_SGES) {
1659                 pr_err("max_send_size %d is too large\n", t->max_send_size);
1660                 return -EINVAL;
1661         }
1662
1663         /*
1664          * allow smb_direct_max_outstanding_rw_ops of in-flight RDMA
1665          * read/writes. HCA guarantees at least max_send_sge of sges for
1666          * a RDMA read/write work request, and if memory registration is used,
1667          * we need reg_mr, local_inv wrs for each read/write.
1668          */
1669         t->max_rdma_rw_size = smb_direct_max_read_write_size;
1670         max_pages = DIV_ROUND_UP(t->max_rdma_rw_size, PAGE_SIZE) + 1;
1671         max_rw_wrs = DIV_ROUND_UP(max_pages, SMB_DIRECT_MAX_SEND_SGES);
1672         max_rw_wrs += rdma_rw_mr_factor(device, t->cm_id->port_num,
1673                         max_pages) * 2;
1674         max_rw_wrs *= smb_direct_max_outstanding_rw_ops;
1675
1676         max_send_wrs = smb_direct_send_credit_target + max_rw_wrs;
1677         if (max_send_wrs > device->attrs.max_cqe ||
1678             max_send_wrs > device->attrs.max_qp_wr) {
1679                 pr_err("consider lowering send_credit_target = %d, or max_outstanding_rw_ops = %d\n",
1680                        smb_direct_send_credit_target,
1681                        smb_direct_max_outstanding_rw_ops);
1682                 pr_err("Possible CQE overrun, device reporting max_cqe %d max_qp_wr %d\n",
1683                        device->attrs.max_cqe, device->attrs.max_qp_wr);
1684                 return -EINVAL;
1685         }
1686
1687         if (smb_direct_receive_credit_max > device->attrs.max_cqe ||
1688             smb_direct_receive_credit_max > device->attrs.max_qp_wr) {
1689                 pr_err("consider lowering receive_credit_max = %d\n",
1690                        smb_direct_receive_credit_max);
1691                 pr_err("Possible CQE overrun, device reporting max_cpe %d max_qp_wr %d\n",
1692                        device->attrs.max_cqe, device->attrs.max_qp_wr);
1693                 return -EINVAL;
1694         }
1695
1696         if (device->attrs.max_send_sge < SMB_DIRECT_MAX_SEND_SGES) {
1697                 pr_err("warning: device max_send_sge = %d too small\n",
1698                        device->attrs.max_send_sge);
1699                 return -EINVAL;
1700         }
1701         if (device->attrs.max_recv_sge < SMB_DIRECT_MAX_RECV_SGES) {
1702                 pr_err("warning: device max_recv_sge = %d too small\n",
1703                        device->attrs.max_recv_sge);
1704                 return -EINVAL;
1705         }
1706
1707         t->recv_credits = 0;
1708         t->count_avail_recvmsg = 0;
1709
1710         t->recv_credit_max = smb_direct_receive_credit_max;
1711         t->recv_credit_target = 10;
1712         t->new_recv_credits = 0;
1713
1714         t->send_credit_target = smb_direct_send_credit_target;
1715         atomic_set(&t->send_credits, 0);
1716         atomic_set(&t->rw_avail_ops, smb_direct_max_outstanding_rw_ops);
1717
1718         t->max_send_size = smb_direct_max_send_size;
1719         t->max_recv_size = smb_direct_max_receive_size;
1720         t->max_fragmented_recv_size = smb_direct_max_fragmented_recv_size;
1721
1722         cap->max_send_wr = max_send_wrs;
1723         cap->max_recv_wr = t->recv_credit_max;
1724         cap->max_send_sge = SMB_DIRECT_MAX_SEND_SGES;
1725         cap->max_recv_sge = SMB_DIRECT_MAX_RECV_SGES;
1726         cap->max_inline_data = 0;
1727         cap->max_rdma_ctxs = 0;
1728         return 0;
1729 }
1730
1731 static void smb_direct_destroy_pools(struct smb_direct_transport *t)
1732 {
1733         struct smb_direct_recvmsg *recvmsg;
1734
1735         while ((recvmsg = get_free_recvmsg(t)))
1736                 mempool_free(recvmsg, t->recvmsg_mempool);
1737         while ((recvmsg = get_empty_recvmsg(t)))
1738                 mempool_free(recvmsg, t->recvmsg_mempool);
1739
1740         mempool_destroy(t->recvmsg_mempool);
1741         t->recvmsg_mempool = NULL;
1742
1743         kmem_cache_destroy(t->recvmsg_cache);
1744         t->recvmsg_cache = NULL;
1745
1746         mempool_destroy(t->sendmsg_mempool);
1747         t->sendmsg_mempool = NULL;
1748
1749         kmem_cache_destroy(t->sendmsg_cache);
1750         t->sendmsg_cache = NULL;
1751 }
1752
1753 static int smb_direct_create_pools(struct smb_direct_transport *t)
1754 {
1755         char name[80];
1756         int i;
1757         struct smb_direct_recvmsg *recvmsg;
1758
1759         snprintf(name, sizeof(name), "smb_direct_rqst_pool_%p", t);
1760         t->sendmsg_cache = kmem_cache_create(name,
1761                                              sizeof(struct smb_direct_sendmsg) +
1762                                               sizeof(struct smb_direct_negotiate_resp),
1763                                              0, SLAB_HWCACHE_ALIGN, NULL);
1764         if (!t->sendmsg_cache)
1765                 return -ENOMEM;
1766
1767         t->sendmsg_mempool = mempool_create(t->send_credit_target,
1768                                             mempool_alloc_slab, mempool_free_slab,
1769                                             t->sendmsg_cache);
1770         if (!t->sendmsg_mempool)
1771                 goto err;
1772
1773         snprintf(name, sizeof(name), "smb_direct_resp_%p", t);
1774         t->recvmsg_cache = kmem_cache_create(name,
1775                                              sizeof(struct smb_direct_recvmsg) +
1776                                               t->max_recv_size,
1777                                              0, SLAB_HWCACHE_ALIGN, NULL);
1778         if (!t->recvmsg_cache)
1779                 goto err;
1780
1781         t->recvmsg_mempool =
1782                 mempool_create(t->recv_credit_max, mempool_alloc_slab,
1783                                mempool_free_slab, t->recvmsg_cache);
1784         if (!t->recvmsg_mempool)
1785                 goto err;
1786
1787         INIT_LIST_HEAD(&t->recvmsg_queue);
1788
1789         for (i = 0; i < t->recv_credit_max; i++) {
1790                 recvmsg = mempool_alloc(t->recvmsg_mempool, GFP_KERNEL);
1791                 if (!recvmsg)
1792                         goto err;
1793                 recvmsg->transport = t;
1794                 list_add(&recvmsg->list, &t->recvmsg_queue);
1795         }
1796         t->count_avail_recvmsg = t->recv_credit_max;
1797
1798         return 0;
1799 err:
1800         smb_direct_destroy_pools(t);
1801         return -ENOMEM;
1802 }
1803
1804 static int smb_direct_create_qpair(struct smb_direct_transport *t,
1805                                    struct ib_qp_cap *cap)
1806 {
1807         int ret;
1808         struct ib_qp_init_attr qp_attr;
1809
1810         t->pd = ib_alloc_pd(t->cm_id->device, 0);
1811         if (IS_ERR(t->pd)) {
1812                 pr_err("Can't create RDMA PD\n");
1813                 ret = PTR_ERR(t->pd);
1814                 t->pd = NULL;
1815                 return ret;
1816         }
1817
1818         t->send_cq = ib_alloc_cq(t->cm_id->device, t,
1819                                  t->send_credit_target, 0, IB_POLL_WORKQUEUE);
1820         if (IS_ERR(t->send_cq)) {
1821                 pr_err("Can't create RDMA send CQ\n");
1822                 ret = PTR_ERR(t->send_cq);
1823                 t->send_cq = NULL;
1824                 goto err;
1825         }
1826
1827         t->recv_cq = ib_alloc_cq(t->cm_id->device, t,
1828                                  cap->max_send_wr + cap->max_rdma_ctxs,
1829                                  0, IB_POLL_WORKQUEUE);
1830         if (IS_ERR(t->recv_cq)) {
1831                 pr_err("Can't create RDMA recv CQ\n");
1832                 ret = PTR_ERR(t->recv_cq);
1833                 t->recv_cq = NULL;
1834                 goto err;
1835         }
1836
1837         memset(&qp_attr, 0, sizeof(qp_attr));
1838         qp_attr.event_handler = smb_direct_qpair_handler;
1839         qp_attr.qp_context = t;
1840         qp_attr.cap = *cap;
1841         qp_attr.sq_sig_type = IB_SIGNAL_REQ_WR;
1842         qp_attr.qp_type = IB_QPT_RC;
1843         qp_attr.send_cq = t->send_cq;
1844         qp_attr.recv_cq = t->recv_cq;
1845         qp_attr.port_num = ~0;
1846
1847         ret = rdma_create_qp(t->cm_id, t->pd, &qp_attr);
1848         if (ret) {
1849                 pr_err("Can't create RDMA QP: %d\n", ret);
1850                 goto err;
1851         }
1852
1853         t->qp = t->cm_id->qp;
1854         t->cm_id->event_handler = smb_direct_cm_handler;
1855
1856         return 0;
1857 err:
1858         if (t->qp) {
1859                 ib_destroy_qp(t->qp);
1860                 t->qp = NULL;
1861         }
1862         if (t->recv_cq) {
1863                 ib_destroy_cq(t->recv_cq);
1864                 t->recv_cq = NULL;
1865         }
1866         if (t->send_cq) {
1867                 ib_destroy_cq(t->send_cq);
1868                 t->send_cq = NULL;
1869         }
1870         if (t->pd) {
1871                 ib_dealloc_pd(t->pd);
1872                 t->pd = NULL;
1873         }
1874         return ret;
1875 }
1876
1877 static int smb_direct_prepare(struct ksmbd_transport *t)
1878 {
1879         struct smb_direct_transport *st = smb_trans_direct_transfort(t);
1880         int ret;
1881         struct ib_qp_cap qp_cap;
1882
1883         ret = smb_direct_init_params(st, &qp_cap);
1884         if (ret) {
1885                 pr_err("Can't configure RDMA parameters\n");
1886                 return ret;
1887         }
1888
1889         ret = smb_direct_create_pools(st);
1890         if (ret) {
1891                 pr_err("Can't init RDMA pool: %d\n", ret);
1892                 return ret;
1893         }
1894
1895         ret = smb_direct_create_qpair(st, &qp_cap);
1896         if (ret) {
1897                 pr_err("Can't accept RDMA client: %d\n", ret);
1898                 return ret;
1899         }
1900
1901         ret = smb_direct_negotiate(st);
1902         if (ret) {
1903                 pr_err("Can't negotiate: %d\n", ret);
1904                 return ret;
1905         }
1906
1907         st->status = SMB_DIRECT_CS_CONNECTED;
1908         return 0;
1909 }
1910
1911 static bool rdma_frwr_is_supported(struct ib_device_attr *attrs)
1912 {
1913         if (!(attrs->device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS))
1914                 return false;
1915         if (attrs->max_fast_reg_page_list_len == 0)
1916                 return false;
1917         return true;
1918 }
1919
1920 static int smb_direct_handle_connect_request(struct rdma_cm_id *new_cm_id)
1921 {
1922         struct smb_direct_transport *t;
1923
1924         if (!rdma_frwr_is_supported(&new_cm_id->device->attrs)) {
1925                 ksmbd_debug(RDMA,
1926                             "Fast Registration Work Requests is not supported. device capabilities=%llx\n",
1927                             new_cm_id->device->attrs.device_cap_flags);
1928                 return -EPROTONOSUPPORT;
1929         }
1930
1931         t = alloc_transport(new_cm_id);
1932         if (!t)
1933                 return -ENOMEM;
1934
1935         KSMBD_TRANS(t)->handler = kthread_run(ksmbd_conn_handler_loop,
1936                                               KSMBD_TRANS(t)->conn, "ksmbd:r%u",
1937                                               SMB_DIRECT_PORT);
1938         if (IS_ERR(KSMBD_TRANS(t)->handler)) {
1939                 int ret = PTR_ERR(KSMBD_TRANS(t)->handler);
1940
1941                 pr_err("Can't start thread\n");
1942                 free_transport(t);
1943                 return ret;
1944         }
1945
1946         return 0;
1947 }
1948
1949 static int smb_direct_listen_handler(struct rdma_cm_id *cm_id,
1950                                      struct rdma_cm_event *event)
1951 {
1952         switch (event->event) {
1953         case RDMA_CM_EVENT_CONNECT_REQUEST: {
1954                 int ret = smb_direct_handle_connect_request(cm_id);
1955
1956                 if (ret) {
1957                         pr_err("Can't create transport: %d\n", ret);
1958                         return ret;
1959                 }
1960
1961                 ksmbd_debug(RDMA, "Received connection request. cm_id=%p\n",
1962                             cm_id);
1963                 break;
1964         }
1965         default:
1966                 pr_err("Unexpected listen event. cm_id=%p, event=%s (%d)\n",
1967                        cm_id, rdma_event_msg(event->event), event->event);
1968                 break;
1969         }
1970         return 0;
1971 }
1972
1973 static int smb_direct_listen(int port)
1974 {
1975         int ret;
1976         struct rdma_cm_id *cm_id;
1977         struct sockaddr_in sin = {
1978                 .sin_family             = AF_INET,
1979                 .sin_addr.s_addr        = htonl(INADDR_ANY),
1980                 .sin_port               = htons(port),
1981         };
1982
1983         cm_id = rdma_create_id(&init_net, smb_direct_listen_handler,
1984                                &smb_direct_listener, RDMA_PS_TCP, IB_QPT_RC);
1985         if (IS_ERR(cm_id)) {
1986                 pr_err("Can't create cm id: %ld\n", PTR_ERR(cm_id));
1987                 return PTR_ERR(cm_id);
1988         }
1989
1990         ret = rdma_bind_addr(cm_id, (struct sockaddr *)&sin);
1991         if (ret) {
1992                 pr_err("Can't bind: %d\n", ret);
1993                 goto err;
1994         }
1995
1996         smb_direct_listener.cm_id = cm_id;
1997
1998         ret = rdma_listen(cm_id, 10);
1999         if (ret) {
2000                 pr_err("Can't listen: %d\n", ret);
2001                 goto err;
2002         }
2003         return 0;
2004 err:
2005         smb_direct_listener.cm_id = NULL;
2006         rdma_destroy_id(cm_id);
2007         return ret;
2008 }
2009
2010 int ksmbd_rdma_init(void)
2011 {
2012         int ret;
2013
2014         smb_direct_listener.cm_id = NULL;
2015
2016         /* When a client is running out of send credits, the credits are
2017          * granted by the server's sending a packet using this queue.
2018          * This avoids the situation that a clients cannot send packets
2019          * for lack of credits
2020          */
2021         smb_direct_wq = alloc_workqueue("ksmbd-smb_direct-wq",
2022                                         WQ_HIGHPRI | WQ_MEM_RECLAIM, 0);
2023         if (!smb_direct_wq)
2024                 return -ENOMEM;
2025
2026         ret = smb_direct_listen(SMB_DIRECT_PORT);
2027         if (ret) {
2028                 destroy_workqueue(smb_direct_wq);
2029                 smb_direct_wq = NULL;
2030                 pr_err("Can't listen: %d\n", ret);
2031                 return ret;
2032         }
2033
2034         ksmbd_debug(RDMA, "init RDMA listener. cm_id=%p\n",
2035                     smb_direct_listener.cm_id);
2036         return 0;
2037 }
2038
2039 int ksmbd_rdma_destroy(void)
2040 {
2041         if (smb_direct_listener.cm_id)
2042                 rdma_destroy_id(smb_direct_listener.cm_id);
2043         smb_direct_listener.cm_id = NULL;
2044
2045         if (smb_direct_wq) {
2046                 flush_workqueue(smb_direct_wq);
2047                 destroy_workqueue(smb_direct_wq);
2048                 smb_direct_wq = NULL;
2049         }
2050         return 0;
2051 }
2052
2053 bool ksmbd_rdma_capable_netdev(struct net_device *netdev)
2054 {
2055         struct ib_device *ibdev;
2056         bool rdma_capable = false;
2057
2058         ibdev = ib_device_get_by_netdev(netdev, RDMA_DRIVER_UNKNOWN);
2059         if (ibdev) {
2060                 if (rdma_frwr_is_supported(&ibdev->attrs))
2061                         rdma_capable = true;
2062                 ib_device_put(ibdev);
2063         }
2064         return rdma_capable;
2065 }
2066
2067 static struct ksmbd_transport_ops ksmbd_smb_direct_transport_ops = {
2068         .prepare        = smb_direct_prepare,
2069         .disconnect     = smb_direct_disconnect,
2070         .writev         = smb_direct_writev,
2071         .read           = smb_direct_read,
2072         .rdma_read      = smb_direct_rdma_read,
2073         .rdma_write     = smb_direct_rdma_write,
2074 };