drivers/nvme/host/tcp.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * NVMe over Fabrics TCP host.
   4  * Copyright (c) 2018 Lightbits Labs. All rights reserved.
   5  */
   6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
   7 #include <linux/module.h>
   8 #include <linux/init.h>
   9 #include <linux/slab.h>
  10 #include <linux/err.h>
  11 #include <linux/nvme-tcp.h>
  12 #include <net/sock.h>
  13 #include <net/tcp.h>
  14 #include <linux/blk-mq.h>
  15 #include <crypto/hash.h>
  16 #include <net/busy_poll.h>
  17
  18 #include "nvme.h"
  19 #include "fabrics.h"
  20
  21 struct nvme_tcp_queue;
  22
  23 /* Define the socket priority to use for connections were it is desirable
  24  * that the NIC consider performing optimized packet processing or filtering.
  25  * A non-zero value being sufficient to indicate general consideration of any
  26  * possible optimization.  Making it a module param allows for alternative
  27  * values that may be unique for some NIC implementations.
  28  */
  29 static int so_priority;
  30 module_param(so_priority, int, 0644);
  31 MODULE_PARM_DESC(so_priority, "nvme tcp socket optimize priority");
  32
  33 #ifdef CONFIG_DEBUG_LOCK_ALLOC
  34 /* lockdep can detect a circular dependency of the form
  35  *   sk_lock -> mmap_lock (page fault) -> fs locks -> sk_lock
  36  * because dependencies are tracked for both nvme-tcp and user contexts. Using
  37  * a separate class prevents lockdep from conflating nvme-tcp socket use with
  38  * user-space socket API use.
  39  */
  40 static struct lock_class_key nvme_tcp_sk_key[2];
  41 static struct lock_class_key nvme_tcp_slock_key[2];
  42
  43 static void nvme_tcp_reclassify_socket(struct socket *sock)
  44 {
  45         struct sock *sk = sock->sk;
  46
  47         if (WARN_ON_ONCE(!sock_allow_reclassification(sk)))
  48                 return;
  49
  50         switch (sk->sk_family) {
  51         case AF_INET:
  52                 sock_lock_init_class_and_name(sk, "slock-AF_INET-NVME",
  53                                               &nvme_tcp_slock_key[0],
  54                                               "sk_lock-AF_INET-NVME",
  55                                               &nvme_tcp_sk_key[0]);
  56                 break;
  57         case AF_INET6:
  58                 sock_lock_init_class_and_name(sk, "slock-AF_INET6-NVME",
  59                                               &nvme_tcp_slock_key[1],
  60                                               "sk_lock-AF_INET6-NVME",
  61                                               &nvme_tcp_sk_key[1]);
  62                 break;
  63         default:
  64                 WARN_ON_ONCE(1);
  65         }
  66 }
  67 #else
  68 static void nvme_tcp_reclassify_socket(struct socket *sock) { }
  69 #endif
  70
  71 enum nvme_tcp_send_state {
  72         NVME_TCP_SEND_CMD_PDU = 0,
  73         NVME_TCP_SEND_H2C_PDU,
  74         NVME_TCP_SEND_DATA,
  75         NVME_TCP_SEND_DDGST,
  76 };
  77
  78 struct nvme_tcp_request {
  79         struct nvme_request     req;
  80         void                    *pdu;
  81         struct nvme_tcp_queue   *queue;
  82         u32                     data_len;
  83         u32                     pdu_len;
  84         u32                     pdu_sent;
  85         u32                     h2cdata_left;
  86         u32                     h2cdata_offset;
  87         u16                     ttag;
  88         __le16                  status;
  89         struct list_head        entry;
  90         struct llist_node       lentry;
  91         __le32                  ddgst;
  92
  93         struct bio              *curr_bio;
  94         struct iov_iter         iter;
  95
  96         /* send state */
  97         size_t                  offset;
  98         size_t                  data_sent;
  99         enum nvme_tcp_send_state state;
 100 };
 101
 102 enum nvme_tcp_queue_flags {
 103         NVME_TCP_Q_ALLOCATED    = 0,
 104         NVME_TCP_Q_LIVE         = 1,
 105         NVME_TCP_Q_POLLING      = 2,
 106 };
 107
 108 enum nvme_tcp_recv_state {
 109         NVME_TCP_RECV_PDU = 0,
 110         NVME_TCP_RECV_DATA,
 111         NVME_TCP_RECV_DDGST,
 112 };
 113
 114 struct nvme_tcp_ctrl;
 115 struct nvme_tcp_queue {
 116         struct socket           *sock;
 117         struct work_struct      io_work;
 118         int                     io_cpu;
 119
 120         struct mutex            queue_lock;
 121         struct mutex            send_mutex;
 122         struct llist_head       req_list;
 123         struct list_head        send_list;
 124         bool                    more_requests;
 125
 126         /* recv state */
 127         void                    *pdu;
 128         int                     pdu_remaining;
 129         int                     pdu_offset;
 130         size_t                  data_remaining;
 131         size_t                  ddgst_remaining;
 132         unsigned int            nr_cqe;
 133
 134         /* send state */
 135         struct nvme_tcp_request *request;
 136
 137         int                     queue_size;
 138         u32                     maxh2cdata;
 139         size_t                  cmnd_capsule_len;
 140         struct nvme_tcp_ctrl    *ctrl;
 141         unsigned long           flags;
 142         bool                    rd_enabled;
 143
 144         bool                    hdr_digest;
 145         bool                    data_digest;
 146         struct ahash_request    *rcv_hash;
 147         struct ahash_request    *snd_hash;
 148         __le32                  exp_ddgst;
 149         __le32                  recv_ddgst;
 150
 151         struct page_frag_cache  pf_cache;
 152
 153         void (*state_change)(struct sock *);
 154         void (*data_ready)(struct sock *);
 155         void (*write_space)(struct sock *);
 156 };
 157
 158 struct nvme_tcp_ctrl {
 159         /* read only in the hot path */
 160         struct nvme_tcp_queue   *queues;
 161         struct blk_mq_tag_set   tag_set;
 162
 163         /* other member variables */
 164         struct list_head        list;
 165         struct blk_mq_tag_set   admin_tag_set;
 166         struct sockaddr_storage addr;
 167         struct sockaddr_storage src_addr;
 168         struct nvme_ctrl        ctrl;
 169
 170         struct work_struct      err_work;
 171         struct delayed_work     connect_work;
 172         struct nvme_tcp_request async_req;
 173         u32                     io_queues[HCTX_MAX_TYPES];
 174 };
 175
 176 static LIST_HEAD(nvme_tcp_ctrl_list);
 177 static DEFINE_MUTEX(nvme_tcp_ctrl_mutex);
 178 static struct workqueue_struct *nvme_tcp_wq;
 179 static const struct blk_mq_ops nvme_tcp_mq_ops;
 180 static const struct blk_mq_ops nvme_tcp_admin_mq_ops;
 181 static int nvme_tcp_try_send(struct nvme_tcp_queue *queue);
 182
 183 static inline struct nvme_tcp_ctrl *to_tcp_ctrl(struct nvme_ctrl *ctrl)
 184 {
 185         return container_of(ctrl, struct nvme_tcp_ctrl, ctrl);
 186 }
 187
 188 static inline int nvme_tcp_queue_id(struct nvme_tcp_queue *queue)
 189 {
 190         return queue - queue->ctrl->queues;
 191 }
 192
 193 static inline struct blk_mq_tags *nvme_tcp_tagset(struct nvme_tcp_queue *queue)
 194 {
 195         u32 queue_idx = nvme_tcp_queue_id(queue);
 196
 197         if (queue_idx == 0)
 198                 return queue->ctrl->admin_tag_set.tags[queue_idx];
 199         return queue->ctrl->tag_set.tags[queue_idx - 1];
 200 }
 201
 202 static inline u8 nvme_tcp_hdgst_len(struct nvme_tcp_queue *queue)
 203 {
 204         return queue->hdr_digest ? NVME_TCP_DIGEST_LENGTH : 0;
 205 }
 206
 207 static inline u8 nvme_tcp_ddgst_len(struct nvme_tcp_queue *queue)
 208 {
 209         return queue->data_digest ? NVME_TCP_DIGEST_LENGTH : 0;
 210 }
 211
 212 static inline size_t nvme_tcp_inline_data_size(struct nvme_tcp_request *req)
 213 {
 214         if (nvme_is_fabrics(req->req.cmd))
 215                 return NVME_TCP_ADMIN_CCSZ;
 216         return req->queue->cmnd_capsule_len - sizeof(struct nvme_command);
 217 }
 218
 219 static inline bool nvme_tcp_async_req(struct nvme_tcp_request *req)
 220 {
 221         return req == &req->queue->ctrl->async_req;
 222 }
 223
 224 static inline bool nvme_tcp_has_inline_data(struct nvme_tcp_request *req)
 225 {
 226         struct request *rq;
 227
 228         if (unlikely(nvme_tcp_async_req(req)))
 229                 return false; /* async events don't have a request */
 230
 231         rq = blk_mq_rq_from_pdu(req);
 232
 233         return rq_data_dir(rq) == WRITE && req->data_len &&
 234                 req->data_len <= nvme_tcp_inline_data_size(req);
 235 }
 236
 237 static inline struct page *nvme_tcp_req_cur_page(struct nvme_tcp_request *req)
 238 {
 239         return req->iter.bvec->bv_page;
 240 }
 241
 242 static inline size_t nvme_tcp_req_cur_offset(struct nvme_tcp_request *req)
 243 {
 244         return req->iter.bvec->bv_offset + req->iter.iov_offset;
 245 }
 246
 247 static inline size_t nvme_tcp_req_cur_length(struct nvme_tcp_request *req)
 248 {
 249         return min_t(size_t, iov_iter_single_seg_count(&req->iter),
 250                         req->pdu_len - req->pdu_sent);
 251 }
 252
 253 static inline size_t nvme_tcp_pdu_data_left(struct nvme_tcp_request *req)
 254 {
 255         return rq_data_dir(blk_mq_rq_from_pdu(req)) == WRITE ?
 256                         req->pdu_len - req->pdu_sent : 0;
 257 }
 258
 259 static inline size_t nvme_tcp_pdu_last_send(struct nvme_tcp_request *req,
 260                 int len)
 261 {
 262         return nvme_tcp_pdu_data_left(req) <= len;
 263 }
 264
 265 static void nvme_tcp_init_iter(struct nvme_tcp_request *req,
 266                 unsigned int dir)
 267 {
 268         struct request *rq = blk_mq_rq_from_pdu(req);
 269         struct bio_vec *vec;
 270         unsigned int size;
 271         int nr_bvec;
 272         size_t offset;
 273
 274         if (rq->rq_flags & RQF_SPECIAL_PAYLOAD) {
 275                 vec = &rq->special_vec;
 276                 nr_bvec = 1;
 277                 size = blk_rq_payload_bytes(rq);
 278                 offset = 0;
 279         } else {
 280                 struct bio *bio = req->curr_bio;
 281                 struct bvec_iter bi;
 282                 struct bio_vec bv;
 283
 284                 vec = __bvec_iter_bvec(bio->bi_io_vec, bio->bi_iter);
 285                 nr_bvec = 0;
 286                 bio_for_each_bvec(bv, bio, bi) {
 287                         nr_bvec++;
 288                 }
 289                 size = bio->bi_iter.bi_size;
 290                 offset = bio->bi_iter.bi_bvec_done;
 291         }
 292
 293         iov_iter_bvec(&req->iter, dir, vec, nr_bvec, size);
 294         req->iter.iov_offset = offset;
 295 }
 296
 297 static inline void nvme_tcp_advance_req(struct nvme_tcp_request *req,
 298                 int len)
 299 {
 300         req->data_sent += len;
 301         req->pdu_sent += len;
 302         iov_iter_advance(&req->iter, len);
 303         if (!iov_iter_count(&req->iter) &&
 304             req->data_sent < req->data_len) {
 305                 req->curr_bio = req->curr_bio->bi_next;
 306                 nvme_tcp_init_iter(req, WRITE);
 307         }
 308 }
 309
 310 static inline void nvme_tcp_send_all(struct nvme_tcp_queue *queue)
 311 {
 312         int ret;
 313
 314         /* drain the send queue as much as we can... */
 315         do {
 316                 ret = nvme_tcp_try_send(queue);
 317         } while (ret > 0);
 318 }
 319
 320 static inline bool nvme_tcp_queue_more(struct nvme_tcp_queue *queue)
 321 {
 322         return !list_empty(&queue->send_list) ||
 323                 !llist_empty(&queue->req_list) || queue->more_requests;
 324 }
 325
 326 static inline void nvme_tcp_queue_request(struct nvme_tcp_request *req,
 327                 bool sync, bool last)
 328 {
 329         struct nvme_tcp_queue *queue = req->queue;
 330         bool empty;
 331
 332         empty = llist_add(&req->lentry, &queue->req_list) &&
 333                 list_empty(&queue->send_list) && !queue->request;
 334
 335         /*
 336          * if we're the first on the send_list and we can try to send
 337          * directly, otherwise queue io_work. Also, only do that if we
 338          * are on the same cpu, so we don't introduce contention.
 339          */
 340         if (queue->io_cpu == raw_smp_processor_id() &&
 341             sync && empty && mutex_trylock(&queue->send_mutex)) {
 342                 queue->more_requests = !last;
 343                 nvme_tcp_send_all(queue);
 344                 queue->more_requests = false;
 345                 mutex_unlock(&queue->send_mutex);
 346         }
 347
 348         if (last && nvme_tcp_queue_more(queue))
 349                 queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work);
 350 }
 351
 352 static void nvme_tcp_process_req_list(struct nvme_tcp_queue *queue)
 353 {
 354         struct nvme_tcp_request *req;
 355         struct llist_node *node;
 356
 357         for (node = llist_del_all(&queue->req_list); node; node = node->next) {
 358                 req = llist_entry(node, struct nvme_tcp_request, lentry);
 359                 list_add(&req->entry, &queue->send_list);
 360         }
 361 }
 362
 363 static inline struct nvme_tcp_request *
 364 nvme_tcp_fetch_request(struct nvme_tcp_queue *queue)
 365 {
 366         struct nvme_tcp_request *req;
 367
 368         req = list_first_entry_or_null(&queue->send_list,
 369                         struct nvme_tcp_request, entry);
 370         if (!req) {
 371                 nvme_tcp_process_req_list(queue);
 372                 req = list_first_entry_or_null(&queue->send_list,
 373                                 struct nvme_tcp_request, entry);
 374                 if (unlikely(!req))
 375                         return NULL;
 376         }
 377
 378         list_del(&req->entry);
 379         return req;
 380 }
 381
 382 static inline void nvme_tcp_ddgst_final(struct ahash_request *hash,
 383                 __le32 *dgst)
 384 {
 385         ahash_request_set_crypt(hash, NULL, (u8 *)dgst, 0);
 386         crypto_ahash_final(hash);
 387 }
 388
 389 static inline void nvme_tcp_ddgst_update(struct ahash_request *hash,
 390                 struct page *page, off_t off, size_t len)
 391 {
 392         struct scatterlist sg;
 393
 394         sg_init_marker(&sg, 1);
 395         sg_set_page(&sg, page, len, off);
 396         ahash_request_set_crypt(hash, &sg, NULL, len);
 397         crypto_ahash_update(hash);
 398 }
 399
 400 static inline void nvme_tcp_hdgst(struct ahash_request *hash,
 401                 void *pdu, size_t len)
 402 {
 403         struct scatterlist sg;
 404
 405         sg_init_one(&sg, pdu, len);
 406         ahash_request_set_crypt(hash, &sg, pdu + len, len);
 407         crypto_ahash_digest(hash);
 408 }
 409
 410 static int nvme_tcp_verify_hdgst(struct nvme_tcp_queue *queue,
 411                 void *pdu, size_t pdu_len)
 412 {
 413         struct nvme_tcp_hdr *hdr = pdu;
 414         __le32 recv_digest;
 415         __le32 exp_digest;
 416
 417         if (unlikely(!(hdr->flags & NVME_TCP_F_HDGST))) {
 418                 dev_err(queue->ctrl->ctrl.device,
 419                         "queue %d: header digest flag is cleared\n",
 420                         nvme_tcp_queue_id(queue));
 421                 return -EPROTO;
 422         }
 423
 424         recv_digest = *(__le32 *)(pdu + hdr->hlen);
 425         nvme_tcp_hdgst(queue->rcv_hash, pdu, pdu_len);
 426         exp_digest = *(__le32 *)(pdu + hdr->hlen);
 427         if (recv_digest != exp_digest) {
 428                 dev_err(queue->ctrl->ctrl.device,
 429                         "header digest error: recv %#x expected %#x\n",
 430                         le32_to_cpu(recv_digest), le32_to_cpu(exp_digest));
 431                 return -EIO;
 432         }
 433
 434         return 0;
 435 }
 436
 437 static int nvme_tcp_check_ddgst(struct nvme_tcp_queue *queue, void *pdu)
 438 {
 439         struct nvme_tcp_hdr *hdr = pdu;
 440         u8 digest_len = nvme_tcp_hdgst_len(queue);
 441         u32 len;
 442
 443         len = le32_to_cpu(hdr->plen) - hdr->hlen -
 444                 ((hdr->flags & NVME_TCP_F_HDGST) ? digest_len : 0);
 445
 446         if (unlikely(len && !(hdr->flags & NVME_TCP_F_DDGST))) {
 447                 dev_err(queue->ctrl->ctrl.device,
 448                         "queue %d: data digest flag is cleared\n",
 449                 nvme_tcp_queue_id(queue));
 450                 return -EPROTO;
 451         }
 452         crypto_ahash_init(queue->rcv_hash);
 453
 454         return 0;
 455 }
 456
 457 static void nvme_tcp_exit_request(struct blk_mq_tag_set *set,
 458                 struct request *rq, unsigned int hctx_idx)
 459 {
 460         struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
 461
 462         page_frag_free(req->pdu);
 463 }
 464
 465 static int nvme_tcp_init_request(struct blk_mq_tag_set *set,
 466                 struct request *rq, unsigned int hctx_idx,
 467                 unsigned int numa_node)
 468 {
 469         struct nvme_tcp_ctrl *ctrl = set->driver_data;
 470         struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
 471         struct nvme_tcp_cmd_pdu *pdu;
 472         int queue_idx = (set == &ctrl->tag_set) ? hctx_idx + 1 : 0;
 473         struct nvme_tcp_queue *queue = &ctrl->queues[queue_idx];
 474         u8 hdgst = nvme_tcp_hdgst_len(queue);
 475
 476         req->pdu = page_frag_alloc(&queue->pf_cache,
 477                 sizeof(struct nvme_tcp_cmd_pdu) + hdgst,
 478                 GFP_KERNEL | __GFP_ZERO);
 479         if (!req->pdu)
 480                 return -ENOMEM;
 481
 482         pdu = req->pdu;
 483         req->queue = queue;
 484         nvme_req(rq)->ctrl = &ctrl->ctrl;
 485         nvme_req(rq)->cmd = &pdu->cmd;
 486
 487         return 0;
 488 }
 489
 490 static int nvme_tcp_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
 491                 unsigned int hctx_idx)
 492 {
 493         struct nvme_tcp_ctrl *ctrl = data;
 494         struct nvme_tcp_queue *queue = &ctrl->queues[hctx_idx + 1];
 495
 496         hctx->driver_data = queue;
 497         return 0;
 498 }
 499
 500 static int nvme_tcp_init_admin_hctx(struct blk_mq_hw_ctx *hctx, void *data,
 501                 unsigned int hctx_idx)
 502 {
 503         struct nvme_tcp_ctrl *ctrl = data;
 504         struct nvme_tcp_queue *queue = &ctrl->queues[0];
 505
 506         hctx->driver_data = queue;
 507         return 0;
 508 }
 509
 510 static enum nvme_tcp_recv_state
 511 nvme_tcp_recv_state(struct nvme_tcp_queue *queue)
 512 {
 513         return  (queue->pdu_remaining) ? NVME_TCP_RECV_PDU :
 514                 (queue->ddgst_remaining) ? NVME_TCP_RECV_DDGST :
 515                 NVME_TCP_RECV_DATA;
 516 }
 517
 518 static void nvme_tcp_init_recv_ctx(struct nvme_tcp_queue *queue)
 519 {
 520         queue->pdu_remaining = sizeof(struct nvme_tcp_rsp_pdu) +
 521                                 nvme_tcp_hdgst_len(queue);
 522         queue->pdu_offset = 0;
 523         queue->data_remaining = -1;
 524         queue->ddgst_remaining = 0;
 525 }
 526
 527 static void nvme_tcp_error_recovery(struct nvme_ctrl *ctrl)
 528 {
 529         if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING))
 530                 return;
 531
 532         dev_warn(ctrl->device, "starting error recovery\n");
 533         queue_work(nvme_reset_wq, &to_tcp_ctrl(ctrl)->err_work);
 534 }
 535
 536 static int nvme_tcp_process_nvme_cqe(struct nvme_tcp_queue *queue,
 537                 struct nvme_completion *cqe)
 538 {
 539         struct nvme_tcp_request *req;
 540         struct request *rq;
 541
 542         rq = nvme_find_rq(nvme_tcp_tagset(queue), cqe->command_id);
 543         if (!rq) {
 544                 dev_err(queue->ctrl->ctrl.device,
 545                         "got bad cqe.command_id %#x on queue %d\n",
 546                         cqe->command_id, nvme_tcp_queue_id(queue));
 547                 nvme_tcp_error_recovery(&queue->ctrl->ctrl);
 548                 return -EINVAL;
 549         }
 550
 551         req = blk_mq_rq_to_pdu(rq);
 552         if (req->status == cpu_to_le16(NVME_SC_SUCCESS))
 553                 req->status = cqe->status;
 554
 555         if (!nvme_try_complete_req(rq, req->status, cqe->result))
 556                 nvme_complete_rq(rq);
 557         queue->nr_cqe++;
 558
 559         return 0;
 560 }
 561
 562 static int nvme_tcp_handle_c2h_data(struct nvme_tcp_queue *queue,
 563                 struct nvme_tcp_data_pdu *pdu)
 564 {
 565         struct request *rq;
 566
 567         rq = nvme_find_rq(nvme_tcp_tagset(queue), pdu->command_id);
 568         if (!rq) {
 569                 dev_err(queue->ctrl->ctrl.device,
 570                         "got bad c2hdata.command_id %#x on queue %d\n",
 571                         pdu->command_id, nvme_tcp_queue_id(queue));
 572                 return -ENOENT;
 573         }
 574
 575         if (!blk_rq_payload_bytes(rq)) {
 576                 dev_err(queue->ctrl->ctrl.device,
 577                         "queue %d tag %#x unexpected data\n",
 578                         nvme_tcp_queue_id(queue), rq->tag);
 579                 return -EIO;
 580         }
 581
 582         queue->data_remaining = le32_to_cpu(pdu->data_length);
 583
 584         if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS &&
 585             unlikely(!(pdu->hdr.flags & NVME_TCP_F_DATA_LAST))) {
 586                 dev_err(queue->ctrl->ctrl.device,
 587                         "queue %d tag %#x SUCCESS set but not last PDU\n",
 588                         nvme_tcp_queue_id(queue), rq->tag);
 589                 nvme_tcp_error_recovery(&queue->ctrl->ctrl);
 590                 return -EPROTO;
 591         }
 592
 593         return 0;
 594 }
 595
 596 static int nvme_tcp_handle_comp(struct nvme_tcp_queue *queue,
 597                 struct nvme_tcp_rsp_pdu *pdu)
 598 {
 599         struct nvme_completion *cqe = &pdu->cqe;
 600         int ret = 0;
 601
 602         /*
 603          * AEN requests are special as they don't time out and can
 604          * survive any kind of queue freeze and often don't respond to
 605          * aborts.  We don't even bother to allocate a struct request
 606          * for them but rather special case them here.
 607          */
 608         if (unlikely(nvme_is_aen_req(nvme_tcp_queue_id(queue),
 609                                      cqe->command_id)))
 610                 nvme_complete_async_event(&queue->ctrl->ctrl, cqe->status,
 611                                 &cqe->result);
 612         else
 613                 ret = nvme_tcp_process_nvme_cqe(queue, cqe);
 614
 615         return ret;
 616 }
 617
 618 static void nvme_tcp_setup_h2c_data_pdu(struct nvme_tcp_request *req)
 619 {
 620         struct nvme_tcp_data_pdu *data = req->pdu;
 621         struct nvme_tcp_queue *queue = req->queue;
 622         struct request *rq = blk_mq_rq_from_pdu(req);
 623         u32 h2cdata_sent = req->pdu_len;
 624         u8 hdgst = nvme_tcp_hdgst_len(queue);
 625         u8 ddgst = nvme_tcp_ddgst_len(queue);
 626
 627         req->state = NVME_TCP_SEND_H2C_PDU;
 628         req->offset = 0;
 629         req->pdu_len = min(req->h2cdata_left, queue->maxh2cdata);
 630         req->pdu_sent = 0;
 631         req->h2cdata_left -= req->pdu_len;
 632         req->h2cdata_offset += h2cdata_sent;
 633
 634         memset(data, 0, sizeof(*data));
 635         data->hdr.type = nvme_tcp_h2c_data;
 636         if (!req->h2cdata_left)
 637                 data->hdr.flags = NVME_TCP_F_DATA_LAST;
 638         if (queue->hdr_digest)
 639                 data->hdr.flags |= NVME_TCP_F_HDGST;
 640         if (queue->data_digest)
 641                 data->hdr.flags |= NVME_TCP_F_DDGST;
 642         data->hdr.hlen = sizeof(*data);
 643         data->hdr.pdo = data->hdr.hlen + hdgst;
 644         data->hdr.plen =
 645                 cpu_to_le32(data->hdr.hlen + hdgst + req->pdu_len + ddgst);
 646         data->ttag = req->ttag;
 647         data->command_id = nvme_cid(rq);
 648         data->data_offset = cpu_to_le32(req->h2cdata_offset);
 649         data->data_length = cpu_to_le32(req->pdu_len);
 650 }
 651
 652 static int nvme_tcp_handle_r2t(struct nvme_tcp_queue *queue,
 653                 struct nvme_tcp_r2t_pdu *pdu)
 654 {
 655         struct nvme_tcp_request *req;
 656         struct request *rq;
 657         u32 r2t_length = le32_to_cpu(pdu->r2t_length);
 658         u32 r2t_offset = le32_to_cpu(pdu->r2t_offset);
 659
 660         rq = nvme_find_rq(nvme_tcp_tagset(queue), pdu->command_id);
 661         if (!rq) {
 662                 dev_err(queue->ctrl->ctrl.device,
 663                         "got bad r2t.command_id %#x on queue %d\n",
 664                         pdu->command_id, nvme_tcp_queue_id(queue));
 665                 return -ENOENT;
 666         }
 667         req = blk_mq_rq_to_pdu(rq);
 668
 669         if (unlikely(!r2t_length)) {
 670                 dev_err(queue->ctrl->ctrl.device,
 671                         "req %d r2t len is %u, probably a bug...\n",
 672                         rq->tag, r2t_length);
 673                 return -EPROTO;
 674         }
 675
 676         if (unlikely(req->data_sent + r2t_length > req->data_len)) {
 677                 dev_err(queue->ctrl->ctrl.device,
 678                         "req %d r2t len %u exceeded data len %u (%zu sent)\n",
 679                         rq->tag, r2t_length, req->data_len, req->data_sent);
 680                 return -EPROTO;
 681         }
 682
 683         if (unlikely(r2t_offset < req->data_sent)) {
 684                 dev_err(queue->ctrl->ctrl.device,
 685                         "req %d unexpected r2t offset %u (expected %zu)\n",
 686                         rq->tag, r2t_offset, req->data_sent);
 687                 return -EPROTO;
 688         }
 689
 690         req->pdu_len = 0;
 691         req->h2cdata_left = r2t_length;
 692         req->h2cdata_offset = r2t_offset;
 693         req->ttag = pdu->ttag;
 694
 695         nvme_tcp_setup_h2c_data_pdu(req);
 696         nvme_tcp_queue_request(req, false, true);
 697
 698         return 0;
 699 }
 700
 701 static int nvme_tcp_recv_pdu(struct nvme_tcp_queue *queue, struct sk_buff *skb,
 702                 unsigned int *offset, size_t *len)
 703 {
 704         struct nvme_tcp_hdr *hdr;
 705         char *pdu = queue->pdu;
 706         size_t rcv_len = min_t(size_t, *len, queue->pdu_remaining);
 707         int ret;
 708
 709         ret = skb_copy_bits(skb, *offset,
 710                 &pdu[queue->pdu_offset], rcv_len);
 711         if (unlikely(ret))
 712                 return ret;
 713
 714         queue->pdu_remaining -= rcv_len;
 715         queue->pdu_offset += rcv_len;
 716         *offset += rcv_len;
 717         *len -= rcv_len;
 718         if (queue->pdu_remaining)
 719                 return 0;
 720
 721         hdr = queue->pdu;
 722         if (queue->hdr_digest) {
 723                 ret = nvme_tcp_verify_hdgst(queue, queue->pdu, hdr->hlen);
 724                 if (unlikely(ret))
 725                         return ret;
 726         }
 727
 728
 729         if (queue->data_digest) {
 730                 ret = nvme_tcp_check_ddgst(queue, queue->pdu);
 731                 if (unlikely(ret))
 732                         return ret;
 733         }
 734
 735         switch (hdr->type) {
 736         case nvme_tcp_c2h_data:
 737                 return nvme_tcp_handle_c2h_data(queue, (void *)queue->pdu);
 738         case nvme_tcp_rsp:
 739                 nvme_tcp_init_recv_ctx(queue);
 740                 return nvme_tcp_handle_comp(queue, (void *)queue->pdu);
 741         case nvme_tcp_r2t:
 742                 nvme_tcp_init_recv_ctx(queue);
 743                 return nvme_tcp_handle_r2t(queue, (void *)queue->pdu);
 744         default:
 745                 dev_err(queue->ctrl->ctrl.device,
 746                         "unsupported pdu type (%d)\n", hdr->type);
 747                 return -EINVAL;
 748         }
 749 }
 750
 751 static inline void nvme_tcp_end_request(struct request *rq, u16 status)
 752 {
 753         union nvme_result res = {};
 754
 755         if (!nvme_try_complete_req(rq, cpu_to_le16(status << 1), res))
 756                 nvme_complete_rq(rq);
 757 }
 758
 759 static int nvme_tcp_recv_data(struct nvme_tcp_queue *queue, struct sk_buff *skb,
 760                               unsigned int *offset, size_t *len)
 761 {
 762         struct nvme_tcp_data_pdu *pdu = (void *)queue->pdu;
 763         struct request *rq =
 764                 nvme_cid_to_rq(nvme_tcp_tagset(queue), pdu->command_id);
 765         struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
 766
 767         while (true) {
 768                 int recv_len, ret;
 769
 770                 recv_len = min_t(size_t, *len, queue->data_remaining);
 771                 if (!recv_len)
 772                         break;
 773
 774                 if (!iov_iter_count(&req->iter)) {
 775                         req->curr_bio = req->curr_bio->bi_next;
 776
 777                         /*
 778                          * If we don`t have any bios it means that controller
 779                          * sent more data than we requested, hence error
 780                          */
 781                         if (!req->curr_bio) {
 782                                 dev_err(queue->ctrl->ctrl.device,
 783                                         "queue %d no space in request %#x",
 784                                         nvme_tcp_queue_id(queue), rq->tag);
 785                                 nvme_tcp_init_recv_ctx(queue);
 786                                 return -EIO;
 787                         }
 788                         nvme_tcp_init_iter(req, READ);
 789                 }
 790
 791                 /* we can read only from what is left in this bio */
 792                 recv_len = min_t(size_t, recv_len,
 793                                 iov_iter_count(&req->iter));
 794
 795                 if (queue->data_digest)
 796                         ret = skb_copy_and_hash_datagram_iter(skb, *offset,
 797                                 &req->iter, recv_len, queue->rcv_hash);
 798                 else
 799                         ret = skb_copy_datagram_iter(skb, *offset,
 800                                         &req->iter, recv_len);
 801                 if (ret) {
 802                         dev_err(queue->ctrl->ctrl.device,
 803                                 "queue %d failed to copy request %#x data",
 804                                 nvme_tcp_queue_id(queue), rq->tag);
 805                         return ret;
 806                 }
 807
 808                 *len -= recv_len;
 809                 *offset += recv_len;
 810                 queue->data_remaining -= recv_len;
 811         }
 812
 813         if (!queue->data_remaining) {
 814                 if (queue->data_digest) {
 815                         nvme_tcp_ddgst_final(queue->rcv_hash, &queue->exp_ddgst);
 816                         queue->ddgst_remaining = NVME_TCP_DIGEST_LENGTH;
 817                 } else {
 818                         if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS) {
 819                                 nvme_tcp_end_request(rq,
 820                                                 le16_to_cpu(req->status));
 821                                 queue->nr_cqe++;
 822                         }
 823                         nvme_tcp_init_recv_ctx(queue);
 824                 }
 825         }
 826
 827         return 0;
 828 }
 829
 830 static int nvme_tcp_recv_ddgst(struct nvme_tcp_queue *queue,
 831                 struct sk_buff *skb, unsigned int *offset, size_t *len)
 832 {
 833         struct nvme_tcp_data_pdu *pdu = (void *)queue->pdu;
 834         char *ddgst = (char *)&queue->recv_ddgst;
 835         size_t recv_len = min_t(size_t, *len, queue->ddgst_remaining);
 836         off_t off = NVME_TCP_DIGEST_LENGTH - queue->ddgst_remaining;
 837         int ret;
 838
 839         ret = skb_copy_bits(skb, *offset, &ddgst[off], recv_len);
 840         if (unlikely(ret))
 841                 return ret;
 842
 843         queue->ddgst_remaining -= recv_len;
 844         *offset += recv_len;
 845         *len -= recv_len;
 846         if (queue->ddgst_remaining)
 847                 return 0;
 848
 849         if (queue->recv_ddgst != queue->exp_ddgst) {
 850                 struct request *rq = nvme_cid_to_rq(nvme_tcp_tagset(queue),
 851                                         pdu->command_id);
 852                 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
 853
 854                 req->status = cpu_to_le16(NVME_SC_DATA_XFER_ERROR);
 855
 856                 dev_err(queue->ctrl->ctrl.device,
 857                         "data digest error: recv %#x expected %#x\n",
 858                         le32_to_cpu(queue->recv_ddgst),
 859                         le32_to_cpu(queue->exp_ddgst));
 860         }
 861
 862         if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS) {
 863                 struct request *rq = nvme_cid_to_rq(nvme_tcp_tagset(queue),
 864                                         pdu->command_id);
 865                 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
 866
 867                 nvme_tcp_end_request(rq, le16_to_cpu(req->status));
 868                 queue->nr_cqe++;
 869         }
 870
 871         nvme_tcp_init_recv_ctx(queue);
 872         return 0;
 873 }
 874
 875 static int nvme_tcp_recv_skb(read_descriptor_t *desc, struct sk_buff *skb,
 876                              unsigned int offset, size_t len)
 877 {
 878         struct nvme_tcp_queue *queue = desc->arg.data;
 879         size_t consumed = len;
 880         int result;
 881
 882         while (len) {
 883                 switch (nvme_tcp_recv_state(queue)) {
 884                 case NVME_TCP_RECV_PDU:
 885                         result = nvme_tcp_recv_pdu(queue, skb, &offset, &len);
 886                         break;
 887                 case NVME_TCP_RECV_DATA:
 888                         result = nvme_tcp_recv_data(queue, skb, &offset, &len);
 889                         break;
 890                 case NVME_TCP_RECV_DDGST:
 891                         result = nvme_tcp_recv_ddgst(queue, skb, &offset, &len);
 892                         break;
 893                 default:
 894                         result = -EFAULT;
 895                 }
 896                 if (result) {
 897                         dev_err(queue->ctrl->ctrl.device,
 898                                 "receive failed:  %d\n", result);
 899                         queue->rd_enabled = false;
 900                         nvme_tcp_error_recovery(&queue->ctrl->ctrl);
 901                         return result;
 902                 }
 903         }
 904
 905         return consumed;
 906 }
 907
 908 static void nvme_tcp_data_ready(struct sock *sk)
 909 {
 910         struct nvme_tcp_queue *queue;
 911
 912         read_lock_bh(&sk->sk_callback_lock);
 913         queue = sk->sk_user_data;
 914         if (likely(queue && queue->rd_enabled) &&
 915             !test_bit(NVME_TCP_Q_POLLING, &queue->flags))
 916                 queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work);
 917         read_unlock_bh(&sk->sk_callback_lock);
 918 }
 919
 920 static void nvme_tcp_write_space(struct sock *sk)
 921 {
 922         struct nvme_tcp_queue *queue;
 923
 924         read_lock_bh(&sk->sk_callback_lock);
 925         queue = sk->sk_user_data;
 926         if (likely(queue && sk_stream_is_writeable(sk))) {
 927                 clear_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
 928                 queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work);
 929         }
 930         read_unlock_bh(&sk->sk_callback_lock);
 931 }
 932
 933 static void nvme_tcp_state_change(struct sock *sk)
 934 {
 935         struct nvme_tcp_queue *queue;
 936
 937         read_lock_bh(&sk->sk_callback_lock);
 938         queue = sk->sk_user_data;
 939         if (!queue)
 940                 goto done;
 941
 942         switch (sk->sk_state) {
 943         case TCP_CLOSE:
 944         case TCP_CLOSE_WAIT:
 945         case TCP_LAST_ACK:
 946         case TCP_FIN_WAIT1:
 947         case TCP_FIN_WAIT2:
 948                 nvme_tcp_error_recovery(&queue->ctrl->ctrl);
 949                 break;
 950         default:
 951                 dev_info(queue->ctrl->ctrl.device,
 952                         "queue %d socket state %d\n",
 953                         nvme_tcp_queue_id(queue), sk->sk_state);
 954         }
 955
 956         queue->state_change(sk);
 957 done:
 958         read_unlock_bh(&sk->sk_callback_lock);
 959 }
 960
 961 static inline void nvme_tcp_done_send_req(struct nvme_tcp_queue *queue)
 962 {
 963         queue->request = NULL;
 964 }
 965
 966 static void nvme_tcp_fail_request(struct nvme_tcp_request *req)
 967 {
 968         if (nvme_tcp_async_req(req)) {
 969                 union nvme_result res = {};
 970
 971                 nvme_complete_async_event(&req->queue->ctrl->ctrl,
 972                                 cpu_to_le16(NVME_SC_HOST_PATH_ERROR), &res);
 973         } else {
 974                 nvme_tcp_end_request(blk_mq_rq_from_pdu(req),
 975                                 NVME_SC_HOST_PATH_ERROR);
 976         }
 977 }
 978
 979 static int nvme_tcp_try_send_data(struct nvme_tcp_request *req)
 980 {
 981         struct nvme_tcp_queue *queue = req->queue;
 982         int req_data_len = req->data_len;
 983         u32 h2cdata_left = req->h2cdata_left;
 984
 985         while (true) {
 986                 struct page *page = nvme_tcp_req_cur_page(req);
 987                 size_t offset = nvme_tcp_req_cur_offset(req);
 988                 size_t len = nvme_tcp_req_cur_length(req);
 989                 bool last = nvme_tcp_pdu_last_send(req, len);
 990                 int req_data_sent = req->data_sent;
 991                 int ret, flags = MSG_DONTWAIT;
 992
 993                 if (last && !queue->data_digest && !nvme_tcp_queue_more(queue))
 994                         flags |= MSG_EOR;
 995                 else
 996                         flags |= MSG_MORE | MSG_SENDPAGE_NOTLAST;
 997
 998                 if (sendpage_ok(page)) {
 999                         ret = kernel_sendpage(queue->sock, page, offset, len,
1000                                         flags);
1001                 } else {
1002                         ret = sock_no_sendpage(queue->sock, page, offset, len,
1003                                         flags);
1004                 }
1005                 if (ret <= 0)
1006                         return ret;
1007
1008                 if (queue->data_digest)
1009                         nvme_tcp_ddgst_update(queue->snd_hash, page,
1010                                         offset, ret);
1011
1012                 /*
1013                  * update the request iterator except for the last payload send
1014                  * in the request where we don't want to modify it as we may
1015                  * compete with the RX path completing the request.
1016                  */
1017                 if (req_data_sent + ret < req_data_len)
1018                         nvme_tcp_advance_req(req, ret);
1019
1020                 /* fully successful last send in current PDU */
1021                 if (last && ret == len) {
1022                         if (queue->data_digest) {
1023                                 nvme_tcp_ddgst_final(queue->snd_hash,
1024                                         &req->ddgst);
1025                                 req->state = NVME_TCP_SEND_DDGST;
1026                                 req->offset = 0;
1027                         } else {
1028                                 if (h2cdata_left)
1029                                         nvme_tcp_setup_h2c_data_pdu(req);
1030                                 else
1031                                         nvme_tcp_done_send_req(queue);
1032                         }
1033                         return 1;
1034                 }
1035         }
1036         return -EAGAIN;
1037 }
1038
1039 static int nvme_tcp_try_send_cmd_pdu(struct nvme_tcp_request *req)
1040 {
1041         struct nvme_tcp_queue *queue = req->queue;
1042         struct nvme_tcp_cmd_pdu *pdu = req->pdu;
1043         bool inline_data = nvme_tcp_has_inline_data(req);
1044         u8 hdgst = nvme_tcp_hdgst_len(queue);
1045         int len = sizeof(*pdu) + hdgst - req->offset;
1046         int flags = MSG_DONTWAIT;
1047         int ret;
1048
1049         if (inline_data || nvme_tcp_queue_more(queue))
1050                 flags |= MSG_MORE | MSG_SENDPAGE_NOTLAST;
1051         else
1052                 flags |= MSG_EOR;
1053
1054         if (queue->hdr_digest && !req->offset)
1055                 nvme_tcp_hdgst(queue->snd_hash, pdu, sizeof(*pdu));
1056
1057         ret = kernel_sendpage(queue->sock, virt_to_page(pdu),
1058                         offset_in_page(pdu) + req->offset, len,  flags);
1059         if (unlikely(ret <= 0))
1060                 return ret;
1061
1062         len -= ret;
1063         if (!len) {
1064                 if (inline_data) {
1065                         req->state = NVME_TCP_SEND_DATA;
1066                         if (queue->data_digest)
1067                                 crypto_ahash_init(queue->snd_hash);
1068                 } else {
1069                         nvme_tcp_done_send_req(queue);
1070                 }
1071                 return 1;
1072         }
1073         req->offset += ret;
1074
1075         return -EAGAIN;
1076 }
1077
1078 static int nvme_tcp_try_send_data_pdu(struct nvme_tcp_request *req)
1079 {
1080         struct nvme_tcp_queue *queue = req->queue;
1081         struct nvme_tcp_data_pdu *pdu = req->pdu;
1082         u8 hdgst = nvme_tcp_hdgst_len(queue);
1083         int len = sizeof(*pdu) - req->offset + hdgst;
1084         int ret;
1085
1086         if (queue->hdr_digest && !req->offset)
1087                 nvme_tcp_hdgst(queue->snd_hash, pdu, sizeof(*pdu));
1088
1089         if (!req->h2cdata_left)
1090                 ret = kernel_sendpage(queue->sock, virt_to_page(pdu),
1091                                 offset_in_page(pdu) + req->offset, len,
1092                                 MSG_DONTWAIT | MSG_MORE | MSG_SENDPAGE_NOTLAST);
1093         else
1094                 ret = sock_no_sendpage(queue->sock, virt_to_page(pdu),
1095                                 offset_in_page(pdu) + req->offset, len,
1096                                 MSG_DONTWAIT | MSG_MORE);
1097         if (unlikely(ret <= 0))
1098                 return ret;
1099
1100         len -= ret;
1101         if (!len) {
1102                 req->state = NVME_TCP_SEND_DATA;
1103                 if (queue->data_digest)
1104                         crypto_ahash_init(queue->snd_hash);
1105                 return 1;
1106         }
1107         req->offset += ret;
1108
1109         return -EAGAIN;
1110 }
1111
1112 static int nvme_tcp_try_send_ddgst(struct nvme_tcp_request *req)
1113 {
1114         struct nvme_tcp_queue *queue = req->queue;
1115         size_t offset = req->offset;
1116         u32 h2cdata_left = req->h2cdata_left;
1117         int ret;
1118         struct msghdr msg = { .msg_flags = MSG_DONTWAIT };
1119         struct kvec iov = {
1120                 .iov_base = (u8 *)&req->ddgst + req->offset,
1121                 .iov_len = NVME_TCP_DIGEST_LENGTH - req->offset
1122         };
1123
1124         if (nvme_tcp_queue_more(queue))
1125                 msg.msg_flags |= MSG_MORE;
1126         else
1127                 msg.msg_flags |= MSG_EOR;
1128
1129         ret = kernel_sendmsg(queue->sock, &msg, &iov, 1, iov.iov_len);
1130         if (unlikely(ret <= 0))
1131                 return ret;
1132
1133         if (offset + ret == NVME_TCP_DIGEST_LENGTH) {
1134                 if (h2cdata_left)
1135                         nvme_tcp_setup_h2c_data_pdu(req);
1136                 else
1137                         nvme_tcp_done_send_req(queue);
1138                 return 1;
1139         }
1140
1141         req->offset += ret;
1142         return -EAGAIN;
1143 }
1144
1145 static int nvme_tcp_try_send(struct nvme_tcp_queue *queue)
1146 {
1147         struct nvme_tcp_request *req;
1148         int ret = 1;
1149
1150         if (!queue->request) {
1151                 queue->request = nvme_tcp_fetch_request(queue);
1152                 if (!queue->request)
1153                         return 0;
1154         }
1155         req = queue->request;
1156
1157         if (req->state == NVME_TCP_SEND_CMD_PDU) {
1158                 ret = nvme_tcp_try_send_cmd_pdu(req);
1159                 if (ret <= 0)
1160                         goto done;
1161                 if (!nvme_tcp_has_inline_data(req))
1162                         return ret;
1163         }
1164
1165         if (req->state == NVME_TCP_SEND_H2C_PDU) {
1166                 ret = nvme_tcp_try_send_data_pdu(req);
1167                 if (ret <= 0)
1168                         goto done;
1169         }
1170
1171         if (req->state == NVME_TCP_SEND_DATA) {
1172                 ret = nvme_tcp_try_send_data(req);
1173                 if (ret <= 0)
1174                         goto done;
1175         }
1176
1177         if (req->state == NVME_TCP_SEND_DDGST)
1178                 ret = nvme_tcp_try_send_ddgst(req);
1179 done:
1180         if (ret == -EAGAIN) {
1181                 ret = 0;
1182         } else if (ret < 0) {
1183                 dev_err(queue->ctrl->ctrl.device,
1184                         "failed to send request %d\n", ret);
1185                 nvme_tcp_fail_request(queue->request);
1186                 nvme_tcp_done_send_req(queue);
1187         }
1188         return ret;
1189 }
1190
1191 static int nvme_tcp_try_recv(struct nvme_tcp_queue *queue)
1192 {
1193         struct socket *sock = queue->sock;
1194         struct sock *sk = sock->sk;
1195         read_descriptor_t rd_desc;
1196         int consumed;
1197
1198         rd_desc.arg.data = queue;
1199         rd_desc.count = 1;
1200         lock_sock(sk);
1201         queue->nr_cqe = 0;
1202         consumed = sock->ops->read_sock(sk, &rd_desc, nvme_tcp_recv_skb);
1203         release_sock(sk);
1204         return consumed;
1205 }
1206
1207 static void nvme_tcp_io_work(struct work_struct *w)
1208 {
1209         struct nvme_tcp_queue *queue =
1210                 container_of(w, struct nvme_tcp_queue, io_work);
1211         unsigned long deadline = jiffies + msecs_to_jiffies(1);
1212
1213         do {
1214                 bool pending = false;
1215                 int result;
1216
1217                 if (mutex_trylock(&queue->send_mutex)) {
1218                         result = nvme_tcp_try_send(queue);
1219                         mutex_unlock(&queue->send_mutex);
1220                         if (result > 0)
1221                                 pending = true;
1222                         else if (unlikely(result < 0))
1223                                 break;
1224                 }
1225
1226                 result = nvme_tcp_try_recv(queue);
1227                 if (result > 0)
1228                         pending = true;
1229                 else if (unlikely(result < 0))
1230                         return;
1231
1232                 if (!pending)
1233                         return;
1234
1235         } while (!time_after(jiffies, deadline)); /* quota is exhausted */
1236
1237         queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work);
1238 }
1239
1240 static void nvme_tcp_free_crypto(struct nvme_tcp_queue *queue)
1241 {
1242         struct crypto_ahash *tfm = crypto_ahash_reqtfm(queue->rcv_hash);
1243
1244         ahash_request_free(queue->rcv_hash);
1245         ahash_request_free(queue->snd_hash);
1246         crypto_free_ahash(tfm);
1247 }
1248
1249 static int nvme_tcp_alloc_crypto(struct nvme_tcp_queue *queue)
1250 {
1251         struct crypto_ahash *tfm;
1252
1253         tfm = crypto_alloc_ahash("crc32c", 0, CRYPTO_ALG_ASYNC);
1254         if (IS_ERR(tfm))
1255                 return PTR_ERR(tfm);
1256
1257         queue->snd_hash = ahash_request_alloc(tfm, GFP_KERNEL);
1258         if (!queue->snd_hash)
1259                 goto free_tfm;
1260         ahash_request_set_callback(queue->snd_hash, 0, NULL, NULL);
1261
1262         queue->rcv_hash = ahash_request_alloc(tfm, GFP_KERNEL);
1263         if (!queue->rcv_hash)
1264                 goto free_snd_hash;
1265         ahash_request_set_callback(queue->rcv_hash, 0, NULL, NULL);
1266
1267         return 0;
1268 free_snd_hash:
1269         ahash_request_free(queue->snd_hash);
1270 free_tfm:
1271         crypto_free_ahash(tfm);
1272         return -ENOMEM;
1273 }
1274
1275 static void nvme_tcp_free_async_req(struct nvme_tcp_ctrl *ctrl)
1276 {
1277         struct nvme_tcp_request *async = &ctrl->async_req;
1278
1279         page_frag_free(async->pdu);
1280 }
1281
1282 static int nvme_tcp_alloc_async_req(struct nvme_tcp_ctrl *ctrl)
1283 {
1284         struct nvme_tcp_queue *queue = &ctrl->queues[0];
1285         struct nvme_tcp_request *async = &ctrl->async_req;
1286         u8 hdgst = nvme_tcp_hdgst_len(queue);
1287
1288         async->pdu = page_frag_alloc(&queue->pf_cache,
1289                 sizeof(struct nvme_tcp_cmd_pdu) + hdgst,
1290                 GFP_KERNEL | __GFP_ZERO);
1291         if (!async->pdu)
1292                 return -ENOMEM;
1293
1294         async->queue = &ctrl->queues[0];
1295         return 0;
1296 }
1297
1298 static void nvme_tcp_free_queue(struct nvme_ctrl *nctrl, int qid)
1299 {
1300         struct page *page;
1301         struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl);
1302         struct nvme_tcp_queue *queue = &ctrl->queues[qid];
1303
1304         if (!test_and_clear_bit(NVME_TCP_Q_ALLOCATED, &queue->flags))
1305                 return;
1306
1307         if (queue->hdr_digest || queue->data_digest)
1308                 nvme_tcp_free_crypto(queue);
1309
1310         if (queue->pf_cache.va) {
1311                 page = virt_to_head_page(queue->pf_cache.va);
1312                 __page_frag_cache_drain(page, queue->pf_cache.pagecnt_bias);
1313                 queue->pf_cache.va = NULL;
1314         }
1315         sock_release(queue->sock);
1316         kfree(queue->pdu);
1317         mutex_destroy(&queue->send_mutex);
1318         mutex_destroy(&queue->queue_lock);
1319 }
1320
1321 static int nvme_tcp_init_connection(struct nvme_tcp_queue *queue)
1322 {
1323         struct nvme_tcp_icreq_pdu *icreq;
1324         struct nvme_tcp_icresp_pdu *icresp;
1325         struct msghdr msg = {};
1326         struct kvec iov;
1327         bool ctrl_hdgst, ctrl_ddgst;
1328         u32 maxh2cdata;
1329         int ret;
1330
1331         icreq = kzalloc(sizeof(*icreq), GFP_KERNEL);
1332         if (!icreq)
1333                 return -ENOMEM;
1334
1335         icresp = kzalloc(sizeof(*icresp), GFP_KERNEL);
1336         if (!icresp) {
1337                 ret = -ENOMEM;
1338                 goto free_icreq;
1339         }
1340
1341         icreq->hdr.type = nvme_tcp_icreq;
1342         icreq->hdr.hlen = sizeof(*icreq);
1343         icreq->hdr.pdo = 0;
1344         icreq->hdr.plen = cpu_to_le32(icreq->hdr.hlen);
1345         icreq->pfv = cpu_to_le16(NVME_TCP_PFV_1_0);
1346         icreq->maxr2t = 0; /* single inflight r2t supported */
1347         icreq->hpda = 0; /* no alignment constraint */
1348         if (queue->hdr_digest)
1349                 icreq->digest |= NVME_TCP_HDR_DIGEST_ENABLE;
1350         if (queue->data_digest)
1351                 icreq->digest |= NVME_TCP_DATA_DIGEST_ENABLE;
1352
1353         iov.iov_base = icreq;
1354         iov.iov_len = sizeof(*icreq);
1355         ret = kernel_sendmsg(queue->sock, &msg, &iov, 1, iov.iov_len);
1356         if (ret < 0)
1357                 goto free_icresp;
1358
1359         memset(&msg, 0, sizeof(msg));
1360         iov.iov_base = icresp;
1361         iov.iov_len = sizeof(*icresp);
1362         ret = kernel_recvmsg(queue->sock, &msg, &iov, 1,
1363                         iov.iov_len, msg.msg_flags);
1364         if (ret < 0)
1365                 goto free_icresp;
1366
1367         ret = -EINVAL;
1368         if (icresp->hdr.type != nvme_tcp_icresp) {
1369                 pr_err("queue %d: bad type returned %d\n",
1370                         nvme_tcp_queue_id(queue), icresp->hdr.type);
1371                 goto free_icresp;
1372         }
1373
1374         if (le32_to_cpu(icresp->hdr.plen) != sizeof(*icresp)) {
1375                 pr_err("queue %d: bad pdu length returned %d\n",
1376                         nvme_tcp_queue_id(queue), icresp->hdr.plen);
1377                 goto free_icresp;
1378         }
1379
1380         if (icresp->pfv != NVME_TCP_PFV_1_0) {
1381                 pr_err("queue %d: bad pfv returned %d\n",
1382                         nvme_tcp_queue_id(queue), icresp->pfv);
1383                 goto free_icresp;
1384         }
1385
1386         ctrl_ddgst = !!(icresp->digest & NVME_TCP_DATA_DIGEST_ENABLE);
1387         if ((queue->data_digest && !ctrl_ddgst) ||
1388             (!queue->data_digest && ctrl_ddgst)) {
1389                 pr_err("queue %d: data digest mismatch host: %s ctrl: %s\n",
1390                         nvme_tcp_queue_id(queue),
1391                         queue->data_digest ? "enabled" : "disabled",
1392                         ctrl_ddgst ? "enabled" : "disabled");
1393                 goto free_icresp;
1394         }
1395
1396         ctrl_hdgst = !!(icresp->digest & NVME_TCP_HDR_DIGEST_ENABLE);
1397         if ((queue->hdr_digest && !ctrl_hdgst) ||
1398             (!queue->hdr_digest && ctrl_hdgst)) {
1399                 pr_err("queue %d: header digest mismatch host: %s ctrl: %s\n",
1400                         nvme_tcp_queue_id(queue),
1401                         queue->hdr_digest ? "enabled" : "disabled",
1402                         ctrl_hdgst ? "enabled" : "disabled");
1403                 goto free_icresp;
1404         }
1405
1406         if (icresp->cpda != 0) {
1407                 pr_err("queue %d: unsupported cpda returned %d\n",
1408                         nvme_tcp_queue_id(queue), icresp->cpda);
1409                 goto free_icresp;
1410         }
1411
1412         maxh2cdata = le32_to_cpu(icresp->maxdata);
1413         if ((maxh2cdata % 4) || (maxh2cdata < NVME_TCP_MIN_MAXH2CDATA)) {
1414                 pr_err("queue %d: invalid maxh2cdata returned %u\n",
1415                        nvme_tcp_queue_id(queue), maxh2cdata);
1416                 goto free_icresp;
1417         }
1418         queue->maxh2cdata = maxh2cdata;
1419
1420         ret = 0;
1421 free_icresp:
1422         kfree(icresp);
1423 free_icreq:
1424         kfree(icreq);
1425         return ret;
1426 }
1427
1428 static bool nvme_tcp_admin_queue(struct nvme_tcp_queue *queue)
1429 {
1430         return nvme_tcp_queue_id(queue) == 0;
1431 }
1432
1433 static bool nvme_tcp_default_queue(struct nvme_tcp_queue *queue)
1434 {
1435         struct nvme_tcp_ctrl *ctrl = queue->ctrl;
1436         int qid = nvme_tcp_queue_id(queue);
1437
1438         return !nvme_tcp_admin_queue(queue) &&
1439                 qid < 1 + ctrl->io_queues[HCTX_TYPE_DEFAULT];
1440 }
1441
1442 static bool nvme_tcp_read_queue(struct nvme_tcp_queue *queue)
1443 {
1444         struct nvme_tcp_ctrl *ctrl = queue->ctrl;
1445         int qid = nvme_tcp_queue_id(queue);
1446
1447         return !nvme_tcp_admin_queue(queue) &&
1448                 !nvme_tcp_default_queue(queue) &&
1449                 qid < 1 + ctrl->io_queues[HCTX_TYPE_DEFAULT] +
1450                           ctrl->io_queues[HCTX_TYPE_READ];
1451 }
1452
1453 static bool nvme_tcp_poll_queue(struct nvme_tcp_queue *queue)
1454 {
1455         struct nvme_tcp_ctrl *ctrl = queue->ctrl;
1456         int qid = nvme_tcp_queue_id(queue);
1457
1458         return !nvme_tcp_admin_queue(queue) &&
1459                 !nvme_tcp_default_queue(queue) &&
1460                 !nvme_tcp_read_queue(queue) &&
1461                 qid < 1 + ctrl->io_queues[HCTX_TYPE_DEFAULT] +
1462                           ctrl->io_queues[HCTX_TYPE_READ] +
1463                           ctrl->io_queues[HCTX_TYPE_POLL];
1464 }
1465
1466 static void nvme_tcp_set_queue_io_cpu(struct nvme_tcp_queue *queue)
1467 {
1468         struct nvme_tcp_ctrl *ctrl = queue->ctrl;
1469         int qid = nvme_tcp_queue_id(queue);
1470         int n = 0;
1471
1472         if (nvme_tcp_default_queue(queue))
1473                 n = qid - 1;
1474         else if (nvme_tcp_read_queue(queue))
1475                 n = qid - ctrl->io_queues[HCTX_TYPE_DEFAULT] - 1;
1476         else if (nvme_tcp_poll_queue(queue))
1477                 n = qid - ctrl->io_queues[HCTX_TYPE_DEFAULT] -
1478                                 ctrl->io_queues[HCTX_TYPE_READ] - 1;
1479         queue->io_cpu = cpumask_next_wrap(n - 1, cpu_online_mask, -1, false);
1480 }
1481
1482 static int nvme_tcp_alloc_queue(struct nvme_ctrl *nctrl,
1483                 int qid, size_t queue_size)
1484 {
1485         struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl);
1486         struct nvme_tcp_queue *queue = &ctrl->queues[qid];
1487         int ret, rcv_pdu_size;
1488
1489         mutex_init(&queue->queue_lock);
1490         queue->ctrl = ctrl;
1491         init_llist_head(&queue->req_list);
1492         INIT_LIST_HEAD(&queue->send_list);
1493         mutex_init(&queue->send_mutex);
1494         INIT_WORK(&queue->io_work, nvme_tcp_io_work);
1495         queue->queue_size = queue_size;
1496
1497         if (qid > 0)
1498                 queue->cmnd_capsule_len = nctrl->ioccsz * 16;
1499         else
1500                 queue->cmnd_capsule_len = sizeof(struct nvme_command) +
1501                                                 NVME_TCP_ADMIN_CCSZ;
1502
1503         ret = sock_create(ctrl->addr.ss_family, SOCK_STREAM,
1504                         IPPROTO_TCP, &queue->sock);
1505         if (ret) {
1506                 dev_err(nctrl->device,
1507                         "failed to create socket: %d\n", ret);
1508                 goto err_destroy_mutex;
1509         }
1510
1511         nvme_tcp_reclassify_socket(queue->sock);
1512
1513         /* Single syn retry */
1514         tcp_sock_set_syncnt(queue->sock->sk, 1);
1515
1516         /* Set TCP no delay */
1517         tcp_sock_set_nodelay(queue->sock->sk);
1518
1519         /*
1520          * Cleanup whatever is sitting in the TCP transmit queue on socket
1521          * close. This is done to prevent stale data from being sent should
1522          * the network connection be restored before TCP times out.
1523          */
1524         sock_no_linger(queue->sock->sk);
1525
1526         if (so_priority > 0)
1527                 sock_set_priority(queue->sock->sk, so_priority);
1528
1529         /* Set socket type of service */
1530         if (nctrl->opts->tos >= 0)
1531                 ip_sock_set_tos(queue->sock->sk, nctrl->opts->tos);
1532
1533         /* Set 10 seconds timeout for icresp recvmsg */
1534         queue->sock->sk->sk_rcvtimeo = 10 * HZ;
1535
1536         queue->sock->sk->sk_allocation = GFP_ATOMIC;
1537         nvme_tcp_set_queue_io_cpu(queue);
1538         queue->request = NULL;
1539         queue->data_remaining = 0;
1540         queue->ddgst_remaining = 0;
1541         queue->pdu_remaining = 0;
1542         queue->pdu_offset = 0;
1543         sk_set_memalloc(queue->sock->sk);
1544
1545         if (nctrl->opts->mask & NVMF_OPT_HOST_TRADDR) {
1546                 ret = kernel_bind(queue->sock, (struct sockaddr *)&ctrl->src_addr,
1547                         sizeof(ctrl->src_addr));
1548                 if (ret) {
1549                         dev_err(nctrl->device,
1550                                 "failed to bind queue %d socket %d\n",
1551                                 qid, ret);
1552                         goto err_sock;
1553                 }
1554         }
1555
1556         if (nctrl->opts->mask & NVMF_OPT_HOST_IFACE) {
1557                 char *iface = nctrl->opts->host_iface;
1558                 sockptr_t optval = KERNEL_SOCKPTR(iface);
1559
1560                 ret = sock_setsockopt(queue->sock, SOL_SOCKET, SO_BINDTODEVICE,
1561                                       optval, strlen(iface));
1562                 if (ret) {
1563                         dev_err(nctrl->device,
1564                           "failed to bind to interface %s queue %d err %d\n",
1565                           iface, qid, ret);
1566                         goto err_sock;
1567                 }
1568         }
1569
1570         queue->hdr_digest = nctrl->opts->hdr_digest;
1571         queue->data_digest = nctrl->opts->data_digest;
1572         if (queue->hdr_digest || queue->data_digest) {
1573                 ret = nvme_tcp_alloc_crypto(queue);
1574                 if (ret) {
1575                         dev_err(nctrl->device,
1576                                 "failed to allocate queue %d crypto\n", qid);
1577                         goto err_sock;
1578                 }
1579         }
1580
1581         rcv_pdu_size = sizeof(struct nvme_tcp_rsp_pdu) +
1582                         nvme_tcp_hdgst_len(queue);
1583         queue->pdu = kmalloc(rcv_pdu_size, GFP_KERNEL);
1584         if (!queue->pdu) {
1585                 ret = -ENOMEM;
1586                 goto err_crypto;
1587         }
1588
1589         dev_dbg(nctrl->device, "connecting queue %d\n",
1590                         nvme_tcp_queue_id(queue));
1591
1592         ret = kernel_connect(queue->sock, (struct sockaddr *)&ctrl->addr,
1593                 sizeof(ctrl->addr), 0);
1594         if (ret) {
1595                 dev_err(nctrl->device,
1596                         "failed to connect socket: %d\n", ret);
1597                 goto err_rcv_pdu;
1598         }
1599
1600         ret = nvme_tcp_init_connection(queue);
1601         if (ret)
1602                 goto err_init_connect;
1603
1604         queue->rd_enabled = true;
1605         set_bit(NVME_TCP_Q_ALLOCATED, &queue->flags);
1606         nvme_tcp_init_recv_ctx(queue);
1607
1608         write_lock_bh(&queue->sock->sk->sk_callback_lock);
1609         queue->sock->sk->sk_user_data = queue;
1610         queue->state_change = queue->sock->sk->sk_state_change;
1611         queue->data_ready = queue->sock->sk->sk_data_ready;
1612         queue->write_space = queue->sock->sk->sk_write_space;
1613         queue->sock->sk->sk_data_ready = nvme_tcp_data_ready;
1614         queue->sock->sk->sk_state_change = nvme_tcp_state_change;
1615         queue->sock->sk->sk_write_space = nvme_tcp_write_space;
1616 #ifdef CONFIG_NET_RX_BUSY_POLL
1617         queue->sock->sk->sk_ll_usec = 1;
1618 #endif
1619         write_unlock_bh(&queue->sock->sk->sk_callback_lock);
1620
1621         return 0;
1622
1623 err_init_connect:
1624         kernel_sock_shutdown(queue->sock, SHUT_RDWR);
1625 err_rcv_pdu:
1626         kfree(queue->pdu);
1627 err_crypto:
1628         if (queue->hdr_digest || queue->data_digest)
1629                 nvme_tcp_free_crypto(queue);
1630 err_sock:
1631         sock_release(queue->sock);
1632         queue->sock = NULL;
1633 err_destroy_mutex:
1634         mutex_destroy(&queue->send_mutex);
1635         mutex_destroy(&queue->queue_lock);
1636         return ret;
1637 }
1638
1639 static void nvme_tcp_restore_sock_calls(struct nvme_tcp_queue *queue)
1640 {
1641         struct socket *sock = queue->sock;
1642
1643         write_lock_bh(&sock->sk->sk_callback_lock);
1644         sock->sk->sk_user_data  = NULL;
1645         sock->sk->sk_data_ready = queue->data_ready;
1646         sock->sk->sk_state_change = queue->state_change;
1647         sock->sk->sk_write_space  = queue->write_space;
1648         write_unlock_bh(&sock->sk->sk_callback_lock);
1649 }
1650
1651 static void __nvme_tcp_stop_queue(struct nvme_tcp_queue *queue)
1652 {
1653         kernel_sock_shutdown(queue->sock, SHUT_RDWR);
1654         nvme_tcp_restore_sock_calls(queue);
1655         cancel_work_sync(&queue->io_work);
1656 }
1657
1658 static void nvme_tcp_stop_queue(struct nvme_ctrl *nctrl, int qid)
1659 {
1660         struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl);
1661         struct nvme_tcp_queue *queue = &ctrl->queues[qid];
1662
1663         mutex_lock(&queue->queue_lock);
1664         if (test_and_clear_bit(NVME_TCP_Q_LIVE, &queue->flags))
1665                 __nvme_tcp_stop_queue(queue);
1666         mutex_unlock(&queue->queue_lock);
1667 }
1668
1669 static int nvme_tcp_start_queue(struct nvme_ctrl *nctrl, int idx)
1670 {
1671         struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl);
1672         int ret;
1673
1674         if (idx)
1675                 ret = nvmf_connect_io_queue(nctrl, idx);
1676         else
1677                 ret = nvmf_connect_admin_queue(nctrl);
1678
1679         if (!ret) {
1680                 set_bit(NVME_TCP_Q_LIVE, &ctrl->queues[idx].flags);
1681         } else {
1682                 if (test_bit(NVME_TCP_Q_ALLOCATED, &ctrl->queues[idx].flags))
1683                         __nvme_tcp_stop_queue(&ctrl->queues[idx]);
1684                 dev_err(nctrl->device,
1685                         "failed to connect queue: %d ret=%d\n", idx, ret);
1686         }
1687         return ret;
1688 }
1689
1690 static int nvme_tcp_alloc_admin_tag_set(struct nvme_ctrl *nctrl)
1691 {
1692         struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl);
1693         struct blk_mq_tag_set *set = &ctrl->admin_tag_set;
1694         int ret;
1695
1696         memset(set, 0, sizeof(*set));
1697         set->ops = &nvme_tcp_admin_mq_ops;
1698         set->queue_depth = NVME_AQ_MQ_TAG_DEPTH;
1699         set->reserved_tags = NVMF_RESERVED_TAGS;
1700         set->numa_node = nctrl->numa_node;
1701         set->flags = BLK_MQ_F_BLOCKING;
1702         set->cmd_size = sizeof(struct nvme_tcp_request);
1703         set->driver_data = ctrl;
1704         set->nr_hw_queues = 1;
1705         set->timeout = NVME_ADMIN_TIMEOUT;
1706         ret = blk_mq_alloc_tag_set(set);
1707         if (!ret)
1708                 nctrl->admin_tagset = set;
1709         return ret;
1710 }
1711
1712 static int nvme_tcp_alloc_tag_set(struct nvme_ctrl *nctrl)
1713 {
1714         struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl);
1715         struct blk_mq_tag_set *set = &ctrl->tag_set;
1716         int ret;
1717
1718         memset(set, 0, sizeof(*set));
1719         set->ops = &nvme_tcp_mq_ops;
1720         set->queue_depth = nctrl->sqsize + 1;
1721         set->reserved_tags = NVMF_RESERVED_TAGS;
1722         set->numa_node = nctrl->numa_node;
1723         set->flags = BLK_MQ_F_SHOULD_MERGE | BLK_MQ_F_BLOCKING;
1724         set->cmd_size = sizeof(struct nvme_tcp_request);
1725         set->driver_data = ctrl;
1726         set->nr_hw_queues = nctrl->queue_count - 1;
1727         set->timeout = NVME_IO_TIMEOUT;
1728         set->nr_maps = nctrl->opts->nr_poll_queues ? HCTX_MAX_TYPES : 2;
1729         ret = blk_mq_alloc_tag_set(set);
1730         if (!ret)
1731                 nctrl->tagset = set;
1732         return ret;
1733 }
1734
1735 static void nvme_tcp_free_admin_queue(struct nvme_ctrl *ctrl)
1736 {
1737         if (to_tcp_ctrl(ctrl)->async_req.pdu) {
1738                 cancel_work_sync(&ctrl->async_event_work);
1739                 nvme_tcp_free_async_req(to_tcp_ctrl(ctrl));
1740                 to_tcp_ctrl(ctrl)->async_req.pdu = NULL;
1741         }
1742
1743         nvme_tcp_free_queue(ctrl, 0);
1744 }
1745
1746 static void nvme_tcp_free_io_queues(struct nvme_ctrl *ctrl)
1747 {
1748         int i;
1749
1750         for (i = 1; i < ctrl->queue_count; i++)
1751                 nvme_tcp_free_queue(ctrl, i);
1752 }
1753
1754 static void nvme_tcp_stop_io_queues(struct nvme_ctrl *ctrl)
1755 {
1756         int i;
1757
1758         for (i = 1; i < ctrl->queue_count; i++)
1759                 nvme_tcp_stop_queue(ctrl, i);
1760 }
1761
1762 static int nvme_tcp_start_io_queues(struct nvme_ctrl *ctrl)
1763 {
1764         int i, ret;
1765
1766         for (i = 1; i < ctrl->queue_count; i++) {
1767                 ret = nvme_tcp_start_queue(ctrl, i);
1768                 if (ret)
1769                         goto out_stop_queues;
1770         }
1771
1772         return 0;
1773
1774 out_stop_queues:
1775         for (i--; i >= 1; i--)
1776                 nvme_tcp_stop_queue(ctrl, i);
1777         return ret;
1778 }
1779
1780 static int nvme_tcp_alloc_admin_queue(struct nvme_ctrl *ctrl)
1781 {
1782         int ret;
1783
1784         ret = nvme_tcp_alloc_queue(ctrl, 0, NVME_AQ_DEPTH);
1785         if (ret)
1786                 return ret;
1787
1788         ret = nvme_tcp_alloc_async_req(to_tcp_ctrl(ctrl));
1789         if (ret)
1790                 goto out_free_queue;
1791
1792         return 0;
1793
1794 out_free_queue:
1795         nvme_tcp_free_queue(ctrl, 0);
1796         return ret;
1797 }
1798
1799 static int __nvme_tcp_alloc_io_queues(struct nvme_ctrl *ctrl)
1800 {
1801         int i, ret;
1802
1803         for (i = 1; i < ctrl->queue_count; i++) {
1804                 ret = nvme_tcp_alloc_queue(ctrl, i, ctrl->sqsize + 1);
1805                 if (ret)
1806                         goto out_free_queues;
1807         }
1808
1809         return 0;
1810
1811 out_free_queues:
1812         for (i--; i >= 1; i--)
1813                 nvme_tcp_free_queue(ctrl, i);
1814
1815         return ret;
1816 }
1817
1818 static unsigned int nvme_tcp_nr_io_queues(struct nvme_ctrl *ctrl)
1819 {
1820         unsigned int nr_io_queues;
1821
1822         nr_io_queues = min(ctrl->opts->nr_io_queues, num_online_cpus());
1823         nr_io_queues += min(ctrl->opts->nr_write_queues, num_online_cpus());
1824         nr_io_queues += min(ctrl->opts->nr_poll_queues, num_online_cpus());
1825
1826         return nr_io_queues;
1827 }
1828
1829 static void nvme_tcp_set_io_queues(struct nvme_ctrl *nctrl,
1830                 unsigned int nr_io_queues)
1831 {
1832         struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl);
1833         struct nvmf_ctrl_options *opts = nctrl->opts;
1834
1835         if (opts->nr_write_queues && opts->nr_io_queues < nr_io_queues) {
1836                 /*
1837                  * separate read/write queues
1838                  * hand out dedicated default queues only after we have
1839                  * sufficient read queues.
1840                  */
1841                 ctrl->io_queues[HCTX_TYPE_READ] = opts->nr_io_queues;
1842                 nr_io_queues -= ctrl->io_queues[HCTX_TYPE_READ];
1843                 ctrl->io_queues[HCTX_TYPE_DEFAULT] =
1844                         min(opts->nr_write_queues, nr_io_queues);
1845                 nr_io_queues -= ctrl->io_queues[HCTX_TYPE_DEFAULT];
1846         } else {
1847                 /*
1848                  * shared read/write queues
1849                  * either no write queues were requested, or we don't have
1850                  * sufficient queue count to have dedicated default queues.
1851                  */
1852                 ctrl->io_queues[HCTX_TYPE_DEFAULT] =
1853                         min(opts->nr_io_queues, nr_io_queues);
1854                 nr_io_queues -= ctrl->io_queues[HCTX_TYPE_DEFAULT];
1855         }
1856
1857         if (opts->nr_poll_queues && nr_io_queues) {
1858                 /* map dedicated poll queues only if we have queues left */
1859                 ctrl->io_queues[HCTX_TYPE_POLL] =
1860                         min(opts->nr_poll_queues, nr_io_queues);
1861         }
1862 }
1863
1864 static int nvme_tcp_alloc_io_queues(struct nvme_ctrl *ctrl)
1865 {
1866         unsigned int nr_io_queues;
1867         int ret;
1868
1869         nr_io_queues = nvme_tcp_nr_io_queues(ctrl);
1870         ret = nvme_set_queue_count(ctrl, &nr_io_queues);
1871         if (ret)
1872                 return ret;
1873
1874         if (nr_io_queues == 0) {
1875                 dev_err(ctrl->device,
1876                         "unable to set any I/O queues\n");
1877                 return -ENOMEM;
1878         }
1879
1880         ctrl->queue_count = nr_io_queues + 1;
1881         dev_info(ctrl->device,
1882                 "creating %d I/O queues.\n", nr_io_queues);
1883
1884         nvme_tcp_set_io_queues(ctrl, nr_io_queues);
1885
1886         return __nvme_tcp_alloc_io_queues(ctrl);
1887 }
1888
1889 static void nvme_tcp_destroy_io_queues(struct nvme_ctrl *ctrl, bool remove)
1890 {
1891         nvme_tcp_stop_io_queues(ctrl);
1892         if (remove) {
1893                 blk_mq_destroy_queue(ctrl->connect_q);
1894                 blk_mq_free_tag_set(ctrl->tagset);
1895         }
1896         nvme_tcp_free_io_queues(ctrl);
1897 }
1898
1899 static int nvme_tcp_configure_io_queues(struct nvme_ctrl *ctrl, bool new)
1900 {
1901         int ret;
1902
1903         ret = nvme_tcp_alloc_io_queues(ctrl);
1904         if (ret)
1905                 return ret;
1906
1907         if (new) {
1908                 ret = nvme_tcp_alloc_tag_set(ctrl);
1909                 if (ret)
1910                         goto out_free_io_queues;
1911
1912                 ret = nvme_ctrl_init_connect_q(ctrl);
1913                 if (ret)
1914                         goto out_free_tag_set;
1915         }
1916
1917         ret = nvme_tcp_start_io_queues(ctrl);
1918         if (ret)
1919                 goto out_cleanup_connect_q;
1920
1921         if (!new) {
1922                 nvme_start_queues(ctrl);
1923                 if (!nvme_wait_freeze_timeout(ctrl, NVME_IO_TIMEOUT)) {
1924                         /*
1925                          * If we timed out waiting for freeze we are likely to
1926                          * be stuck.  Fail the controller initialization just
1927                          * to be safe.
1928                          */
1929                         ret = -ENODEV;
1930                         goto out_wait_freeze_timed_out;
1931                 }
1932                 blk_mq_update_nr_hw_queues(ctrl->tagset,
1933                         ctrl->queue_count - 1);
1934                 nvme_unfreeze(ctrl);
1935         }
1936
1937         return 0;
1938
1939 out_wait_freeze_timed_out:
1940         nvme_stop_queues(ctrl);
1941         nvme_sync_io_queues(ctrl);
1942         nvme_tcp_stop_io_queues(ctrl);
1943 out_cleanup_connect_q:
1944         nvme_cancel_tagset(ctrl);
1945         if (new)
1946                 blk_mq_destroy_queue(ctrl->connect_q);
1947 out_free_tag_set:
1948         if (new)
1949                 blk_mq_free_tag_set(ctrl->tagset);
1950 out_free_io_queues:
1951         nvme_tcp_free_io_queues(ctrl);
1952         return ret;
1953 }
1954
1955 static void nvme_tcp_destroy_admin_queue(struct nvme_ctrl *ctrl, bool remove)
1956 {
1957         nvme_tcp_stop_queue(ctrl, 0);
1958         if (remove) {
1959                 blk_mq_destroy_queue(ctrl->admin_q);
1960                 blk_mq_destroy_queue(ctrl->fabrics_q);
1961                 blk_mq_free_tag_set(ctrl->admin_tagset);
1962         }
1963         nvme_tcp_free_admin_queue(ctrl);
1964 }
1965
1966 static int nvme_tcp_configure_admin_queue(struct nvme_ctrl *ctrl, bool new)
1967 {
1968         int error;
1969
1970         error = nvme_tcp_alloc_admin_queue(ctrl);
1971         if (error)
1972                 return error;
1973
1974         if (new) {
1975                 error = nvme_tcp_alloc_admin_tag_set(ctrl);
1976                 if (error)
1977                         goto out_free_queue;
1978
1979                 ctrl->fabrics_q = blk_mq_init_queue(ctrl->admin_tagset);
1980                 if (IS_ERR(ctrl->fabrics_q)) {
1981                         error = PTR_ERR(ctrl->fabrics_q);
1982                         goto out_free_tagset;
1983                 }
1984
1985                 ctrl->admin_q = blk_mq_init_queue(ctrl->admin_tagset);
1986                 if (IS_ERR(ctrl->admin_q)) {
1987                         error = PTR_ERR(ctrl->admin_q);
1988                         goto out_cleanup_fabrics_q;
1989                 }
1990         }
1991
1992         error = nvme_tcp_start_queue(ctrl, 0);
1993         if (error)
1994                 goto out_cleanup_queue;
1995
1996         error = nvme_enable_ctrl(ctrl);
1997         if (error)
1998                 goto out_stop_queue;
1999
2000         nvme_start_admin_queue(ctrl);
2001
2002         error = nvme_init_ctrl_finish(ctrl);
2003         if (error)
2004                 goto out_quiesce_queue;
2005
2006         return 0;
2007
2008 out_quiesce_queue:
2009         nvme_stop_admin_queue(ctrl);
2010         blk_sync_queue(ctrl->admin_q);
2011 out_stop_queue:
2012         nvme_tcp_stop_queue(ctrl, 0);
2013         nvme_cancel_admin_tagset(ctrl);
2014 out_cleanup_queue:
2015         if (new)
2016                 blk_mq_destroy_queue(ctrl->admin_q);
2017 out_cleanup_fabrics_q:
2018         if (new)
2019                 blk_mq_destroy_queue(ctrl->fabrics_q);
2020 out_free_tagset:
2021         if (new)
2022                 blk_mq_free_tag_set(ctrl->admin_tagset);
2023 out_free_queue:
2024         nvme_tcp_free_admin_queue(ctrl);
2025         return error;
2026 }
2027
2028 static void nvme_tcp_teardown_admin_queue(struct nvme_ctrl *ctrl,
2029                 bool remove)
2030 {
2031         nvme_stop_admin_queue(ctrl);
2032         blk_sync_queue(ctrl->admin_q);
2033         nvme_tcp_stop_queue(ctrl, 0);
2034         nvme_cancel_admin_tagset(ctrl);
2035         if (remove)
2036                 nvme_start_admin_queue(ctrl);
2037         nvme_tcp_destroy_admin_queue(ctrl, remove);
2038 }
2039
2040 static void nvme_tcp_teardown_io_queues(struct nvme_ctrl *ctrl,
2041                 bool remove)
2042 {
2043         if (ctrl->queue_count <= 1)
2044                 return;
2045         nvme_stop_admin_queue(ctrl);
2046         nvme_start_freeze(ctrl);
2047         nvme_stop_queues(ctrl);
2048         nvme_sync_io_queues(ctrl);
2049         nvme_tcp_stop_io_queues(ctrl);
2050         nvme_cancel_tagset(ctrl);
2051         if (remove)
2052                 nvme_start_queues(ctrl);
2053         nvme_tcp_destroy_io_queues(ctrl, remove);
2054 }
2055
2056 static void nvme_tcp_reconnect_or_remove(struct nvme_ctrl *ctrl)
2057 {
2058         /* If we are resetting/deleting then do nothing */
2059         if (ctrl->state != NVME_CTRL_CONNECTING) {
2060                 WARN_ON_ONCE(ctrl->state == NVME_CTRL_NEW ||
2061                         ctrl->state == NVME_CTRL_LIVE);
2062                 return;
2063         }
2064
2065         if (nvmf_should_reconnect(ctrl)) {
2066                 dev_info(ctrl->device, "Reconnecting in %d seconds...\n",
2067                         ctrl->opts->reconnect_delay);
2068                 queue_delayed_work(nvme_wq, &to_tcp_ctrl(ctrl)->connect_work,
2069                                 ctrl->opts->reconnect_delay * HZ);
2070         } else {
2071                 dev_info(ctrl->device, "Removing controller...\n");
2072                 nvme_delete_ctrl(ctrl);
2073         }
2074 }
2075
2076 static int nvme_tcp_setup_ctrl(struct nvme_ctrl *ctrl, bool new)
2077 {
2078         struct nvmf_ctrl_options *opts = ctrl->opts;
2079         int ret;
2080
2081         ret = nvme_tcp_configure_admin_queue(ctrl, new);
2082         if (ret)
2083                 return ret;
2084
2085         if (ctrl->icdoff) {
2086                 ret = -EOPNOTSUPP;
2087                 dev_err(ctrl->device, "icdoff is not supported!\n");
2088                 goto destroy_admin;
2089         }
2090
2091         if (!nvme_ctrl_sgl_supported(ctrl)) {
2092                 ret = -EOPNOTSUPP;
2093                 dev_err(ctrl->device, "Mandatory sgls are not supported!\n");
2094                 goto destroy_admin;
2095         }
2096
2097         if (opts->queue_size > ctrl->sqsize + 1)
2098                 dev_warn(ctrl->device,
2099                         "queue_size %zu > ctrl sqsize %u, clamping down\n",
2100                         opts->queue_size, ctrl->sqsize + 1);
2101
2102         if (ctrl->sqsize + 1 > ctrl->maxcmd) {
2103                 dev_warn(ctrl->device,
2104                         "sqsize %u > ctrl maxcmd %u, clamping down\n",
2105                         ctrl->sqsize + 1, ctrl->maxcmd);
2106                 ctrl->sqsize = ctrl->maxcmd - 1;
2107         }
2108
2109         if (ctrl->queue_count > 1) {
2110                 ret = nvme_tcp_configure_io_queues(ctrl, new);
2111                 if (ret)
2112                         goto destroy_admin;
2113         }
2114
2115         if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_LIVE)) {
2116                 /*
2117                  * state change failure is ok if we started ctrl delete,
2118                  * unless we're during creation of a new controller to
2119                  * avoid races with teardown flow.
2120                  */
2121                 WARN_ON_ONCE(ctrl->state != NVME_CTRL_DELETING &&
2122                              ctrl->state != NVME_CTRL_DELETING_NOIO);
2123                 WARN_ON_ONCE(new);
2124                 ret = -EINVAL;
2125                 goto destroy_io;
2126         }
2127
2128         nvme_start_ctrl(ctrl);
2129         return 0;
2130
2131 destroy_io:
2132         if (ctrl->queue_count > 1) {
2133                 nvme_stop_queues(ctrl);
2134                 nvme_sync_io_queues(ctrl);
2135                 nvme_tcp_stop_io_queues(ctrl);
2136                 nvme_cancel_tagset(ctrl);
2137                 nvme_tcp_destroy_io_queues(ctrl, new);
2138         }
2139 destroy_admin:
2140         nvme_stop_admin_queue(ctrl);
2141         blk_sync_queue(ctrl->admin_q);
2142         nvme_tcp_stop_queue(ctrl, 0);
2143         nvme_cancel_admin_tagset(ctrl);
2144         nvme_tcp_destroy_admin_queue(ctrl, new);
2145         return ret;
2146 }
2147
2148 static void nvme_tcp_reconnect_ctrl_work(struct work_struct *work)
2149 {
2150         struct nvme_tcp_ctrl *tcp_ctrl = container_of(to_delayed_work(work),
2151                         struct nvme_tcp_ctrl, connect_work);
2152         struct nvme_ctrl *ctrl = &tcp_ctrl->ctrl;
2153
2154         ++ctrl->nr_reconnects;
2155
2156         if (nvme_tcp_setup_ctrl(ctrl, false))
2157                 goto requeue;
2158
2159         dev_info(ctrl->device, "Successfully reconnected (%d attempt)\n",
2160                         ctrl->nr_reconnects);
2161
2162         ctrl->nr_reconnects = 0;
2163
2164         return;
2165
2166 requeue:
2167         dev_info(ctrl->device, "Failed reconnect attempt %d\n",
2168                         ctrl->nr_reconnects);
2169         nvme_tcp_reconnect_or_remove(ctrl);
2170 }
2171
2172 static void nvme_tcp_error_recovery_work(struct work_struct *work)
2173 {
2174         struct nvme_tcp_ctrl *tcp_ctrl = container_of(work,
2175                                 struct nvme_tcp_ctrl, err_work);
2176         struct nvme_ctrl *ctrl = &tcp_ctrl->ctrl;
2177
2178         nvme_auth_stop(ctrl);
2179         nvme_stop_keep_alive(ctrl);
2180         flush_work(&ctrl->async_event_work);
2181         nvme_tcp_teardown_io_queues(ctrl, false);
2182         /* unquiesce to fail fast pending requests */
2183         nvme_start_queues(ctrl);
2184         nvme_tcp_teardown_admin_queue(ctrl, false);
2185         nvme_start_admin_queue(ctrl);
2186
2187         if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_CONNECTING)) {
2188                 /* state change failure is ok if we started ctrl delete */
2189                 WARN_ON_ONCE(ctrl->state != NVME_CTRL_DELETING &&
2190                              ctrl->state != NVME_CTRL_DELETING_NOIO);
2191                 return;
2192         }
2193
2194         nvme_tcp_reconnect_or_remove(ctrl);
2195 }
2196
2197 static void nvme_tcp_teardown_ctrl(struct nvme_ctrl *ctrl, bool shutdown)
2198 {
2199         nvme_tcp_teardown_io_queues(ctrl, shutdown);
2200         nvme_stop_admin_queue(ctrl);
2201         if (shutdown)
2202                 nvme_shutdown_ctrl(ctrl);
2203         else
2204                 nvme_disable_ctrl(ctrl);
2205         nvme_tcp_teardown_admin_queue(ctrl, shutdown);
2206 }
2207
2208 static void nvme_tcp_delete_ctrl(struct nvme_ctrl *ctrl)
2209 {
2210         nvme_tcp_teardown_ctrl(ctrl, true);
2211 }
2212
2213 static void nvme_reset_ctrl_work(struct work_struct *work)
2214 {
2215         struct nvme_ctrl *ctrl =
2216                 container_of(work, struct nvme_ctrl, reset_work);
2217
2218         nvme_stop_ctrl(ctrl);
2219         nvme_tcp_teardown_ctrl(ctrl, false);
2220
2221         if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_CONNECTING)) {
2222                 /* state change failure is ok if we started ctrl delete */
2223                 WARN_ON_ONCE(ctrl->state != NVME_CTRL_DELETING &&
2224                              ctrl->state != NVME_CTRL_DELETING_NOIO);
2225                 return;
2226         }
2227
2228         if (nvme_tcp_setup_ctrl(ctrl, false))
2229                 goto out_fail;
2230
2231         return;
2232
2233 out_fail:
2234         ++ctrl->nr_reconnects;
2235         nvme_tcp_reconnect_or_remove(ctrl);
2236 }
2237
2238 static void nvme_tcp_stop_ctrl(struct nvme_ctrl *ctrl)
2239 {
2240         cancel_work_sync(&to_tcp_ctrl(ctrl)->err_work);
2241         cancel_delayed_work_sync(&to_tcp_ctrl(ctrl)->connect_work);
2242 }
2243
2244 static void nvme_tcp_free_ctrl(struct nvme_ctrl *nctrl)
2245 {
2246         struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl);
2247
2248         if (list_empty(&ctrl->list))
2249                 goto free_ctrl;
2250
2251         mutex_lock(&nvme_tcp_ctrl_mutex);
2252         list_del(&ctrl->list);
2253         mutex_unlock(&nvme_tcp_ctrl_mutex);
2254
2255         nvmf_free_options(nctrl->opts);
2256 free_ctrl:
2257         kfree(ctrl->queues);
2258         kfree(ctrl);
2259 }
2260
2261 static void nvme_tcp_set_sg_null(struct nvme_command *c)
2262 {
2263         struct nvme_sgl_desc *sg = &c->common.dptr.sgl;
2264
2265         sg->addr = 0;
2266         sg->length = 0;
2267         sg->type = (NVME_TRANSPORT_SGL_DATA_DESC << 4) |
2268                         NVME_SGL_FMT_TRANSPORT_A;
2269 }
2270
2271 static void nvme_tcp_set_sg_inline(struct nvme_tcp_queue *queue,
2272                 struct nvme_command *c, u32 data_len)
2273 {
2274         struct nvme_sgl_desc *sg = &c->common.dptr.sgl;
2275
2276         sg->addr = cpu_to_le64(queue->ctrl->ctrl.icdoff);
2277         sg->length = cpu_to_le32(data_len);
2278         sg->type = (NVME_SGL_FMT_DATA_DESC << 4) | NVME_SGL_FMT_OFFSET;
2279 }
2280
2281 static void nvme_tcp_set_sg_host_data(struct nvme_command *c,
2282                 u32 data_len)
2283 {
2284         struct nvme_sgl_desc *sg = &c->common.dptr.sgl;
2285
2286         sg->addr = 0;
2287         sg->length = cpu_to_le32(data_len);
2288         sg->type = (NVME_TRANSPORT_SGL_DATA_DESC << 4) |
2289                         NVME_SGL_FMT_TRANSPORT_A;
2290 }
2291
2292 static void nvme_tcp_submit_async_event(struct nvme_ctrl *arg)
2293 {
2294         struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(arg);
2295         struct nvme_tcp_queue *queue = &ctrl->queues[0];
2296         struct nvme_tcp_cmd_pdu *pdu = ctrl->async_req.pdu;
2297         struct nvme_command *cmd = &pdu->cmd;
2298         u8 hdgst = nvme_tcp_hdgst_len(queue);
2299
2300         memset(pdu, 0, sizeof(*pdu));
2301         pdu->hdr.type = nvme_tcp_cmd;
2302         if (queue->hdr_digest)
2303                 pdu->hdr.flags |= NVME_TCP_F_HDGST;
2304         pdu->hdr.hlen = sizeof(*pdu);
2305         pdu->hdr.plen = cpu_to_le32(pdu->hdr.hlen + hdgst);
2306
2307         cmd->common.opcode = nvme_admin_async_event;
2308         cmd->common.command_id = NVME_AQ_BLK_MQ_DEPTH;
2309         cmd->common.flags |= NVME_CMD_SGL_METABUF;
2310         nvme_tcp_set_sg_null(cmd);
2311
2312         ctrl->async_req.state = NVME_TCP_SEND_CMD_PDU;
2313         ctrl->async_req.offset = 0;
2314         ctrl->async_req.curr_bio = NULL;
2315         ctrl->async_req.data_len = 0;
2316
2317         nvme_tcp_queue_request(&ctrl->async_req, true, true);
2318 }
2319
2320 static void nvme_tcp_complete_timed_out(struct request *rq)
2321 {
2322         struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
2323         struct nvme_ctrl *ctrl = &req->queue->ctrl->ctrl;
2324
2325         nvme_tcp_stop_queue(ctrl, nvme_tcp_queue_id(req->queue));
2326         nvmf_complete_timed_out_request(rq);
2327 }
2328
2329 static enum blk_eh_timer_return nvme_tcp_timeout(struct request *rq)
2330 {
2331         struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
2332         struct nvme_ctrl *ctrl = &req->queue->ctrl->ctrl;
2333         struct nvme_tcp_cmd_pdu *pdu = req->pdu;
2334
2335         dev_warn(ctrl->device,
2336                 "queue %d: timeout request %#x type %d\n",
2337                 nvme_tcp_queue_id(req->queue), rq->tag, pdu->hdr.type);
2338
2339         if (ctrl->state != NVME_CTRL_LIVE) {
2340                 /*
2341                  * If we are resetting, connecting or deleting we should
2342                  * complete immediately because we may block controller
2343                  * teardown or setup sequence
2344                  * - ctrl disable/shutdown fabrics requests
2345                  * - connect requests
2346                  * - initialization admin requests
2347                  * - I/O requests that entered after unquiescing and
2348                  *   the controller stopped responding
2349                  *
2350                  * All other requests should be cancelled by the error
2351                  * recovery work, so it's fine that we fail it here.
2352                  */
2353                 nvme_tcp_complete_timed_out(rq);
2354                 return BLK_EH_DONE;
2355         }
2356
2357         /*
2358          * LIVE state should trigger the normal error recovery which will
2359          * handle completing this request.
2360          */
2361         nvme_tcp_error_recovery(ctrl);
2362         return BLK_EH_RESET_TIMER;
2363 }
2364
2365 static blk_status_t nvme_tcp_map_data(struct nvme_tcp_queue *queue,
2366                         struct request *rq)
2367 {
2368         struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
2369         struct nvme_tcp_cmd_pdu *pdu = req->pdu;
2370         struct nvme_command *c = &pdu->cmd;
2371
2372         c->common.flags |= NVME_CMD_SGL_METABUF;
2373
2374         if (!blk_rq_nr_phys_segments(rq))
2375                 nvme_tcp_set_sg_null(c);
2376         else if (rq_data_dir(rq) == WRITE &&
2377             req->data_len <= nvme_tcp_inline_data_size(req))
2378                 nvme_tcp_set_sg_inline(queue, c, req->data_len);
2379         else
2380                 nvme_tcp_set_sg_host_data(c, req->data_len);
2381
2382         return 0;
2383 }
2384
2385 static blk_status_t nvme_tcp_setup_cmd_pdu(struct nvme_ns *ns,
2386                 struct request *rq)
2387 {
2388         struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
2389         struct nvme_tcp_cmd_pdu *pdu = req->pdu;
2390         struct nvme_tcp_queue *queue = req->queue;
2391         u8 hdgst = nvme_tcp_hdgst_len(queue), ddgst = 0;
2392         blk_status_t ret;
2393
2394         ret = nvme_setup_cmd(ns, rq);
2395         if (ret)
2396                 return ret;
2397
2398         req->state = NVME_TCP_SEND_CMD_PDU;
2399         req->status = cpu_to_le16(NVME_SC_SUCCESS);
2400         req->offset = 0;
2401         req->data_sent = 0;
2402         req->pdu_len = 0;
2403         req->pdu_sent = 0;
2404         req->h2cdata_left = 0;
2405         req->data_len = blk_rq_nr_phys_segments(rq) ?
2406                                 blk_rq_payload_bytes(rq) : 0;
2407         req->curr_bio = rq->bio;
2408         if (req->curr_bio && req->data_len)
2409                 nvme_tcp_init_iter(req, rq_data_dir(rq));
2410
2411         if (rq_data_dir(rq) == WRITE &&
2412             req->data_len <= nvme_tcp_inline_data_size(req))
2413                 req->pdu_len = req->data_len;
2414
2415         pdu->hdr.type = nvme_tcp_cmd;
2416         pdu->hdr.flags = 0;
2417         if (queue->hdr_digest)
2418                 pdu->hdr.flags |= NVME_TCP_F_HDGST;
2419         if (queue->data_digest && req->pdu_len) {
2420                 pdu->hdr.flags |= NVME_TCP_F_DDGST;
2421                 ddgst = nvme_tcp_ddgst_len(queue);
2422         }
2423         pdu->hdr.hlen = sizeof(*pdu);
2424         pdu->hdr.pdo = req->pdu_len ? pdu->hdr.hlen + hdgst : 0;
2425         pdu->hdr.plen =
2426                 cpu_to_le32(pdu->hdr.hlen + hdgst + req->pdu_len + ddgst);
2427
2428         ret = nvme_tcp_map_data(queue, rq);
2429         if (unlikely(ret)) {
2430                 nvme_cleanup_cmd(rq);
2431                 dev_err(queue->ctrl->ctrl.device,
2432                         "Failed to map data (%d)\n", ret);
2433                 return ret;
2434         }
2435
2436         return 0;
2437 }
2438
2439 static void nvme_tcp_commit_rqs(struct blk_mq_hw_ctx *hctx)
2440 {
2441         struct nvme_tcp_queue *queue = hctx->driver_data;
2442
2443         if (!llist_empty(&queue->req_list))
2444                 queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work);
2445 }
2446
2447 static blk_status_t nvme_tcp_queue_rq(struct blk_mq_hw_ctx *hctx,
2448                 const struct blk_mq_queue_data *bd)
2449 {
2450         struct nvme_ns *ns = hctx->queue->queuedata;
2451         struct nvme_tcp_queue *queue = hctx->driver_data;
2452         struct request *rq = bd->rq;
2453         struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
2454         bool queue_ready = test_bit(NVME_TCP_Q_LIVE, &queue->flags);
2455         blk_status_t ret;
2456
2457         if (!nvme_check_ready(&queue->ctrl->ctrl, rq, queue_ready))
2458                 return nvme_fail_nonready_command(&queue->ctrl->ctrl, rq);
2459
2460         ret = nvme_tcp_setup_cmd_pdu(ns, rq);
2461         if (unlikely(ret))
2462                 return ret;
2463
2464         blk_mq_start_request(rq);
2465
2466         nvme_tcp_queue_request(req, true, bd->last);
2467
2468         return BLK_STS_OK;
2469 }
2470
2471 static int nvme_tcp_map_queues(struct blk_mq_tag_set *set)
2472 {
2473         struct nvme_tcp_ctrl *ctrl = set->driver_data;
2474         struct nvmf_ctrl_options *opts = ctrl->ctrl.opts;
2475
2476         if (opts->nr_write_queues && ctrl->io_queues[HCTX_TYPE_READ]) {
2477                 /* separate read/write queues */
2478                 set->map[HCTX_TYPE_DEFAULT].nr_queues =
2479                         ctrl->io_queues[HCTX_TYPE_DEFAULT];
2480                 set->map[HCTX_TYPE_DEFAULT].queue_offset = 0;
2481                 set->map[HCTX_TYPE_READ].nr_queues =
2482                         ctrl->io_queues[HCTX_TYPE_READ];
2483                 set->map[HCTX_TYPE_READ].queue_offset =
2484                         ctrl->io_queues[HCTX_TYPE_DEFAULT];
2485         } else {
2486                 /* shared read/write queues */
2487                 set->map[HCTX_TYPE_DEFAULT].nr_queues =
2488                         ctrl->io_queues[HCTX_TYPE_DEFAULT];
2489                 set->map[HCTX_TYPE_DEFAULT].queue_offset = 0;
2490                 set->map[HCTX_TYPE_READ].nr_queues =
2491                         ctrl->io_queues[HCTX_TYPE_DEFAULT];
2492                 set->map[HCTX_TYPE_READ].queue_offset = 0;
2493         }
2494         blk_mq_map_queues(&set->map[HCTX_TYPE_DEFAULT]);
2495         blk_mq_map_queues(&set->map[HCTX_TYPE_READ]);
2496
2497         if (opts->nr_poll_queues && ctrl->io_queues[HCTX_TYPE_POLL]) {
2498                 /* map dedicated poll queues only if we have queues left */
2499                 set->map[HCTX_TYPE_POLL].nr_queues =
2500                                 ctrl->io_queues[HCTX_TYPE_POLL];
2501                 set->map[HCTX_TYPE_POLL].queue_offset =
2502                         ctrl->io_queues[HCTX_TYPE_DEFAULT] +
2503                         ctrl->io_queues[HCTX_TYPE_READ];
2504                 blk_mq_map_queues(&set->map[HCTX_TYPE_POLL]);
2505         }
2506
2507         dev_info(ctrl->ctrl.device,
2508                 "mapped %d/%d/%d default/read/poll queues.\n",
2509                 ctrl->io_queues[HCTX_TYPE_DEFAULT],
2510                 ctrl->io_queues[HCTX_TYPE_READ],
2511                 ctrl->io_queues[HCTX_TYPE_POLL]);
2512
2513         return 0;
2514 }
2515
2516 static int nvme_tcp_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob)
2517 {
2518         struct nvme_tcp_queue *queue = hctx->driver_data;
2519         struct sock *sk = queue->sock->sk;
2520
2521         if (!test_bit(NVME_TCP_Q_LIVE, &queue->flags))
2522                 return 0;
2523
2524         set_bit(NVME_TCP_Q_POLLING, &queue->flags);
2525         if (sk_can_busy_loop(sk) && skb_queue_empty_lockless(&sk->sk_receive_queue))
2526                 sk_busy_loop(sk, true);
2527         nvme_tcp_try_recv(queue);
2528         clear_bit(NVME_TCP_Q_POLLING, &queue->flags);
2529         return queue->nr_cqe;
2530 }
2531
2532 static const struct blk_mq_ops nvme_tcp_mq_ops = {
2533         .queue_rq       = nvme_tcp_queue_rq,
2534         .commit_rqs     = nvme_tcp_commit_rqs,
2535         .complete       = nvme_complete_rq,
2536         .init_request   = nvme_tcp_init_request,
2537         .exit_request   = nvme_tcp_exit_request,
2538         .init_hctx      = nvme_tcp_init_hctx,
2539         .timeout        = nvme_tcp_timeout,
2540         .map_queues     = nvme_tcp_map_queues,
2541         .poll           = nvme_tcp_poll,
2542 };
2543
2544 static const struct blk_mq_ops nvme_tcp_admin_mq_ops = {
2545         .queue_rq       = nvme_tcp_queue_rq,
2546         .complete       = nvme_complete_rq,
2547         .init_request   = nvme_tcp_init_request,
2548         .exit_request   = nvme_tcp_exit_request,
2549         .init_hctx      = nvme_tcp_init_admin_hctx,
2550         .timeout        = nvme_tcp_timeout,
2551 };
2552
2553 static const struct nvme_ctrl_ops nvme_tcp_ctrl_ops = {
2554         .name                   = "tcp",
2555         .module                 = THIS_MODULE,
2556         .flags                  = NVME_F_FABRICS,
2557         .reg_read32             = nvmf_reg_read32,
2558         .reg_read64             = nvmf_reg_read64,
2559         .reg_write32            = nvmf_reg_write32,
2560         .free_ctrl              = nvme_tcp_free_ctrl,
2561         .submit_async_event     = nvme_tcp_submit_async_event,
2562         .delete_ctrl            = nvme_tcp_delete_ctrl,
2563         .get_address            = nvmf_get_address,
2564         .stop_ctrl              = nvme_tcp_stop_ctrl,
2565 };
2566
2567 static bool
2568 nvme_tcp_existing_controller(struct nvmf_ctrl_options *opts)
2569 {
2570         struct nvme_tcp_ctrl *ctrl;
2571         bool found = false;
2572
2573         mutex_lock(&nvme_tcp_ctrl_mutex);
2574         list_for_each_entry(ctrl, &nvme_tcp_ctrl_list, list) {
2575                 found = nvmf_ip_options_match(&ctrl->ctrl, opts);
2576                 if (found)
2577                         break;
2578         }
2579         mutex_unlock(&nvme_tcp_ctrl_mutex);
2580
2581         return found;
2582 }
2583
2584 static struct nvme_ctrl *nvme_tcp_create_ctrl(struct device *dev,
2585                 struct nvmf_ctrl_options *opts)
2586 {
2587         struct nvme_tcp_ctrl *ctrl;
2588         int ret;
2589
2590         ctrl = kzalloc(sizeof(*ctrl), GFP_KERNEL);
2591         if (!ctrl)
2592                 return ERR_PTR(-ENOMEM);
2593
2594         INIT_LIST_HEAD(&ctrl->list);
2595         ctrl->ctrl.opts = opts;
2596         ctrl->ctrl.queue_count = opts->nr_io_queues + opts->nr_write_queues +
2597                                 opts->nr_poll_queues + 1;
2598         ctrl->ctrl.sqsize = opts->queue_size - 1;
2599         ctrl->ctrl.kato = opts->kato;
2600
2601         INIT_DELAYED_WORK(&ctrl->connect_work,
2602                         nvme_tcp_reconnect_ctrl_work);
2603         INIT_WORK(&ctrl->err_work, nvme_tcp_error_recovery_work);
2604         INIT_WORK(&ctrl->ctrl.reset_work, nvme_reset_ctrl_work);
2605
2606         if (!(opts->mask & NVMF_OPT_TRSVCID)) {
2607                 opts->trsvcid =
2608                         kstrdup(__stringify(NVME_TCP_DISC_PORT), GFP_KERNEL);
2609                 if (!opts->trsvcid) {
2610                         ret = -ENOMEM;
2611                         goto out_free_ctrl;
2612                 }
2613                 opts->mask |= NVMF_OPT_TRSVCID;
2614         }
2615
2616         ret = inet_pton_with_scope(&init_net, AF_UNSPEC,
2617                         opts->traddr, opts->trsvcid, &ctrl->addr);
2618         if (ret) {
2619                 pr_err("malformed address passed: %s:%s\n",
2620                         opts->traddr, opts->trsvcid);
2621                 goto out_free_ctrl;
2622         }
2623
2624         if (opts->mask & NVMF_OPT_HOST_TRADDR) {
2625                 ret = inet_pton_with_scope(&init_net, AF_UNSPEC,
2626                         opts->host_traddr, NULL, &ctrl->src_addr);
2627                 if (ret) {
2628                         pr_err("malformed src address passed: %s\n",
2629                                opts->host_traddr);
2630                         goto out_free_ctrl;
2631                 }
2632         }
2633
2634         if (opts->mask & NVMF_OPT_HOST_IFACE) {
2635                 if (!__dev_get_by_name(&init_net, opts->host_iface)) {
2636                         pr_err("invalid interface passed: %s\n",
2637                                opts->host_iface);
2638                         ret = -ENODEV;
2639                         goto out_free_ctrl;
2640                 }
2641         }
2642
2643         if (!opts->duplicate_connect && nvme_tcp_existing_controller(opts)) {
2644                 ret = -EALREADY;
2645                 goto out_free_ctrl;
2646         }
2647
2648         ctrl->queues = kcalloc(ctrl->ctrl.queue_count, sizeof(*ctrl->queues),
2649                                 GFP_KERNEL);
2650         if (!ctrl->queues) {
2651                 ret = -ENOMEM;
2652                 goto out_free_ctrl;
2653         }
2654
2655         ret = nvme_init_ctrl(&ctrl->ctrl, dev, &nvme_tcp_ctrl_ops, 0);
2656         if (ret)
2657                 goto out_kfree_queues;
2658
2659         if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_CONNECTING)) {
2660                 WARN_ON_ONCE(1);
2661                 ret = -EINTR;
2662                 goto out_uninit_ctrl;
2663         }
2664
2665         ret = nvme_tcp_setup_ctrl(&ctrl->ctrl, true);
2666         if (ret)
2667                 goto out_uninit_ctrl;
2668
2669         dev_info(ctrl->ctrl.device, "new ctrl: NQN \"%s\", addr %pISp\n",
2670                 nvmf_ctrl_subsysnqn(&ctrl->ctrl), &ctrl->addr);
2671
2672         mutex_lock(&nvme_tcp_ctrl_mutex);
2673         list_add_tail(&ctrl->list, &nvme_tcp_ctrl_list);
2674         mutex_unlock(&nvme_tcp_ctrl_mutex);
2675
2676         return &ctrl->ctrl;
2677
2678 out_uninit_ctrl:
2679         nvme_uninit_ctrl(&ctrl->ctrl);
2680         nvme_put_ctrl(&ctrl->ctrl);
2681         if (ret > 0)
2682                 ret = -EIO;
2683         return ERR_PTR(ret);
2684 out_kfree_queues:
2685         kfree(ctrl->queues);
2686 out_free_ctrl:
2687         kfree(ctrl);
2688         return ERR_PTR(ret);
2689 }
2690
2691 static struct nvmf_transport_ops nvme_tcp_transport = {
2692         .name           = "tcp",
2693         .module         = THIS_MODULE,
2694         .required_opts  = NVMF_OPT_TRADDR,
2695         .allowed_opts   = NVMF_OPT_TRSVCID | NVMF_OPT_RECONNECT_DELAY |
2696                           NVMF_OPT_HOST_TRADDR | NVMF_OPT_CTRL_LOSS_TMO |
2697                           NVMF_OPT_HDR_DIGEST | NVMF_OPT_DATA_DIGEST |
2698                           NVMF_OPT_NR_WRITE_QUEUES | NVMF_OPT_NR_POLL_QUEUES |
2699                           NVMF_OPT_TOS | NVMF_OPT_HOST_IFACE,
2700         .create_ctrl    = nvme_tcp_create_ctrl,
2701 };
2702
2703 static int __init nvme_tcp_init_module(void)
2704 {
2705         nvme_tcp_wq = alloc_workqueue("nvme_tcp_wq",
2706                         WQ_MEM_RECLAIM | WQ_HIGHPRI, 0);
2707         if (!nvme_tcp_wq)
2708                 return -ENOMEM;
2709
2710         nvmf_register_transport(&nvme_tcp_transport);
2711         return 0;
2712 }
2713
2714 static void __exit nvme_tcp_cleanup_module(void)
2715 {
2716         struct nvme_tcp_ctrl *ctrl;
2717
2718         nvmf_unregister_transport(&nvme_tcp_transport);
2719
2720         mutex_lock(&nvme_tcp_ctrl_mutex);
2721         list_for_each_entry(ctrl, &nvme_tcp_ctrl_list, list)
2722                 nvme_delete_ctrl(&ctrl->ctrl);
2723         mutex_unlock(&nvme_tcp_ctrl_mutex);
2724         flush_workqueue(nvme_delete_wq);
2725
2726         destroy_workqueue(nvme_tcp_wq);
2727 }
2728
2729 module_init(nvme_tcp_init_module);
2730 module_exit(nvme_tcp_cleanup_module);
2731
2732 MODULE_LICENSE("GPL v2");