drivers/nvme/host/tcp.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * NVMe over Fabrics TCP host.
   4  * Copyright (c) 2018 Lightbits Labs. All rights reserved.
   5  */
   6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
   7 #include <linux/module.h>
   8 #include <linux/init.h>
   9 #include <linux/slab.h>
  10 #include <linux/err.h>
  11 #include <linux/nvme-tcp.h>
  12 #include <net/sock.h>
  13 #include <net/tcp.h>
  14 #include <linux/blk-mq.h>
  15 #include <crypto/hash.h>
  16 #include <net/busy_poll.h>
  17
  18 #include "nvme.h"
  19 #include "fabrics.h"
  20
  21 struct nvme_tcp_queue;
  22
  23 /* Define the socket priority to use for connections were it is desirable
  24  * that the NIC consider performing optimized packet processing or filtering.
  25  * A non-zero value being sufficient to indicate general consideration of any
  26  * possible optimization.  Making it a module param allows for alternative
  27  * values that may be unique for some NIC implementations.
  28  */
  29 static int so_priority;
  30 module_param(so_priority, int, 0644);
  31 MODULE_PARM_DESC(so_priority, "nvme tcp socket optimize priority");
  32
  33 #ifdef CONFIG_DEBUG_LOCK_ALLOC
  34 /* lockdep can detect a circular dependency of the form
  35  *   sk_lock -> mmap_lock (page fault) -> fs locks -> sk_lock
  36  * because dependencies are tracked for both nvme-tcp and user contexts. Using
  37  * a separate class prevents lockdep from conflating nvme-tcp socket use with
  38  * user-space socket API use.
  39  */
  40 static struct lock_class_key nvme_tcp_sk_key[2];
  41 static struct lock_class_key nvme_tcp_slock_key[2];
  42
  43 static void nvme_tcp_reclassify_socket(struct socket *sock)
  44 {
  45         struct sock *sk = sock->sk;
  46
  47         if (WARN_ON_ONCE(!sock_allow_reclassification(sk)))
  48                 return;
  49
  50         switch (sk->sk_family) {
  51         case AF_INET:
  52                 sock_lock_init_class_and_name(sk, "slock-AF_INET-NVME",
  53                                               &nvme_tcp_slock_key[0],
  54                                               "sk_lock-AF_INET-NVME",
  55                                               &nvme_tcp_sk_key[0]);
  56                 break;
  57         case AF_INET6:
  58                 sock_lock_init_class_and_name(sk, "slock-AF_INET6-NVME",
  59                                               &nvme_tcp_slock_key[1],
  60                                               "sk_lock-AF_INET6-NVME",
  61                                               &nvme_tcp_sk_key[1]);
  62                 break;
  63         default:
  64                 WARN_ON_ONCE(1);
  65         }
  66 }
  67 #else
  68 static void nvme_tcp_reclassify_socket(struct socket *sock) { }
  69 #endif
  70
  71 enum nvme_tcp_send_state {
  72         NVME_TCP_SEND_CMD_PDU = 0,
  73         NVME_TCP_SEND_H2C_PDU,
  74         NVME_TCP_SEND_DATA,
  75         NVME_TCP_SEND_DDGST,
  76 };
  77
  78 struct nvme_tcp_request {
  79         struct nvme_request     req;
  80         void                    *pdu;
  81         struct nvme_tcp_queue   *queue;
  82         u32                     data_len;
  83         u32                     pdu_len;
  84         u32                     pdu_sent;
  85         u32                     h2cdata_left;
  86         u32                     h2cdata_offset;
  87         u16                     ttag;
  88         __le16                  status;
  89         struct list_head        entry;
  90         struct llist_node       lentry;
  91         __le32                  ddgst;
  92
  93         struct bio              *curr_bio;
  94         struct iov_iter         iter;
  95
  96         /* send state */
  97         size_t                  offset;
  98         size_t                  data_sent;
  99         enum nvme_tcp_send_state state;
 100 };
 101
 102 enum nvme_tcp_queue_flags {
 103         NVME_TCP_Q_ALLOCATED    = 0,
 104         NVME_TCP_Q_LIVE         = 1,
 105         NVME_TCP_Q_POLLING      = 2,
 106 };
 107
 108 enum nvme_tcp_recv_state {
 109         NVME_TCP_RECV_PDU = 0,
 110         NVME_TCP_RECV_DATA,
 111         NVME_TCP_RECV_DDGST,
 112 };
 113
 114 struct nvme_tcp_ctrl;
 115 struct nvme_tcp_queue {
 116         struct socket           *sock;
 117         struct work_struct      io_work;
 118         int                     io_cpu;
 119
 120         struct mutex            queue_lock;
 121         struct mutex            send_mutex;
 122         struct llist_head       req_list;
 123         struct list_head        send_list;
 124         bool                    more_requests;
 125
 126         /* recv state */
 127         void                    *pdu;
 128         int                     pdu_remaining;
 129         int                     pdu_offset;
 130         size_t                  data_remaining;
 131         size_t                  ddgst_remaining;
 132         unsigned int            nr_cqe;
 133
 134         /* send state */
 135         struct nvme_tcp_request *request;
 136
 137         int                     queue_size;
 138         u32                     maxh2cdata;
 139         size_t                  cmnd_capsule_len;
 140         struct nvme_tcp_ctrl    *ctrl;
 141         unsigned long           flags;
 142         bool                    rd_enabled;
 143
 144         bool                    hdr_digest;
 145         bool                    data_digest;
 146         struct ahash_request    *rcv_hash;
 147         struct ahash_request    *snd_hash;
 148         __le32                  exp_ddgst;
 149         __le32                  recv_ddgst;
 150
 151         struct page_frag_cache  pf_cache;
 152
 153         void (*state_change)(struct sock *);
 154         void (*data_ready)(struct sock *);
 155         void (*write_space)(struct sock *);
 156 };
 157
 158 struct nvme_tcp_ctrl {
 159         /* read only in the hot path */
 160         struct nvme_tcp_queue   *queues;
 161         struct blk_mq_tag_set   tag_set;
 162
 163         /* other member variables */
 164         struct list_head        list;
 165         struct blk_mq_tag_set   admin_tag_set;
 166         struct sockaddr_storage addr;
 167         struct sockaddr_storage src_addr;
 168         struct nvme_ctrl        ctrl;
 169
 170         struct work_struct      err_work;
 171         struct delayed_work     connect_work;
 172         struct nvme_tcp_request async_req;
 173         u32                     io_queues[HCTX_MAX_TYPES];
 174 };
 175
 176 static LIST_HEAD(nvme_tcp_ctrl_list);
 177 static DEFINE_MUTEX(nvme_tcp_ctrl_mutex);
 178 static struct workqueue_struct *nvme_tcp_wq;
 179 static const struct blk_mq_ops nvme_tcp_mq_ops;
 180 static const struct blk_mq_ops nvme_tcp_admin_mq_ops;
 181 static int nvme_tcp_try_send(struct nvme_tcp_queue *queue);
 182
 183 static inline struct nvme_tcp_ctrl *to_tcp_ctrl(struct nvme_ctrl *ctrl)
 184 {
 185         return container_of(ctrl, struct nvme_tcp_ctrl, ctrl);
 186 }
 187
 188 static inline int nvme_tcp_queue_id(struct nvme_tcp_queue *queue)
 189 {
 190         return queue - queue->ctrl->queues;
 191 }
 192
 193 static inline struct blk_mq_tags *nvme_tcp_tagset(struct nvme_tcp_queue *queue)
 194 {
 195         u32 queue_idx = nvme_tcp_queue_id(queue);
 196
 197         if (queue_idx == 0)
 198                 return queue->ctrl->admin_tag_set.tags[queue_idx];
 199         return queue->ctrl->tag_set.tags[queue_idx - 1];
 200 }
 201
 202 static inline u8 nvme_tcp_hdgst_len(struct nvme_tcp_queue *queue)
 203 {
 204         return queue->hdr_digest ? NVME_TCP_DIGEST_LENGTH : 0;
 205 }
 206
 207 static inline u8 nvme_tcp_ddgst_len(struct nvme_tcp_queue *queue)
 208 {
 209         return queue->data_digest ? NVME_TCP_DIGEST_LENGTH : 0;
 210 }
 211
 212 static inline size_t nvme_tcp_inline_data_size(struct nvme_tcp_queue *queue)
 213 {
 214         return queue->cmnd_capsule_len - sizeof(struct nvme_command);
 215 }
 216
 217 static inline bool nvme_tcp_async_req(struct nvme_tcp_request *req)
 218 {
 219         return req == &req->queue->ctrl->async_req;
 220 }
 221
 222 static inline bool nvme_tcp_has_inline_data(struct nvme_tcp_request *req)
 223 {
 224         struct request *rq;
 225
 226         if (unlikely(nvme_tcp_async_req(req)))
 227                 return false; /* async events don't have a request */
 228
 229         rq = blk_mq_rq_from_pdu(req);
 230
 231         return rq_data_dir(rq) == WRITE && req->data_len &&
 232                 req->data_len <= nvme_tcp_inline_data_size(req->queue);
 233 }
 234
 235 static inline struct page *nvme_tcp_req_cur_page(struct nvme_tcp_request *req)
 236 {
 237         return req->iter.bvec->bv_page;
 238 }
 239
 240 static inline size_t nvme_tcp_req_cur_offset(struct nvme_tcp_request *req)
 241 {
 242         return req->iter.bvec->bv_offset + req->iter.iov_offset;
 243 }
 244
 245 static inline size_t nvme_tcp_req_cur_length(struct nvme_tcp_request *req)
 246 {
 247         return min_t(size_t, iov_iter_single_seg_count(&req->iter),
 248                         req->pdu_len - req->pdu_sent);
 249 }
 250
 251 static inline size_t nvme_tcp_pdu_data_left(struct nvme_tcp_request *req)
 252 {
 253         return rq_data_dir(blk_mq_rq_from_pdu(req)) == WRITE ?
 254                         req->pdu_len - req->pdu_sent : 0;
 255 }
 256
 257 static inline size_t nvme_tcp_pdu_last_send(struct nvme_tcp_request *req,
 258                 int len)
 259 {
 260         return nvme_tcp_pdu_data_left(req) <= len;
 261 }
 262
 263 static void nvme_tcp_init_iter(struct nvme_tcp_request *req,
 264                 unsigned int dir)
 265 {
 266         struct request *rq = blk_mq_rq_from_pdu(req);
 267         struct bio_vec *vec;
 268         unsigned int size;
 269         int nr_bvec;
 270         size_t offset;
 271
 272         if (rq->rq_flags & RQF_SPECIAL_PAYLOAD) {
 273                 vec = &rq->special_vec;
 274                 nr_bvec = 1;
 275                 size = blk_rq_payload_bytes(rq);
 276                 offset = 0;
 277         } else {
 278                 struct bio *bio = req->curr_bio;
 279                 struct bvec_iter bi;
 280                 struct bio_vec bv;
 281
 282                 vec = __bvec_iter_bvec(bio->bi_io_vec, bio->bi_iter);
 283                 nr_bvec = 0;
 284                 bio_for_each_bvec(bv, bio, bi) {
 285                         nr_bvec++;
 286                 }
 287                 size = bio->bi_iter.bi_size;
 288                 offset = bio->bi_iter.bi_bvec_done;
 289         }
 290
 291         iov_iter_bvec(&req->iter, dir, vec, nr_bvec, size);
 292         req->iter.iov_offset = offset;
 293 }
 294
 295 static inline void nvme_tcp_advance_req(struct nvme_tcp_request *req,
 296                 int len)
 297 {
 298         req->data_sent += len;
 299         req->pdu_sent += len;
 300         iov_iter_advance(&req->iter, len);
 301         if (!iov_iter_count(&req->iter) &&
 302             req->data_sent < req->data_len) {
 303                 req->curr_bio = req->curr_bio->bi_next;
 304                 nvme_tcp_init_iter(req, WRITE);
 305         }
 306 }
 307
 308 static inline void nvme_tcp_send_all(struct nvme_tcp_queue *queue)
 309 {
 310         int ret;
 311
 312         /* drain the send queue as much as we can... */
 313         do {
 314                 ret = nvme_tcp_try_send(queue);
 315         } while (ret > 0);
 316 }
 317
 318 static inline bool nvme_tcp_queue_more(struct nvme_tcp_queue *queue)
 319 {
 320         return !list_empty(&queue->send_list) ||
 321                 !llist_empty(&queue->req_list) || queue->more_requests;
 322 }
 323
 324 static inline void nvme_tcp_queue_request(struct nvme_tcp_request *req,
 325                 bool sync, bool last)
 326 {
 327         struct nvme_tcp_queue *queue = req->queue;
 328         bool empty;
 329
 330         empty = llist_add(&req->lentry, &queue->req_list) &&
 331                 list_empty(&queue->send_list) && !queue->request;
 332
 333         /*
 334          * if we're the first on the send_list and we can try to send
 335          * directly, otherwise queue io_work. Also, only do that if we
 336          * are on the same cpu, so we don't introduce contention.
 337          */
 338         if (queue->io_cpu == raw_smp_processor_id() &&
 339             sync && empty && mutex_trylock(&queue->send_mutex)) {
 340                 queue->more_requests = !last;
 341                 nvme_tcp_send_all(queue);
 342                 queue->more_requests = false;
 343                 mutex_unlock(&queue->send_mutex);
 344         }
 345
 346         if (last && nvme_tcp_queue_more(queue))
 347                 queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work);
 348 }
 349
 350 static void nvme_tcp_process_req_list(struct nvme_tcp_queue *queue)
 351 {
 352         struct nvme_tcp_request *req;
 353         struct llist_node *node;
 354
 355         for (node = llist_del_all(&queue->req_list); node; node = node->next) {
 356                 req = llist_entry(node, struct nvme_tcp_request, lentry);
 357                 list_add(&req->entry, &queue->send_list);
 358         }
 359 }
 360
 361 static inline struct nvme_tcp_request *
 362 nvme_tcp_fetch_request(struct nvme_tcp_queue *queue)
 363 {
 364         struct nvme_tcp_request *req;
 365
 366         req = list_first_entry_or_null(&queue->send_list,
 367                         struct nvme_tcp_request, entry);
 368         if (!req) {
 369                 nvme_tcp_process_req_list(queue);
 370                 req = list_first_entry_or_null(&queue->send_list,
 371                                 struct nvme_tcp_request, entry);
 372                 if (unlikely(!req))
 373                         return NULL;
 374         }
 375
 376         list_del(&req->entry);
 377         return req;
 378 }
 379
 380 static inline void nvme_tcp_ddgst_final(struct ahash_request *hash,
 381                 __le32 *dgst)
 382 {
 383         ahash_request_set_crypt(hash, NULL, (u8 *)dgst, 0);
 384         crypto_ahash_final(hash);
 385 }
 386
 387 static inline void nvme_tcp_ddgst_update(struct ahash_request *hash,
 388                 struct page *page, off_t off, size_t len)
 389 {
 390         struct scatterlist sg;
 391
 392         sg_init_marker(&sg, 1);
 393         sg_set_page(&sg, page, len, off);
 394         ahash_request_set_crypt(hash, &sg, NULL, len);
 395         crypto_ahash_update(hash);
 396 }
 397
 398 static inline void nvme_tcp_hdgst(struct ahash_request *hash,
 399                 void *pdu, size_t len)
 400 {
 401         struct scatterlist sg;
 402
 403         sg_init_one(&sg, pdu, len);
 404         ahash_request_set_crypt(hash, &sg, pdu + len, len);
 405         crypto_ahash_digest(hash);
 406 }
 407
 408 static int nvme_tcp_verify_hdgst(struct nvme_tcp_queue *queue,
 409                 void *pdu, size_t pdu_len)
 410 {
 411         struct nvme_tcp_hdr *hdr = pdu;
 412         __le32 recv_digest;
 413         __le32 exp_digest;
 414
 415         if (unlikely(!(hdr->flags & NVME_TCP_F_HDGST))) {
 416                 dev_err(queue->ctrl->ctrl.device,
 417                         "queue %d: header digest flag is cleared\n",
 418                         nvme_tcp_queue_id(queue));
 419                 return -EPROTO;
 420         }
 421
 422         recv_digest = *(__le32 *)(pdu + hdr->hlen);
 423         nvme_tcp_hdgst(queue->rcv_hash, pdu, pdu_len);
 424         exp_digest = *(__le32 *)(pdu + hdr->hlen);
 425         if (recv_digest != exp_digest) {
 426                 dev_err(queue->ctrl->ctrl.device,
 427                         "header digest error: recv %#x expected %#x\n",
 428                         le32_to_cpu(recv_digest), le32_to_cpu(exp_digest));
 429                 return -EIO;
 430         }
 431
 432         return 0;
 433 }
 434
 435 static int nvme_tcp_check_ddgst(struct nvme_tcp_queue *queue, void *pdu)
 436 {
 437         struct nvme_tcp_hdr *hdr = pdu;
 438         u8 digest_len = nvme_tcp_hdgst_len(queue);
 439         u32 len;
 440
 441         len = le32_to_cpu(hdr->plen) - hdr->hlen -
 442                 ((hdr->flags & NVME_TCP_F_HDGST) ? digest_len : 0);
 443
 444         if (unlikely(len && !(hdr->flags & NVME_TCP_F_DDGST))) {
 445                 dev_err(queue->ctrl->ctrl.device,
 446                         "queue %d: data digest flag is cleared\n",
 447                 nvme_tcp_queue_id(queue));
 448                 return -EPROTO;
 449         }
 450         crypto_ahash_init(queue->rcv_hash);
 451
 452         return 0;
 453 }
 454
 455 static void nvme_tcp_exit_request(struct blk_mq_tag_set *set,
 456                 struct request *rq, unsigned int hctx_idx)
 457 {
 458         struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
 459
 460         page_frag_free(req->pdu);
 461 }
 462
 463 static int nvme_tcp_init_request(struct blk_mq_tag_set *set,
 464                 struct request *rq, unsigned int hctx_idx,
 465                 unsigned int numa_node)
 466 {
 467         struct nvme_tcp_ctrl *ctrl = set->driver_data;
 468         struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
 469         struct nvme_tcp_cmd_pdu *pdu;
 470         int queue_idx = (set == &ctrl->tag_set) ? hctx_idx + 1 : 0;
 471         struct nvme_tcp_queue *queue = &ctrl->queues[queue_idx];
 472         u8 hdgst = nvme_tcp_hdgst_len(queue);
 473
 474         req->pdu = page_frag_alloc(&queue->pf_cache,
 475                 sizeof(struct nvme_tcp_cmd_pdu) + hdgst,
 476                 GFP_KERNEL | __GFP_ZERO);
 477         if (!req->pdu)
 478                 return -ENOMEM;
 479
 480         pdu = req->pdu;
 481         req->queue = queue;
 482         nvme_req(rq)->ctrl = &ctrl->ctrl;
 483         nvme_req(rq)->cmd = &pdu->cmd;
 484
 485         return 0;
 486 }
 487
 488 static int nvme_tcp_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
 489                 unsigned int hctx_idx)
 490 {
 491         struct nvme_tcp_ctrl *ctrl = data;
 492         struct nvme_tcp_queue *queue = &ctrl->queues[hctx_idx + 1];
 493
 494         hctx->driver_data = queue;
 495         return 0;
 496 }
 497
 498 static int nvme_tcp_init_admin_hctx(struct blk_mq_hw_ctx *hctx, void *data,
 499                 unsigned int hctx_idx)
 500 {
 501         struct nvme_tcp_ctrl *ctrl = data;
 502         struct nvme_tcp_queue *queue = &ctrl->queues[0];
 503
 504         hctx->driver_data = queue;
 505         return 0;
 506 }
 507
 508 static enum nvme_tcp_recv_state
 509 nvme_tcp_recv_state(struct nvme_tcp_queue *queue)
 510 {
 511         return  (queue->pdu_remaining) ? NVME_TCP_RECV_PDU :
 512                 (queue->ddgst_remaining) ? NVME_TCP_RECV_DDGST :
 513                 NVME_TCP_RECV_DATA;
 514 }
 515
 516 static void nvme_tcp_init_recv_ctx(struct nvme_tcp_queue *queue)
 517 {
 518         queue->pdu_remaining = sizeof(struct nvme_tcp_rsp_pdu) +
 519                                 nvme_tcp_hdgst_len(queue);
 520         queue->pdu_offset = 0;
 521         queue->data_remaining = -1;
 522         queue->ddgst_remaining = 0;
 523 }
 524
 525 static void nvme_tcp_error_recovery(struct nvme_ctrl *ctrl)
 526 {
 527         if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING))
 528                 return;
 529
 530         dev_warn(ctrl->device, "starting error recovery\n");
 531         queue_work(nvme_reset_wq, &to_tcp_ctrl(ctrl)->err_work);
 532 }
 533
 534 static int nvme_tcp_process_nvme_cqe(struct nvme_tcp_queue *queue,
 535                 struct nvme_completion *cqe)
 536 {
 537         struct nvme_tcp_request *req;
 538         struct request *rq;
 539
 540         rq = nvme_find_rq(nvme_tcp_tagset(queue), cqe->command_id);
 541         if (!rq) {
 542                 dev_err(queue->ctrl->ctrl.device,
 543                         "got bad cqe.command_id %#x on queue %d\n",
 544                         cqe->command_id, nvme_tcp_queue_id(queue));
 545                 nvme_tcp_error_recovery(&queue->ctrl->ctrl);
 546                 return -EINVAL;
 547         }
 548
 549         req = blk_mq_rq_to_pdu(rq);
 550         if (req->status == cpu_to_le16(NVME_SC_SUCCESS))
 551                 req->status = cqe->status;
 552
 553         if (!nvme_try_complete_req(rq, req->status, cqe->result))
 554                 nvme_complete_rq(rq);
 555         queue->nr_cqe++;
 556
 557         return 0;
 558 }
 559
 560 static int nvme_tcp_handle_c2h_data(struct nvme_tcp_queue *queue,
 561                 struct nvme_tcp_data_pdu *pdu)
 562 {
 563         struct request *rq;
 564
 565         rq = nvme_find_rq(nvme_tcp_tagset(queue), pdu->command_id);
 566         if (!rq) {
 567                 dev_err(queue->ctrl->ctrl.device,
 568                         "got bad c2hdata.command_id %#x on queue %d\n",
 569                         pdu->command_id, nvme_tcp_queue_id(queue));
 570                 return -ENOENT;
 571         }
 572
 573         if (!blk_rq_payload_bytes(rq)) {
 574                 dev_err(queue->ctrl->ctrl.device,
 575                         "queue %d tag %#x unexpected data\n",
 576                         nvme_tcp_queue_id(queue), rq->tag);
 577                 return -EIO;
 578         }
 579
 580         queue->data_remaining = le32_to_cpu(pdu->data_length);
 581
 582         if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS &&
 583             unlikely(!(pdu->hdr.flags & NVME_TCP_F_DATA_LAST))) {
 584                 dev_err(queue->ctrl->ctrl.device,
 585                         "queue %d tag %#x SUCCESS set but not last PDU\n",
 586                         nvme_tcp_queue_id(queue), rq->tag);
 587                 nvme_tcp_error_recovery(&queue->ctrl->ctrl);
 588                 return -EPROTO;
 589         }
 590
 591         return 0;
 592 }
 593
 594 static int nvme_tcp_handle_comp(struct nvme_tcp_queue *queue,
 595                 struct nvme_tcp_rsp_pdu *pdu)
 596 {
 597         struct nvme_completion *cqe = &pdu->cqe;
 598         int ret = 0;
 599
 600         /*
 601          * AEN requests are special as they don't time out and can
 602          * survive any kind of queue freeze and often don't respond to
 603          * aborts.  We don't even bother to allocate a struct request
 604          * for them but rather special case them here.
 605          */
 606         if (unlikely(nvme_is_aen_req(nvme_tcp_queue_id(queue),
 607                                      cqe->command_id)))
 608                 nvme_complete_async_event(&queue->ctrl->ctrl, cqe->status,
 609                                 &cqe->result);
 610         else
 611                 ret = nvme_tcp_process_nvme_cqe(queue, cqe);
 612
 613         return ret;
 614 }
 615
 616 static void nvme_tcp_setup_h2c_data_pdu(struct nvme_tcp_request *req)
 617 {
 618         struct nvme_tcp_data_pdu *data = req->pdu;
 619         struct nvme_tcp_queue *queue = req->queue;
 620         struct request *rq = blk_mq_rq_from_pdu(req);
 621         u32 h2cdata_sent = req->pdu_len;
 622         u8 hdgst = nvme_tcp_hdgst_len(queue);
 623         u8 ddgst = nvme_tcp_ddgst_len(queue);
 624
 625         req->state = NVME_TCP_SEND_H2C_PDU;
 626         req->offset = 0;
 627         req->pdu_len = min(req->h2cdata_left, queue->maxh2cdata);
 628         req->pdu_sent = 0;
 629         req->h2cdata_left -= req->pdu_len;
 630         req->h2cdata_offset += h2cdata_sent;
 631
 632         memset(data, 0, sizeof(*data));
 633         data->hdr.type = nvme_tcp_h2c_data;
 634         if (!req->h2cdata_left)
 635                 data->hdr.flags = NVME_TCP_F_DATA_LAST;
 636         if (queue->hdr_digest)
 637                 data->hdr.flags |= NVME_TCP_F_HDGST;
 638         if (queue->data_digest)
 639                 data->hdr.flags |= NVME_TCP_F_DDGST;
 640         data->hdr.hlen = sizeof(*data);
 641         data->hdr.pdo = data->hdr.hlen + hdgst;
 642         data->hdr.plen =
 643                 cpu_to_le32(data->hdr.hlen + hdgst + req->pdu_len + ddgst);
 644         data->ttag = req->ttag;
 645         data->command_id = nvme_cid(rq);
 646         data->data_offset = cpu_to_le32(req->h2cdata_offset);
 647         data->data_length = cpu_to_le32(req->pdu_len);
 648 }
 649
 650 static int nvme_tcp_handle_r2t(struct nvme_tcp_queue *queue,
 651                 struct nvme_tcp_r2t_pdu *pdu)
 652 {
 653         struct nvme_tcp_request *req;
 654         struct request *rq;
 655         u32 r2t_length = le32_to_cpu(pdu->r2t_length);
 656         u32 r2t_offset = le32_to_cpu(pdu->r2t_offset);
 657
 658         rq = nvme_find_rq(nvme_tcp_tagset(queue), pdu->command_id);
 659         if (!rq) {
 660                 dev_err(queue->ctrl->ctrl.device,
 661                         "got bad r2t.command_id %#x on queue %d\n",
 662                         pdu->command_id, nvme_tcp_queue_id(queue));
 663                 return -ENOENT;
 664         }
 665         req = blk_mq_rq_to_pdu(rq);
 666
 667         if (unlikely(!r2t_length)) {
 668                 dev_err(queue->ctrl->ctrl.device,
 669                         "req %d r2t len is %u, probably a bug...\n",
 670                         rq->tag, r2t_length);
 671                 return -EPROTO;
 672         }
 673
 674         if (unlikely(req->data_sent + r2t_length > req->data_len)) {
 675                 dev_err(queue->ctrl->ctrl.device,
 676                         "req %d r2t len %u exceeded data len %u (%zu sent)\n",
 677                         rq->tag, r2t_length, req->data_len, req->data_sent);
 678                 return -EPROTO;
 679         }
 680
 681         if (unlikely(r2t_offset < req->data_sent)) {
 682                 dev_err(queue->ctrl->ctrl.device,
 683                         "req %d unexpected r2t offset %u (expected %zu)\n",
 684                         rq->tag, r2t_offset, req->data_sent);
 685                 return -EPROTO;
 686         }
 687
 688         req->pdu_len = 0;
 689         req->h2cdata_left = r2t_length;
 690         req->h2cdata_offset = r2t_offset;
 691         req->ttag = pdu->ttag;
 692
 693         nvme_tcp_setup_h2c_data_pdu(req);
 694         nvme_tcp_queue_request(req, false, true);
 695
 696         return 0;
 697 }
 698
 699 static int nvme_tcp_recv_pdu(struct nvme_tcp_queue *queue, struct sk_buff *skb,
 700                 unsigned int *offset, size_t *len)
 701 {
 702         struct nvme_tcp_hdr *hdr;
 703         char *pdu = queue->pdu;
 704         size_t rcv_len = min_t(size_t, *len, queue->pdu_remaining);
 705         int ret;
 706
 707         ret = skb_copy_bits(skb, *offset,
 708                 &pdu[queue->pdu_offset], rcv_len);
 709         if (unlikely(ret))
 710                 return ret;
 711
 712         queue->pdu_remaining -= rcv_len;
 713         queue->pdu_offset += rcv_len;
 714         *offset += rcv_len;
 715         *len -= rcv_len;
 716         if (queue->pdu_remaining)
 717                 return 0;
 718
 719         hdr = queue->pdu;
 720         if (queue->hdr_digest) {
 721                 ret = nvme_tcp_verify_hdgst(queue, queue->pdu, hdr->hlen);
 722                 if (unlikely(ret))
 723                         return ret;
 724         }
 725
 726
 727         if (queue->data_digest) {
 728                 ret = nvme_tcp_check_ddgst(queue, queue->pdu);
 729                 if (unlikely(ret))
 730                         return ret;
 731         }
 732
 733         switch (hdr->type) {
 734         case nvme_tcp_c2h_data:
 735                 return nvme_tcp_handle_c2h_data(queue, (void *)queue->pdu);
 736         case nvme_tcp_rsp:
 737                 nvme_tcp_init_recv_ctx(queue);
 738                 return nvme_tcp_handle_comp(queue, (void *)queue->pdu);
 739         case nvme_tcp_r2t:
 740                 nvme_tcp_init_recv_ctx(queue);
 741                 return nvme_tcp_handle_r2t(queue, (void *)queue->pdu);
 742         default:
 743                 dev_err(queue->ctrl->ctrl.device,
 744                         "unsupported pdu type (%d)\n", hdr->type);
 745                 return -EINVAL;
 746         }
 747 }
 748
 749 static inline void nvme_tcp_end_request(struct request *rq, u16 status)
 750 {
 751         union nvme_result res = {};
 752
 753         if (!nvme_try_complete_req(rq, cpu_to_le16(status << 1), res))
 754                 nvme_complete_rq(rq);
 755 }
 756
 757 static int nvme_tcp_recv_data(struct nvme_tcp_queue *queue, struct sk_buff *skb,
 758                               unsigned int *offset, size_t *len)
 759 {
 760         struct nvme_tcp_data_pdu *pdu = (void *)queue->pdu;
 761         struct request *rq =
 762                 nvme_cid_to_rq(nvme_tcp_tagset(queue), pdu->command_id);
 763         struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
 764
 765         while (true) {
 766                 int recv_len, ret;
 767
 768                 recv_len = min_t(size_t, *len, queue->data_remaining);
 769                 if (!recv_len)
 770                         break;
 771
 772                 if (!iov_iter_count(&req->iter)) {
 773                         req->curr_bio = req->curr_bio->bi_next;
 774
 775                         /*
 776                          * If we don`t have any bios it means that controller
 777                          * sent more data than we requested, hence error
 778                          */
 779                         if (!req->curr_bio) {
 780                                 dev_err(queue->ctrl->ctrl.device,
 781                                         "queue %d no space in request %#x",
 782                                         nvme_tcp_queue_id(queue), rq->tag);
 783                                 nvme_tcp_init_recv_ctx(queue);
 784                                 return -EIO;
 785                         }
 786                         nvme_tcp_init_iter(req, READ);
 787                 }
 788
 789                 /* we can read only from what is left in this bio */
 790                 recv_len = min_t(size_t, recv_len,
 791                                 iov_iter_count(&req->iter));
 792
 793                 if (queue->data_digest)
 794                         ret = skb_copy_and_hash_datagram_iter(skb, *offset,
 795                                 &req->iter, recv_len, queue->rcv_hash);
 796                 else
 797                         ret = skb_copy_datagram_iter(skb, *offset,
 798                                         &req->iter, recv_len);
 799                 if (ret) {
 800                         dev_err(queue->ctrl->ctrl.device,
 801                                 "queue %d failed to copy request %#x data",
 802                                 nvme_tcp_queue_id(queue), rq->tag);
 803                         return ret;
 804                 }
 805
 806                 *len -= recv_len;
 807                 *offset += recv_len;
 808                 queue->data_remaining -= recv_len;
 809         }
 810
 811         if (!queue->data_remaining) {
 812                 if (queue->data_digest) {
 813                         nvme_tcp_ddgst_final(queue->rcv_hash, &queue->exp_ddgst);
 814                         queue->ddgst_remaining = NVME_TCP_DIGEST_LENGTH;
 815                 } else {
 816                         if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS) {
 817                                 nvme_tcp_end_request(rq,
 818                                                 le16_to_cpu(req->status));
 819                                 queue->nr_cqe++;
 820                         }
 821                         nvme_tcp_init_recv_ctx(queue);
 822                 }
 823         }
 824
 825         return 0;
 826 }
 827
 828 static int nvme_tcp_recv_ddgst(struct nvme_tcp_queue *queue,
 829                 struct sk_buff *skb, unsigned int *offset, size_t *len)
 830 {
 831         struct nvme_tcp_data_pdu *pdu = (void *)queue->pdu;
 832         char *ddgst = (char *)&queue->recv_ddgst;
 833         size_t recv_len = min_t(size_t, *len, queue->ddgst_remaining);
 834         off_t off = NVME_TCP_DIGEST_LENGTH - queue->ddgst_remaining;
 835         int ret;
 836
 837         ret = skb_copy_bits(skb, *offset, &ddgst[off], recv_len);
 838         if (unlikely(ret))
 839                 return ret;
 840
 841         queue->ddgst_remaining -= recv_len;
 842         *offset += recv_len;
 843         *len -= recv_len;
 844         if (queue->ddgst_remaining)
 845                 return 0;
 846
 847         if (queue->recv_ddgst != queue->exp_ddgst) {
 848                 struct request *rq = nvme_cid_to_rq(nvme_tcp_tagset(queue),
 849                                         pdu->command_id);
 850                 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
 851
 852                 req->status = cpu_to_le16(NVME_SC_DATA_XFER_ERROR);
 853
 854                 dev_err(queue->ctrl->ctrl.device,
 855                         "data digest error: recv %#x expected %#x\n",
 856                         le32_to_cpu(queue->recv_ddgst),
 857                         le32_to_cpu(queue->exp_ddgst));
 858         }
 859
 860         if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS) {
 861                 struct request *rq = nvme_cid_to_rq(nvme_tcp_tagset(queue),
 862                                         pdu->command_id);
 863                 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
 864
 865                 nvme_tcp_end_request(rq, le16_to_cpu(req->status));
 866                 queue->nr_cqe++;
 867         }
 868
 869         nvme_tcp_init_recv_ctx(queue);
 870         return 0;
 871 }
 872
 873 static int nvme_tcp_recv_skb(read_descriptor_t *desc, struct sk_buff *skb,
 874                              unsigned int offset, size_t len)
 875 {
 876         struct nvme_tcp_queue *queue = desc->arg.data;
 877         size_t consumed = len;
 878         int result;
 879
 880         while (len) {
 881                 switch (nvme_tcp_recv_state(queue)) {
 882                 case NVME_TCP_RECV_PDU:
 883                         result = nvme_tcp_recv_pdu(queue, skb, &offset, &len);
 884                         break;
 885                 case NVME_TCP_RECV_DATA:
 886                         result = nvme_tcp_recv_data(queue, skb, &offset, &len);
 887                         break;
 888                 case NVME_TCP_RECV_DDGST:
 889                         result = nvme_tcp_recv_ddgst(queue, skb, &offset, &len);
 890                         break;
 891                 default:
 892                         result = -EFAULT;
 893                 }
 894                 if (result) {
 895                         dev_err(queue->ctrl->ctrl.device,
 896                                 "receive failed:  %d\n", result);
 897                         queue->rd_enabled = false;
 898                         nvme_tcp_error_recovery(&queue->ctrl->ctrl);
 899                         return result;
 900                 }
 901         }
 902
 903         return consumed;
 904 }
 905
 906 static void nvme_tcp_data_ready(struct sock *sk)
 907 {
 908         struct nvme_tcp_queue *queue;
 909
 910         read_lock_bh(&sk->sk_callback_lock);
 911         queue = sk->sk_user_data;
 912         if (likely(queue && queue->rd_enabled) &&
 913             !test_bit(NVME_TCP_Q_POLLING, &queue->flags))
 914                 queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work);
 915         read_unlock_bh(&sk->sk_callback_lock);
 916 }
 917
 918 static void nvme_tcp_write_space(struct sock *sk)
 919 {
 920         struct nvme_tcp_queue *queue;
 921
 922         read_lock_bh(&sk->sk_callback_lock);
 923         queue = sk->sk_user_data;
 924         if (likely(queue && sk_stream_is_writeable(sk))) {
 925                 clear_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
 926                 queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work);
 927         }
 928         read_unlock_bh(&sk->sk_callback_lock);
 929 }
 930
 931 static void nvme_tcp_state_change(struct sock *sk)
 932 {
 933         struct nvme_tcp_queue *queue;
 934
 935         read_lock_bh(&sk->sk_callback_lock);
 936         queue = sk->sk_user_data;
 937         if (!queue)
 938                 goto done;
 939
 940         switch (sk->sk_state) {
 941         case TCP_CLOSE:
 942         case TCP_CLOSE_WAIT:
 943         case TCP_LAST_ACK:
 944         case TCP_FIN_WAIT1:
 945         case TCP_FIN_WAIT2:
 946                 nvme_tcp_error_recovery(&queue->ctrl->ctrl);
 947                 break;
 948         default:
 949                 dev_info(queue->ctrl->ctrl.device,
 950                         "queue %d socket state %d\n",
 951                         nvme_tcp_queue_id(queue), sk->sk_state);
 952         }
 953
 954         queue->state_change(sk);
 955 done:
 956         read_unlock_bh(&sk->sk_callback_lock);
 957 }
 958
 959 static inline void nvme_tcp_done_send_req(struct nvme_tcp_queue *queue)
 960 {
 961         queue->request = NULL;
 962 }
 963
 964 static void nvme_tcp_fail_request(struct nvme_tcp_request *req)
 965 {
 966         if (nvme_tcp_async_req(req)) {
 967                 union nvme_result res = {};
 968
 969                 nvme_complete_async_event(&req->queue->ctrl->ctrl,
 970                                 cpu_to_le16(NVME_SC_HOST_PATH_ERROR), &res);
 971         } else {
 972                 nvme_tcp_end_request(blk_mq_rq_from_pdu(req),
 973                                 NVME_SC_HOST_PATH_ERROR);
 974         }
 975 }
 976
 977 static int nvme_tcp_try_send_data(struct nvme_tcp_request *req)
 978 {
 979         struct nvme_tcp_queue *queue = req->queue;
 980         int req_data_len = req->data_len;
 981         u32 h2cdata_left = req->h2cdata_left;
 982
 983         while (true) {
 984                 struct page *page = nvme_tcp_req_cur_page(req);
 985                 size_t offset = nvme_tcp_req_cur_offset(req);
 986                 size_t len = nvme_tcp_req_cur_length(req);
 987                 bool last = nvme_tcp_pdu_last_send(req, len);
 988                 int req_data_sent = req->data_sent;
 989                 int ret, flags = MSG_DONTWAIT;
 990
 991                 if (last && !queue->data_digest && !nvme_tcp_queue_more(queue))
 992                         flags |= MSG_EOR;
 993                 else
 994                         flags |= MSG_MORE | MSG_SENDPAGE_NOTLAST;
 995
 996                 if (sendpage_ok(page)) {
 997                         ret = kernel_sendpage(queue->sock, page, offset, len,
 998                                         flags);
 999                 } else {
1000                         ret = sock_no_sendpage(queue->sock, page, offset, len,
1001                                         flags);
1002                 }
1003                 if (ret <= 0)
1004                         return ret;
1005
1006                 if (queue->data_digest)
1007                         nvme_tcp_ddgst_update(queue->snd_hash, page,
1008                                         offset, ret);
1009
1010                 /*
1011                  * update the request iterator except for the last payload send
1012                  * in the request where we don't want to modify it as we may
1013                  * compete with the RX path completing the request.
1014                  */
1015                 if (req_data_sent + ret < req_data_len)
1016                         nvme_tcp_advance_req(req, ret);
1017
1018                 /* fully successful last send in current PDU */
1019                 if (last && ret == len) {
1020                         if (queue->data_digest) {
1021                                 nvme_tcp_ddgst_final(queue->snd_hash,
1022                                         &req->ddgst);
1023                                 req->state = NVME_TCP_SEND_DDGST;
1024                                 req->offset = 0;
1025                         } else {
1026                                 if (h2cdata_left)
1027                                         nvme_tcp_setup_h2c_data_pdu(req);
1028                                 else
1029                                         nvme_tcp_done_send_req(queue);
1030                         }
1031                         return 1;
1032                 }
1033         }
1034         return -EAGAIN;
1035 }
1036
1037 static int nvme_tcp_try_send_cmd_pdu(struct nvme_tcp_request *req)
1038 {
1039         struct nvme_tcp_queue *queue = req->queue;
1040         struct nvme_tcp_cmd_pdu *pdu = req->pdu;
1041         bool inline_data = nvme_tcp_has_inline_data(req);
1042         u8 hdgst = nvme_tcp_hdgst_len(queue);
1043         int len = sizeof(*pdu) + hdgst - req->offset;
1044         int flags = MSG_DONTWAIT;
1045         int ret;
1046
1047         if (inline_data || nvme_tcp_queue_more(queue))
1048                 flags |= MSG_MORE | MSG_SENDPAGE_NOTLAST;
1049         else
1050                 flags |= MSG_EOR;
1051
1052         if (queue->hdr_digest && !req->offset)
1053                 nvme_tcp_hdgst(queue->snd_hash, pdu, sizeof(*pdu));
1054
1055         ret = kernel_sendpage(queue->sock, virt_to_page(pdu),
1056                         offset_in_page(pdu) + req->offset, len,  flags);
1057         if (unlikely(ret <= 0))
1058                 return ret;
1059
1060         len -= ret;
1061         if (!len) {
1062                 if (inline_data) {
1063                         req->state = NVME_TCP_SEND_DATA;
1064                         if (queue->data_digest)
1065                                 crypto_ahash_init(queue->snd_hash);
1066                 } else {
1067                         nvme_tcp_done_send_req(queue);
1068                 }
1069                 return 1;
1070         }
1071         req->offset += ret;
1072
1073         return -EAGAIN;
1074 }
1075
1076 static int nvme_tcp_try_send_data_pdu(struct nvme_tcp_request *req)
1077 {
1078         struct nvme_tcp_queue *queue = req->queue;
1079         struct nvme_tcp_data_pdu *pdu = req->pdu;
1080         u8 hdgst = nvme_tcp_hdgst_len(queue);
1081         int len = sizeof(*pdu) - req->offset + hdgst;
1082         int ret;
1083
1084         if (queue->hdr_digest && !req->offset)
1085                 nvme_tcp_hdgst(queue->snd_hash, pdu, sizeof(*pdu));
1086
1087         if (!req->h2cdata_left)
1088                 ret = kernel_sendpage(queue->sock, virt_to_page(pdu),
1089                                 offset_in_page(pdu) + req->offset, len,
1090                                 MSG_DONTWAIT | MSG_MORE | MSG_SENDPAGE_NOTLAST);
1091         else
1092                 ret = sock_no_sendpage(queue->sock, virt_to_page(pdu),
1093                                 offset_in_page(pdu) + req->offset, len,
1094                                 MSG_DONTWAIT | MSG_MORE);
1095         if (unlikely(ret <= 0))
1096                 return ret;
1097
1098         len -= ret;
1099         if (!len) {
1100                 req->state = NVME_TCP_SEND_DATA;
1101                 if (queue->data_digest)
1102                         crypto_ahash_init(queue->snd_hash);
1103                 return 1;
1104         }
1105         req->offset += ret;
1106
1107         return -EAGAIN;
1108 }
1109
1110 static int nvme_tcp_try_send_ddgst(struct nvme_tcp_request *req)
1111 {
1112         struct nvme_tcp_queue *queue = req->queue;
1113         size_t offset = req->offset;
1114         u32 h2cdata_left = req->h2cdata_left;
1115         int ret;
1116         struct msghdr msg = { .msg_flags = MSG_DONTWAIT };
1117         struct kvec iov = {
1118                 .iov_base = (u8 *)&req->ddgst + req->offset,
1119                 .iov_len = NVME_TCP_DIGEST_LENGTH - req->offset
1120         };
1121
1122         if (nvme_tcp_queue_more(queue))
1123                 msg.msg_flags |= MSG_MORE;
1124         else
1125                 msg.msg_flags |= MSG_EOR;
1126
1127         ret = kernel_sendmsg(queue->sock, &msg, &iov, 1, iov.iov_len);
1128         if (unlikely(ret <= 0))
1129                 return ret;
1130
1131         if (offset + ret == NVME_TCP_DIGEST_LENGTH) {
1132                 if (h2cdata_left)
1133                         nvme_tcp_setup_h2c_data_pdu(req);
1134                 else
1135                         nvme_tcp_done_send_req(queue);
1136                 return 1;
1137         }
1138
1139         req->offset += ret;
1140         return -EAGAIN;
1141 }
1142
1143 static int nvme_tcp_try_send(struct nvme_tcp_queue *queue)
1144 {
1145         struct nvme_tcp_request *req;
1146         int ret = 1;
1147
1148         if (!queue->request) {
1149                 queue->request = nvme_tcp_fetch_request(queue);
1150                 if (!queue->request)
1151                         return 0;
1152         }
1153         req = queue->request;
1154
1155         if (req->state == NVME_TCP_SEND_CMD_PDU) {
1156                 ret = nvme_tcp_try_send_cmd_pdu(req);
1157                 if (ret <= 0)
1158                         goto done;
1159                 if (!nvme_tcp_has_inline_data(req))
1160                         return ret;
1161         }
1162
1163         if (req->state == NVME_TCP_SEND_H2C_PDU) {
1164                 ret = nvme_tcp_try_send_data_pdu(req);
1165                 if (ret <= 0)
1166                         goto done;
1167         }
1168
1169         if (req->state == NVME_TCP_SEND_DATA) {
1170                 ret = nvme_tcp_try_send_data(req);
1171                 if (ret <= 0)
1172                         goto done;
1173         }
1174
1175         if (req->state == NVME_TCP_SEND_DDGST)
1176                 ret = nvme_tcp_try_send_ddgst(req);
1177 done:
1178         if (ret == -EAGAIN) {
1179                 ret = 0;
1180         } else if (ret < 0) {
1181                 dev_err(queue->ctrl->ctrl.device,
1182                         "failed to send request %d\n", ret);
1183                 nvme_tcp_fail_request(queue->request);
1184                 nvme_tcp_done_send_req(queue);
1185         }
1186         return ret;
1187 }
1188
1189 static int nvme_tcp_try_recv(struct nvme_tcp_queue *queue)
1190 {
1191         struct socket *sock = queue->sock;
1192         struct sock *sk = sock->sk;
1193         read_descriptor_t rd_desc;
1194         int consumed;
1195
1196         rd_desc.arg.data = queue;
1197         rd_desc.count = 1;
1198         lock_sock(sk);
1199         queue->nr_cqe = 0;
1200         consumed = sock->ops->read_sock(sk, &rd_desc, nvme_tcp_recv_skb);
1201         release_sock(sk);
1202         return consumed;
1203 }
1204
1205 static void nvme_tcp_io_work(struct work_struct *w)
1206 {
1207         struct nvme_tcp_queue *queue =
1208                 container_of(w, struct nvme_tcp_queue, io_work);
1209         unsigned long deadline = jiffies + msecs_to_jiffies(1);
1210
1211         do {
1212                 bool pending = false;
1213                 int result;
1214
1215                 if (mutex_trylock(&queue->send_mutex)) {
1216                         result = nvme_tcp_try_send(queue);
1217                         mutex_unlock(&queue->send_mutex);
1218                         if (result > 0)
1219                                 pending = true;
1220                         else if (unlikely(result < 0))
1221                                 break;
1222                 }
1223
1224                 result = nvme_tcp_try_recv(queue);
1225                 if (result > 0)
1226                         pending = true;
1227                 else if (unlikely(result < 0))
1228                         return;
1229
1230                 if (!pending)
1231                         return;
1232
1233         } while (!time_after(jiffies, deadline)); /* quota is exhausted */
1234
1235         queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work);
1236 }
1237
1238 static void nvme_tcp_free_crypto(struct nvme_tcp_queue *queue)
1239 {
1240         struct crypto_ahash *tfm = crypto_ahash_reqtfm(queue->rcv_hash);
1241
1242         ahash_request_free(queue->rcv_hash);
1243         ahash_request_free(queue->snd_hash);
1244         crypto_free_ahash(tfm);
1245 }
1246
1247 static int nvme_tcp_alloc_crypto(struct nvme_tcp_queue *queue)
1248 {
1249         struct crypto_ahash *tfm;
1250
1251         tfm = crypto_alloc_ahash("crc32c", 0, CRYPTO_ALG_ASYNC);
1252         if (IS_ERR(tfm))
1253                 return PTR_ERR(tfm);
1254
1255         queue->snd_hash = ahash_request_alloc(tfm, GFP_KERNEL);
1256         if (!queue->snd_hash)
1257                 goto free_tfm;
1258         ahash_request_set_callback(queue->snd_hash, 0, NULL, NULL);
1259
1260         queue->rcv_hash = ahash_request_alloc(tfm, GFP_KERNEL);
1261         if (!queue->rcv_hash)
1262                 goto free_snd_hash;
1263         ahash_request_set_callback(queue->rcv_hash, 0, NULL, NULL);
1264
1265         return 0;
1266 free_snd_hash:
1267         ahash_request_free(queue->snd_hash);
1268 free_tfm:
1269         crypto_free_ahash(tfm);
1270         return -ENOMEM;
1271 }
1272
1273 static void nvme_tcp_free_async_req(struct nvme_tcp_ctrl *ctrl)
1274 {
1275         struct nvme_tcp_request *async = &ctrl->async_req;
1276
1277         page_frag_free(async->pdu);
1278 }
1279
1280 static int nvme_tcp_alloc_async_req(struct nvme_tcp_ctrl *ctrl)
1281 {
1282         struct nvme_tcp_queue *queue = &ctrl->queues[0];
1283         struct nvme_tcp_request *async = &ctrl->async_req;
1284         u8 hdgst = nvme_tcp_hdgst_len(queue);
1285
1286         async->pdu = page_frag_alloc(&queue->pf_cache,
1287                 sizeof(struct nvme_tcp_cmd_pdu) + hdgst,
1288                 GFP_KERNEL | __GFP_ZERO);
1289         if (!async->pdu)
1290                 return -ENOMEM;
1291
1292         async->queue = &ctrl->queues[0];
1293         return 0;
1294 }
1295
1296 static void nvme_tcp_free_queue(struct nvme_ctrl *nctrl, int qid)
1297 {
1298         struct page *page;
1299         struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl);
1300         struct nvme_tcp_queue *queue = &ctrl->queues[qid];
1301
1302         if (!test_and_clear_bit(NVME_TCP_Q_ALLOCATED, &queue->flags))
1303                 return;
1304
1305         if (queue->hdr_digest || queue->data_digest)
1306                 nvme_tcp_free_crypto(queue);
1307
1308         if (queue->pf_cache.va) {
1309                 page = virt_to_head_page(queue->pf_cache.va);
1310                 __page_frag_cache_drain(page, queue->pf_cache.pagecnt_bias);
1311                 queue->pf_cache.va = NULL;
1312         }
1313         sock_release(queue->sock);
1314         kfree(queue->pdu);
1315         mutex_destroy(&queue->send_mutex);
1316         mutex_destroy(&queue->queue_lock);
1317 }
1318
1319 static int nvme_tcp_init_connection(struct nvme_tcp_queue *queue)
1320 {
1321         struct nvme_tcp_icreq_pdu *icreq;
1322         struct nvme_tcp_icresp_pdu *icresp;
1323         struct msghdr msg = {};
1324         struct kvec iov;
1325         bool ctrl_hdgst, ctrl_ddgst;
1326         u32 maxh2cdata;
1327         int ret;
1328
1329         icreq = kzalloc(sizeof(*icreq), GFP_KERNEL);
1330         if (!icreq)
1331                 return -ENOMEM;
1332
1333         icresp = kzalloc(sizeof(*icresp), GFP_KERNEL);
1334         if (!icresp) {
1335                 ret = -ENOMEM;
1336                 goto free_icreq;
1337         }
1338
1339         icreq->hdr.type = nvme_tcp_icreq;
1340         icreq->hdr.hlen = sizeof(*icreq);
1341         icreq->hdr.pdo = 0;
1342         icreq->hdr.plen = cpu_to_le32(icreq->hdr.hlen);
1343         icreq->pfv = cpu_to_le16(NVME_TCP_PFV_1_0);
1344         icreq->maxr2t = 0; /* single inflight r2t supported */
1345         icreq->hpda = 0; /* no alignment constraint */
1346         if (queue->hdr_digest)
1347                 icreq->digest |= NVME_TCP_HDR_DIGEST_ENABLE;
1348         if (queue->data_digest)
1349                 icreq->digest |= NVME_TCP_DATA_DIGEST_ENABLE;
1350
1351         iov.iov_base = icreq;
1352         iov.iov_len = sizeof(*icreq);
1353         ret = kernel_sendmsg(queue->sock, &msg, &iov, 1, iov.iov_len);
1354         if (ret < 0)
1355                 goto free_icresp;
1356
1357         memset(&msg, 0, sizeof(msg));
1358         iov.iov_base = icresp;
1359         iov.iov_len = sizeof(*icresp);
1360         ret = kernel_recvmsg(queue->sock, &msg, &iov, 1,
1361                         iov.iov_len, msg.msg_flags);
1362         if (ret < 0)
1363                 goto free_icresp;
1364
1365         ret = -EINVAL;
1366         if (icresp->hdr.type != nvme_tcp_icresp) {
1367                 pr_err("queue %d: bad type returned %d\n",
1368                         nvme_tcp_queue_id(queue), icresp->hdr.type);
1369                 goto free_icresp;
1370         }
1371
1372         if (le32_to_cpu(icresp->hdr.plen) != sizeof(*icresp)) {
1373                 pr_err("queue %d: bad pdu length returned %d\n",
1374                         nvme_tcp_queue_id(queue), icresp->hdr.plen);
1375                 goto free_icresp;
1376         }
1377
1378         if (icresp->pfv != NVME_TCP_PFV_1_0) {
1379                 pr_err("queue %d: bad pfv returned %d\n",
1380                         nvme_tcp_queue_id(queue), icresp->pfv);
1381                 goto free_icresp;
1382         }
1383
1384         ctrl_ddgst = !!(icresp->digest & NVME_TCP_DATA_DIGEST_ENABLE);
1385         if ((queue->data_digest && !ctrl_ddgst) ||
1386             (!queue->data_digest && ctrl_ddgst)) {
1387                 pr_err("queue %d: data digest mismatch host: %s ctrl: %s\n",
1388                         nvme_tcp_queue_id(queue),
1389                         queue->data_digest ? "enabled" : "disabled",
1390                         ctrl_ddgst ? "enabled" : "disabled");
1391                 goto free_icresp;
1392         }
1393
1394         ctrl_hdgst = !!(icresp->digest & NVME_TCP_HDR_DIGEST_ENABLE);
1395         if ((queue->hdr_digest && !ctrl_hdgst) ||
1396             (!queue->hdr_digest && ctrl_hdgst)) {
1397                 pr_err("queue %d: header digest mismatch host: %s ctrl: %s\n",
1398                         nvme_tcp_queue_id(queue),
1399                         queue->hdr_digest ? "enabled" : "disabled",
1400                         ctrl_hdgst ? "enabled" : "disabled");
1401                 goto free_icresp;
1402         }
1403
1404         if (icresp->cpda != 0) {
1405                 pr_err("queue %d: unsupported cpda returned %d\n",
1406                         nvme_tcp_queue_id(queue), icresp->cpda);
1407                 goto free_icresp;
1408         }
1409
1410         maxh2cdata = le32_to_cpu(icresp->maxdata);
1411         if ((maxh2cdata % 4) || (maxh2cdata < NVME_TCP_MIN_MAXH2CDATA)) {
1412                 pr_err("queue %d: invalid maxh2cdata returned %u\n",
1413                        nvme_tcp_queue_id(queue), maxh2cdata);
1414                 goto free_icresp;
1415         }
1416         queue->maxh2cdata = maxh2cdata;
1417
1418         ret = 0;
1419 free_icresp:
1420         kfree(icresp);
1421 free_icreq:
1422         kfree(icreq);
1423         return ret;
1424 }
1425
1426 static bool nvme_tcp_admin_queue(struct nvme_tcp_queue *queue)
1427 {
1428         return nvme_tcp_queue_id(queue) == 0;
1429 }
1430
1431 static bool nvme_tcp_default_queue(struct nvme_tcp_queue *queue)
1432 {
1433         struct nvme_tcp_ctrl *ctrl = queue->ctrl;
1434         int qid = nvme_tcp_queue_id(queue);
1435
1436         return !nvme_tcp_admin_queue(queue) &&
1437                 qid < 1 + ctrl->io_queues[HCTX_TYPE_DEFAULT];
1438 }
1439
1440 static bool nvme_tcp_read_queue(struct nvme_tcp_queue *queue)
1441 {
1442         struct nvme_tcp_ctrl *ctrl = queue->ctrl;
1443         int qid = nvme_tcp_queue_id(queue);
1444
1445         return !nvme_tcp_admin_queue(queue) &&
1446                 !nvme_tcp_default_queue(queue) &&
1447                 qid < 1 + ctrl->io_queues[HCTX_TYPE_DEFAULT] +
1448                           ctrl->io_queues[HCTX_TYPE_READ];
1449 }
1450
1451 static bool nvme_tcp_poll_queue(struct nvme_tcp_queue *queue)
1452 {
1453         struct nvme_tcp_ctrl *ctrl = queue->ctrl;
1454         int qid = nvme_tcp_queue_id(queue);
1455
1456         return !nvme_tcp_admin_queue(queue) &&
1457                 !nvme_tcp_default_queue(queue) &&
1458                 !nvme_tcp_read_queue(queue) &&
1459                 qid < 1 + ctrl->io_queues[HCTX_TYPE_DEFAULT] +
1460                           ctrl->io_queues[HCTX_TYPE_READ] +
1461                           ctrl->io_queues[HCTX_TYPE_POLL];
1462 }
1463
1464 static void nvme_tcp_set_queue_io_cpu(struct nvme_tcp_queue *queue)
1465 {
1466         struct nvme_tcp_ctrl *ctrl = queue->ctrl;
1467         int qid = nvme_tcp_queue_id(queue);
1468         int n = 0;
1469
1470         if (nvme_tcp_default_queue(queue))
1471                 n = qid - 1;
1472         else if (nvme_tcp_read_queue(queue))
1473                 n = qid - ctrl->io_queues[HCTX_TYPE_DEFAULT] - 1;
1474         else if (nvme_tcp_poll_queue(queue))
1475                 n = qid - ctrl->io_queues[HCTX_TYPE_DEFAULT] -
1476                                 ctrl->io_queues[HCTX_TYPE_READ] - 1;
1477         queue->io_cpu = cpumask_next_wrap(n - 1, cpu_online_mask, -1, false);
1478 }
1479
1480 static int nvme_tcp_alloc_queue(struct nvme_ctrl *nctrl,
1481                 int qid, size_t queue_size)
1482 {
1483         struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl);
1484         struct nvme_tcp_queue *queue = &ctrl->queues[qid];
1485         int ret, rcv_pdu_size;
1486
1487         mutex_init(&queue->queue_lock);
1488         queue->ctrl = ctrl;
1489         init_llist_head(&queue->req_list);
1490         INIT_LIST_HEAD(&queue->send_list);
1491         mutex_init(&queue->send_mutex);
1492         INIT_WORK(&queue->io_work, nvme_tcp_io_work);
1493         queue->queue_size = queue_size;
1494
1495         if (qid > 0)
1496                 queue->cmnd_capsule_len = nctrl->ioccsz * 16;
1497         else
1498                 queue->cmnd_capsule_len = sizeof(struct nvme_command) +
1499                                                 NVME_TCP_ADMIN_CCSZ;
1500
1501         ret = sock_create(ctrl->addr.ss_family, SOCK_STREAM,
1502                         IPPROTO_TCP, &queue->sock);
1503         if (ret) {
1504                 dev_err(nctrl->device,
1505                         "failed to create socket: %d\n", ret);
1506                 goto err_destroy_mutex;
1507         }
1508
1509         nvme_tcp_reclassify_socket(queue->sock);
1510
1511         /* Single syn retry */
1512         tcp_sock_set_syncnt(queue->sock->sk, 1);
1513
1514         /* Set TCP no delay */
1515         tcp_sock_set_nodelay(queue->sock->sk);
1516
1517         /*
1518          * Cleanup whatever is sitting in the TCP transmit queue on socket
1519          * close. This is done to prevent stale data from being sent should
1520          * the network connection be restored before TCP times out.
1521          */
1522         sock_no_linger(queue->sock->sk);
1523
1524         if (so_priority > 0)
1525                 sock_set_priority(queue->sock->sk, so_priority);
1526
1527         /* Set socket type of service */
1528         if (nctrl->opts->tos >= 0)
1529                 ip_sock_set_tos(queue->sock->sk, nctrl->opts->tos);
1530
1531         /* Set 10 seconds timeout for icresp recvmsg */
1532         queue->sock->sk->sk_rcvtimeo = 10 * HZ;
1533
1534         queue->sock->sk->sk_allocation = GFP_ATOMIC;
1535         nvme_tcp_set_queue_io_cpu(queue);
1536         queue->request = NULL;
1537         queue->data_remaining = 0;
1538         queue->ddgst_remaining = 0;
1539         queue->pdu_remaining = 0;
1540         queue->pdu_offset = 0;
1541         sk_set_memalloc(queue->sock->sk);
1542
1543         if (nctrl->opts->mask & NVMF_OPT_HOST_TRADDR) {
1544                 ret = kernel_bind(queue->sock, (struct sockaddr *)&ctrl->src_addr,
1545                         sizeof(ctrl->src_addr));
1546                 if (ret) {
1547                         dev_err(nctrl->device,
1548                                 "failed to bind queue %d socket %d\n",
1549                                 qid, ret);
1550                         goto err_sock;
1551                 }
1552         }
1553
1554         if (nctrl->opts->mask & NVMF_OPT_HOST_IFACE) {
1555                 char *iface = nctrl->opts->host_iface;
1556                 sockptr_t optval = KERNEL_SOCKPTR(iface);
1557
1558                 ret = sock_setsockopt(queue->sock, SOL_SOCKET, SO_BINDTODEVICE,
1559                                       optval, strlen(iface));
1560                 if (ret) {
1561                         dev_err(nctrl->device,
1562                           "failed to bind to interface %s queue %d err %d\n",
1563                           iface, qid, ret);
1564                         goto err_sock;
1565                 }
1566         }
1567
1568         queue->hdr_digest = nctrl->opts->hdr_digest;
1569         queue->data_digest = nctrl->opts->data_digest;
1570         if (queue->hdr_digest || queue->data_digest) {
1571                 ret = nvme_tcp_alloc_crypto(queue);
1572                 if (ret) {
1573                         dev_err(nctrl->device,
1574                                 "failed to allocate queue %d crypto\n", qid);
1575                         goto err_sock;
1576                 }
1577         }
1578
1579         rcv_pdu_size = sizeof(struct nvme_tcp_rsp_pdu) +
1580                         nvme_tcp_hdgst_len(queue);
1581         queue->pdu = kmalloc(rcv_pdu_size, GFP_KERNEL);
1582         if (!queue->pdu) {
1583                 ret = -ENOMEM;
1584                 goto err_crypto;
1585         }
1586
1587         dev_dbg(nctrl->device, "connecting queue %d\n",
1588                         nvme_tcp_queue_id(queue));
1589
1590         ret = kernel_connect(queue->sock, (struct sockaddr *)&ctrl->addr,
1591                 sizeof(ctrl->addr), 0);
1592         if (ret) {
1593                 dev_err(nctrl->device,
1594                         "failed to connect socket: %d\n", ret);
1595                 goto err_rcv_pdu;
1596         }
1597
1598         ret = nvme_tcp_init_connection(queue);
1599         if (ret)
1600                 goto err_init_connect;
1601
1602         queue->rd_enabled = true;
1603         set_bit(NVME_TCP_Q_ALLOCATED, &queue->flags);
1604         nvme_tcp_init_recv_ctx(queue);
1605
1606         write_lock_bh(&queue->sock->sk->sk_callback_lock);
1607         queue->sock->sk->sk_user_data = queue;
1608         queue->state_change = queue->sock->sk->sk_state_change;
1609         queue->data_ready = queue->sock->sk->sk_data_ready;
1610         queue->write_space = queue->sock->sk->sk_write_space;
1611         queue->sock->sk->sk_data_ready = nvme_tcp_data_ready;
1612         queue->sock->sk->sk_state_change = nvme_tcp_state_change;
1613         queue->sock->sk->sk_write_space = nvme_tcp_write_space;
1614 #ifdef CONFIG_NET_RX_BUSY_POLL
1615         queue->sock->sk->sk_ll_usec = 1;
1616 #endif
1617         write_unlock_bh(&queue->sock->sk->sk_callback_lock);
1618
1619         return 0;
1620
1621 err_init_connect:
1622         kernel_sock_shutdown(queue->sock, SHUT_RDWR);
1623 err_rcv_pdu:
1624         kfree(queue->pdu);
1625 err_crypto:
1626         if (queue->hdr_digest || queue->data_digest)
1627                 nvme_tcp_free_crypto(queue);
1628 err_sock:
1629         sock_release(queue->sock);
1630         queue->sock = NULL;
1631 err_destroy_mutex:
1632         mutex_destroy(&queue->send_mutex);
1633         mutex_destroy(&queue->queue_lock);
1634         return ret;
1635 }
1636
1637 static void nvme_tcp_restore_sock_calls(struct nvme_tcp_queue *queue)
1638 {
1639         struct socket *sock = queue->sock;
1640
1641         write_lock_bh(&sock->sk->sk_callback_lock);
1642         sock->sk->sk_user_data  = NULL;
1643         sock->sk->sk_data_ready = queue->data_ready;
1644         sock->sk->sk_state_change = queue->state_change;
1645         sock->sk->sk_write_space  = queue->write_space;
1646         write_unlock_bh(&sock->sk->sk_callback_lock);
1647 }
1648
1649 static void __nvme_tcp_stop_queue(struct nvme_tcp_queue *queue)
1650 {
1651         kernel_sock_shutdown(queue->sock, SHUT_RDWR);
1652         nvme_tcp_restore_sock_calls(queue);
1653         cancel_work_sync(&queue->io_work);
1654 }
1655
1656 static void nvme_tcp_stop_queue(struct nvme_ctrl *nctrl, int qid)
1657 {
1658         struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl);
1659         struct nvme_tcp_queue *queue = &ctrl->queues[qid];
1660
1661         mutex_lock(&queue->queue_lock);
1662         if (test_and_clear_bit(NVME_TCP_Q_LIVE, &queue->flags))
1663                 __nvme_tcp_stop_queue(queue);
1664         mutex_unlock(&queue->queue_lock);
1665 }
1666
1667 static int nvme_tcp_start_queue(struct nvme_ctrl *nctrl, int idx)
1668 {
1669         struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl);
1670         int ret;
1671
1672         if (idx)
1673                 ret = nvmf_connect_io_queue(nctrl, idx);
1674         else
1675                 ret = nvmf_connect_admin_queue(nctrl);
1676
1677         if (!ret) {
1678                 set_bit(NVME_TCP_Q_LIVE, &ctrl->queues[idx].flags);
1679         } else {
1680                 if (test_bit(NVME_TCP_Q_ALLOCATED, &ctrl->queues[idx].flags))
1681                         __nvme_tcp_stop_queue(&ctrl->queues[idx]);
1682                 dev_err(nctrl->device,
1683                         "failed to connect queue: %d ret=%d\n", idx, ret);
1684         }
1685         return ret;
1686 }
1687
1688 static struct blk_mq_tag_set *nvme_tcp_alloc_tagset(struct nvme_ctrl *nctrl,
1689                 bool admin)
1690 {
1691         struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl);
1692         struct blk_mq_tag_set *set;
1693         int ret;
1694
1695         if (admin) {
1696                 set = &ctrl->admin_tag_set;
1697                 memset(set, 0, sizeof(*set));
1698                 set->ops = &nvme_tcp_admin_mq_ops;
1699                 set->queue_depth = NVME_AQ_MQ_TAG_DEPTH;
1700                 set->reserved_tags = NVMF_RESERVED_TAGS;
1701                 set->numa_node = nctrl->numa_node;
1702                 set->flags = BLK_MQ_F_BLOCKING;
1703                 set->cmd_size = sizeof(struct nvme_tcp_request);
1704                 set->driver_data = ctrl;
1705                 set->nr_hw_queues = 1;
1706                 set->timeout = NVME_ADMIN_TIMEOUT;
1707         } else {
1708                 set = &ctrl->tag_set;
1709                 memset(set, 0, sizeof(*set));
1710                 set->ops = &nvme_tcp_mq_ops;
1711                 set->queue_depth = nctrl->sqsize + 1;
1712                 set->reserved_tags = NVMF_RESERVED_TAGS;
1713                 set->numa_node = nctrl->numa_node;
1714                 set->flags = BLK_MQ_F_SHOULD_MERGE | BLK_MQ_F_BLOCKING;
1715                 set->cmd_size = sizeof(struct nvme_tcp_request);
1716                 set->driver_data = ctrl;
1717                 set->nr_hw_queues = nctrl->queue_count - 1;
1718                 set->timeout = NVME_IO_TIMEOUT;
1719                 set->nr_maps = nctrl->opts->nr_poll_queues ? HCTX_MAX_TYPES : 2;
1720         }
1721
1722         ret = blk_mq_alloc_tag_set(set);
1723         if (ret)
1724                 return ERR_PTR(ret);
1725
1726         return set;
1727 }
1728
1729 static void nvme_tcp_free_admin_queue(struct nvme_ctrl *ctrl)
1730 {
1731         if (to_tcp_ctrl(ctrl)->async_req.pdu) {
1732                 cancel_work_sync(&ctrl->async_event_work);
1733                 nvme_tcp_free_async_req(to_tcp_ctrl(ctrl));
1734                 to_tcp_ctrl(ctrl)->async_req.pdu = NULL;
1735         }
1736
1737         nvme_tcp_free_queue(ctrl, 0);
1738 }
1739
1740 static void nvme_tcp_free_io_queues(struct nvme_ctrl *ctrl)
1741 {
1742         int i;
1743
1744         for (i = 1; i < ctrl->queue_count; i++)
1745                 nvme_tcp_free_queue(ctrl, i);
1746 }
1747
1748 static void nvme_tcp_stop_io_queues(struct nvme_ctrl *ctrl)
1749 {
1750         int i;
1751
1752         for (i = 1; i < ctrl->queue_count; i++)
1753                 nvme_tcp_stop_queue(ctrl, i);
1754 }
1755
1756 static int nvme_tcp_start_io_queues(struct nvme_ctrl *ctrl)
1757 {
1758         int i, ret;
1759
1760         for (i = 1; i < ctrl->queue_count; i++) {
1761                 ret = nvme_tcp_start_queue(ctrl, i);
1762                 if (ret)
1763                         goto out_stop_queues;
1764         }
1765
1766         return 0;
1767
1768 out_stop_queues:
1769         for (i--; i >= 1; i--)
1770                 nvme_tcp_stop_queue(ctrl, i);
1771         return ret;
1772 }
1773
1774 static int nvme_tcp_alloc_admin_queue(struct nvme_ctrl *ctrl)
1775 {
1776         int ret;
1777
1778         ret = nvme_tcp_alloc_queue(ctrl, 0, NVME_AQ_DEPTH);
1779         if (ret)
1780                 return ret;
1781
1782         ret = nvme_tcp_alloc_async_req(to_tcp_ctrl(ctrl));
1783         if (ret)
1784                 goto out_free_queue;
1785
1786         return 0;
1787
1788 out_free_queue:
1789         nvme_tcp_free_queue(ctrl, 0);
1790         return ret;
1791 }
1792
1793 static int __nvme_tcp_alloc_io_queues(struct nvme_ctrl *ctrl)
1794 {
1795         int i, ret;
1796
1797         for (i = 1; i < ctrl->queue_count; i++) {
1798                 ret = nvme_tcp_alloc_queue(ctrl, i, ctrl->sqsize + 1);
1799                 if (ret)
1800                         goto out_free_queues;
1801         }
1802
1803         return 0;
1804
1805 out_free_queues:
1806         for (i--; i >= 1; i--)
1807                 nvme_tcp_free_queue(ctrl, i);
1808
1809         return ret;
1810 }
1811
1812 static unsigned int nvme_tcp_nr_io_queues(struct nvme_ctrl *ctrl)
1813 {
1814         unsigned int nr_io_queues;
1815
1816         nr_io_queues = min(ctrl->opts->nr_io_queues, num_online_cpus());
1817         nr_io_queues += min(ctrl->opts->nr_write_queues, num_online_cpus());
1818         nr_io_queues += min(ctrl->opts->nr_poll_queues, num_online_cpus());
1819
1820         return nr_io_queues;
1821 }
1822
1823 static void nvme_tcp_set_io_queues(struct nvme_ctrl *nctrl,
1824                 unsigned int nr_io_queues)
1825 {
1826         struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl);
1827         struct nvmf_ctrl_options *opts = nctrl->opts;
1828
1829         if (opts->nr_write_queues && opts->nr_io_queues < nr_io_queues) {
1830                 /*
1831                  * separate read/write queues
1832                  * hand out dedicated default queues only after we have
1833                  * sufficient read queues.
1834                  */
1835                 ctrl->io_queues[HCTX_TYPE_READ] = opts->nr_io_queues;
1836                 nr_io_queues -= ctrl->io_queues[HCTX_TYPE_READ];
1837                 ctrl->io_queues[HCTX_TYPE_DEFAULT] =
1838                         min(opts->nr_write_queues, nr_io_queues);
1839                 nr_io_queues -= ctrl->io_queues[HCTX_TYPE_DEFAULT];
1840         } else {
1841                 /*
1842                  * shared read/write queues
1843                  * either no write queues were requested, or we don't have
1844                  * sufficient queue count to have dedicated default queues.
1845                  */
1846                 ctrl->io_queues[HCTX_TYPE_DEFAULT] =
1847                         min(opts->nr_io_queues, nr_io_queues);
1848                 nr_io_queues -= ctrl->io_queues[HCTX_TYPE_DEFAULT];
1849         }
1850
1851         if (opts->nr_poll_queues && nr_io_queues) {
1852                 /* map dedicated poll queues only if we have queues left */
1853                 ctrl->io_queues[HCTX_TYPE_POLL] =
1854                         min(opts->nr_poll_queues, nr_io_queues);
1855         }
1856 }
1857
1858 static int nvme_tcp_alloc_io_queues(struct nvme_ctrl *ctrl)
1859 {
1860         unsigned int nr_io_queues;
1861         int ret;
1862
1863         nr_io_queues = nvme_tcp_nr_io_queues(ctrl);
1864         ret = nvme_set_queue_count(ctrl, &nr_io_queues);
1865         if (ret)
1866                 return ret;
1867
1868         if (nr_io_queues == 0) {
1869                 dev_err(ctrl->device,
1870                         "unable to set any I/O queues\n");
1871                 return -ENOMEM;
1872         }
1873
1874         ctrl->queue_count = nr_io_queues + 1;
1875         dev_info(ctrl->device,
1876                 "creating %d I/O queues.\n", nr_io_queues);
1877
1878         nvme_tcp_set_io_queues(ctrl, nr_io_queues);
1879
1880         return __nvme_tcp_alloc_io_queues(ctrl);
1881 }
1882
1883 static void nvme_tcp_destroy_io_queues(struct nvme_ctrl *ctrl, bool remove)
1884 {
1885         nvme_tcp_stop_io_queues(ctrl);
1886         if (remove) {
1887                 blk_cleanup_queue(ctrl->connect_q);
1888                 blk_mq_free_tag_set(ctrl->tagset);
1889         }
1890         nvme_tcp_free_io_queues(ctrl);
1891 }
1892
1893 static int nvme_tcp_configure_io_queues(struct nvme_ctrl *ctrl, bool new)
1894 {
1895         int ret;
1896
1897         ret = nvme_tcp_alloc_io_queues(ctrl);
1898         if (ret)
1899                 return ret;
1900
1901         if (new) {
1902                 ctrl->tagset = nvme_tcp_alloc_tagset(ctrl, false);
1903                 if (IS_ERR(ctrl->tagset)) {
1904                         ret = PTR_ERR(ctrl->tagset);
1905                         goto out_free_io_queues;
1906                 }
1907
1908                 ret = nvme_ctrl_init_connect_q(ctrl);
1909                 if (ret)
1910                         goto out_free_tag_set;
1911         }
1912
1913         ret = nvme_tcp_start_io_queues(ctrl);
1914         if (ret)
1915                 goto out_cleanup_connect_q;
1916
1917         if (!new) {
1918                 nvme_start_queues(ctrl);
1919                 if (!nvme_wait_freeze_timeout(ctrl, NVME_IO_TIMEOUT)) {
1920                         /*
1921                          * If we timed out waiting for freeze we are likely to
1922                          * be stuck.  Fail the controller initialization just
1923                          * to be safe.
1924                          */
1925                         ret = -ENODEV;
1926                         goto out_wait_freeze_timed_out;
1927                 }
1928                 blk_mq_update_nr_hw_queues(ctrl->tagset,
1929                         ctrl->queue_count - 1);
1930                 nvme_unfreeze(ctrl);
1931         }
1932
1933         return 0;
1934
1935 out_wait_freeze_timed_out:
1936         nvme_stop_queues(ctrl);
1937         nvme_sync_io_queues(ctrl);
1938         nvme_tcp_stop_io_queues(ctrl);
1939 out_cleanup_connect_q:
1940         nvme_cancel_tagset(ctrl);
1941         if (new)
1942                 blk_cleanup_queue(ctrl->connect_q);
1943 out_free_tag_set:
1944         if (new)
1945                 blk_mq_free_tag_set(ctrl->tagset);
1946 out_free_io_queues:
1947         nvme_tcp_free_io_queues(ctrl);
1948         return ret;
1949 }
1950
1951 static void nvme_tcp_destroy_admin_queue(struct nvme_ctrl *ctrl, bool remove)
1952 {
1953         nvme_tcp_stop_queue(ctrl, 0);
1954         if (remove) {
1955                 blk_cleanup_queue(ctrl->admin_q);
1956                 blk_cleanup_queue(ctrl->fabrics_q);
1957                 blk_mq_free_tag_set(ctrl->admin_tagset);
1958         }
1959         nvme_tcp_free_admin_queue(ctrl);
1960 }
1961
1962 static int nvme_tcp_configure_admin_queue(struct nvme_ctrl *ctrl, bool new)
1963 {
1964         int error;
1965
1966         error = nvme_tcp_alloc_admin_queue(ctrl);
1967         if (error)
1968                 return error;
1969
1970         if (new) {
1971                 ctrl->admin_tagset = nvme_tcp_alloc_tagset(ctrl, true);
1972                 if (IS_ERR(ctrl->admin_tagset)) {
1973                         error = PTR_ERR(ctrl->admin_tagset);
1974                         goto out_free_queue;
1975                 }
1976
1977                 ctrl->fabrics_q = blk_mq_init_queue(ctrl->admin_tagset);
1978                 if (IS_ERR(ctrl->fabrics_q)) {
1979                         error = PTR_ERR(ctrl->fabrics_q);
1980                         goto out_free_tagset;
1981                 }
1982
1983                 ctrl->admin_q = blk_mq_init_queue(ctrl->admin_tagset);
1984                 if (IS_ERR(ctrl->admin_q)) {
1985                         error = PTR_ERR(ctrl->admin_q);
1986                         goto out_cleanup_fabrics_q;
1987                 }
1988         }
1989
1990         error = nvme_tcp_start_queue(ctrl, 0);
1991         if (error)
1992                 goto out_cleanup_queue;
1993
1994         error = nvme_enable_ctrl(ctrl);
1995         if (error)
1996                 goto out_stop_queue;
1997
1998         nvme_start_admin_queue(ctrl);
1999
2000         error = nvme_init_ctrl_finish(ctrl);
2001         if (error)
2002                 goto out_quiesce_queue;
2003
2004         return 0;
2005
2006 out_quiesce_queue:
2007         nvme_stop_admin_queue(ctrl);
2008         blk_sync_queue(ctrl->admin_q);
2009 out_stop_queue:
2010         nvme_tcp_stop_queue(ctrl, 0);
2011         nvme_cancel_admin_tagset(ctrl);
2012 out_cleanup_queue:
2013         if (new)
2014                 blk_cleanup_queue(ctrl->admin_q);
2015 out_cleanup_fabrics_q:
2016         if (new)
2017                 blk_cleanup_queue(ctrl->fabrics_q);
2018 out_free_tagset:
2019         if (new)
2020                 blk_mq_free_tag_set(ctrl->admin_tagset);
2021 out_free_queue:
2022         nvme_tcp_free_admin_queue(ctrl);
2023         return error;
2024 }
2025
2026 static void nvme_tcp_teardown_admin_queue(struct nvme_ctrl *ctrl,
2027                 bool remove)
2028 {
2029         nvme_stop_admin_queue(ctrl);
2030         blk_sync_queue(ctrl->admin_q);
2031         nvme_tcp_stop_queue(ctrl, 0);
2032         nvme_cancel_admin_tagset(ctrl);
2033         if (remove)
2034                 nvme_start_admin_queue(ctrl);
2035         nvme_tcp_destroy_admin_queue(ctrl, remove);
2036 }
2037
2038 static void nvme_tcp_teardown_io_queues(struct nvme_ctrl *ctrl,
2039                 bool remove)
2040 {
2041         if (ctrl->queue_count <= 1)
2042                 return;
2043         nvme_stop_admin_queue(ctrl);
2044         nvme_start_freeze(ctrl);
2045         nvme_stop_queues(ctrl);
2046         nvme_sync_io_queues(ctrl);
2047         nvme_tcp_stop_io_queues(ctrl);
2048         nvme_cancel_tagset(ctrl);
2049         if (remove)
2050                 nvme_start_queues(ctrl);
2051         nvme_tcp_destroy_io_queues(ctrl, remove);
2052 }
2053
2054 static void nvme_tcp_reconnect_or_remove(struct nvme_ctrl *ctrl)
2055 {
2056         /* If we are resetting/deleting then do nothing */
2057         if (ctrl->state != NVME_CTRL_CONNECTING) {
2058                 WARN_ON_ONCE(ctrl->state == NVME_CTRL_NEW ||
2059                         ctrl->state == NVME_CTRL_LIVE);
2060                 return;
2061         }
2062
2063         if (nvmf_should_reconnect(ctrl)) {
2064                 dev_info(ctrl->device, "Reconnecting in %d seconds...\n",
2065                         ctrl->opts->reconnect_delay);
2066                 queue_delayed_work(nvme_wq, &to_tcp_ctrl(ctrl)->connect_work,
2067                                 ctrl->opts->reconnect_delay * HZ);
2068         } else {
2069                 dev_info(ctrl->device, "Removing controller...\n");
2070                 nvme_delete_ctrl(ctrl);
2071         }
2072 }
2073
2074 static int nvme_tcp_setup_ctrl(struct nvme_ctrl *ctrl, bool new)
2075 {
2076         struct nvmf_ctrl_options *opts = ctrl->opts;
2077         int ret;
2078
2079         ret = nvme_tcp_configure_admin_queue(ctrl, new);
2080         if (ret)
2081                 return ret;
2082
2083         if (ctrl->icdoff) {
2084                 ret = -EOPNOTSUPP;
2085                 dev_err(ctrl->device, "icdoff is not supported!\n");
2086                 goto destroy_admin;
2087         }
2088
2089         if (!nvme_ctrl_sgl_supported(ctrl)) {
2090                 ret = -EOPNOTSUPP;
2091                 dev_err(ctrl->device, "Mandatory sgls are not supported!\n");
2092                 goto destroy_admin;
2093         }
2094
2095         if (opts->queue_size > ctrl->sqsize + 1)
2096                 dev_warn(ctrl->device,
2097                         "queue_size %zu > ctrl sqsize %u, clamping down\n",
2098                         opts->queue_size, ctrl->sqsize + 1);
2099
2100         if (ctrl->sqsize + 1 > ctrl->maxcmd) {
2101                 dev_warn(ctrl->device,
2102                         "sqsize %u > ctrl maxcmd %u, clamping down\n",
2103                         ctrl->sqsize + 1, ctrl->maxcmd);
2104                 ctrl->sqsize = ctrl->maxcmd - 1;
2105         }
2106
2107         if (ctrl->queue_count > 1) {
2108                 ret = nvme_tcp_configure_io_queues(ctrl, new);
2109                 if (ret)
2110                         goto destroy_admin;
2111         }
2112
2113         if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_LIVE)) {
2114                 /*
2115                  * state change failure is ok if we started ctrl delete,
2116                  * unless we're during creation of a new controller to
2117                  * avoid races with teardown flow.
2118                  */
2119                 WARN_ON_ONCE(ctrl->state != NVME_CTRL_DELETING &&
2120                              ctrl->state != NVME_CTRL_DELETING_NOIO);
2121                 WARN_ON_ONCE(new);
2122                 ret = -EINVAL;
2123                 goto destroy_io;
2124         }
2125
2126         nvme_start_ctrl(ctrl);
2127         return 0;
2128
2129 destroy_io:
2130         if (ctrl->queue_count > 1) {
2131                 nvme_stop_queues(ctrl);
2132                 nvme_sync_io_queues(ctrl);
2133                 nvme_tcp_stop_io_queues(ctrl);
2134                 nvme_cancel_tagset(ctrl);
2135                 nvme_tcp_destroy_io_queues(ctrl, new);
2136         }
2137 destroy_admin:
2138         nvme_stop_admin_queue(ctrl);
2139         blk_sync_queue(ctrl->admin_q);
2140         nvme_tcp_stop_queue(ctrl, 0);
2141         nvme_cancel_admin_tagset(ctrl);
2142         nvme_tcp_destroy_admin_queue(ctrl, new);
2143         return ret;
2144 }
2145
2146 static void nvme_tcp_reconnect_ctrl_work(struct work_struct *work)
2147 {
2148         struct nvme_tcp_ctrl *tcp_ctrl = container_of(to_delayed_work(work),
2149                         struct nvme_tcp_ctrl, connect_work);
2150         struct nvme_ctrl *ctrl = &tcp_ctrl->ctrl;
2151
2152         ++ctrl->nr_reconnects;
2153
2154         if (nvme_tcp_setup_ctrl(ctrl, false))
2155                 goto requeue;
2156
2157         dev_info(ctrl->device, "Successfully reconnected (%d attempt)\n",
2158                         ctrl->nr_reconnects);
2159
2160         ctrl->nr_reconnects = 0;
2161
2162         return;
2163
2164 requeue:
2165         dev_info(ctrl->device, "Failed reconnect attempt %d\n",
2166                         ctrl->nr_reconnects);
2167         nvme_tcp_reconnect_or_remove(ctrl);
2168 }
2169
2170 static void nvme_tcp_error_recovery_work(struct work_struct *work)
2171 {
2172         struct nvme_tcp_ctrl *tcp_ctrl = container_of(work,
2173                                 struct nvme_tcp_ctrl, err_work);
2174         struct nvme_ctrl *ctrl = &tcp_ctrl->ctrl;
2175
2176         nvme_stop_keep_alive(ctrl);
2177         flush_work(&ctrl->async_event_work);
2178         nvme_tcp_teardown_io_queues(ctrl, false);
2179         /* unquiesce to fail fast pending requests */
2180         nvme_start_queues(ctrl);
2181         nvme_tcp_teardown_admin_queue(ctrl, false);
2182         nvme_start_admin_queue(ctrl);
2183
2184         if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_CONNECTING)) {
2185                 /* state change failure is ok if we started ctrl delete */
2186                 WARN_ON_ONCE(ctrl->state != NVME_CTRL_DELETING &&
2187                              ctrl->state != NVME_CTRL_DELETING_NOIO);
2188                 return;
2189         }
2190
2191         nvme_tcp_reconnect_or_remove(ctrl);
2192 }
2193
2194 static void nvme_tcp_teardown_ctrl(struct nvme_ctrl *ctrl, bool shutdown)
2195 {
2196         nvme_tcp_teardown_io_queues(ctrl, shutdown);
2197         nvme_stop_admin_queue(ctrl);
2198         if (shutdown)
2199                 nvme_shutdown_ctrl(ctrl);
2200         else
2201                 nvme_disable_ctrl(ctrl);
2202         nvme_tcp_teardown_admin_queue(ctrl, shutdown);
2203 }
2204
2205 static void nvme_tcp_delete_ctrl(struct nvme_ctrl *ctrl)
2206 {
2207         nvme_tcp_teardown_ctrl(ctrl, true);
2208 }
2209
2210 static void nvme_reset_ctrl_work(struct work_struct *work)
2211 {
2212         struct nvme_ctrl *ctrl =
2213                 container_of(work, struct nvme_ctrl, reset_work);
2214
2215         nvme_stop_ctrl(ctrl);
2216         nvme_tcp_teardown_ctrl(ctrl, false);
2217
2218         if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_CONNECTING)) {
2219                 /* state change failure is ok if we started ctrl delete */
2220                 WARN_ON_ONCE(ctrl->state != NVME_CTRL_DELETING &&
2221                              ctrl->state != NVME_CTRL_DELETING_NOIO);
2222                 return;
2223         }
2224
2225         if (nvme_tcp_setup_ctrl(ctrl, false))
2226                 goto out_fail;
2227
2228         return;
2229
2230 out_fail:
2231         ++ctrl->nr_reconnects;
2232         nvme_tcp_reconnect_or_remove(ctrl);
2233 }
2234
2235 static void nvme_tcp_stop_ctrl(struct nvme_ctrl *ctrl)
2236 {
2237         cancel_work_sync(&to_tcp_ctrl(ctrl)->err_work);
2238         cancel_delayed_work_sync(&to_tcp_ctrl(ctrl)->connect_work);
2239 }
2240
2241 static void nvme_tcp_free_ctrl(struct nvme_ctrl *nctrl)
2242 {
2243         struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl);
2244
2245         if (list_empty(&ctrl->list))
2246                 goto free_ctrl;
2247
2248         mutex_lock(&nvme_tcp_ctrl_mutex);
2249         list_del(&ctrl->list);
2250         mutex_unlock(&nvme_tcp_ctrl_mutex);
2251
2252         nvmf_free_options(nctrl->opts);
2253 free_ctrl:
2254         kfree(ctrl->queues);
2255         kfree(ctrl);
2256 }
2257
2258 static void nvme_tcp_set_sg_null(struct nvme_command *c)
2259 {
2260         struct nvme_sgl_desc *sg = &c->common.dptr.sgl;
2261
2262         sg->addr = 0;
2263         sg->length = 0;
2264         sg->type = (NVME_TRANSPORT_SGL_DATA_DESC << 4) |
2265                         NVME_SGL_FMT_TRANSPORT_A;
2266 }
2267
2268 static void nvme_tcp_set_sg_inline(struct nvme_tcp_queue *queue,
2269                 struct nvme_command *c, u32 data_len)
2270 {
2271         struct nvme_sgl_desc *sg = &c->common.dptr.sgl;
2272
2273         sg->addr = cpu_to_le64(queue->ctrl->ctrl.icdoff);
2274         sg->length = cpu_to_le32(data_len);
2275         sg->type = (NVME_SGL_FMT_DATA_DESC << 4) | NVME_SGL_FMT_OFFSET;
2276 }
2277
2278 static void nvme_tcp_set_sg_host_data(struct nvme_command *c,
2279                 u32 data_len)
2280 {
2281         struct nvme_sgl_desc *sg = &c->common.dptr.sgl;
2282
2283         sg->addr = 0;
2284         sg->length = cpu_to_le32(data_len);
2285         sg->type = (NVME_TRANSPORT_SGL_DATA_DESC << 4) |
2286                         NVME_SGL_FMT_TRANSPORT_A;
2287 }
2288
2289 static void nvme_tcp_submit_async_event(struct nvme_ctrl *arg)
2290 {
2291         struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(arg);
2292         struct nvme_tcp_queue *queue = &ctrl->queues[0];
2293         struct nvme_tcp_cmd_pdu *pdu = ctrl->async_req.pdu;
2294         struct nvme_command *cmd = &pdu->cmd;
2295         u8 hdgst = nvme_tcp_hdgst_len(queue);
2296
2297         memset(pdu, 0, sizeof(*pdu));
2298         pdu->hdr.type = nvme_tcp_cmd;
2299         if (queue->hdr_digest)
2300                 pdu->hdr.flags |= NVME_TCP_F_HDGST;
2301         pdu->hdr.hlen = sizeof(*pdu);
2302         pdu->hdr.plen = cpu_to_le32(pdu->hdr.hlen + hdgst);
2303
2304         cmd->common.opcode = nvme_admin_async_event;
2305         cmd->common.command_id = NVME_AQ_BLK_MQ_DEPTH;
2306         cmd->common.flags |= NVME_CMD_SGL_METABUF;
2307         nvme_tcp_set_sg_null(cmd);
2308
2309         ctrl->async_req.state = NVME_TCP_SEND_CMD_PDU;
2310         ctrl->async_req.offset = 0;
2311         ctrl->async_req.curr_bio = NULL;
2312         ctrl->async_req.data_len = 0;
2313
2314         nvme_tcp_queue_request(&ctrl->async_req, true, true);
2315 }
2316
2317 static void nvme_tcp_complete_timed_out(struct request *rq)
2318 {
2319         struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
2320         struct nvme_ctrl *ctrl = &req->queue->ctrl->ctrl;
2321
2322         nvme_tcp_stop_queue(ctrl, nvme_tcp_queue_id(req->queue));
2323         nvmf_complete_timed_out_request(rq);
2324 }
2325
2326 static enum blk_eh_timer_return
2327 nvme_tcp_timeout(struct request *rq, bool reserved)
2328 {
2329         struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
2330         struct nvme_ctrl *ctrl = &req->queue->ctrl->ctrl;
2331         struct nvme_tcp_cmd_pdu *pdu = req->pdu;
2332
2333         dev_warn(ctrl->device,
2334                 "queue %d: timeout request %#x type %d\n",
2335                 nvme_tcp_queue_id(req->queue), rq->tag, pdu->hdr.type);
2336
2337         if (ctrl->state != NVME_CTRL_LIVE) {
2338                 /*
2339                  * If we are resetting, connecting or deleting we should
2340                  * complete immediately because we may block controller
2341                  * teardown or setup sequence
2342                  * - ctrl disable/shutdown fabrics requests
2343                  * - connect requests
2344                  * - initialization admin requests
2345                  * - I/O requests that entered after unquiescing and
2346                  *   the controller stopped responding
2347                  *
2348                  * All other requests should be cancelled by the error
2349                  * recovery work, so it's fine that we fail it here.
2350                  */
2351                 nvme_tcp_complete_timed_out(rq);
2352                 return BLK_EH_DONE;
2353         }
2354
2355         /*
2356          * LIVE state should trigger the normal error recovery which will
2357          * handle completing this request.
2358          */
2359         nvme_tcp_error_recovery(ctrl);
2360         return BLK_EH_RESET_TIMER;
2361 }
2362
2363 static blk_status_t nvme_tcp_map_data(struct nvme_tcp_queue *queue,
2364                         struct request *rq)
2365 {
2366         struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
2367         struct nvme_tcp_cmd_pdu *pdu = req->pdu;
2368         struct nvme_command *c = &pdu->cmd;
2369
2370         c->common.flags |= NVME_CMD_SGL_METABUF;
2371
2372         if (!blk_rq_nr_phys_segments(rq))
2373                 nvme_tcp_set_sg_null(c);
2374         else if (rq_data_dir(rq) == WRITE &&
2375             req->data_len <= nvme_tcp_inline_data_size(queue))
2376                 nvme_tcp_set_sg_inline(queue, c, req->data_len);
2377         else
2378                 nvme_tcp_set_sg_host_data(c, req->data_len);
2379
2380         return 0;
2381 }
2382
2383 static blk_status_t nvme_tcp_setup_cmd_pdu(struct nvme_ns *ns,
2384                 struct request *rq)
2385 {
2386         struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
2387         struct nvme_tcp_cmd_pdu *pdu = req->pdu;
2388         struct nvme_tcp_queue *queue = req->queue;
2389         u8 hdgst = nvme_tcp_hdgst_len(queue), ddgst = 0;
2390         blk_status_t ret;
2391
2392         ret = nvme_setup_cmd(ns, rq);
2393         if (ret)
2394                 return ret;
2395
2396         req->state = NVME_TCP_SEND_CMD_PDU;
2397         req->status = cpu_to_le16(NVME_SC_SUCCESS);
2398         req->offset = 0;
2399         req->data_sent = 0;
2400         req->pdu_len = 0;
2401         req->pdu_sent = 0;
2402         req->h2cdata_left = 0;
2403         req->data_len = blk_rq_nr_phys_segments(rq) ?
2404                                 blk_rq_payload_bytes(rq) : 0;
2405         req->curr_bio = rq->bio;
2406         if (req->curr_bio && req->data_len)
2407                 nvme_tcp_init_iter(req, rq_data_dir(rq));
2408
2409         if (rq_data_dir(rq) == WRITE &&
2410             req->data_len <= nvme_tcp_inline_data_size(queue))
2411                 req->pdu_len = req->data_len;
2412
2413         pdu->hdr.type = nvme_tcp_cmd;
2414         pdu->hdr.flags = 0;
2415         if (queue->hdr_digest)
2416                 pdu->hdr.flags |= NVME_TCP_F_HDGST;
2417         if (queue->data_digest && req->pdu_len) {
2418                 pdu->hdr.flags |= NVME_TCP_F_DDGST;
2419                 ddgst = nvme_tcp_ddgst_len(queue);
2420         }
2421         pdu->hdr.hlen = sizeof(*pdu);
2422         pdu->hdr.pdo = req->pdu_len ? pdu->hdr.hlen + hdgst : 0;
2423         pdu->hdr.plen =
2424                 cpu_to_le32(pdu->hdr.hlen + hdgst + req->pdu_len + ddgst);
2425
2426         ret = nvme_tcp_map_data(queue, rq);
2427         if (unlikely(ret)) {
2428                 nvme_cleanup_cmd(rq);
2429                 dev_err(queue->ctrl->ctrl.device,
2430                         "Failed to map data (%d)\n", ret);
2431                 return ret;
2432         }
2433
2434         return 0;
2435 }
2436
2437 static void nvme_tcp_commit_rqs(struct blk_mq_hw_ctx *hctx)
2438 {
2439         struct nvme_tcp_queue *queue = hctx->driver_data;
2440
2441         if (!llist_empty(&queue->req_list))
2442                 queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work);
2443 }
2444
2445 static blk_status_t nvme_tcp_queue_rq(struct blk_mq_hw_ctx *hctx,
2446                 const struct blk_mq_queue_data *bd)
2447 {
2448         struct nvme_ns *ns = hctx->queue->queuedata;
2449         struct nvme_tcp_queue *queue = hctx->driver_data;
2450         struct request *rq = bd->rq;
2451         struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
2452         bool queue_ready = test_bit(NVME_TCP_Q_LIVE, &queue->flags);
2453         blk_status_t ret;
2454
2455         if (!nvme_check_ready(&queue->ctrl->ctrl, rq, queue_ready))
2456                 return nvme_fail_nonready_command(&queue->ctrl->ctrl, rq);
2457
2458         ret = nvme_tcp_setup_cmd_pdu(ns, rq);
2459         if (unlikely(ret))
2460                 return ret;
2461
2462         blk_mq_start_request(rq);
2463
2464         nvme_tcp_queue_request(req, true, bd->last);
2465
2466         return BLK_STS_OK;
2467 }
2468
2469 static int nvme_tcp_map_queues(struct blk_mq_tag_set *set)
2470 {
2471         struct nvme_tcp_ctrl *ctrl = set->driver_data;
2472         struct nvmf_ctrl_options *opts = ctrl->ctrl.opts;
2473
2474         if (opts->nr_write_queues && ctrl->io_queues[HCTX_TYPE_READ]) {
2475                 /* separate read/write queues */
2476                 set->map[HCTX_TYPE_DEFAULT].nr_queues =
2477                         ctrl->io_queues[HCTX_TYPE_DEFAULT];
2478                 set->map[HCTX_TYPE_DEFAULT].queue_offset = 0;
2479                 set->map[HCTX_TYPE_READ].nr_queues =
2480                         ctrl->io_queues[HCTX_TYPE_READ];
2481                 set->map[HCTX_TYPE_READ].queue_offset =
2482                         ctrl->io_queues[HCTX_TYPE_DEFAULT];
2483         } else {
2484                 /* shared read/write queues */
2485                 set->map[HCTX_TYPE_DEFAULT].nr_queues =
2486                         ctrl->io_queues[HCTX_TYPE_DEFAULT];
2487                 set->map[HCTX_TYPE_DEFAULT].queue_offset = 0;
2488                 set->map[HCTX_TYPE_READ].nr_queues =
2489                         ctrl->io_queues[HCTX_TYPE_DEFAULT];
2490                 set->map[HCTX_TYPE_READ].queue_offset = 0;
2491         }
2492         blk_mq_map_queues(&set->map[HCTX_TYPE_DEFAULT]);
2493         blk_mq_map_queues(&set->map[HCTX_TYPE_READ]);
2494
2495         if (opts->nr_poll_queues && ctrl->io_queues[HCTX_TYPE_POLL]) {
2496                 /* map dedicated poll queues only if we have queues left */
2497                 set->map[HCTX_TYPE_POLL].nr_queues =
2498                                 ctrl->io_queues[HCTX_TYPE_POLL];
2499                 set->map[HCTX_TYPE_POLL].queue_offset =
2500                         ctrl->io_queues[HCTX_TYPE_DEFAULT] +
2501                         ctrl->io_queues[HCTX_TYPE_READ];
2502                 blk_mq_map_queues(&set->map[HCTX_TYPE_POLL]);
2503         }
2504
2505         dev_info(ctrl->ctrl.device,
2506                 "mapped %d/%d/%d default/read/poll queues.\n",
2507                 ctrl->io_queues[HCTX_TYPE_DEFAULT],
2508                 ctrl->io_queues[HCTX_TYPE_READ],
2509                 ctrl->io_queues[HCTX_TYPE_POLL]);
2510
2511         return 0;
2512 }
2513
2514 static int nvme_tcp_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob)
2515 {
2516         struct nvme_tcp_queue *queue = hctx->driver_data;
2517         struct sock *sk = queue->sock->sk;
2518
2519         if (!test_bit(NVME_TCP_Q_LIVE, &queue->flags))
2520                 return 0;
2521
2522         set_bit(NVME_TCP_Q_POLLING, &queue->flags);
2523         if (sk_can_busy_loop(sk) && skb_queue_empty_lockless(&sk->sk_receive_queue))
2524                 sk_busy_loop(sk, true);
2525         nvme_tcp_try_recv(queue);
2526         clear_bit(NVME_TCP_Q_POLLING, &queue->flags);
2527         return queue->nr_cqe;
2528 }
2529
2530 static const struct blk_mq_ops nvme_tcp_mq_ops = {
2531         .queue_rq       = nvme_tcp_queue_rq,
2532         .commit_rqs     = nvme_tcp_commit_rqs,
2533         .complete       = nvme_complete_rq,
2534         .init_request   = nvme_tcp_init_request,
2535         .exit_request   = nvme_tcp_exit_request,
2536         .init_hctx      = nvme_tcp_init_hctx,
2537         .timeout        = nvme_tcp_timeout,
2538         .map_queues     = nvme_tcp_map_queues,
2539         .poll           = nvme_tcp_poll,
2540 };
2541
2542 static const struct blk_mq_ops nvme_tcp_admin_mq_ops = {
2543         .queue_rq       = nvme_tcp_queue_rq,
2544         .complete       = nvme_complete_rq,
2545         .init_request   = nvme_tcp_init_request,
2546         .exit_request   = nvme_tcp_exit_request,
2547         .init_hctx      = nvme_tcp_init_admin_hctx,
2548         .timeout        = nvme_tcp_timeout,
2549 };
2550
2551 static const struct nvme_ctrl_ops nvme_tcp_ctrl_ops = {
2552         .name                   = "tcp",
2553         .module                 = THIS_MODULE,
2554         .flags                  = NVME_F_FABRICS,
2555         .reg_read32             = nvmf_reg_read32,
2556         .reg_read64             = nvmf_reg_read64,
2557         .reg_write32            = nvmf_reg_write32,
2558         .free_ctrl              = nvme_tcp_free_ctrl,
2559         .submit_async_event     = nvme_tcp_submit_async_event,
2560         .delete_ctrl            = nvme_tcp_delete_ctrl,
2561         .get_address            = nvmf_get_address,
2562         .stop_ctrl              = nvme_tcp_stop_ctrl,
2563 };
2564
2565 static bool
2566 nvme_tcp_existing_controller(struct nvmf_ctrl_options *opts)
2567 {
2568         struct nvme_tcp_ctrl *ctrl;
2569         bool found = false;
2570
2571         mutex_lock(&nvme_tcp_ctrl_mutex);
2572         list_for_each_entry(ctrl, &nvme_tcp_ctrl_list, list) {
2573                 found = nvmf_ip_options_match(&ctrl->ctrl, opts);
2574                 if (found)
2575                         break;
2576         }
2577         mutex_unlock(&nvme_tcp_ctrl_mutex);
2578
2579         return found;
2580 }
2581
2582 static struct nvme_ctrl *nvme_tcp_create_ctrl(struct device *dev,
2583                 struct nvmf_ctrl_options *opts)
2584 {
2585         struct nvme_tcp_ctrl *ctrl;
2586         int ret;
2587
2588         ctrl = kzalloc(sizeof(*ctrl), GFP_KERNEL);
2589         if (!ctrl)
2590                 return ERR_PTR(-ENOMEM);
2591
2592         INIT_LIST_HEAD(&ctrl->list);
2593         ctrl->ctrl.opts = opts;
2594         ctrl->ctrl.queue_count = opts->nr_io_queues + opts->nr_write_queues +
2595                                 opts->nr_poll_queues + 1;
2596         ctrl->ctrl.sqsize = opts->queue_size - 1;
2597         ctrl->ctrl.kato = opts->kato;
2598
2599         INIT_DELAYED_WORK(&ctrl->connect_work,
2600                         nvme_tcp_reconnect_ctrl_work);
2601         INIT_WORK(&ctrl->err_work, nvme_tcp_error_recovery_work);
2602         INIT_WORK(&ctrl->ctrl.reset_work, nvme_reset_ctrl_work);
2603
2604         if (!(opts->mask & NVMF_OPT_TRSVCID)) {
2605                 opts->trsvcid =
2606                         kstrdup(__stringify(NVME_TCP_DISC_PORT), GFP_KERNEL);
2607                 if (!opts->trsvcid) {
2608                         ret = -ENOMEM;
2609                         goto out_free_ctrl;
2610                 }
2611                 opts->mask |= NVMF_OPT_TRSVCID;
2612         }
2613
2614         ret = inet_pton_with_scope(&init_net, AF_UNSPEC,
2615                         opts->traddr, opts->trsvcid, &ctrl->addr);
2616         if (ret) {
2617                 pr_err("malformed address passed: %s:%s\n",
2618                         opts->traddr, opts->trsvcid);
2619                 goto out_free_ctrl;
2620         }
2621
2622         if (opts->mask & NVMF_OPT_HOST_TRADDR) {
2623                 ret = inet_pton_with_scope(&init_net, AF_UNSPEC,
2624                         opts->host_traddr, NULL, &ctrl->src_addr);
2625                 if (ret) {
2626                         pr_err("malformed src address passed: %s\n",
2627                                opts->host_traddr);
2628                         goto out_free_ctrl;
2629                 }
2630         }
2631
2632         if (opts->mask & NVMF_OPT_HOST_IFACE) {
2633                 if (!__dev_get_by_name(&init_net, opts->host_iface)) {
2634                         pr_err("invalid interface passed: %s\n",
2635                                opts->host_iface);
2636                         ret = -ENODEV;
2637                         goto out_free_ctrl;
2638                 }
2639         }
2640
2641         if (!opts->duplicate_connect && nvme_tcp_existing_controller(opts)) {
2642                 ret = -EALREADY;
2643                 goto out_free_ctrl;
2644         }
2645
2646         ctrl->queues = kcalloc(ctrl->ctrl.queue_count, sizeof(*ctrl->queues),
2647                                 GFP_KERNEL);
2648         if (!ctrl->queues) {
2649                 ret = -ENOMEM;
2650                 goto out_free_ctrl;
2651         }
2652
2653         ret = nvme_init_ctrl(&ctrl->ctrl, dev, &nvme_tcp_ctrl_ops, 0);
2654         if (ret)
2655                 goto out_kfree_queues;
2656
2657         if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_CONNECTING)) {
2658                 WARN_ON_ONCE(1);
2659                 ret = -EINTR;
2660                 goto out_uninit_ctrl;
2661         }
2662
2663         ret = nvme_tcp_setup_ctrl(&ctrl->ctrl, true);
2664         if (ret)
2665                 goto out_uninit_ctrl;
2666
2667         dev_info(ctrl->ctrl.device, "new ctrl: NQN \"%s\", addr %pISp\n",
2668                 nvmf_ctrl_subsysnqn(&ctrl->ctrl), &ctrl->addr);
2669
2670         mutex_lock(&nvme_tcp_ctrl_mutex);
2671         list_add_tail(&ctrl->list, &nvme_tcp_ctrl_list);
2672         mutex_unlock(&nvme_tcp_ctrl_mutex);
2673
2674         return &ctrl->ctrl;
2675
2676 out_uninit_ctrl:
2677         nvme_uninit_ctrl(&ctrl->ctrl);
2678         nvme_put_ctrl(&ctrl->ctrl);
2679         if (ret > 0)
2680                 ret = -EIO;
2681         return ERR_PTR(ret);
2682 out_kfree_queues:
2683         kfree(ctrl->queues);
2684 out_free_ctrl:
2685         kfree(ctrl);
2686         return ERR_PTR(ret);
2687 }
2688
2689 static struct nvmf_transport_ops nvme_tcp_transport = {
2690         .name           = "tcp",
2691         .module         = THIS_MODULE,
2692         .required_opts  = NVMF_OPT_TRADDR,
2693         .allowed_opts   = NVMF_OPT_TRSVCID | NVMF_OPT_RECONNECT_DELAY |
2694                           NVMF_OPT_HOST_TRADDR | NVMF_OPT_CTRL_LOSS_TMO |
2695                           NVMF_OPT_HDR_DIGEST | NVMF_OPT_DATA_DIGEST |
2696                           NVMF_OPT_NR_WRITE_QUEUES | NVMF_OPT_NR_POLL_QUEUES |
2697                           NVMF_OPT_TOS | NVMF_OPT_HOST_IFACE,
2698         .create_ctrl    = nvme_tcp_create_ctrl,
2699 };
2700
2701 static int __init nvme_tcp_init_module(void)
2702 {
2703         nvme_tcp_wq = alloc_workqueue("nvme_tcp_wq",
2704                         WQ_MEM_RECLAIM | WQ_HIGHPRI, 0);
2705         if (!nvme_tcp_wq)
2706                 return -ENOMEM;
2707
2708         nvmf_register_transport(&nvme_tcp_transport);
2709         return 0;
2710 }
2711
2712 static void __exit nvme_tcp_cleanup_module(void)
2713 {
2714         struct nvme_tcp_ctrl *ctrl;
2715
2716         nvmf_unregister_transport(&nvme_tcp_transport);
2717
2718         mutex_lock(&nvme_tcp_ctrl_mutex);
2719         list_for_each_entry(ctrl, &nvme_tcp_ctrl_list, list)
2720                 nvme_delete_ctrl(&ctrl->ctrl);
2721         mutex_unlock(&nvme_tcp_ctrl_mutex);
2722         flush_workqueue(nvme_delete_wq);
2723
2724         destroy_workqueue(nvme_tcp_wq);
2725 }
2726
2727 module_init(nvme_tcp_init_module);
2728 module_exit(nvme_tcp_cleanup_module);
2729
2730 MODULE_LICENSE("GPL v2");