drivers/vhost/scsi.c

   1 // SPDX-License-Identifier: GPL-2.0+
   2 /*******************************************************************************
   3  * Vhost kernel TCM fabric driver for virtio SCSI initiators
   4  *
   5  * (C) Copyright 2010-2013 Datera, Inc.
   6  * (C) Copyright 2010-2012 IBM Corp.
   7  *
   8  * Authors: Nicholas A. Bellinger <nab@daterainc.com>
   9  *          Stefan Hajnoczi <stefanha@linux.vnet.ibm.com>
  10  ****************************************************************************/
  11
  12 #include <linux/module.h>
  13 #include <linux/moduleparam.h>
  14 #include <generated/utsrelease.h>
  15 #include <linux/utsname.h>
  16 #include <linux/init.h>
  17 #include <linux/slab.h>
  18 #include <linux/kthread.h>
  19 #include <linux/types.h>
  20 #include <linux/string.h>
  21 #include <linux/configfs.h>
  22 #include <linux/ctype.h>
  23 #include <linux/compat.h>
  24 #include <linux/eventfd.h>
  25 #include <linux/fs.h>
  26 #include <linux/vmalloc.h>
  27 #include <linux/miscdevice.h>
  28 #include <linux/blk_types.h>
  29 #include <linux/bio.h>
  30 #include <asm/unaligned.h>
  31 #include <scsi/scsi_common.h>
  32 #include <scsi/scsi_proto.h>
  33 #include <target/target_core_base.h>
  34 #include <target/target_core_fabric.h>
  35 #include <linux/vhost.h>
  36 #include <linux/virtio_scsi.h>
  37 #include <linux/llist.h>
  38 #include <linux/bitmap.h>
  39
  40 #include "vhost.h"
  41
  42 #define VHOST_SCSI_VERSION  "v0.1"
  43 #define VHOST_SCSI_NAMELEN 256
  44 #define VHOST_SCSI_MAX_CDB_SIZE 32
  45 #define VHOST_SCSI_PREALLOC_SGLS 2048
  46 #define VHOST_SCSI_PREALLOC_UPAGES 2048
  47 #define VHOST_SCSI_PREALLOC_PROT_SGLS 2048
  48
  49 /* Max number of requests before requeueing the job.
  50  * Using this limit prevents one virtqueue from starving others with
  51  * request.
  52  */
  53 #define VHOST_SCSI_WEIGHT 256
  54
  55 struct vhost_scsi_inflight {
  56         /* Wait for the flush operation to finish */
  57         struct completion comp;
  58         /* Refcount for the inflight reqs */
  59         struct kref kref;
  60 };
  61
  62 struct vhost_scsi_cmd {
  63         /* Descriptor from vhost_get_vq_desc() for virt_queue segment */
  64         int tvc_vq_desc;
  65         /* virtio-scsi initiator task attribute */
  66         int tvc_task_attr;
  67         /* virtio-scsi response incoming iovecs */
  68         int tvc_in_iovs;
  69         /* virtio-scsi initiator data direction */
  70         enum dma_data_direction tvc_data_direction;
  71         /* Expected data transfer length from virtio-scsi header */
  72         u32 tvc_exp_data_len;
  73         /* The Tag from include/linux/virtio_scsi.h:struct virtio_scsi_cmd_req */
  74         u64 tvc_tag;
  75         /* The number of scatterlists associated with this cmd */
  76         u32 tvc_sgl_count;
  77         u32 tvc_prot_sgl_count;
  78         /* Saved unpacked SCSI LUN for vhost_scsi_target_queue_cmd() */
  79         u32 tvc_lun;
  80         u32 copied_iov:1;
  81         const void *saved_iter_addr;
  82         struct iov_iter saved_iter;
  83         /* Pointer to the SGL formatted memory from virtio-scsi */
  84         struct scatterlist *tvc_sgl;
  85         struct scatterlist *tvc_prot_sgl;
  86         struct page **tvc_upages;
  87         /* Pointer to response header iovec */
  88         struct iovec *tvc_resp_iov;
  89         /* Pointer to vhost_scsi for our device */
  90         struct vhost_scsi *tvc_vhost;
  91         /* Pointer to vhost_virtqueue for the cmd */
  92         struct vhost_virtqueue *tvc_vq;
  93         /* Pointer to vhost nexus memory */
  94         struct vhost_scsi_nexus *tvc_nexus;
  95         /* The TCM I/O descriptor that is accessed via container_of() */
  96         struct se_cmd tvc_se_cmd;
  97         /* Copy of the incoming SCSI command descriptor block (CDB) */
  98         unsigned char tvc_cdb[VHOST_SCSI_MAX_CDB_SIZE];
  99         /* Sense buffer that will be mapped into outgoing status */
 100         unsigned char tvc_sense_buf[TRANSPORT_SENSE_BUFFER];
 101         /* Completed commands list, serviced from vhost worker thread */
 102         struct llist_node tvc_completion_list;
 103         /* Used to track inflight cmd */
 104         struct vhost_scsi_inflight *inflight;
 105 };
 106
 107 struct vhost_scsi_nexus {
 108         /* Pointer to TCM session for I_T Nexus */
 109         struct se_session *tvn_se_sess;
 110 };
 111
 112 struct vhost_scsi_tpg {
 113         /* Vhost port target portal group tag for TCM */
 114         u16 tport_tpgt;
 115         /* Used to track number of TPG Port/Lun Links wrt to explict I_T Nexus shutdown */
 116         int tv_tpg_port_count;
 117         /* Used for vhost_scsi device reference to tpg_nexus, protected by tv_tpg_mutex */
 118         int tv_tpg_vhost_count;
 119         /* Used for enabling T10-PI with legacy devices */
 120         int tv_fabric_prot_type;
 121         /* list for vhost_scsi_list */
 122         struct list_head tv_tpg_list;
 123         /* Used to protect access for tpg_nexus */
 124         struct mutex tv_tpg_mutex;
 125         /* Pointer to the TCM VHost I_T Nexus for this TPG endpoint */
 126         struct vhost_scsi_nexus *tpg_nexus;
 127         /* Pointer back to vhost_scsi_tport */
 128         struct vhost_scsi_tport *tport;
 129         /* Returned by vhost_scsi_make_tpg() */
 130         struct se_portal_group se_tpg;
 131         /* Pointer back to vhost_scsi, protected by tv_tpg_mutex */
 132         struct vhost_scsi *vhost_scsi;
 133 };
 134
 135 struct vhost_scsi_tport {
 136         /* SCSI protocol the tport is providing */
 137         u8 tport_proto_id;
 138         /* Binary World Wide unique Port Name for Vhost Target port */
 139         u64 tport_wwpn;
 140         /* ASCII formatted WWPN for Vhost Target port */
 141         char tport_name[VHOST_SCSI_NAMELEN];
 142         /* Returned by vhost_scsi_make_tport() */
 143         struct se_wwn tport_wwn;
 144 };
 145
 146 struct vhost_scsi_evt {
 147         /* event to be sent to guest */
 148         struct virtio_scsi_event event;
 149         /* event list, serviced from vhost worker thread */
 150         struct llist_node list;
 151 };
 152
 153 enum {
 154         VHOST_SCSI_VQ_CTL = 0,
 155         VHOST_SCSI_VQ_EVT = 1,
 156         VHOST_SCSI_VQ_IO = 2,
 157 };
 158
 159 /* Note: can't set VIRTIO_F_VERSION_1 yet, since that implies ANY_LAYOUT. */
 160 enum {
 161         VHOST_SCSI_FEATURES = VHOST_FEATURES | (1ULL << VIRTIO_SCSI_F_HOTPLUG) |
 162                                                (1ULL << VIRTIO_SCSI_F_T10_PI)
 163 };
 164
 165 #define VHOST_SCSI_MAX_TARGET   256
 166 #define VHOST_SCSI_MAX_IO_VQ    1024
 167 #define VHOST_SCSI_MAX_EVENT    128
 168
 169 static unsigned vhost_scsi_max_io_vqs = 128;
 170 module_param_named(max_io_vqs, vhost_scsi_max_io_vqs, uint, 0644);
 171 MODULE_PARM_DESC(max_io_vqs, "Set the max number of IO virtqueues a vhost scsi device can support. The default is 128. The max is 1024.");
 172
 173 struct vhost_scsi_virtqueue {
 174         struct vhost_virtqueue vq;
 175         struct vhost_scsi *vs;
 176         /*
 177          * Reference counting for inflight reqs, used for flush operation. At
 178          * each time, one reference tracks new commands submitted, while we
 179          * wait for another one to reach 0.
 180          */
 181         struct vhost_scsi_inflight inflights[2];
 182         /*
 183          * Indicate current inflight in use, protected by vq->mutex.
 184          * Writers must also take dev mutex and flush under it.
 185          */
 186         int inflight_idx;
 187         struct vhost_scsi_cmd *scsi_cmds;
 188         struct sbitmap scsi_tags;
 189         int max_cmds;
 190
 191         struct vhost_work completion_work;
 192         struct llist_head completion_list;
 193 };
 194
 195 struct vhost_scsi {
 196         /* Protected by vhost_scsi->dev.mutex */
 197         struct vhost_scsi_tpg **vs_tpg;
 198         char vs_vhost_wwpn[TRANSPORT_IQN_LEN];
 199
 200         struct vhost_dev dev;
 201         struct vhost_scsi_virtqueue *vqs;
 202         struct vhost_scsi_inflight **old_inflight;
 203
 204         struct vhost_work vs_event_work; /* evt injection work item */
 205         struct llist_head vs_event_list; /* evt injection queue */
 206
 207         bool vs_events_missed; /* any missed events, protected by vq->mutex */
 208         int vs_events_nr; /* num of pending events, protected by vq->mutex */
 209 };
 210
 211 struct vhost_scsi_tmf {
 212         struct vhost_work vwork;
 213         struct vhost_scsi *vhost;
 214         struct vhost_scsi_virtqueue *svq;
 215
 216         struct se_cmd se_cmd;
 217         u8 scsi_resp;
 218         struct vhost_scsi_inflight *inflight;
 219         struct iovec resp_iov;
 220         int in_iovs;
 221         int vq_desc;
 222 };
 223
 224 /*
 225  * Context for processing request and control queue operations.
 226  */
 227 struct vhost_scsi_ctx {
 228         int head;
 229         unsigned int out, in;
 230         size_t req_size, rsp_size;
 231         size_t out_size, in_size;
 232         u8 *target, *lunp;
 233         void *req;
 234         struct iov_iter out_iter;
 235 };
 236
 237 /*
 238  * Global mutex to protect vhost_scsi TPG list for vhost IOCTLs and LIO
 239  * configfs management operations.
 240  */
 241 static DEFINE_MUTEX(vhost_scsi_mutex);
 242 static LIST_HEAD(vhost_scsi_list);
 243
 244 static void vhost_scsi_done_inflight(struct kref *kref)
 245 {
 246         struct vhost_scsi_inflight *inflight;
 247
 248         inflight = container_of(kref, struct vhost_scsi_inflight, kref);
 249         complete(&inflight->comp);
 250 }
 251
 252 static void vhost_scsi_init_inflight(struct vhost_scsi *vs,
 253                                     struct vhost_scsi_inflight *old_inflight[])
 254 {
 255         struct vhost_scsi_inflight *new_inflight;
 256         struct vhost_virtqueue *vq;
 257         int idx, i;
 258
 259         for (i = 0; i < vs->dev.nvqs;  i++) {
 260                 vq = &vs->vqs[i].vq;
 261
 262                 mutex_lock(&vq->mutex);
 263
 264                 /* store old infight */
 265                 idx = vs->vqs[i].inflight_idx;
 266                 if (old_inflight)
 267                         old_inflight[i] = &vs->vqs[i].inflights[idx];
 268
 269                 /* setup new infight */
 270                 vs->vqs[i].inflight_idx = idx ^ 1;
 271                 new_inflight = &vs->vqs[i].inflights[idx ^ 1];
 272                 kref_init(&new_inflight->kref);
 273                 init_completion(&new_inflight->comp);
 274
 275                 mutex_unlock(&vq->mutex);
 276         }
 277 }
 278
 279 static struct vhost_scsi_inflight *
 280 vhost_scsi_get_inflight(struct vhost_virtqueue *vq)
 281 {
 282         struct vhost_scsi_inflight *inflight;
 283         struct vhost_scsi_virtqueue *svq;
 284
 285         svq = container_of(vq, struct vhost_scsi_virtqueue, vq);
 286         inflight = &svq->inflights[svq->inflight_idx];
 287         kref_get(&inflight->kref);
 288
 289         return inflight;
 290 }
 291
 292 static void vhost_scsi_put_inflight(struct vhost_scsi_inflight *inflight)
 293 {
 294         kref_put(&inflight->kref, vhost_scsi_done_inflight);
 295 }
 296
 297 static int vhost_scsi_check_true(struct se_portal_group *se_tpg)
 298 {
 299         return 1;
 300 }
 301
 302 static char *vhost_scsi_get_fabric_wwn(struct se_portal_group *se_tpg)
 303 {
 304         struct vhost_scsi_tpg *tpg = container_of(se_tpg,
 305                                 struct vhost_scsi_tpg, se_tpg);
 306         struct vhost_scsi_tport *tport = tpg->tport;
 307
 308         return &tport->tport_name[0];
 309 }
 310
 311 static u16 vhost_scsi_get_tpgt(struct se_portal_group *se_tpg)
 312 {
 313         struct vhost_scsi_tpg *tpg = container_of(se_tpg,
 314                                 struct vhost_scsi_tpg, se_tpg);
 315         return tpg->tport_tpgt;
 316 }
 317
 318 static int vhost_scsi_check_prot_fabric_only(struct se_portal_group *se_tpg)
 319 {
 320         struct vhost_scsi_tpg *tpg = container_of(se_tpg,
 321                                 struct vhost_scsi_tpg, se_tpg);
 322
 323         return tpg->tv_fabric_prot_type;
 324 }
 325
 326 static void vhost_scsi_release_cmd_res(struct se_cmd *se_cmd)
 327 {
 328         struct vhost_scsi_cmd *tv_cmd = container_of(se_cmd,
 329                                 struct vhost_scsi_cmd, tvc_se_cmd);
 330         struct vhost_scsi_virtqueue *svq = container_of(tv_cmd->tvc_vq,
 331                                 struct vhost_scsi_virtqueue, vq);
 332         struct vhost_scsi_inflight *inflight = tv_cmd->inflight;
 333         int i;
 334
 335         if (tv_cmd->tvc_sgl_count) {
 336                 for (i = 0; i < tv_cmd->tvc_sgl_count; i++) {
 337                         if (tv_cmd->copied_iov)
 338                                 __free_page(sg_page(&tv_cmd->tvc_sgl[i]));
 339                         else
 340                                 put_page(sg_page(&tv_cmd->tvc_sgl[i]));
 341                 }
 342                 kfree(tv_cmd->saved_iter_addr);
 343         }
 344         if (tv_cmd->tvc_prot_sgl_count) {
 345                 for (i = 0; i < tv_cmd->tvc_prot_sgl_count; i++)
 346                         put_page(sg_page(&tv_cmd->tvc_prot_sgl[i]));
 347         }
 348
 349         sbitmap_clear_bit(&svq->scsi_tags, se_cmd->map_tag);
 350         vhost_scsi_put_inflight(inflight);
 351 }
 352
 353 static void vhost_scsi_release_tmf_res(struct vhost_scsi_tmf *tmf)
 354 {
 355         struct vhost_scsi_inflight *inflight = tmf->inflight;
 356
 357         kfree(tmf);
 358         vhost_scsi_put_inflight(inflight);
 359 }
 360
 361 static void vhost_scsi_release_cmd(struct se_cmd *se_cmd)
 362 {
 363         if (se_cmd->se_cmd_flags & SCF_SCSI_TMR_CDB) {
 364                 struct vhost_scsi_tmf *tmf = container_of(se_cmd,
 365                                         struct vhost_scsi_tmf, se_cmd);
 366                 struct vhost_virtqueue *vq = &tmf->svq->vq;
 367
 368                 vhost_vq_work_queue(vq, &tmf->vwork);
 369         } else {
 370                 struct vhost_scsi_cmd *cmd = container_of(se_cmd,
 371                                         struct vhost_scsi_cmd, tvc_se_cmd);
 372                 struct vhost_scsi_virtqueue *svq =  container_of(cmd->tvc_vq,
 373                                         struct vhost_scsi_virtqueue, vq);
 374
 375                 llist_add(&cmd->tvc_completion_list, &svq->completion_list);
 376                 vhost_vq_work_queue(&svq->vq, &svq->completion_work);
 377         }
 378 }
 379
 380 static int vhost_scsi_write_pending(struct se_cmd *se_cmd)
 381 {
 382         /* Go ahead and process the write immediately */
 383         target_execute_cmd(se_cmd);
 384         return 0;
 385 }
 386
 387 static int vhost_scsi_queue_data_in(struct se_cmd *se_cmd)
 388 {
 389         transport_generic_free_cmd(se_cmd, 0);
 390         return 0;
 391 }
 392
 393 static int vhost_scsi_queue_status(struct se_cmd *se_cmd)
 394 {
 395         transport_generic_free_cmd(se_cmd, 0);
 396         return 0;
 397 }
 398
 399 static void vhost_scsi_queue_tm_rsp(struct se_cmd *se_cmd)
 400 {
 401         struct vhost_scsi_tmf *tmf = container_of(se_cmd, struct vhost_scsi_tmf,
 402                                                   se_cmd);
 403
 404         tmf->scsi_resp = se_cmd->se_tmr_req->response;
 405         transport_generic_free_cmd(&tmf->se_cmd, 0);
 406 }
 407
 408 static void vhost_scsi_aborted_task(struct se_cmd *se_cmd)
 409 {
 410         return;
 411 }
 412
 413 static void vhost_scsi_free_evt(struct vhost_scsi *vs, struct vhost_scsi_evt *evt)
 414 {
 415         vs->vs_events_nr--;
 416         kfree(evt);
 417 }
 418
 419 static struct vhost_scsi_evt *
 420 vhost_scsi_allocate_evt(struct vhost_scsi *vs,
 421                        u32 event, u32 reason)
 422 {
 423         struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;
 424         struct vhost_scsi_evt *evt;
 425
 426         if (vs->vs_events_nr > VHOST_SCSI_MAX_EVENT) {
 427                 vs->vs_events_missed = true;
 428                 return NULL;
 429         }
 430
 431         evt = kzalloc(sizeof(*evt), GFP_KERNEL);
 432         if (!evt) {
 433                 vq_err(vq, "Failed to allocate vhost_scsi_evt\n");
 434                 vs->vs_events_missed = true;
 435                 return NULL;
 436         }
 437
 438         evt->event.event = cpu_to_vhost32(vq, event);
 439         evt->event.reason = cpu_to_vhost32(vq, reason);
 440         vs->vs_events_nr++;
 441
 442         return evt;
 443 }
 444
 445 static int vhost_scsi_check_stop_free(struct se_cmd *se_cmd)
 446 {
 447         return target_put_sess_cmd(se_cmd);
 448 }
 449
 450 static void
 451 vhost_scsi_do_evt_work(struct vhost_scsi *vs, struct vhost_scsi_evt *evt)
 452 {
 453         struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;
 454         struct virtio_scsi_event *event = &evt->event;
 455         struct virtio_scsi_event __user *eventp;
 456         unsigned out, in;
 457         int head, ret;
 458
 459         if (!vhost_vq_get_backend(vq)) {
 460                 vs->vs_events_missed = true;
 461                 return;
 462         }
 463
 464 again:
 465         vhost_disable_notify(&vs->dev, vq);
 466         head = vhost_get_vq_desc(vq, vq->iov,
 467                         ARRAY_SIZE(vq->iov), &out, &in,
 468                         NULL, NULL);
 469         if (head < 0) {
 470                 vs->vs_events_missed = true;
 471                 return;
 472         }
 473         if (head == vq->num) {
 474                 if (vhost_enable_notify(&vs->dev, vq))
 475                         goto again;
 476                 vs->vs_events_missed = true;
 477                 return;
 478         }
 479
 480         if ((vq->iov[out].iov_len != sizeof(struct virtio_scsi_event))) {
 481                 vq_err(vq, "Expecting virtio_scsi_event, got %zu bytes\n",
 482                                 vq->iov[out].iov_len);
 483                 vs->vs_events_missed = true;
 484                 return;
 485         }
 486
 487         if (vs->vs_events_missed) {
 488                 event->event |= cpu_to_vhost32(vq, VIRTIO_SCSI_T_EVENTS_MISSED);
 489                 vs->vs_events_missed = false;
 490         }
 491
 492         eventp = vq->iov[out].iov_base;
 493         ret = __copy_to_user(eventp, event, sizeof(*event));
 494         if (!ret)
 495                 vhost_add_used_and_signal(&vs->dev, vq, head, 0);
 496         else
 497                 vq_err(vq, "Faulted on vhost_scsi_send_event\n");
 498 }
 499
 500 static void vhost_scsi_evt_work(struct vhost_work *work)
 501 {
 502         struct vhost_scsi *vs = container_of(work, struct vhost_scsi,
 503                                         vs_event_work);
 504         struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;
 505         struct vhost_scsi_evt *evt, *t;
 506         struct llist_node *llnode;
 507
 508         mutex_lock(&vq->mutex);
 509         llnode = llist_del_all(&vs->vs_event_list);
 510         llist_for_each_entry_safe(evt, t, llnode, list) {
 511                 vhost_scsi_do_evt_work(vs, evt);
 512                 vhost_scsi_free_evt(vs, evt);
 513         }
 514         mutex_unlock(&vq->mutex);
 515 }
 516
 517 static int vhost_scsi_copy_sgl_to_iov(struct vhost_scsi_cmd *cmd)
 518 {
 519         struct iov_iter *iter = &cmd->saved_iter;
 520         struct scatterlist *sg = cmd->tvc_sgl;
 521         struct page *page;
 522         size_t len;
 523         int i;
 524
 525         for (i = 0; i < cmd->tvc_sgl_count; i++) {
 526                 page = sg_page(&sg[i]);
 527                 len = sg[i].length;
 528
 529                 if (copy_page_to_iter(page, 0, len, iter) != len) {
 530                         pr_err("Could not copy data while handling misaligned cmd. Error %zu\n",
 531                                len);
 532                         return -1;
 533                 }
 534         }
 535
 536         return 0;
 537 }
 538
 539 /* Fill in status and signal that we are done processing this command
 540  *
 541  * This is scheduled in the vhost work queue so we are called with the owner
 542  * process mm and can access the vring.
 543  */
 544 static void vhost_scsi_complete_cmd_work(struct vhost_work *work)
 545 {
 546         struct vhost_scsi_virtqueue *svq = container_of(work,
 547                                 struct vhost_scsi_virtqueue, completion_work);
 548         struct virtio_scsi_cmd_resp v_rsp;
 549         struct vhost_scsi_cmd *cmd, *t;
 550         struct llist_node *llnode;
 551         struct se_cmd *se_cmd;
 552         struct iov_iter iov_iter;
 553         bool signal = false;
 554         int ret;
 555
 556         llnode = llist_del_all(&svq->completion_list);
 557         llist_for_each_entry_safe(cmd, t, llnode, tvc_completion_list) {
 558                 se_cmd = &cmd->tvc_se_cmd;
 559
 560                 pr_debug("%s tv_cmd %p resid %u status %#02x\n", __func__,
 561                         cmd, se_cmd->residual_count, se_cmd->scsi_status);
 562                 memset(&v_rsp, 0, sizeof(v_rsp));
 563
 564                 if (cmd->saved_iter_addr && vhost_scsi_copy_sgl_to_iov(cmd)) {
 565                         v_rsp.response = VIRTIO_SCSI_S_BAD_TARGET;
 566                 } else {
 567                         v_rsp.resid = cpu_to_vhost32(cmd->tvc_vq,
 568                                                      se_cmd->residual_count);
 569                         /* TODO is status_qualifier field needed? */
 570                         v_rsp.status = se_cmd->scsi_status;
 571                         v_rsp.sense_len = cpu_to_vhost32(cmd->tvc_vq,
 572                                                          se_cmd->scsi_sense_length);
 573                         memcpy(v_rsp.sense, cmd->tvc_sense_buf,
 574                                se_cmd->scsi_sense_length);
 575                 }
 576
 577                 iov_iter_init(&iov_iter, ITER_DEST, cmd->tvc_resp_iov,
 578                               cmd->tvc_in_iovs, sizeof(v_rsp));
 579                 ret = copy_to_iter(&v_rsp, sizeof(v_rsp), &iov_iter);
 580                 if (likely(ret == sizeof(v_rsp))) {
 581                         signal = true;
 582
 583                         vhost_add_used(cmd->tvc_vq, cmd->tvc_vq_desc, 0);
 584                 } else
 585                         pr_err("Faulted on virtio_scsi_cmd_resp\n");
 586
 587                 vhost_scsi_release_cmd_res(se_cmd);
 588         }
 589
 590         if (signal)
 591                 vhost_signal(&svq->vs->dev, &svq->vq);
 592 }
 593
 594 static struct vhost_scsi_cmd *
 595 vhost_scsi_get_cmd(struct vhost_virtqueue *vq, struct vhost_scsi_tpg *tpg,
 596                    unsigned char *cdb, u64 scsi_tag, u16 lun, u8 task_attr,
 597                    u32 exp_data_len, int data_direction)
 598 {
 599         struct vhost_scsi_virtqueue *svq = container_of(vq,
 600                                         struct vhost_scsi_virtqueue, vq);
 601         struct vhost_scsi_cmd *cmd;
 602         struct vhost_scsi_nexus *tv_nexus;
 603         struct scatterlist *sg, *prot_sg;
 604         struct iovec *tvc_resp_iov;
 605         struct page **pages;
 606         int tag;
 607
 608         tv_nexus = tpg->tpg_nexus;
 609         if (!tv_nexus) {
 610                 pr_err("Unable to locate active struct vhost_scsi_nexus\n");
 611                 return ERR_PTR(-EIO);
 612         }
 613
 614         tag = sbitmap_get(&svq->scsi_tags);
 615         if (tag < 0) {
 616                 pr_err("Unable to obtain tag for vhost_scsi_cmd\n");
 617                 return ERR_PTR(-ENOMEM);
 618         }
 619
 620         cmd = &svq->scsi_cmds[tag];
 621         sg = cmd->tvc_sgl;
 622         prot_sg = cmd->tvc_prot_sgl;
 623         pages = cmd->tvc_upages;
 624         tvc_resp_iov = cmd->tvc_resp_iov;
 625         memset(cmd, 0, sizeof(*cmd));
 626         cmd->tvc_sgl = sg;
 627         cmd->tvc_prot_sgl = prot_sg;
 628         cmd->tvc_upages = pages;
 629         cmd->tvc_se_cmd.map_tag = tag;
 630         cmd->tvc_tag = scsi_tag;
 631         cmd->tvc_lun = lun;
 632         cmd->tvc_task_attr = task_attr;
 633         cmd->tvc_exp_data_len = exp_data_len;
 634         cmd->tvc_data_direction = data_direction;
 635         cmd->tvc_nexus = tv_nexus;
 636         cmd->inflight = vhost_scsi_get_inflight(vq);
 637         cmd->tvc_resp_iov = tvc_resp_iov;
 638
 639         memcpy(cmd->tvc_cdb, cdb, VHOST_SCSI_MAX_CDB_SIZE);
 640
 641         return cmd;
 642 }
 643
 644 /*
 645  * Map a user memory range into a scatterlist
 646  *
 647  * Returns the number of scatterlist entries used or -errno on error.
 648  */
 649 static int
 650 vhost_scsi_map_to_sgl(struct vhost_scsi_cmd *cmd,
 651                       struct iov_iter *iter,
 652                       struct scatterlist *sgl,
 653                       bool is_prot)
 654 {
 655         struct page **pages = cmd->tvc_upages;
 656         struct scatterlist *sg = sgl;
 657         ssize_t bytes, mapped_bytes;
 658         size_t offset, mapped_offset;
 659         unsigned int npages = 0;
 660
 661         bytes = iov_iter_get_pages2(iter, pages, LONG_MAX,
 662                                 VHOST_SCSI_PREALLOC_UPAGES, &offset);
 663         /* No pages were pinned */
 664         if (bytes <= 0)
 665                 return bytes < 0 ? bytes : -EFAULT;
 666
 667         mapped_bytes = bytes;
 668         mapped_offset = offset;
 669
 670         while (bytes) {
 671                 unsigned n = min_t(unsigned, PAGE_SIZE - offset, bytes);
 672                 /*
 673                  * The block layer requires bios/requests to be a multiple of
 674                  * 512 bytes, but Windows can send us vecs that are misaligned.
 675                  * This can result in bios and later requests with misaligned
 676                  * sizes if we have to break up a cmd/scatterlist into multiple
 677                  * bios.
 678                  *
 679                  * We currently only break up a command into multiple bios if
 680                  * we hit the vec/seg limit, so check if our sgl_count is
 681                  * greater than the max and if a vec in the cmd has a
 682                  * misaligned offset/size.
 683                  */
 684                 if (!is_prot &&
 685                     (offset & (SECTOR_SIZE - 1) || n & (SECTOR_SIZE - 1)) &&
 686                     cmd->tvc_sgl_count > BIO_MAX_VECS) {
 687                         WARN_ONCE(true,
 688                                   "vhost-scsi detected misaligned IO. Performance may be degraded.");
 689                         goto revert_iter_get_pages;
 690                 }
 691
 692                 sg_set_page(sg++, pages[npages++], n, offset);
 693                 bytes -= n;
 694                 offset = 0;
 695         }
 696
 697         return npages;
 698
 699 revert_iter_get_pages:
 700         iov_iter_revert(iter, mapped_bytes);
 701
 702         npages = 0;
 703         while (mapped_bytes) {
 704                 unsigned int n = min_t(unsigned int, PAGE_SIZE - mapped_offset,
 705                                        mapped_bytes);
 706
 707                 put_page(pages[npages++]);
 708
 709                 mapped_bytes -= n;
 710                 mapped_offset = 0;
 711         }
 712
 713         return -EINVAL;
 714 }
 715
 716 static int
 717 vhost_scsi_calc_sgls(struct iov_iter *iter, size_t bytes, int max_sgls)
 718 {
 719         int sgl_count = 0;
 720
 721         if (!iter || !iter_iov(iter)) {
 722                 pr_err("%s: iter->iov is NULL, but expected bytes: %zu"
 723                        " present\n", __func__, bytes);
 724                 return -EINVAL;
 725         }
 726
 727         sgl_count = iov_iter_npages(iter, 0xffff);
 728         if (sgl_count > max_sgls) {
 729                 pr_err("%s: requested sgl_count: %d exceeds pre-allocated"
 730                        " max_sgls: %d\n", __func__, sgl_count, max_sgls);
 731                 return -EINVAL;
 732         }
 733         return sgl_count;
 734 }
 735
 736 static int
 737 vhost_scsi_copy_iov_to_sgl(struct vhost_scsi_cmd *cmd, struct iov_iter *iter,
 738                            struct scatterlist *sg, int sg_count)
 739 {
 740         size_t len = iov_iter_count(iter);
 741         unsigned int nbytes = 0;
 742         struct page *page;
 743         int i;
 744
 745         if (cmd->tvc_data_direction == DMA_FROM_DEVICE) {
 746                 cmd->saved_iter_addr = dup_iter(&cmd->saved_iter, iter,
 747                                                 GFP_KERNEL);
 748                 if (!cmd->saved_iter_addr)
 749                         return -ENOMEM;
 750         }
 751
 752         for (i = 0; i < sg_count; i++) {
 753                 page = alloc_page(GFP_KERNEL);
 754                 if (!page) {
 755                         i--;
 756                         goto err;
 757                 }
 758
 759                 nbytes = min_t(unsigned int, PAGE_SIZE, len);
 760                 sg_set_page(&sg[i], page, nbytes, 0);
 761
 762                 if (cmd->tvc_data_direction == DMA_TO_DEVICE &&
 763                     copy_page_from_iter(page, 0, nbytes, iter) != nbytes)
 764                         goto err;
 765
 766                 len -= nbytes;
 767         }
 768
 769         cmd->copied_iov = 1;
 770         return 0;
 771
 772 err:
 773         pr_err("Could not read %u bytes while handling misaligned cmd\n",
 774                nbytes);
 775
 776         for (; i >= 0; i--)
 777                 __free_page(sg_page(&sg[i]));
 778         kfree(cmd->saved_iter_addr);
 779         return -ENOMEM;
 780 }
 781
 782 static int
 783 vhost_scsi_iov_to_sgl(struct vhost_scsi_cmd *cmd, struct iov_iter *iter,
 784                       struct scatterlist *sg, int sg_count, bool is_prot)
 785 {
 786         struct scatterlist *p = sg;
 787         size_t revert_bytes;
 788         int ret;
 789
 790         while (iov_iter_count(iter)) {
 791                 ret = vhost_scsi_map_to_sgl(cmd, iter, sg, is_prot);
 792                 if (ret < 0) {
 793                         revert_bytes = 0;
 794
 795                         while (p < sg) {
 796                                 struct page *page = sg_page(p);
 797
 798                                 if (page) {
 799                                         put_page(page);
 800                                         revert_bytes += p->length;
 801                                 }
 802                                 p++;
 803                         }
 804
 805                         iov_iter_revert(iter, revert_bytes);
 806                         return ret;
 807                 }
 808                 sg += ret;
 809         }
 810
 811         return 0;
 812 }
 813
 814 static int
 815 vhost_scsi_mapal(struct vhost_scsi_cmd *cmd,
 816                  size_t prot_bytes, struct iov_iter *prot_iter,
 817                  size_t data_bytes, struct iov_iter *data_iter)
 818 {
 819         int sgl_count, ret;
 820
 821         if (prot_bytes) {
 822                 sgl_count = vhost_scsi_calc_sgls(prot_iter, prot_bytes,
 823                                                  VHOST_SCSI_PREALLOC_PROT_SGLS);
 824                 if (sgl_count < 0)
 825                         return sgl_count;
 826
 827                 sg_init_table(cmd->tvc_prot_sgl, sgl_count);
 828                 cmd->tvc_prot_sgl_count = sgl_count;
 829                 pr_debug("%s prot_sg %p prot_sgl_count %u\n", __func__,
 830                          cmd->tvc_prot_sgl, cmd->tvc_prot_sgl_count);
 831
 832                 ret = vhost_scsi_iov_to_sgl(cmd, prot_iter, cmd->tvc_prot_sgl,
 833                                             cmd->tvc_prot_sgl_count, true);
 834                 if (ret < 0) {
 835                         cmd->tvc_prot_sgl_count = 0;
 836                         return ret;
 837                 }
 838         }
 839         sgl_count = vhost_scsi_calc_sgls(data_iter, data_bytes,
 840                                          VHOST_SCSI_PREALLOC_SGLS);
 841         if (sgl_count < 0)
 842                 return sgl_count;
 843
 844         sg_init_table(cmd->tvc_sgl, sgl_count);
 845         cmd->tvc_sgl_count = sgl_count;
 846         pr_debug("%s data_sg %p data_sgl_count %u\n", __func__,
 847                   cmd->tvc_sgl, cmd->tvc_sgl_count);
 848
 849         ret = vhost_scsi_iov_to_sgl(cmd, data_iter, cmd->tvc_sgl,
 850                                     cmd->tvc_sgl_count, false);
 851         if (ret == -EINVAL) {
 852                 sg_init_table(cmd->tvc_sgl, cmd->tvc_sgl_count);
 853                 ret = vhost_scsi_copy_iov_to_sgl(cmd, data_iter, cmd->tvc_sgl,
 854                                                  cmd->tvc_sgl_count);
 855         }
 856
 857         if (ret < 0) {
 858                 cmd->tvc_sgl_count = 0;
 859                 return ret;
 860         }
 861         return 0;
 862 }
 863
 864 static int vhost_scsi_to_tcm_attr(int attr)
 865 {
 866         switch (attr) {
 867         case VIRTIO_SCSI_S_SIMPLE:
 868                 return TCM_SIMPLE_TAG;
 869         case VIRTIO_SCSI_S_ORDERED:
 870                 return TCM_ORDERED_TAG;
 871         case VIRTIO_SCSI_S_HEAD:
 872                 return TCM_HEAD_TAG;
 873         case VIRTIO_SCSI_S_ACA:
 874                 return TCM_ACA_TAG;
 875         default:
 876                 break;
 877         }
 878         return TCM_SIMPLE_TAG;
 879 }
 880
 881 static void vhost_scsi_target_queue_cmd(struct vhost_scsi_cmd *cmd)
 882 {
 883         struct se_cmd *se_cmd = &cmd->tvc_se_cmd;
 884         struct vhost_scsi_nexus *tv_nexus;
 885         struct scatterlist *sg_ptr, *sg_prot_ptr = NULL;
 886
 887         /* FIXME: BIDI operation */
 888         if (cmd->tvc_sgl_count) {
 889                 sg_ptr = cmd->tvc_sgl;
 890
 891                 if (cmd->tvc_prot_sgl_count)
 892                         sg_prot_ptr = cmd->tvc_prot_sgl;
 893                 else
 894                         se_cmd->prot_pto = true;
 895         } else {
 896                 sg_ptr = NULL;
 897         }
 898         tv_nexus = cmd->tvc_nexus;
 899
 900         se_cmd->tag = 0;
 901         target_init_cmd(se_cmd, tv_nexus->tvn_se_sess, &cmd->tvc_sense_buf[0],
 902                         cmd->tvc_lun, cmd->tvc_exp_data_len,
 903                         vhost_scsi_to_tcm_attr(cmd->tvc_task_attr),
 904                         cmd->tvc_data_direction, TARGET_SCF_ACK_KREF);
 905
 906         if (target_submit_prep(se_cmd, cmd->tvc_cdb, sg_ptr,
 907                                cmd->tvc_sgl_count, NULL, 0, sg_prot_ptr,
 908                                cmd->tvc_prot_sgl_count, GFP_KERNEL))
 909                 return;
 910
 911         target_queue_submission(se_cmd);
 912 }
 913
 914 static void
 915 vhost_scsi_send_bad_target(struct vhost_scsi *vs,
 916                            struct vhost_virtqueue *vq,
 917                            int head, unsigned out)
 918 {
 919         struct virtio_scsi_cmd_resp __user *resp;
 920         struct virtio_scsi_cmd_resp rsp;
 921         int ret;
 922
 923         memset(&rsp, 0, sizeof(rsp));
 924         rsp.response = VIRTIO_SCSI_S_BAD_TARGET;
 925         resp = vq->iov[out].iov_base;
 926         ret = __copy_to_user(resp, &rsp, sizeof(rsp));
 927         if (!ret)
 928                 vhost_add_used_and_signal(&vs->dev, vq, head, 0);
 929         else
 930                 pr_err("Faulted on virtio_scsi_cmd_resp\n");
 931 }
 932
 933 static int
 934 vhost_scsi_get_desc(struct vhost_scsi *vs, struct vhost_virtqueue *vq,
 935                     struct vhost_scsi_ctx *vc)
 936 {
 937         int ret = -ENXIO;
 938
 939         vc->head = vhost_get_vq_desc(vq, vq->iov,
 940                                      ARRAY_SIZE(vq->iov), &vc->out, &vc->in,
 941                                      NULL, NULL);
 942
 943         pr_debug("vhost_get_vq_desc: head: %d, out: %u in: %u\n",
 944                  vc->head, vc->out, vc->in);
 945
 946         /* On error, stop handling until the next kick. */
 947         if (unlikely(vc->head < 0))
 948                 goto done;
 949
 950         /* Nothing new?  Wait for eventfd to tell us they refilled. */
 951         if (vc->head == vq->num) {
 952                 if (unlikely(vhost_enable_notify(&vs->dev, vq))) {
 953                         vhost_disable_notify(&vs->dev, vq);
 954                         ret = -EAGAIN;
 955                 }
 956                 goto done;
 957         }
 958
 959         /*
 960          * Get the size of request and response buffers.
 961          * FIXME: Not correct for BIDI operation
 962          */
 963         vc->out_size = iov_length(vq->iov, vc->out);
 964         vc->in_size = iov_length(&vq->iov[vc->out], vc->in);
 965
 966         /*
 967          * Copy over the virtio-scsi request header, which for a
 968          * ANY_LAYOUT enabled guest may span multiple iovecs, or a
 969          * single iovec may contain both the header + outgoing
 970          * WRITE payloads.
 971          *
 972          * copy_from_iter() will advance out_iter, so that it will
 973          * point at the start of the outgoing WRITE payload, if
 974          * DMA_TO_DEVICE is set.
 975          */
 976         iov_iter_init(&vc->out_iter, ITER_SOURCE, vq->iov, vc->out, vc->out_size);
 977         ret = 0;
 978
 979 done:
 980         return ret;
 981 }
 982
 983 static int
 984 vhost_scsi_chk_size(struct vhost_virtqueue *vq, struct vhost_scsi_ctx *vc)
 985 {
 986         if (unlikely(vc->in_size < vc->rsp_size)) {
 987                 vq_err(vq,
 988                        "Response buf too small, need min %zu bytes got %zu",
 989                        vc->rsp_size, vc->in_size);
 990                 return -EINVAL;
 991         } else if (unlikely(vc->out_size < vc->req_size)) {
 992                 vq_err(vq,
 993                        "Request buf too small, need min %zu bytes got %zu",
 994                        vc->req_size, vc->out_size);
 995                 return -EIO;
 996         }
 997
 998         return 0;
 999 }
1000
1001 static int
1002 vhost_scsi_get_req(struct vhost_virtqueue *vq, struct vhost_scsi_ctx *vc,
1003                    struct vhost_scsi_tpg **tpgp)
1004 {
1005         int ret = -EIO;
1006
1007         if (unlikely(!copy_from_iter_full(vc->req, vc->req_size,
1008                                           &vc->out_iter))) {
1009                 vq_err(vq, "Faulted on copy_from_iter_full\n");
1010         } else if (unlikely(*vc->lunp != 1)) {
1011                 /* virtio-scsi spec requires byte 0 of the lun to be 1 */
1012                 vq_err(vq, "Illegal virtio-scsi lun: %u\n", *vc->lunp);
1013         } else {
1014                 struct vhost_scsi_tpg **vs_tpg, *tpg;
1015
1016                 vs_tpg = vhost_vq_get_backend(vq);      /* validated at handler entry */
1017
1018                 tpg = READ_ONCE(vs_tpg[*vc->target]);
1019                 if (unlikely(!tpg)) {
1020                         vq_err(vq, "Target 0x%x does not exist\n", *vc->target);
1021                 } else {
1022                         if (tpgp)
1023                                 *tpgp = tpg;
1024                         ret = 0;
1025                 }
1026         }
1027
1028         return ret;
1029 }
1030
1031 static u16 vhost_buf_to_lun(u8 *lun_buf)
1032 {
1033         return ((lun_buf[2] << 8) | lun_buf[3]) & 0x3FFF;
1034 }
1035
1036 static void
1037 vhost_scsi_handle_vq(struct vhost_scsi *vs, struct vhost_virtqueue *vq)
1038 {
1039         struct vhost_scsi_tpg **vs_tpg, *tpg;
1040         struct virtio_scsi_cmd_req v_req;
1041         struct virtio_scsi_cmd_req_pi v_req_pi;
1042         struct vhost_scsi_ctx vc;
1043         struct vhost_scsi_cmd *cmd;
1044         struct iov_iter in_iter, prot_iter, data_iter;
1045         u64 tag;
1046         u32 exp_data_len, data_direction;
1047         int ret, prot_bytes, i, c = 0;
1048         u16 lun;
1049         u8 task_attr;
1050         bool t10_pi = vhost_has_feature(vq, VIRTIO_SCSI_F_T10_PI);
1051         void *cdb;
1052
1053         mutex_lock(&vq->mutex);
1054         /*
1055          * We can handle the vq only after the endpoint is setup by calling the
1056          * VHOST_SCSI_SET_ENDPOINT ioctl.
1057          */
1058         vs_tpg = vhost_vq_get_backend(vq);
1059         if (!vs_tpg)
1060                 goto out;
1061
1062         memset(&vc, 0, sizeof(vc));
1063         vc.rsp_size = sizeof(struct virtio_scsi_cmd_resp);
1064
1065         vhost_disable_notify(&vs->dev, vq);
1066
1067         do {
1068                 ret = vhost_scsi_get_desc(vs, vq, &vc);
1069                 if (ret)
1070                         goto err;
1071
1072                 /*
1073                  * Setup pointers and values based upon different virtio-scsi
1074                  * request header if T10_PI is enabled in KVM guest.
1075                  */
1076                 if (t10_pi) {
1077                         vc.req = &v_req_pi;
1078                         vc.req_size = sizeof(v_req_pi);
1079                         vc.lunp = &v_req_pi.lun[0];
1080                         vc.target = &v_req_pi.lun[1];
1081                 } else {
1082                         vc.req = &v_req;
1083                         vc.req_size = sizeof(v_req);
1084                         vc.lunp = &v_req.lun[0];
1085                         vc.target = &v_req.lun[1];
1086                 }
1087
1088                 /*
1089                  * Validate the size of request and response buffers.
1090                  * Check for a sane response buffer so we can report
1091                  * early errors back to the guest.
1092                  */
1093                 ret = vhost_scsi_chk_size(vq, &vc);
1094                 if (ret)
1095                         goto err;
1096
1097                 ret = vhost_scsi_get_req(vq, &vc, &tpg);
1098                 if (ret)
1099                         goto err;
1100
1101                 ret = -EIO;     /* bad target on any error from here on */
1102
1103                 /*
1104                  * Determine data_direction by calculating the total outgoing
1105                  * iovec sizes + incoming iovec sizes vs. virtio-scsi request +
1106                  * response headers respectively.
1107                  *
1108                  * For DMA_TO_DEVICE this is out_iter, which is already pointing
1109                  * to the right place.
1110                  *
1111                  * For DMA_FROM_DEVICE, the iovec will be just past the end
1112                  * of the virtio-scsi response header in either the same
1113                  * or immediately following iovec.
1114                  *
1115                  * Any associated T10_PI bytes for the outgoing / incoming
1116                  * payloads are included in calculation of exp_data_len here.
1117                  */
1118                 prot_bytes = 0;
1119
1120                 if (vc.out_size > vc.req_size) {
1121                         data_direction = DMA_TO_DEVICE;
1122                         exp_data_len = vc.out_size - vc.req_size;
1123                         data_iter = vc.out_iter;
1124                 } else if (vc.in_size > vc.rsp_size) {
1125                         data_direction = DMA_FROM_DEVICE;
1126                         exp_data_len = vc.in_size - vc.rsp_size;
1127
1128                         iov_iter_init(&in_iter, ITER_DEST, &vq->iov[vc.out], vc.in,
1129                                       vc.rsp_size + exp_data_len);
1130                         iov_iter_advance(&in_iter, vc.rsp_size);
1131                         data_iter = in_iter;
1132                 } else {
1133                         data_direction = DMA_NONE;
1134                         exp_data_len = 0;
1135                 }
1136                 /*
1137                  * If T10_PI header + payload is present, setup prot_iter values
1138                  * and recalculate data_iter for vhost_scsi_mapal() mapping to
1139                  * host scatterlists via get_user_pages_fast().
1140                  */
1141                 if (t10_pi) {
1142                         if (v_req_pi.pi_bytesout) {
1143                                 if (data_direction != DMA_TO_DEVICE) {
1144                                         vq_err(vq, "Received non zero pi_bytesout,"
1145                                                 " but wrong data_direction\n");
1146                                         goto err;
1147                                 }
1148                                 prot_bytes = vhost32_to_cpu(vq, v_req_pi.pi_bytesout);
1149                         } else if (v_req_pi.pi_bytesin) {
1150                                 if (data_direction != DMA_FROM_DEVICE) {
1151                                         vq_err(vq, "Received non zero pi_bytesin,"
1152                                                 " but wrong data_direction\n");
1153                                         goto err;
1154                                 }
1155                                 prot_bytes = vhost32_to_cpu(vq, v_req_pi.pi_bytesin);
1156                         }
1157                         /*
1158                          * Set prot_iter to data_iter and truncate it to
1159                          * prot_bytes, and advance data_iter past any
1160                          * preceeding prot_bytes that may be present.
1161                          *
1162                          * Also fix up the exp_data_len to reflect only the
1163                          * actual data payload length.
1164                          */
1165                         if (prot_bytes) {
1166                                 exp_data_len -= prot_bytes;
1167                                 prot_iter = data_iter;
1168                                 iov_iter_truncate(&prot_iter, prot_bytes);
1169                                 iov_iter_advance(&data_iter, prot_bytes);
1170                         }
1171                         tag = vhost64_to_cpu(vq, v_req_pi.tag);
1172                         task_attr = v_req_pi.task_attr;
1173                         cdb = &v_req_pi.cdb[0];
1174                         lun = vhost_buf_to_lun(v_req_pi.lun);
1175                 } else {
1176                         tag = vhost64_to_cpu(vq, v_req.tag);
1177                         task_attr = v_req.task_attr;
1178                         cdb = &v_req.cdb[0];
1179                         lun = vhost_buf_to_lun(v_req.lun);
1180                 }
1181                 /*
1182                  * Check that the received CDB size does not exceeded our
1183                  * hardcoded max for vhost-scsi, then get a pre-allocated
1184                  * cmd descriptor for the new virtio-scsi tag.
1185                  *
1186                  * TODO what if cdb was too small for varlen cdb header?
1187                  */
1188                 if (unlikely(scsi_command_size(cdb) > VHOST_SCSI_MAX_CDB_SIZE)) {
1189                         vq_err(vq, "Received SCSI CDB with command_size: %d that"
1190                                 " exceeds SCSI_MAX_VARLEN_CDB_SIZE: %d\n",
1191                                 scsi_command_size(cdb), VHOST_SCSI_MAX_CDB_SIZE);
1192                                 goto err;
1193                 }
1194                 cmd = vhost_scsi_get_cmd(vq, tpg, cdb, tag, lun, task_attr,
1195                                          exp_data_len + prot_bytes,
1196                                          data_direction);
1197                 if (IS_ERR(cmd)) {
1198                         vq_err(vq, "vhost_scsi_get_cmd failed %ld\n",
1199                                PTR_ERR(cmd));
1200                         goto err;
1201                 }
1202                 cmd->tvc_vhost = vs;
1203                 cmd->tvc_vq = vq;
1204                 for (i = 0; i < vc.in ; i++)
1205                         cmd->tvc_resp_iov[i] = vq->iov[vc.out + i];
1206                 cmd->tvc_in_iovs = vc.in;
1207
1208                 pr_debug("vhost_scsi got command opcode: %#02x, lun: %d\n",
1209                          cmd->tvc_cdb[0], cmd->tvc_lun);
1210                 pr_debug("cmd: %p exp_data_len: %d, prot_bytes: %d data_direction:"
1211                          " %d\n", cmd, exp_data_len, prot_bytes, data_direction);
1212
1213                 if (data_direction != DMA_NONE) {
1214                         if (unlikely(vhost_scsi_mapal(cmd, prot_bytes,
1215                                                       &prot_iter, exp_data_len,
1216                                                       &data_iter))) {
1217                                 vq_err(vq, "Failed to map iov to sgl\n");
1218                                 vhost_scsi_release_cmd_res(&cmd->tvc_se_cmd);
1219                                 goto err;
1220                         }
1221                 }
1222                 /*
1223                  * Save the descriptor from vhost_get_vq_desc() to be used to
1224                  * complete the virtio-scsi request in TCM callback context via
1225                  * vhost_scsi_queue_data_in() and vhost_scsi_queue_status()
1226                  */
1227                 cmd->tvc_vq_desc = vc.head;
1228                 vhost_scsi_target_queue_cmd(cmd);
1229                 ret = 0;
1230 err:
1231                 /*
1232                  * ENXIO:  No more requests, or read error, wait for next kick
1233                  * EINVAL: Invalid response buffer, drop the request
1234                  * EIO:    Respond with bad target
1235                  * EAGAIN: Pending request
1236                  */
1237                 if (ret == -ENXIO)
1238                         break;
1239                 else if (ret == -EIO)
1240                         vhost_scsi_send_bad_target(vs, vq, vc.head, vc.out);
1241         } while (likely(!vhost_exceeds_weight(vq, ++c, 0)));
1242 out:
1243         mutex_unlock(&vq->mutex);
1244 }
1245
1246 static void
1247 vhost_scsi_send_tmf_resp(struct vhost_scsi *vs, struct vhost_virtqueue *vq,
1248                          int in_iovs, int vq_desc, struct iovec *resp_iov,
1249                          int tmf_resp_code)
1250 {
1251         struct virtio_scsi_ctrl_tmf_resp rsp;
1252         struct iov_iter iov_iter;
1253         int ret;
1254
1255         pr_debug("%s\n", __func__);
1256         memset(&rsp, 0, sizeof(rsp));
1257         rsp.response = tmf_resp_code;
1258
1259         iov_iter_init(&iov_iter, ITER_DEST, resp_iov, in_iovs, sizeof(rsp));
1260
1261         ret = copy_to_iter(&rsp, sizeof(rsp), &iov_iter);
1262         if (likely(ret == sizeof(rsp)))
1263                 vhost_add_used_and_signal(&vs->dev, vq, vq_desc, 0);
1264         else
1265                 pr_err("Faulted on virtio_scsi_ctrl_tmf_resp\n");
1266 }
1267
1268 static void vhost_scsi_tmf_resp_work(struct vhost_work *work)
1269 {
1270         struct vhost_scsi_tmf *tmf = container_of(work, struct vhost_scsi_tmf,
1271                                                   vwork);
1272         struct vhost_virtqueue *ctl_vq, *vq;
1273         int resp_code, i;
1274
1275         if (tmf->scsi_resp == TMR_FUNCTION_COMPLETE) {
1276                 /*
1277                  * Flush IO vqs that don't share a worker with the ctl to make
1278                  * sure they have sent their responses before us.
1279                  */
1280                 ctl_vq = &tmf->vhost->vqs[VHOST_SCSI_VQ_CTL].vq;
1281                 for (i = VHOST_SCSI_VQ_IO; i < tmf->vhost->dev.nvqs; i++) {
1282                         vq = &tmf->vhost->vqs[i].vq;
1283
1284                         if (vhost_vq_is_setup(vq) &&
1285                             vq->worker != ctl_vq->worker)
1286                                 vhost_vq_flush(vq);
1287                 }
1288
1289                 resp_code = VIRTIO_SCSI_S_FUNCTION_SUCCEEDED;
1290         } else {
1291                 resp_code = VIRTIO_SCSI_S_FUNCTION_REJECTED;
1292         }
1293
1294         vhost_scsi_send_tmf_resp(tmf->vhost, &tmf->svq->vq, tmf->in_iovs,
1295                                  tmf->vq_desc, &tmf->resp_iov, resp_code);
1296         vhost_scsi_release_tmf_res(tmf);
1297 }
1298
1299 static void
1300 vhost_scsi_handle_tmf(struct vhost_scsi *vs, struct vhost_scsi_tpg *tpg,
1301                       struct vhost_virtqueue *vq,
1302                       struct virtio_scsi_ctrl_tmf_req *vtmf,
1303                       struct vhost_scsi_ctx *vc)
1304 {
1305         struct vhost_scsi_virtqueue *svq = container_of(vq,
1306                                         struct vhost_scsi_virtqueue, vq);
1307         struct vhost_scsi_tmf *tmf;
1308
1309         if (vhost32_to_cpu(vq, vtmf->subtype) !=
1310             VIRTIO_SCSI_T_TMF_LOGICAL_UNIT_RESET)
1311                 goto send_reject;
1312
1313         if (!tpg->tpg_nexus || !tpg->tpg_nexus->tvn_se_sess) {
1314                 pr_err("Unable to locate active struct vhost_scsi_nexus for LUN RESET.\n");
1315                 goto send_reject;
1316         }
1317
1318         tmf = kzalloc(sizeof(*tmf), GFP_KERNEL);
1319         if (!tmf)
1320                 goto send_reject;
1321
1322         vhost_work_init(&tmf->vwork, vhost_scsi_tmf_resp_work);
1323         tmf->vhost = vs;
1324         tmf->svq = svq;
1325         tmf->resp_iov = vq->iov[vc->out];
1326         tmf->vq_desc = vc->head;
1327         tmf->in_iovs = vc->in;
1328         tmf->inflight = vhost_scsi_get_inflight(vq);
1329
1330         if (target_submit_tmr(&tmf->se_cmd, tpg->tpg_nexus->tvn_se_sess, NULL,
1331                               vhost_buf_to_lun(vtmf->lun), NULL,
1332                               TMR_LUN_RESET, GFP_KERNEL, 0,
1333                               TARGET_SCF_ACK_KREF) < 0) {
1334                 vhost_scsi_release_tmf_res(tmf);
1335                 goto send_reject;
1336         }
1337
1338         return;
1339
1340 send_reject:
1341         vhost_scsi_send_tmf_resp(vs, vq, vc->in, vc->head, &vq->iov[vc->out],
1342                                  VIRTIO_SCSI_S_FUNCTION_REJECTED);
1343 }
1344
1345 static void
1346 vhost_scsi_send_an_resp(struct vhost_scsi *vs,
1347                         struct vhost_virtqueue *vq,
1348                         struct vhost_scsi_ctx *vc)
1349 {
1350         struct virtio_scsi_ctrl_an_resp rsp;
1351         struct iov_iter iov_iter;
1352         int ret;
1353
1354         pr_debug("%s\n", __func__);
1355         memset(&rsp, 0, sizeof(rsp));   /* event_actual = 0 */
1356         rsp.response = VIRTIO_SCSI_S_OK;
1357
1358         iov_iter_init(&iov_iter, ITER_DEST, &vq->iov[vc->out], vc->in, sizeof(rsp));
1359
1360         ret = copy_to_iter(&rsp, sizeof(rsp), &iov_iter);
1361         if (likely(ret == sizeof(rsp)))
1362                 vhost_add_used_and_signal(&vs->dev, vq, vc->head, 0);
1363         else
1364                 pr_err("Faulted on virtio_scsi_ctrl_an_resp\n");
1365 }
1366
1367 static void
1368 vhost_scsi_ctl_handle_vq(struct vhost_scsi *vs, struct vhost_virtqueue *vq)
1369 {
1370         struct vhost_scsi_tpg *tpg;
1371         union {
1372                 __virtio32 type;
1373                 struct virtio_scsi_ctrl_an_req an;
1374                 struct virtio_scsi_ctrl_tmf_req tmf;
1375         } v_req;
1376         struct vhost_scsi_ctx vc;
1377         size_t typ_size;
1378         int ret, c = 0;
1379
1380         mutex_lock(&vq->mutex);
1381         /*
1382          * We can handle the vq only after the endpoint is setup by calling the
1383          * VHOST_SCSI_SET_ENDPOINT ioctl.
1384          */
1385         if (!vhost_vq_get_backend(vq))
1386                 goto out;
1387
1388         memset(&vc, 0, sizeof(vc));
1389
1390         vhost_disable_notify(&vs->dev, vq);
1391
1392         do {
1393                 ret = vhost_scsi_get_desc(vs, vq, &vc);
1394                 if (ret)
1395                         goto err;
1396
1397                 /*
1398                  * Get the request type first in order to setup
1399                  * other parameters dependent on the type.
1400                  */
1401                 vc.req = &v_req.type;
1402                 typ_size = sizeof(v_req.type);
1403
1404                 if (unlikely(!copy_from_iter_full(vc.req, typ_size,
1405                                                   &vc.out_iter))) {
1406                         vq_err(vq, "Faulted on copy_from_iter tmf type\n");
1407                         /*
1408                          * The size of the response buffer depends on the
1409                          * request type and must be validated against it.
1410                          * Since the request type is not known, don't send
1411                          * a response.
1412                          */
1413                         continue;
1414                 }
1415
1416                 switch (vhost32_to_cpu(vq, v_req.type)) {
1417                 case VIRTIO_SCSI_T_TMF:
1418                         vc.req = &v_req.tmf;
1419                         vc.req_size = sizeof(struct virtio_scsi_ctrl_tmf_req);
1420                         vc.rsp_size = sizeof(struct virtio_scsi_ctrl_tmf_resp);
1421                         vc.lunp = &v_req.tmf.lun[0];
1422                         vc.target = &v_req.tmf.lun[1];
1423                         break;
1424                 case VIRTIO_SCSI_T_AN_QUERY:
1425                 case VIRTIO_SCSI_T_AN_SUBSCRIBE:
1426                         vc.req = &v_req.an;
1427                         vc.req_size = sizeof(struct virtio_scsi_ctrl_an_req);
1428                         vc.rsp_size = sizeof(struct virtio_scsi_ctrl_an_resp);
1429                         vc.lunp = &v_req.an.lun[0];
1430                         vc.target = NULL;
1431                         break;
1432                 default:
1433                         vq_err(vq, "Unknown control request %d", v_req.type);
1434                         continue;
1435                 }
1436
1437                 /*
1438                  * Validate the size of request and response buffers.
1439                  * Check for a sane response buffer so we can report
1440                  * early errors back to the guest.
1441                  */
1442                 ret = vhost_scsi_chk_size(vq, &vc);
1443                 if (ret)
1444                         goto err;
1445
1446                 /*
1447                  * Get the rest of the request now that its size is known.
1448                  */
1449                 vc.req += typ_size;
1450                 vc.req_size -= typ_size;
1451
1452                 ret = vhost_scsi_get_req(vq, &vc, &tpg);
1453                 if (ret)
1454                         goto err;
1455
1456                 if (v_req.type == VIRTIO_SCSI_T_TMF)
1457                         vhost_scsi_handle_tmf(vs, tpg, vq, &v_req.tmf, &vc);
1458                 else
1459                         vhost_scsi_send_an_resp(vs, vq, &vc);
1460 err:
1461                 /*
1462                  * ENXIO:  No more requests, or read error, wait for next kick
1463                  * EINVAL: Invalid response buffer, drop the request
1464                  * EIO:    Respond with bad target
1465                  * EAGAIN: Pending request
1466                  */
1467                 if (ret == -ENXIO)
1468                         break;
1469                 else if (ret == -EIO)
1470                         vhost_scsi_send_bad_target(vs, vq, vc.head, vc.out);
1471         } while (likely(!vhost_exceeds_weight(vq, ++c, 0)));
1472 out:
1473         mutex_unlock(&vq->mutex);
1474 }
1475
1476 static void vhost_scsi_ctl_handle_kick(struct vhost_work *work)
1477 {
1478         struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,
1479                                                 poll.work);
1480         struct vhost_scsi *vs = container_of(vq->dev, struct vhost_scsi, dev);
1481
1482         pr_debug("%s: The handling func for control queue.\n", __func__);
1483         vhost_scsi_ctl_handle_vq(vs, vq);
1484 }
1485
1486 static void
1487 vhost_scsi_send_evt(struct vhost_scsi *vs, struct vhost_virtqueue *vq,
1488                     struct vhost_scsi_tpg *tpg, struct se_lun *lun,
1489                     u32 event, u32 reason)
1490 {
1491         struct vhost_scsi_evt *evt;
1492
1493         evt = vhost_scsi_allocate_evt(vs, event, reason);
1494         if (!evt)
1495                 return;
1496
1497         if (tpg && lun) {
1498                 /* TODO: share lun setup code with virtio-scsi.ko */
1499                 /*
1500                  * Note: evt->event is zeroed when we allocate it and
1501                  * lun[4-7] need to be zero according to virtio-scsi spec.
1502                  */
1503                 evt->event.lun[0] = 0x01;
1504                 evt->event.lun[1] = tpg->tport_tpgt;
1505                 if (lun->unpacked_lun >= 256)
1506                         evt->event.lun[2] = lun->unpacked_lun >> 8 | 0x40 ;
1507                 evt->event.lun[3] = lun->unpacked_lun & 0xFF;
1508         }
1509
1510         llist_add(&evt->list, &vs->vs_event_list);
1511         vhost_vq_work_queue(vq, &vs->vs_event_work);
1512 }
1513
1514 static void vhost_scsi_evt_handle_kick(struct vhost_work *work)
1515 {
1516         struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,
1517                                                 poll.work);
1518         struct vhost_scsi *vs = container_of(vq->dev, struct vhost_scsi, dev);
1519
1520         mutex_lock(&vq->mutex);
1521         if (!vhost_vq_get_backend(vq))
1522                 goto out;
1523
1524         if (vs->vs_events_missed)
1525                 vhost_scsi_send_evt(vs, vq, NULL, NULL, VIRTIO_SCSI_T_NO_EVENT,
1526                                     0);
1527 out:
1528         mutex_unlock(&vq->mutex);
1529 }
1530
1531 static void vhost_scsi_handle_kick(struct vhost_work *work)
1532 {
1533         struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,
1534                                                 poll.work);
1535         struct vhost_scsi *vs = container_of(vq->dev, struct vhost_scsi, dev);
1536
1537         vhost_scsi_handle_vq(vs, vq);
1538 }
1539
1540 /* Callers must hold dev mutex */
1541 static void vhost_scsi_flush(struct vhost_scsi *vs)
1542 {
1543         int i;
1544
1545         /* Init new inflight and remember the old inflight */
1546         vhost_scsi_init_inflight(vs, vs->old_inflight);
1547
1548         /*
1549          * The inflight->kref was initialized to 1. We decrement it here to
1550          * indicate the start of the flush operation so that it will reach 0
1551          * when all the reqs are finished.
1552          */
1553         for (i = 0; i < vs->dev.nvqs; i++)
1554                 kref_put(&vs->old_inflight[i]->kref, vhost_scsi_done_inflight);
1555
1556         /* Flush both the vhost poll and vhost work */
1557         vhost_dev_flush(&vs->dev);
1558
1559         /* Wait for all reqs issued before the flush to be finished */
1560         for (i = 0; i < vs->dev.nvqs; i++)
1561                 wait_for_completion(&vs->old_inflight[i]->comp);
1562 }
1563
1564 static void vhost_scsi_destroy_vq_cmds(struct vhost_virtqueue *vq)
1565 {
1566         struct vhost_scsi_virtqueue *svq = container_of(vq,
1567                                         struct vhost_scsi_virtqueue, vq);
1568         struct vhost_scsi_cmd *tv_cmd;
1569         unsigned int i;
1570
1571         if (!svq->scsi_cmds)
1572                 return;
1573
1574         for (i = 0; i < svq->max_cmds; i++) {
1575                 tv_cmd = &svq->scsi_cmds[i];
1576
1577                 kfree(tv_cmd->tvc_sgl);
1578                 kfree(tv_cmd->tvc_prot_sgl);
1579                 kfree(tv_cmd->tvc_upages);
1580                 kfree(tv_cmd->tvc_resp_iov);
1581         }
1582
1583         sbitmap_free(&svq->scsi_tags);
1584         kfree(svq->scsi_cmds);
1585         svq->scsi_cmds = NULL;
1586 }
1587
1588 static int vhost_scsi_setup_vq_cmds(struct vhost_virtqueue *vq, int max_cmds)
1589 {
1590         struct vhost_scsi_virtqueue *svq = container_of(vq,
1591                                         struct vhost_scsi_virtqueue, vq);
1592         struct vhost_scsi_cmd *tv_cmd;
1593         unsigned int i;
1594
1595         if (svq->scsi_cmds)
1596                 return 0;
1597
1598         if (sbitmap_init_node(&svq->scsi_tags, max_cmds, -1, GFP_KERNEL,
1599                               NUMA_NO_NODE, false, true))
1600                 return -ENOMEM;
1601         svq->max_cmds = max_cmds;
1602
1603         svq->scsi_cmds = kcalloc(max_cmds, sizeof(*tv_cmd), GFP_KERNEL);
1604         if (!svq->scsi_cmds) {
1605                 sbitmap_free(&svq->scsi_tags);
1606                 return -ENOMEM;
1607         }
1608
1609         for (i = 0; i < max_cmds; i++) {
1610                 tv_cmd = &svq->scsi_cmds[i];
1611
1612                 tv_cmd->tvc_sgl = kcalloc(VHOST_SCSI_PREALLOC_SGLS,
1613                                           sizeof(struct scatterlist),
1614                                           GFP_KERNEL);
1615                 if (!tv_cmd->tvc_sgl) {
1616                         pr_err("Unable to allocate tv_cmd->tvc_sgl\n");
1617                         goto out;
1618                 }
1619
1620                 tv_cmd->tvc_upages = kcalloc(VHOST_SCSI_PREALLOC_UPAGES,
1621                                              sizeof(struct page *),
1622                                              GFP_KERNEL);
1623                 if (!tv_cmd->tvc_upages) {
1624                         pr_err("Unable to allocate tv_cmd->tvc_upages\n");
1625                         goto out;
1626                 }
1627
1628                 tv_cmd->tvc_resp_iov = kcalloc(UIO_MAXIOV,
1629                                                sizeof(struct iovec),
1630                                                GFP_KERNEL);
1631                 if (!tv_cmd->tvc_resp_iov) {
1632                         pr_err("Unable to allocate tv_cmd->tvc_resp_iov\n");
1633                         goto out;
1634                 }
1635
1636                 tv_cmd->tvc_prot_sgl = kcalloc(VHOST_SCSI_PREALLOC_PROT_SGLS,
1637                                                sizeof(struct scatterlist),
1638                                                GFP_KERNEL);
1639                 if (!tv_cmd->tvc_prot_sgl) {
1640                         pr_err("Unable to allocate tv_cmd->tvc_prot_sgl\n");
1641                         goto out;
1642                 }
1643         }
1644         return 0;
1645 out:
1646         vhost_scsi_destroy_vq_cmds(vq);
1647         return -ENOMEM;
1648 }
1649
1650 /*
1651  * Called from vhost_scsi_ioctl() context to walk the list of available
1652  * vhost_scsi_tpg with an active struct vhost_scsi_nexus
1653  *
1654  *  The lock nesting rule is:
1655  *    vs->dev.mutex -> vhost_scsi_mutex -> tpg->tv_tpg_mutex -> vq->mutex
1656  */
1657 static int
1658 vhost_scsi_set_endpoint(struct vhost_scsi *vs,
1659                         struct vhost_scsi_target *t)
1660 {
1661         struct se_portal_group *se_tpg;
1662         struct vhost_scsi_tport *tv_tport;
1663         struct vhost_scsi_tpg *tpg;
1664         struct vhost_scsi_tpg **vs_tpg;
1665         struct vhost_virtqueue *vq;
1666         int index, ret, i, len;
1667         bool match = false;
1668
1669         mutex_lock(&vs->dev.mutex);
1670
1671         /* Verify that ring has been setup correctly. */
1672         for (index = 0; index < vs->dev.nvqs; ++index) {
1673                 /* Verify that ring has been setup correctly. */
1674                 if (!vhost_vq_access_ok(&vs->vqs[index].vq)) {
1675                         ret = -EFAULT;
1676                         goto out;
1677                 }
1678         }
1679
1680         len = sizeof(vs_tpg[0]) * VHOST_SCSI_MAX_TARGET;
1681         vs_tpg = kzalloc(len, GFP_KERNEL);
1682         if (!vs_tpg) {
1683                 ret = -ENOMEM;
1684                 goto out;
1685         }
1686         if (vs->vs_tpg)
1687                 memcpy(vs_tpg, vs->vs_tpg, len);
1688
1689         mutex_lock(&vhost_scsi_mutex);
1690         list_for_each_entry(tpg, &vhost_scsi_list, tv_tpg_list) {
1691                 mutex_lock(&tpg->tv_tpg_mutex);
1692                 if (!tpg->tpg_nexus) {
1693                         mutex_unlock(&tpg->tv_tpg_mutex);
1694                         continue;
1695                 }
1696                 if (tpg->tv_tpg_vhost_count != 0) {
1697                         mutex_unlock(&tpg->tv_tpg_mutex);
1698                         continue;
1699                 }
1700                 tv_tport = tpg->tport;
1701
1702                 if (!strcmp(tv_tport->tport_name, t->vhost_wwpn)) {
1703                         if (vs->vs_tpg && vs->vs_tpg[tpg->tport_tpgt]) {
1704                                 mutex_unlock(&tpg->tv_tpg_mutex);
1705                                 mutex_unlock(&vhost_scsi_mutex);
1706                                 ret = -EEXIST;
1707                                 goto undepend;
1708                         }
1709                         /*
1710                          * In order to ensure individual vhost-scsi configfs
1711                          * groups cannot be removed while in use by vhost ioctl,
1712                          * go ahead and take an explicit se_tpg->tpg_group.cg_item
1713                          * dependency now.
1714                          */
1715                         se_tpg = &tpg->se_tpg;
1716                         ret = target_depend_item(&se_tpg->tpg_group.cg_item);
1717                         if (ret) {
1718                                 pr_warn("target_depend_item() failed: %d\n", ret);
1719                                 mutex_unlock(&tpg->tv_tpg_mutex);
1720                                 mutex_unlock(&vhost_scsi_mutex);
1721                                 goto undepend;
1722                         }
1723                         tpg->tv_tpg_vhost_count++;
1724                         tpg->vhost_scsi = vs;
1725                         vs_tpg[tpg->tport_tpgt] = tpg;
1726                         match = true;
1727                 }
1728                 mutex_unlock(&tpg->tv_tpg_mutex);
1729         }
1730         mutex_unlock(&vhost_scsi_mutex);
1731
1732         if (match) {
1733                 memcpy(vs->vs_vhost_wwpn, t->vhost_wwpn,
1734                        sizeof(vs->vs_vhost_wwpn));
1735
1736                 for (i = VHOST_SCSI_VQ_IO; i < vs->dev.nvqs; i++) {
1737                         vq = &vs->vqs[i].vq;
1738                         if (!vhost_vq_is_setup(vq))
1739                                 continue;
1740
1741                         ret = vhost_scsi_setup_vq_cmds(vq, vq->num);
1742                         if (ret)
1743                                 goto destroy_vq_cmds;
1744                 }
1745
1746                 for (i = 0; i < vs->dev.nvqs; i++) {
1747                         vq = &vs->vqs[i].vq;
1748                         mutex_lock(&vq->mutex);
1749                         vhost_vq_set_backend(vq, vs_tpg);
1750                         vhost_vq_init_access(vq);
1751                         mutex_unlock(&vq->mutex);
1752                 }
1753                 ret = 0;
1754         } else {
1755                 ret = -EEXIST;
1756         }
1757
1758         /*
1759          * Act as synchronize_rcu to make sure access to
1760          * old vs->vs_tpg is finished.
1761          */
1762         vhost_scsi_flush(vs);
1763         kfree(vs->vs_tpg);
1764         vs->vs_tpg = vs_tpg;
1765         goto out;
1766
1767 destroy_vq_cmds:
1768         for (i--; i >= VHOST_SCSI_VQ_IO; i--) {
1769                 if (!vhost_vq_get_backend(&vs->vqs[i].vq))
1770                         vhost_scsi_destroy_vq_cmds(&vs->vqs[i].vq);
1771         }
1772 undepend:
1773         for (i = 0; i < VHOST_SCSI_MAX_TARGET; i++) {
1774                 tpg = vs_tpg[i];
1775                 if (tpg) {
1776                         mutex_lock(&tpg->tv_tpg_mutex);
1777                         tpg->vhost_scsi = NULL;
1778                         tpg->tv_tpg_vhost_count--;
1779                         mutex_unlock(&tpg->tv_tpg_mutex);
1780                         target_undepend_item(&tpg->se_tpg.tpg_group.cg_item);
1781                 }
1782         }
1783         kfree(vs_tpg);
1784 out:
1785         mutex_unlock(&vs->dev.mutex);
1786         return ret;
1787 }
1788
1789 static int
1790 vhost_scsi_clear_endpoint(struct vhost_scsi *vs,
1791                           struct vhost_scsi_target *t)
1792 {
1793         struct se_portal_group *se_tpg;
1794         struct vhost_scsi_tport *tv_tport;
1795         struct vhost_scsi_tpg *tpg;
1796         struct vhost_virtqueue *vq;
1797         bool match = false;
1798         int index, ret, i;
1799         u8 target;
1800
1801         mutex_lock(&vs->dev.mutex);
1802         /* Verify that ring has been setup correctly. */
1803         for (index = 0; index < vs->dev.nvqs; ++index) {
1804                 if (!vhost_vq_access_ok(&vs->vqs[index].vq)) {
1805                         ret = -EFAULT;
1806                         goto err_dev;
1807                 }
1808         }
1809
1810         if (!vs->vs_tpg) {
1811                 ret = 0;
1812                 goto err_dev;
1813         }
1814
1815         for (i = 0; i < VHOST_SCSI_MAX_TARGET; i++) {
1816                 target = i;
1817                 tpg = vs->vs_tpg[target];
1818                 if (!tpg)
1819                         continue;
1820
1821                 tv_tport = tpg->tport;
1822                 if (!tv_tport) {
1823                         ret = -ENODEV;
1824                         goto err_dev;
1825                 }
1826
1827                 if (strcmp(tv_tport->tport_name, t->vhost_wwpn)) {
1828                         pr_warn("tv_tport->tport_name: %s, tpg->tport_tpgt: %hu"
1829                                 " does not match t->vhost_wwpn: %s, t->vhost_tpgt: %hu\n",
1830                                 tv_tport->tport_name, tpg->tport_tpgt,
1831                                 t->vhost_wwpn, t->vhost_tpgt);
1832                         ret = -EINVAL;
1833                         goto err_dev;
1834                 }
1835                 match = true;
1836         }
1837         if (!match)
1838                 goto free_vs_tpg;
1839
1840         /* Prevent new cmds from starting and accessing the tpgs/sessions */
1841         for (i = 0; i < vs->dev.nvqs; i++) {
1842                 vq = &vs->vqs[i].vq;
1843                 mutex_lock(&vq->mutex);
1844                 vhost_vq_set_backend(vq, NULL);
1845                 mutex_unlock(&vq->mutex);
1846         }
1847         /* Make sure cmds are not running before tearing them down. */
1848         vhost_scsi_flush(vs);
1849
1850         for (i = 0; i < vs->dev.nvqs; i++) {
1851                 vq = &vs->vqs[i].vq;
1852                 vhost_scsi_destroy_vq_cmds(vq);
1853         }
1854
1855         /*
1856          * We can now release our hold on the tpg and sessions and userspace
1857          * can free them after this point.
1858          */
1859         for (i = 0; i < VHOST_SCSI_MAX_TARGET; i++) {
1860                 target = i;
1861                 tpg = vs->vs_tpg[target];
1862                 if (!tpg)
1863                         continue;
1864
1865                 mutex_lock(&tpg->tv_tpg_mutex);
1866
1867                 tpg->tv_tpg_vhost_count--;
1868                 tpg->vhost_scsi = NULL;
1869                 vs->vs_tpg[target] = NULL;
1870
1871                 mutex_unlock(&tpg->tv_tpg_mutex);
1872
1873                 se_tpg = &tpg->se_tpg;
1874                 target_undepend_item(&se_tpg->tpg_group.cg_item);
1875         }
1876
1877 free_vs_tpg:
1878         /*
1879          * Act as synchronize_rcu to make sure access to
1880          * old vs->vs_tpg is finished.
1881          */
1882         vhost_scsi_flush(vs);
1883         kfree(vs->vs_tpg);
1884         vs->vs_tpg = NULL;
1885         WARN_ON(vs->vs_events_nr);
1886         mutex_unlock(&vs->dev.mutex);
1887         return 0;
1888
1889 err_dev:
1890         mutex_unlock(&vs->dev.mutex);
1891         return ret;
1892 }
1893
1894 static int vhost_scsi_set_features(struct vhost_scsi *vs, u64 features)
1895 {
1896         struct vhost_virtqueue *vq;
1897         int i;
1898
1899         if (features & ~VHOST_SCSI_FEATURES)
1900                 return -EOPNOTSUPP;
1901
1902         mutex_lock(&vs->dev.mutex);
1903         if ((features & (1 << VHOST_F_LOG_ALL)) &&
1904             !vhost_log_access_ok(&vs->dev)) {
1905                 mutex_unlock(&vs->dev.mutex);
1906                 return -EFAULT;
1907         }
1908
1909         for (i = 0; i < vs->dev.nvqs; i++) {
1910                 vq = &vs->vqs[i].vq;
1911                 mutex_lock(&vq->mutex);
1912                 vq->acked_features = features;
1913                 mutex_unlock(&vq->mutex);
1914         }
1915         mutex_unlock(&vs->dev.mutex);
1916         return 0;
1917 }
1918
1919 static int vhost_scsi_open(struct inode *inode, struct file *f)
1920 {
1921         struct vhost_scsi_virtqueue *svq;
1922         struct vhost_scsi *vs;
1923         struct vhost_virtqueue **vqs;
1924         int r = -ENOMEM, i, nvqs = vhost_scsi_max_io_vqs;
1925
1926         vs = kvzalloc(sizeof(*vs), GFP_KERNEL);
1927         if (!vs)
1928                 goto err_vs;
1929
1930         if (nvqs > VHOST_SCSI_MAX_IO_VQ) {
1931                 pr_err("Invalid max_io_vqs of %d. Using %d.\n", nvqs,
1932                        VHOST_SCSI_MAX_IO_VQ);
1933                 nvqs = VHOST_SCSI_MAX_IO_VQ;
1934         } else if (nvqs == 0) {
1935                 pr_err("Invalid max_io_vqs of %d. Using 1.\n", nvqs);
1936                 nvqs = 1;
1937         }
1938         nvqs += VHOST_SCSI_VQ_IO;
1939
1940         vs->old_inflight = kmalloc_array(nvqs, sizeof(*vs->old_inflight),
1941                                          GFP_KERNEL | __GFP_ZERO);
1942         if (!vs->old_inflight)
1943                 goto err_inflight;
1944
1945         vs->vqs = kmalloc_array(nvqs, sizeof(*vs->vqs),
1946                                 GFP_KERNEL | __GFP_ZERO);
1947         if (!vs->vqs)
1948                 goto err_vqs;
1949
1950         vqs = kmalloc_array(nvqs, sizeof(*vqs), GFP_KERNEL);
1951         if (!vqs)
1952                 goto err_local_vqs;
1953
1954         vhost_work_init(&vs->vs_event_work, vhost_scsi_evt_work);
1955
1956         vs->vs_events_nr = 0;
1957         vs->vs_events_missed = false;
1958
1959         vqs[VHOST_SCSI_VQ_CTL] = &vs->vqs[VHOST_SCSI_VQ_CTL].vq;
1960         vqs[VHOST_SCSI_VQ_EVT] = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;
1961         vs->vqs[VHOST_SCSI_VQ_CTL].vq.handle_kick = vhost_scsi_ctl_handle_kick;
1962         vs->vqs[VHOST_SCSI_VQ_EVT].vq.handle_kick = vhost_scsi_evt_handle_kick;
1963         for (i = VHOST_SCSI_VQ_IO; i < nvqs; i++) {
1964                 svq = &vs->vqs[i];
1965
1966                 vqs[i] = &svq->vq;
1967                 svq->vs = vs;
1968                 init_llist_head(&svq->completion_list);
1969                 vhost_work_init(&svq->completion_work,
1970                                 vhost_scsi_complete_cmd_work);
1971                 svq->vq.handle_kick = vhost_scsi_handle_kick;
1972         }
1973         vhost_dev_init(&vs->dev, vqs, nvqs, UIO_MAXIOV,
1974                        VHOST_SCSI_WEIGHT, 0, true, NULL);
1975
1976         vhost_scsi_init_inflight(vs, NULL);
1977
1978         f->private_data = vs;
1979         return 0;
1980
1981 err_local_vqs:
1982         kfree(vs->vqs);
1983 err_vqs:
1984         kfree(vs->old_inflight);
1985 err_inflight:
1986         kvfree(vs);
1987 err_vs:
1988         return r;
1989 }
1990
1991 static int vhost_scsi_release(struct inode *inode, struct file *f)
1992 {
1993         struct vhost_scsi *vs = f->private_data;
1994         struct vhost_scsi_target t;
1995
1996         mutex_lock(&vs->dev.mutex);
1997         memcpy(t.vhost_wwpn, vs->vs_vhost_wwpn, sizeof(t.vhost_wwpn));
1998         mutex_unlock(&vs->dev.mutex);
1999         vhost_scsi_clear_endpoint(vs, &t);
2000         vhost_dev_stop(&vs->dev);
2001         vhost_dev_cleanup(&vs->dev);
2002         kfree(vs->dev.vqs);
2003         kfree(vs->vqs);
2004         kfree(vs->old_inflight);
2005         kvfree(vs);
2006         return 0;
2007 }
2008
2009 static long
2010 vhost_scsi_ioctl(struct file *f,
2011                  unsigned int ioctl,
2012                  unsigned long arg)
2013 {
2014         struct vhost_scsi *vs = f->private_data;
2015         struct vhost_scsi_target backend;
2016         void __user *argp = (void __user *)arg;
2017         u64 __user *featurep = argp;
2018         u32 __user *eventsp = argp;
2019         u32 events_missed;
2020         u64 features;
2021         int r, abi_version = VHOST_SCSI_ABI_VERSION;
2022         struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;
2023
2024         switch (ioctl) {
2025         case VHOST_SCSI_SET_ENDPOINT:
2026                 if (copy_from_user(&backend, argp, sizeof backend))
2027                         return -EFAULT;
2028                 if (backend.reserved != 0)
2029                         return -EOPNOTSUPP;
2030
2031                 return vhost_scsi_set_endpoint(vs, &backend);
2032         case VHOST_SCSI_CLEAR_ENDPOINT:
2033                 if (copy_from_user(&backend, argp, sizeof backend))
2034                         return -EFAULT;
2035                 if (backend.reserved != 0)
2036                         return -EOPNOTSUPP;
2037
2038                 return vhost_scsi_clear_endpoint(vs, &backend);
2039         case VHOST_SCSI_GET_ABI_VERSION:
2040                 if (copy_to_user(argp, &abi_version, sizeof abi_version))
2041                         return -EFAULT;
2042                 return 0;
2043         case VHOST_SCSI_SET_EVENTS_MISSED:
2044                 if (get_user(events_missed, eventsp))
2045                         return -EFAULT;
2046                 mutex_lock(&vq->mutex);
2047                 vs->vs_events_missed = events_missed;
2048                 mutex_unlock(&vq->mutex);
2049                 return 0;
2050         case VHOST_SCSI_GET_EVENTS_MISSED:
2051                 mutex_lock(&vq->mutex);
2052                 events_missed = vs->vs_events_missed;
2053                 mutex_unlock(&vq->mutex);
2054                 if (put_user(events_missed, eventsp))
2055                         return -EFAULT;
2056                 return 0;
2057         case VHOST_GET_FEATURES:
2058                 features = VHOST_SCSI_FEATURES;
2059                 if (copy_to_user(featurep, &features, sizeof features))
2060                         return -EFAULT;
2061                 return 0;
2062         case VHOST_SET_FEATURES:
2063                 if (copy_from_user(&features, featurep, sizeof features))
2064                         return -EFAULT;
2065                 return vhost_scsi_set_features(vs, features);
2066         case VHOST_NEW_WORKER:
2067         case VHOST_FREE_WORKER:
2068         case VHOST_ATTACH_VRING_WORKER:
2069         case VHOST_GET_VRING_WORKER:
2070                 mutex_lock(&vs->dev.mutex);
2071                 r = vhost_worker_ioctl(&vs->dev, ioctl, argp);
2072                 mutex_unlock(&vs->dev.mutex);
2073                 return r;
2074         default:
2075                 mutex_lock(&vs->dev.mutex);
2076                 r = vhost_dev_ioctl(&vs->dev, ioctl, argp);
2077                 /* TODO: flush backend after dev ioctl. */
2078                 if (r == -ENOIOCTLCMD)
2079                         r = vhost_vring_ioctl(&vs->dev, ioctl, argp);
2080                 mutex_unlock(&vs->dev.mutex);
2081                 return r;
2082         }
2083 }
2084
2085 static const struct file_operations vhost_scsi_fops = {
2086         .owner          = THIS_MODULE,
2087         .release        = vhost_scsi_release,
2088         .unlocked_ioctl = vhost_scsi_ioctl,
2089         .compat_ioctl   = compat_ptr_ioctl,
2090         .open           = vhost_scsi_open,
2091         .llseek         = noop_llseek,
2092 };
2093
2094 static struct miscdevice vhost_scsi_misc = {
2095         MISC_DYNAMIC_MINOR,
2096         "vhost-scsi",
2097         &vhost_scsi_fops,
2098 };
2099
2100 static int __init vhost_scsi_register(void)
2101 {
2102         return misc_register(&vhost_scsi_misc);
2103 }
2104
2105 static void vhost_scsi_deregister(void)
2106 {
2107         misc_deregister(&vhost_scsi_misc);
2108 }
2109
2110 static char *vhost_scsi_dump_proto_id(struct vhost_scsi_tport *tport)
2111 {
2112         switch (tport->tport_proto_id) {
2113         case SCSI_PROTOCOL_SAS:
2114                 return "SAS";
2115         case SCSI_PROTOCOL_FCP:
2116                 return "FCP";
2117         case SCSI_PROTOCOL_ISCSI:
2118                 return "iSCSI";
2119         default:
2120                 break;
2121         }
2122
2123         return "Unknown";
2124 }
2125
2126 static void
2127 vhost_scsi_do_plug(struct vhost_scsi_tpg *tpg,
2128                   struct se_lun *lun, bool plug)
2129 {
2130
2131         struct vhost_scsi *vs = tpg->vhost_scsi;
2132         struct vhost_virtqueue *vq;
2133         u32 reason;
2134
2135         if (!vs)
2136                 return;
2137
2138         if (plug)
2139                 reason = VIRTIO_SCSI_EVT_RESET_RESCAN;
2140         else
2141                 reason = VIRTIO_SCSI_EVT_RESET_REMOVED;
2142
2143         vq = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;
2144         mutex_lock(&vq->mutex);
2145         /*
2146          * We can't queue events if the backend has been cleared, because
2147          * we could end up queueing an event after the flush.
2148          */
2149         if (!vhost_vq_get_backend(vq))
2150                 goto unlock;
2151
2152         if (vhost_has_feature(vq, VIRTIO_SCSI_F_HOTPLUG))
2153                 vhost_scsi_send_evt(vs, vq, tpg, lun,
2154                                    VIRTIO_SCSI_T_TRANSPORT_RESET, reason);
2155 unlock:
2156         mutex_unlock(&vq->mutex);
2157 }
2158
2159 static void vhost_scsi_hotplug(struct vhost_scsi_tpg *tpg, struct se_lun *lun)
2160 {
2161         vhost_scsi_do_plug(tpg, lun, true);
2162 }
2163
2164 static void vhost_scsi_hotunplug(struct vhost_scsi_tpg *tpg, struct se_lun *lun)
2165 {
2166         vhost_scsi_do_plug(tpg, lun, false);
2167 }
2168
2169 static int vhost_scsi_port_link(struct se_portal_group *se_tpg,
2170                                struct se_lun *lun)
2171 {
2172         struct vhost_scsi_tpg *tpg = container_of(se_tpg,
2173                                 struct vhost_scsi_tpg, se_tpg);
2174
2175         mutex_lock(&tpg->tv_tpg_mutex);
2176         tpg->tv_tpg_port_count++;
2177         vhost_scsi_hotplug(tpg, lun);
2178         mutex_unlock(&tpg->tv_tpg_mutex);
2179
2180         return 0;
2181 }
2182
2183 static void vhost_scsi_port_unlink(struct se_portal_group *se_tpg,
2184                                   struct se_lun *lun)
2185 {
2186         struct vhost_scsi_tpg *tpg = container_of(se_tpg,
2187                                 struct vhost_scsi_tpg, se_tpg);
2188
2189         mutex_lock(&tpg->tv_tpg_mutex);
2190         tpg->tv_tpg_port_count--;
2191         vhost_scsi_hotunplug(tpg, lun);
2192         mutex_unlock(&tpg->tv_tpg_mutex);
2193 }
2194
2195 static ssize_t vhost_scsi_tpg_attrib_fabric_prot_type_store(
2196                 struct config_item *item, const char *page, size_t count)
2197 {
2198         struct se_portal_group *se_tpg = attrib_to_tpg(item);
2199         struct vhost_scsi_tpg *tpg = container_of(se_tpg,
2200                                 struct vhost_scsi_tpg, se_tpg);
2201         unsigned long val;
2202         int ret = kstrtoul(page, 0, &val);
2203
2204         if (ret) {
2205                 pr_err("kstrtoul() returned %d for fabric_prot_type\n", ret);
2206                 return ret;
2207         }
2208         if (val != 0 && val != 1 && val != 3) {
2209                 pr_err("Invalid vhost_scsi fabric_prot_type: %lu\n", val);
2210                 return -EINVAL;
2211         }
2212         tpg->tv_fabric_prot_type = val;
2213
2214         return count;
2215 }
2216
2217 static ssize_t vhost_scsi_tpg_attrib_fabric_prot_type_show(
2218                 struct config_item *item, char *page)
2219 {
2220         struct se_portal_group *se_tpg = attrib_to_tpg(item);
2221         struct vhost_scsi_tpg *tpg = container_of(se_tpg,
2222                                 struct vhost_scsi_tpg, se_tpg);
2223
2224         return sysfs_emit(page, "%d\n", tpg->tv_fabric_prot_type);
2225 }
2226
2227 CONFIGFS_ATTR(vhost_scsi_tpg_attrib_, fabric_prot_type);
2228
2229 static struct configfs_attribute *vhost_scsi_tpg_attrib_attrs[] = {
2230         &vhost_scsi_tpg_attrib_attr_fabric_prot_type,
2231         NULL,
2232 };
2233
2234 static int vhost_scsi_make_nexus(struct vhost_scsi_tpg *tpg,
2235                                 const char *name)
2236 {
2237         struct vhost_scsi_nexus *tv_nexus;
2238
2239         mutex_lock(&tpg->tv_tpg_mutex);
2240         if (tpg->tpg_nexus) {
2241                 mutex_unlock(&tpg->tv_tpg_mutex);
2242                 pr_debug("tpg->tpg_nexus already exists\n");
2243                 return -EEXIST;
2244         }
2245
2246         tv_nexus = kzalloc(sizeof(*tv_nexus), GFP_KERNEL);
2247         if (!tv_nexus) {
2248                 mutex_unlock(&tpg->tv_tpg_mutex);
2249                 pr_err("Unable to allocate struct vhost_scsi_nexus\n");
2250                 return -ENOMEM;
2251         }
2252         /*
2253          * Since we are running in 'demo mode' this call with generate a
2254          * struct se_node_acl for the vhost_scsi struct se_portal_group with
2255          * the SCSI Initiator port name of the passed configfs group 'name'.
2256          */
2257         tv_nexus->tvn_se_sess = target_setup_session(&tpg->se_tpg, 0, 0,
2258                                         TARGET_PROT_DIN_PASS | TARGET_PROT_DOUT_PASS,
2259                                         (unsigned char *)name, tv_nexus, NULL);
2260         if (IS_ERR(tv_nexus->tvn_se_sess)) {
2261                 mutex_unlock(&tpg->tv_tpg_mutex);
2262                 kfree(tv_nexus);
2263                 return -ENOMEM;
2264         }
2265         tpg->tpg_nexus = tv_nexus;
2266
2267         mutex_unlock(&tpg->tv_tpg_mutex);
2268         return 0;
2269 }
2270
2271 static int vhost_scsi_drop_nexus(struct vhost_scsi_tpg *tpg)
2272 {
2273         struct se_session *se_sess;
2274         struct vhost_scsi_nexus *tv_nexus;
2275
2276         mutex_lock(&tpg->tv_tpg_mutex);
2277         tv_nexus = tpg->tpg_nexus;
2278         if (!tv_nexus) {
2279                 mutex_unlock(&tpg->tv_tpg_mutex);
2280                 return -ENODEV;
2281         }
2282
2283         se_sess = tv_nexus->tvn_se_sess;
2284         if (!se_sess) {
2285                 mutex_unlock(&tpg->tv_tpg_mutex);
2286                 return -ENODEV;
2287         }
2288
2289         if (tpg->tv_tpg_port_count != 0) {
2290                 mutex_unlock(&tpg->tv_tpg_mutex);
2291                 pr_err("Unable to remove TCM_vhost I_T Nexus with"
2292                         " active TPG port count: %d\n",
2293                         tpg->tv_tpg_port_count);
2294                 return -EBUSY;
2295         }
2296
2297         if (tpg->tv_tpg_vhost_count != 0) {
2298                 mutex_unlock(&tpg->tv_tpg_mutex);
2299                 pr_err("Unable to remove TCM_vhost I_T Nexus with"
2300                         " active TPG vhost count: %d\n",
2301                         tpg->tv_tpg_vhost_count);
2302                 return -EBUSY;
2303         }
2304
2305         pr_debug("TCM_vhost_ConfigFS: Removing I_T Nexus to emulated"
2306                 " %s Initiator Port: %s\n", vhost_scsi_dump_proto_id(tpg->tport),
2307                 tv_nexus->tvn_se_sess->se_node_acl->initiatorname);
2308
2309         /*
2310          * Release the SCSI I_T Nexus to the emulated vhost Target Port
2311          */
2312         target_remove_session(se_sess);
2313         tpg->tpg_nexus = NULL;
2314         mutex_unlock(&tpg->tv_tpg_mutex);
2315
2316         kfree(tv_nexus);
2317         return 0;
2318 }
2319
2320 static ssize_t vhost_scsi_tpg_nexus_show(struct config_item *item, char *page)
2321 {
2322         struct se_portal_group *se_tpg = to_tpg(item);
2323         struct vhost_scsi_tpg *tpg = container_of(se_tpg,
2324                                 struct vhost_scsi_tpg, se_tpg);
2325         struct vhost_scsi_nexus *tv_nexus;
2326         ssize_t ret;
2327
2328         mutex_lock(&tpg->tv_tpg_mutex);
2329         tv_nexus = tpg->tpg_nexus;
2330         if (!tv_nexus) {
2331                 mutex_unlock(&tpg->tv_tpg_mutex);
2332                 return -ENODEV;
2333         }
2334         ret = sysfs_emit(page, "%s\n",
2335                         tv_nexus->tvn_se_sess->se_node_acl->initiatorname);
2336         mutex_unlock(&tpg->tv_tpg_mutex);
2337
2338         return ret;
2339 }
2340
2341 static ssize_t vhost_scsi_tpg_nexus_store(struct config_item *item,
2342                 const char *page, size_t count)
2343 {
2344         struct se_portal_group *se_tpg = to_tpg(item);
2345         struct vhost_scsi_tpg *tpg = container_of(se_tpg,
2346                                 struct vhost_scsi_tpg, se_tpg);
2347         struct vhost_scsi_tport *tport_wwn = tpg->tport;
2348         unsigned char i_port[VHOST_SCSI_NAMELEN], *ptr, *port_ptr;
2349         int ret;
2350         /*
2351          * Shutdown the active I_T nexus if 'NULL' is passed..
2352          */
2353         if (!strncmp(page, "NULL", 4)) {
2354                 ret = vhost_scsi_drop_nexus(tpg);
2355                 return (!ret) ? count : ret;
2356         }
2357         /*
2358          * Otherwise make sure the passed virtual Initiator port WWN matches
2359          * the fabric protocol_id set in vhost_scsi_make_tport(), and call
2360          * vhost_scsi_make_nexus().
2361          */
2362         if (strlen(page) >= VHOST_SCSI_NAMELEN) {
2363                 pr_err("Emulated NAA Sas Address: %s, exceeds"
2364                                 " max: %d\n", page, VHOST_SCSI_NAMELEN);
2365                 return -EINVAL;
2366         }
2367         snprintf(&i_port[0], VHOST_SCSI_NAMELEN, "%s", page);
2368
2369         ptr = strstr(i_port, "naa.");
2370         if (ptr) {
2371                 if (tport_wwn->tport_proto_id != SCSI_PROTOCOL_SAS) {
2372                         pr_err("Passed SAS Initiator Port %s does not"
2373                                 " match target port protoid: %s\n", i_port,
2374                                 vhost_scsi_dump_proto_id(tport_wwn));
2375                         return -EINVAL;
2376                 }
2377                 port_ptr = &i_port[0];
2378                 goto check_newline;
2379         }
2380         ptr = strstr(i_port, "fc.");
2381         if (ptr) {
2382                 if (tport_wwn->tport_proto_id != SCSI_PROTOCOL_FCP) {
2383                         pr_err("Passed FCP Initiator Port %s does not"
2384                                 " match target port protoid: %s\n", i_port,
2385                                 vhost_scsi_dump_proto_id(tport_wwn));
2386                         return -EINVAL;
2387                 }
2388                 port_ptr = &i_port[3]; /* Skip over "fc." */
2389                 goto check_newline;
2390         }
2391         ptr = strstr(i_port, "iqn.");
2392         if (ptr) {
2393                 if (tport_wwn->tport_proto_id != SCSI_PROTOCOL_ISCSI) {
2394                         pr_err("Passed iSCSI Initiator Port %s does not"
2395                                 " match target port protoid: %s\n", i_port,
2396                                 vhost_scsi_dump_proto_id(tport_wwn));
2397                         return -EINVAL;
2398                 }
2399                 port_ptr = &i_port[0];
2400                 goto check_newline;
2401         }
2402         pr_err("Unable to locate prefix for emulated Initiator Port:"
2403                         " %s\n", i_port);
2404         return -EINVAL;
2405         /*
2406          * Clear any trailing newline for the NAA WWN
2407          */
2408 check_newline:
2409         if (i_port[strlen(i_port)-1] == '\n')
2410                 i_port[strlen(i_port)-1] = '\0';
2411
2412         ret = vhost_scsi_make_nexus(tpg, port_ptr);
2413         if (ret < 0)
2414                 return ret;
2415
2416         return count;
2417 }
2418
2419 CONFIGFS_ATTR(vhost_scsi_tpg_, nexus);
2420
2421 static struct configfs_attribute *vhost_scsi_tpg_attrs[] = {
2422         &vhost_scsi_tpg_attr_nexus,
2423         NULL,
2424 };
2425
2426 static struct se_portal_group *
2427 vhost_scsi_make_tpg(struct se_wwn *wwn, const char *name)
2428 {
2429         struct vhost_scsi_tport *tport = container_of(wwn,
2430                         struct vhost_scsi_tport, tport_wwn);
2431
2432         struct vhost_scsi_tpg *tpg;
2433         u16 tpgt;
2434         int ret;
2435
2436         if (strstr(name, "tpgt_") != name)
2437                 return ERR_PTR(-EINVAL);
2438         if (kstrtou16(name + 5, 10, &tpgt) || tpgt >= VHOST_SCSI_MAX_TARGET)
2439                 return ERR_PTR(-EINVAL);
2440
2441         tpg = kzalloc(sizeof(*tpg), GFP_KERNEL);
2442         if (!tpg) {
2443                 pr_err("Unable to allocate struct vhost_scsi_tpg");
2444                 return ERR_PTR(-ENOMEM);
2445         }
2446         mutex_init(&tpg->tv_tpg_mutex);
2447         INIT_LIST_HEAD(&tpg->tv_tpg_list);
2448         tpg->tport = tport;
2449         tpg->tport_tpgt = tpgt;
2450
2451         ret = core_tpg_register(wwn, &tpg->se_tpg, tport->tport_proto_id);
2452         if (ret < 0) {
2453                 kfree(tpg);
2454                 return NULL;
2455         }
2456         mutex_lock(&vhost_scsi_mutex);
2457         list_add_tail(&tpg->tv_tpg_list, &vhost_scsi_list);
2458         mutex_unlock(&vhost_scsi_mutex);
2459
2460         return &tpg->se_tpg;
2461 }
2462
2463 static void vhost_scsi_drop_tpg(struct se_portal_group *se_tpg)
2464 {
2465         struct vhost_scsi_tpg *tpg = container_of(se_tpg,
2466                                 struct vhost_scsi_tpg, se_tpg);
2467
2468         mutex_lock(&vhost_scsi_mutex);
2469         list_del(&tpg->tv_tpg_list);
2470         mutex_unlock(&vhost_scsi_mutex);
2471         /*
2472          * Release the virtual I_T Nexus for this vhost TPG
2473          */
2474         vhost_scsi_drop_nexus(tpg);
2475         /*
2476          * Deregister the se_tpg from TCM..
2477          */
2478         core_tpg_deregister(se_tpg);
2479         kfree(tpg);
2480 }
2481
2482 static struct se_wwn *
2483 vhost_scsi_make_tport(struct target_fabric_configfs *tf,
2484                      struct config_group *group,
2485                      const char *name)
2486 {
2487         struct vhost_scsi_tport *tport;
2488         char *ptr;
2489         u64 wwpn = 0;
2490         int off = 0;
2491
2492         /* if (vhost_scsi_parse_wwn(name, &wwpn, 1) < 0)
2493                 return ERR_PTR(-EINVAL); */
2494
2495         tport = kzalloc(sizeof(*tport), GFP_KERNEL);
2496         if (!tport) {
2497                 pr_err("Unable to allocate struct vhost_scsi_tport");
2498                 return ERR_PTR(-ENOMEM);
2499         }
2500         tport->tport_wwpn = wwpn;
2501         /*
2502          * Determine the emulated Protocol Identifier and Target Port Name
2503          * based on the incoming configfs directory name.
2504          */
2505         ptr = strstr(name, "naa.");
2506         if (ptr) {
2507                 tport->tport_proto_id = SCSI_PROTOCOL_SAS;
2508                 goto check_len;
2509         }
2510         ptr = strstr(name, "fc.");
2511         if (ptr) {
2512                 tport->tport_proto_id = SCSI_PROTOCOL_FCP;
2513                 off = 3; /* Skip over "fc." */
2514                 goto check_len;
2515         }
2516         ptr = strstr(name, "iqn.");
2517         if (ptr) {
2518                 tport->tport_proto_id = SCSI_PROTOCOL_ISCSI;
2519                 goto check_len;
2520         }
2521
2522         pr_err("Unable to locate prefix for emulated Target Port:"
2523                         " %s\n", name);
2524         kfree(tport);
2525         return ERR_PTR(-EINVAL);
2526
2527 check_len:
2528         if (strlen(name) >= VHOST_SCSI_NAMELEN) {
2529                 pr_err("Emulated %s Address: %s, exceeds"
2530                         " max: %d\n", name, vhost_scsi_dump_proto_id(tport),
2531                         VHOST_SCSI_NAMELEN);
2532                 kfree(tport);
2533                 return ERR_PTR(-EINVAL);
2534         }
2535         snprintf(&tport->tport_name[0], VHOST_SCSI_NAMELEN, "%s", &name[off]);
2536
2537         pr_debug("TCM_VHost_ConfigFS: Allocated emulated Target"
2538                 " %s Address: %s\n", vhost_scsi_dump_proto_id(tport), name);
2539
2540         return &tport->tport_wwn;
2541 }
2542
2543 static void vhost_scsi_drop_tport(struct se_wwn *wwn)
2544 {
2545         struct vhost_scsi_tport *tport = container_of(wwn,
2546                                 struct vhost_scsi_tport, tport_wwn);
2547
2548         pr_debug("TCM_VHost_ConfigFS: Deallocating emulated Target"
2549                 " %s Address: %s\n", vhost_scsi_dump_proto_id(tport),
2550                 tport->tport_name);
2551
2552         kfree(tport);
2553 }
2554
2555 static ssize_t
2556 vhost_scsi_wwn_version_show(struct config_item *item, char *page)
2557 {
2558         return sysfs_emit(page, "TCM_VHOST fabric module %s on %s/%s"
2559                 "on "UTS_RELEASE"\n", VHOST_SCSI_VERSION, utsname()->sysname,
2560                 utsname()->machine);
2561 }
2562
2563 CONFIGFS_ATTR_RO(vhost_scsi_wwn_, version);
2564
2565 static struct configfs_attribute *vhost_scsi_wwn_attrs[] = {
2566         &vhost_scsi_wwn_attr_version,
2567         NULL,
2568 };
2569
2570 static const struct target_core_fabric_ops vhost_scsi_ops = {
2571         .module                         = THIS_MODULE,
2572         .fabric_name                    = "vhost",
2573         .max_data_sg_nents              = VHOST_SCSI_PREALLOC_SGLS,
2574         .tpg_get_wwn                    = vhost_scsi_get_fabric_wwn,
2575         .tpg_get_tag                    = vhost_scsi_get_tpgt,
2576         .tpg_check_demo_mode            = vhost_scsi_check_true,
2577         .tpg_check_demo_mode_cache      = vhost_scsi_check_true,
2578         .tpg_check_prot_fabric_only     = vhost_scsi_check_prot_fabric_only,
2579         .release_cmd                    = vhost_scsi_release_cmd,
2580         .check_stop_free                = vhost_scsi_check_stop_free,
2581         .sess_get_initiator_sid         = NULL,
2582         .write_pending                  = vhost_scsi_write_pending,
2583         .queue_data_in                  = vhost_scsi_queue_data_in,
2584         .queue_status                   = vhost_scsi_queue_status,
2585         .queue_tm_rsp                   = vhost_scsi_queue_tm_rsp,
2586         .aborted_task                   = vhost_scsi_aborted_task,
2587         /*
2588          * Setup callers for generic logic in target_core_fabric_configfs.c
2589          */
2590         .fabric_make_wwn                = vhost_scsi_make_tport,
2591         .fabric_drop_wwn                = vhost_scsi_drop_tport,
2592         .fabric_make_tpg                = vhost_scsi_make_tpg,
2593         .fabric_drop_tpg                = vhost_scsi_drop_tpg,
2594         .fabric_post_link               = vhost_scsi_port_link,
2595         .fabric_pre_unlink              = vhost_scsi_port_unlink,
2596
2597         .tfc_wwn_attrs                  = vhost_scsi_wwn_attrs,
2598         .tfc_tpg_base_attrs             = vhost_scsi_tpg_attrs,
2599         .tfc_tpg_attrib_attrs           = vhost_scsi_tpg_attrib_attrs,
2600 };
2601
2602 static int __init vhost_scsi_init(void)
2603 {
2604         int ret = -ENOMEM;
2605
2606         pr_debug("TCM_VHOST fabric module %s on %s/%s"
2607                 " on "UTS_RELEASE"\n", VHOST_SCSI_VERSION, utsname()->sysname,
2608                 utsname()->machine);
2609
2610         ret = vhost_scsi_register();
2611         if (ret < 0)
2612                 goto out;
2613
2614         ret = target_register_template(&vhost_scsi_ops);
2615         if (ret < 0)
2616                 goto out_vhost_scsi_deregister;
2617
2618         return 0;
2619
2620 out_vhost_scsi_deregister:
2621         vhost_scsi_deregister();
2622 out:
2623         return ret;
2624 };
2625
2626 static void vhost_scsi_exit(void)
2627 {
2628         target_unregister_template(&vhost_scsi_ops);
2629         vhost_scsi_deregister();
2630 };
2631
2632 MODULE_DESCRIPTION("VHOST_SCSI series fabric driver");
2633 MODULE_ALIAS("tcm_vhost");
2634 MODULE_LICENSE("GPL");
2635 module_init(vhost_scsi_init);
2636 module_exit(vhost_scsi_exit);