drivers/vhost/scsi.c

   1 /*******************************************************************************
   2  * Vhost kernel TCM fabric driver for virtio SCSI initiators
   3  *
   4  * (C) Copyright 2010-2013 Datera, Inc.
   5  * (C) Copyright 2010-2012 IBM Corp.
   6  *
   7  * Licensed to the Linux Foundation under the General Public License (GPL) version 2.
   8  *
   9  * Authors: Nicholas A. Bellinger <nab@daterainc.com>
  10  *          Stefan Hajnoczi <stefanha@linux.vnet.ibm.com>
  11  *
  12  * This program is free software; you can redistribute it and/or modify
  13  * it under the terms of the GNU General Public License as published by
  14  * the Free Software Foundation; either version 2 of the License, or
  15  * (at your option) any later version.
  16  *
  17  * This program is distributed in the hope that it will be useful,
  18  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  19  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  20  * GNU General Public License for more details.
  21  *
  22  ****************************************************************************/
  23
  24 #include <linux/module.h>
  25 #include <linux/moduleparam.h>
  26 #include <generated/utsrelease.h>
  27 #include <linux/utsname.h>
  28 #include <linux/init.h>
  29 #include <linux/slab.h>
  30 #include <linux/kthread.h>
  31 #include <linux/types.h>
  32 #include <linux/string.h>
  33 #include <linux/configfs.h>
  34 #include <linux/ctype.h>
  35 #include <linux/compat.h>
  36 #include <linux/eventfd.h>
  37 #include <linux/fs.h>
  38 #include <linux/vmalloc.h>
  39 #include <linux/miscdevice.h>
  40 #include <asm/unaligned.h>
  41 #include <scsi/scsi_common.h>
  42 #include <scsi/scsi_proto.h>
  43 #include <target/target_core_base.h>
  44 #include <target/target_core_fabric.h>
  45 #include <linux/vhost.h>
  46 #include <linux/virtio_scsi.h>
  47 #include <linux/llist.h>
  48 #include <linux/bitmap.h>
  49
  50 #include "vhost.h"
  51
  52 #define VHOST_SCSI_VERSION  "v0.1"
  53 #define VHOST_SCSI_NAMELEN 256
  54 #define VHOST_SCSI_MAX_CDB_SIZE 32
  55 #define VHOST_SCSI_DEFAULT_TAGS 256
  56 #define VHOST_SCSI_PREALLOC_SGLS 2048
  57 #define VHOST_SCSI_PREALLOC_UPAGES 2048
  58 #define VHOST_SCSI_PREALLOC_PROT_SGLS 2048
  59
  60 struct vhost_scsi_inflight {
  61         /* Wait for the flush operation to finish */
  62         struct completion comp;
  63         /* Refcount for the inflight reqs */
  64         struct kref kref;
  65 };
  66
  67 struct vhost_scsi_cmd {
  68         /* Descriptor from vhost_get_vq_desc() for virt_queue segment */
  69         int tvc_vq_desc;
  70         /* virtio-scsi initiator task attribute */
  71         int tvc_task_attr;
  72         /* virtio-scsi response incoming iovecs */
  73         int tvc_in_iovs;
  74         /* virtio-scsi initiator data direction */
  75         enum dma_data_direction tvc_data_direction;
  76         /* Expected data transfer length from virtio-scsi header */
  77         u32 tvc_exp_data_len;
  78         /* The Tag from include/linux/virtio_scsi.h:struct virtio_scsi_cmd_req */
  79         u64 tvc_tag;
  80         /* The number of scatterlists associated with this cmd */
  81         u32 tvc_sgl_count;
  82         u32 tvc_prot_sgl_count;
  83         /* Saved unpacked SCSI LUN for vhost_scsi_submission_work() */
  84         u32 tvc_lun;
  85         /* Pointer to the SGL formatted memory from virtio-scsi */
  86         struct scatterlist *tvc_sgl;
  87         struct scatterlist *tvc_prot_sgl;
  88         struct page **tvc_upages;
  89         /* Pointer to response header iovec */
  90         struct iovec tvc_resp_iov;
  91         /* Pointer to vhost_scsi for our device */
  92         struct vhost_scsi *tvc_vhost;
  93         /* Pointer to vhost_virtqueue for the cmd */
  94         struct vhost_virtqueue *tvc_vq;
  95         /* Pointer to vhost nexus memory */
  96         struct vhost_scsi_nexus *tvc_nexus;
  97         /* The TCM I/O descriptor that is accessed via container_of() */
  98         struct se_cmd tvc_se_cmd;
  99         /* work item used for cmwq dispatch to vhost_scsi_submission_work() */
 100         struct work_struct work;
 101         /* Copy of the incoming SCSI command descriptor block (CDB) */
 102         unsigned char tvc_cdb[VHOST_SCSI_MAX_CDB_SIZE];
 103         /* Sense buffer that will be mapped into outgoing status */
 104         unsigned char tvc_sense_buf[TRANSPORT_SENSE_BUFFER];
 105         /* Completed commands list, serviced from vhost worker thread */
 106         struct llist_node tvc_completion_list;
 107         /* Used to track inflight cmd */
 108         struct vhost_scsi_inflight *inflight;
 109 };
 110
 111 struct vhost_scsi_nexus {
 112         /* Pointer to TCM session for I_T Nexus */
 113         struct se_session *tvn_se_sess;
 114 };
 115
 116 struct vhost_scsi_tpg {
 117         /* Vhost port target portal group tag for TCM */
 118         u16 tport_tpgt;
 119         /* Used to track number of TPG Port/Lun Links wrt to explict I_T Nexus shutdown */
 120         int tv_tpg_port_count;
 121         /* Used for vhost_scsi device reference to tpg_nexus, protected by tv_tpg_mutex */
 122         int tv_tpg_vhost_count;
 123         /* Used for enabling T10-PI with legacy devices */
 124         int tv_fabric_prot_type;
 125         /* list for vhost_scsi_list */
 126         struct list_head tv_tpg_list;
 127         /* Used to protect access for tpg_nexus */
 128         struct mutex tv_tpg_mutex;
 129         /* Pointer to the TCM VHost I_T Nexus for this TPG endpoint */
 130         struct vhost_scsi_nexus *tpg_nexus;
 131         /* Pointer back to vhost_scsi_tport */
 132         struct vhost_scsi_tport *tport;
 133         /* Returned by vhost_scsi_make_tpg() */
 134         struct se_portal_group se_tpg;
 135         /* Pointer back to vhost_scsi, protected by tv_tpg_mutex */
 136         struct vhost_scsi *vhost_scsi;
 137 };
 138
 139 struct vhost_scsi_tport {
 140         /* SCSI protocol the tport is providing */
 141         u8 tport_proto_id;
 142         /* Binary World Wide unique Port Name for Vhost Target port */
 143         u64 tport_wwpn;
 144         /* ASCII formatted WWPN for Vhost Target port */
 145         char tport_name[VHOST_SCSI_NAMELEN];
 146         /* Returned by vhost_scsi_make_tport() */
 147         struct se_wwn tport_wwn;
 148 };
 149
 150 struct vhost_scsi_evt {
 151         /* event to be sent to guest */
 152         struct virtio_scsi_event event;
 153         /* event list, serviced from vhost worker thread */
 154         struct llist_node list;
 155 };
 156
 157 enum {
 158         VHOST_SCSI_VQ_CTL = 0,
 159         VHOST_SCSI_VQ_EVT = 1,
 160         VHOST_SCSI_VQ_IO = 2,
 161 };
 162
 163 /* Note: can't set VIRTIO_F_VERSION_1 yet, since that implies ANY_LAYOUT. */
 164 enum {
 165         VHOST_SCSI_FEATURES = VHOST_FEATURES | (1ULL << VIRTIO_SCSI_F_HOTPLUG) |
 166                                                (1ULL << VIRTIO_SCSI_F_T10_PI)
 167 };
 168
 169 #define VHOST_SCSI_MAX_TARGET   256
 170 #define VHOST_SCSI_MAX_VQ       128
 171 #define VHOST_SCSI_MAX_EVENT    128
 172
 173 struct vhost_scsi_virtqueue {
 174         struct vhost_virtqueue vq;
 175         /*
 176          * Reference counting for inflight reqs, used for flush operation. At
 177          * each time, one reference tracks new commands submitted, while we
 178          * wait for another one to reach 0.
 179          */
 180         struct vhost_scsi_inflight inflights[2];
 181         /*
 182          * Indicate current inflight in use, protected by vq->mutex.
 183          * Writers must also take dev mutex and flush under it.
 184          */
 185         int inflight_idx;
 186 };
 187
 188 struct vhost_scsi {
 189         /* Protected by vhost_scsi->dev.mutex */
 190         struct vhost_scsi_tpg **vs_tpg;
 191         char vs_vhost_wwpn[TRANSPORT_IQN_LEN];
 192
 193         struct vhost_dev dev;
 194         struct vhost_scsi_virtqueue vqs[VHOST_SCSI_MAX_VQ];
 195
 196         struct vhost_work vs_completion_work; /* cmd completion work item */
 197         struct llist_head vs_completion_list; /* cmd completion queue */
 198
 199         struct vhost_work vs_event_work; /* evt injection work item */
 200         struct llist_head vs_event_list; /* evt injection queue */
 201
 202         bool vs_events_missed; /* any missed events, protected by vq->mutex */
 203         int vs_events_nr; /* num of pending events, protected by vq->mutex */
 204 };
 205
 206 /*
 207  * Context for processing request and control queue operations.
 208  */
 209 struct vhost_scsi_ctx {
 210         int head;
 211         unsigned int out, in;
 212         size_t req_size, rsp_size;
 213         size_t out_size, in_size;
 214         u8 *target, *lunp;
 215         void *req;
 216         struct iov_iter out_iter;
 217 };
 218
 219 static struct workqueue_struct *vhost_scsi_workqueue;
 220
 221 /* Global spinlock to protect vhost_scsi TPG list for vhost IOCTL access */
 222 static DEFINE_MUTEX(vhost_scsi_mutex);
 223 static LIST_HEAD(vhost_scsi_list);
 224
 225 static void vhost_scsi_done_inflight(struct kref *kref)
 226 {
 227         struct vhost_scsi_inflight *inflight;
 228
 229         inflight = container_of(kref, struct vhost_scsi_inflight, kref);
 230         complete(&inflight->comp);
 231 }
 232
 233 static void vhost_scsi_init_inflight(struct vhost_scsi *vs,
 234                                     struct vhost_scsi_inflight *old_inflight[])
 235 {
 236         struct vhost_scsi_inflight *new_inflight;
 237         struct vhost_virtqueue *vq;
 238         int idx, i;
 239
 240         for (i = 0; i < VHOST_SCSI_MAX_VQ; i++) {
 241                 vq = &vs->vqs[i].vq;
 242
 243                 mutex_lock(&vq->mutex);
 244
 245                 /* store old infight */
 246                 idx = vs->vqs[i].inflight_idx;
 247                 if (old_inflight)
 248                         old_inflight[i] = &vs->vqs[i].inflights[idx];
 249
 250                 /* setup new infight */
 251                 vs->vqs[i].inflight_idx = idx ^ 1;
 252                 new_inflight = &vs->vqs[i].inflights[idx ^ 1];
 253                 kref_init(&new_inflight->kref);
 254                 init_completion(&new_inflight->comp);
 255
 256                 mutex_unlock(&vq->mutex);
 257         }
 258 }
 259
 260 static struct vhost_scsi_inflight *
 261 vhost_scsi_get_inflight(struct vhost_virtqueue *vq)
 262 {
 263         struct vhost_scsi_inflight *inflight;
 264         struct vhost_scsi_virtqueue *svq;
 265
 266         svq = container_of(vq, struct vhost_scsi_virtqueue, vq);
 267         inflight = &svq->inflights[svq->inflight_idx];
 268         kref_get(&inflight->kref);
 269
 270         return inflight;
 271 }
 272
 273 static void vhost_scsi_put_inflight(struct vhost_scsi_inflight *inflight)
 274 {
 275         kref_put(&inflight->kref, vhost_scsi_done_inflight);
 276 }
 277
 278 static int vhost_scsi_check_true(struct se_portal_group *se_tpg)
 279 {
 280         return 1;
 281 }
 282
 283 static int vhost_scsi_check_false(struct se_portal_group *se_tpg)
 284 {
 285         return 0;
 286 }
 287
 288 static char *vhost_scsi_get_fabric_wwn(struct se_portal_group *se_tpg)
 289 {
 290         struct vhost_scsi_tpg *tpg = container_of(se_tpg,
 291                                 struct vhost_scsi_tpg, se_tpg);
 292         struct vhost_scsi_tport *tport = tpg->tport;
 293
 294         return &tport->tport_name[0];
 295 }
 296
 297 static u16 vhost_scsi_get_tpgt(struct se_portal_group *se_tpg)
 298 {
 299         struct vhost_scsi_tpg *tpg = container_of(se_tpg,
 300                                 struct vhost_scsi_tpg, se_tpg);
 301         return tpg->tport_tpgt;
 302 }
 303
 304 static int vhost_scsi_check_prot_fabric_only(struct se_portal_group *se_tpg)
 305 {
 306         struct vhost_scsi_tpg *tpg = container_of(se_tpg,
 307                                 struct vhost_scsi_tpg, se_tpg);
 308
 309         return tpg->tv_fabric_prot_type;
 310 }
 311
 312 static u32 vhost_scsi_tpg_get_inst_index(struct se_portal_group *se_tpg)
 313 {
 314         return 1;
 315 }
 316
 317 static void vhost_scsi_release_cmd(struct se_cmd *se_cmd)
 318 {
 319         struct vhost_scsi_cmd *tv_cmd = container_of(se_cmd,
 320                                 struct vhost_scsi_cmd, tvc_se_cmd);
 321         struct se_session *se_sess = tv_cmd->tvc_nexus->tvn_se_sess;
 322         int i;
 323
 324         if (tv_cmd->tvc_sgl_count) {
 325                 for (i = 0; i < tv_cmd->tvc_sgl_count; i++)
 326                         put_page(sg_page(&tv_cmd->tvc_sgl[i]));
 327         }
 328         if (tv_cmd->tvc_prot_sgl_count) {
 329                 for (i = 0; i < tv_cmd->tvc_prot_sgl_count; i++)
 330                         put_page(sg_page(&tv_cmd->tvc_prot_sgl[i]));
 331         }
 332
 333         vhost_scsi_put_inflight(tv_cmd->inflight);
 334         target_free_tag(se_sess, se_cmd);
 335 }
 336
 337 static u32 vhost_scsi_sess_get_index(struct se_session *se_sess)
 338 {
 339         return 0;
 340 }
 341
 342 static int vhost_scsi_write_pending(struct se_cmd *se_cmd)
 343 {
 344         /* Go ahead and process the write immediately */
 345         target_execute_cmd(se_cmd);
 346         return 0;
 347 }
 348
 349 static void vhost_scsi_set_default_node_attrs(struct se_node_acl *nacl)
 350 {
 351         return;
 352 }
 353
 354 static int vhost_scsi_get_cmd_state(struct se_cmd *se_cmd)
 355 {
 356         return 0;
 357 }
 358
 359 static void vhost_scsi_complete_cmd(struct vhost_scsi_cmd *cmd)
 360 {
 361         struct vhost_scsi *vs = cmd->tvc_vhost;
 362
 363         llist_add(&cmd->tvc_completion_list, &vs->vs_completion_list);
 364
 365         vhost_work_queue(&vs->dev, &vs->vs_completion_work);
 366 }
 367
 368 static int vhost_scsi_queue_data_in(struct se_cmd *se_cmd)
 369 {
 370         struct vhost_scsi_cmd *cmd = container_of(se_cmd,
 371                                 struct vhost_scsi_cmd, tvc_se_cmd);
 372         vhost_scsi_complete_cmd(cmd);
 373         return 0;
 374 }
 375
 376 static int vhost_scsi_queue_status(struct se_cmd *se_cmd)
 377 {
 378         struct vhost_scsi_cmd *cmd = container_of(se_cmd,
 379                                 struct vhost_scsi_cmd, tvc_se_cmd);
 380         vhost_scsi_complete_cmd(cmd);
 381         return 0;
 382 }
 383
 384 static void vhost_scsi_queue_tm_rsp(struct se_cmd *se_cmd)
 385 {
 386         return;
 387 }
 388
 389 static void vhost_scsi_aborted_task(struct se_cmd *se_cmd)
 390 {
 391         return;
 392 }
 393
 394 static void vhost_scsi_free_evt(struct vhost_scsi *vs, struct vhost_scsi_evt *evt)
 395 {
 396         vs->vs_events_nr--;
 397         kfree(evt);
 398 }
 399
 400 static struct vhost_scsi_evt *
 401 vhost_scsi_allocate_evt(struct vhost_scsi *vs,
 402                        u32 event, u32 reason)
 403 {
 404         struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;
 405         struct vhost_scsi_evt *evt;
 406
 407         if (vs->vs_events_nr > VHOST_SCSI_MAX_EVENT) {
 408                 vs->vs_events_missed = true;
 409                 return NULL;
 410         }
 411
 412         evt = kzalloc(sizeof(*evt), GFP_KERNEL);
 413         if (!evt) {
 414                 vq_err(vq, "Failed to allocate vhost_scsi_evt\n");
 415                 vs->vs_events_missed = true;
 416                 return NULL;
 417         }
 418
 419         evt->event.event = cpu_to_vhost32(vq, event);
 420         evt->event.reason = cpu_to_vhost32(vq, reason);
 421         vs->vs_events_nr++;
 422
 423         return evt;
 424 }
 425
 426 static void vhost_scsi_free_cmd(struct vhost_scsi_cmd *cmd)
 427 {
 428         struct se_cmd *se_cmd = &cmd->tvc_se_cmd;
 429
 430         /* TODO locking against target/backend threads? */
 431         transport_generic_free_cmd(se_cmd, 0);
 432
 433 }
 434
 435 static int vhost_scsi_check_stop_free(struct se_cmd *se_cmd)
 436 {
 437         return target_put_sess_cmd(se_cmd);
 438 }
 439
 440 static void
 441 vhost_scsi_do_evt_work(struct vhost_scsi *vs, struct vhost_scsi_evt *evt)
 442 {
 443         struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;
 444         struct virtio_scsi_event *event = &evt->event;
 445         struct virtio_scsi_event __user *eventp;
 446         unsigned out, in;
 447         int head, ret;
 448
 449         if (!vq->private_data) {
 450                 vs->vs_events_missed = true;
 451                 return;
 452         }
 453
 454 again:
 455         vhost_disable_notify(&vs->dev, vq);
 456         head = vhost_get_vq_desc(vq, vq->iov,
 457                         ARRAY_SIZE(vq->iov), &out, &in,
 458                         NULL, NULL);
 459         if (head < 0) {
 460                 vs->vs_events_missed = true;
 461                 return;
 462         }
 463         if (head == vq->num) {
 464                 if (vhost_enable_notify(&vs->dev, vq))
 465                         goto again;
 466                 vs->vs_events_missed = true;
 467                 return;
 468         }
 469
 470         if ((vq->iov[out].iov_len != sizeof(struct virtio_scsi_event))) {
 471                 vq_err(vq, "Expecting virtio_scsi_event, got %zu bytes\n",
 472                                 vq->iov[out].iov_len);
 473                 vs->vs_events_missed = true;
 474                 return;
 475         }
 476
 477         if (vs->vs_events_missed) {
 478                 event->event |= cpu_to_vhost32(vq, VIRTIO_SCSI_T_EVENTS_MISSED);
 479                 vs->vs_events_missed = false;
 480         }
 481
 482         eventp = vq->iov[out].iov_base;
 483         ret = __copy_to_user(eventp, event, sizeof(*event));
 484         if (!ret)
 485                 vhost_add_used_and_signal(&vs->dev, vq, head, 0);
 486         else
 487                 vq_err(vq, "Faulted on vhost_scsi_send_event\n");
 488 }
 489
 490 static void vhost_scsi_evt_work(struct vhost_work *work)
 491 {
 492         struct vhost_scsi *vs = container_of(work, struct vhost_scsi,
 493                                         vs_event_work);
 494         struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;
 495         struct vhost_scsi_evt *evt, *t;
 496         struct llist_node *llnode;
 497
 498         mutex_lock(&vq->mutex);
 499         llnode = llist_del_all(&vs->vs_event_list);
 500         llist_for_each_entry_safe(evt, t, llnode, list) {
 501                 vhost_scsi_do_evt_work(vs, evt);
 502                 vhost_scsi_free_evt(vs, evt);
 503         }
 504         mutex_unlock(&vq->mutex);
 505 }
 506
 507 /* Fill in status and signal that we are done processing this command
 508  *
 509  * This is scheduled in the vhost work queue so we are called with the owner
 510  * process mm and can access the vring.
 511  */
 512 static void vhost_scsi_complete_cmd_work(struct vhost_work *work)
 513 {
 514         struct vhost_scsi *vs = container_of(work, struct vhost_scsi,
 515                                         vs_completion_work);
 516         DECLARE_BITMAP(signal, VHOST_SCSI_MAX_VQ);
 517         struct virtio_scsi_cmd_resp v_rsp;
 518         struct vhost_scsi_cmd *cmd, *t;
 519         struct llist_node *llnode;
 520         struct se_cmd *se_cmd;
 521         struct iov_iter iov_iter;
 522         int ret, vq;
 523
 524         bitmap_zero(signal, VHOST_SCSI_MAX_VQ);
 525         llnode = llist_del_all(&vs->vs_completion_list);
 526         llist_for_each_entry_safe(cmd, t, llnode, tvc_completion_list) {
 527                 se_cmd = &cmd->tvc_se_cmd;
 528
 529                 pr_debug("%s tv_cmd %p resid %u status %#02x\n", __func__,
 530                         cmd, se_cmd->residual_count, se_cmd->scsi_status);
 531
 532                 memset(&v_rsp, 0, sizeof(v_rsp));
 533                 v_rsp.resid = cpu_to_vhost32(cmd->tvc_vq, se_cmd->residual_count);
 534                 /* TODO is status_qualifier field needed? */
 535                 v_rsp.status = se_cmd->scsi_status;
 536                 v_rsp.sense_len = cpu_to_vhost32(cmd->tvc_vq,
 537                                                  se_cmd->scsi_sense_length);
 538                 memcpy(v_rsp.sense, cmd->tvc_sense_buf,
 539                        se_cmd->scsi_sense_length);
 540
 541                 iov_iter_init(&iov_iter, READ, &cmd->tvc_resp_iov,
 542                               cmd->tvc_in_iovs, sizeof(v_rsp));
 543                 ret = copy_to_iter(&v_rsp, sizeof(v_rsp), &iov_iter);
 544                 if (likely(ret == sizeof(v_rsp))) {
 545                         struct vhost_scsi_virtqueue *q;
 546                         vhost_add_used(cmd->tvc_vq, cmd->tvc_vq_desc, 0);
 547                         q = container_of(cmd->tvc_vq, struct vhost_scsi_virtqueue, vq);
 548                         vq = q - vs->vqs;
 549                         __set_bit(vq, signal);
 550                 } else
 551                         pr_err("Faulted on virtio_scsi_cmd_resp\n");
 552
 553                 vhost_scsi_free_cmd(cmd);
 554         }
 555
 556         vq = -1;
 557         while ((vq = find_next_bit(signal, VHOST_SCSI_MAX_VQ, vq + 1))
 558                 < VHOST_SCSI_MAX_VQ)
 559                 vhost_signal(&vs->dev, &vs->vqs[vq].vq);
 560 }
 561
 562 static struct vhost_scsi_cmd *
 563 vhost_scsi_get_tag(struct vhost_virtqueue *vq, struct vhost_scsi_tpg *tpg,
 564                    unsigned char *cdb, u64 scsi_tag, u16 lun, u8 task_attr,
 565                    u32 exp_data_len, int data_direction)
 566 {
 567         struct vhost_scsi_cmd *cmd;
 568         struct vhost_scsi_nexus *tv_nexus;
 569         struct se_session *se_sess;
 570         struct scatterlist *sg, *prot_sg;
 571         struct page **pages;
 572         int tag, cpu;
 573
 574         tv_nexus = tpg->tpg_nexus;
 575         if (!tv_nexus) {
 576                 pr_err("Unable to locate active struct vhost_scsi_nexus\n");
 577                 return ERR_PTR(-EIO);
 578         }
 579         se_sess = tv_nexus->tvn_se_sess;
 580
 581         tag = sbitmap_queue_get(&se_sess->sess_tag_pool, &cpu);
 582         if (tag < 0) {
 583                 pr_err("Unable to obtain tag for vhost_scsi_cmd\n");
 584                 return ERR_PTR(-ENOMEM);
 585         }
 586
 587         cmd = &((struct vhost_scsi_cmd *)se_sess->sess_cmd_map)[tag];
 588         sg = cmd->tvc_sgl;
 589         prot_sg = cmd->tvc_prot_sgl;
 590         pages = cmd->tvc_upages;
 591         memset(cmd, 0, sizeof(*cmd));
 592         cmd->tvc_sgl = sg;
 593         cmd->tvc_prot_sgl = prot_sg;
 594         cmd->tvc_upages = pages;
 595         cmd->tvc_se_cmd.map_tag = tag;
 596         cmd->tvc_se_cmd.map_cpu = cpu;
 597         cmd->tvc_tag = scsi_tag;
 598         cmd->tvc_lun = lun;
 599         cmd->tvc_task_attr = task_attr;
 600         cmd->tvc_exp_data_len = exp_data_len;
 601         cmd->tvc_data_direction = data_direction;
 602         cmd->tvc_nexus = tv_nexus;
 603         cmd->inflight = vhost_scsi_get_inflight(vq);
 604
 605         memcpy(cmd->tvc_cdb, cdb, VHOST_SCSI_MAX_CDB_SIZE);
 606
 607         return cmd;
 608 }
 609
 610 /*
 611  * Map a user memory range into a scatterlist
 612  *
 613  * Returns the number of scatterlist entries used or -errno on error.
 614  */
 615 static int
 616 vhost_scsi_map_to_sgl(struct vhost_scsi_cmd *cmd,
 617                       struct iov_iter *iter,
 618                       struct scatterlist *sgl,
 619                       bool write)
 620 {
 621         struct page **pages = cmd->tvc_upages;
 622         struct scatterlist *sg = sgl;
 623         ssize_t bytes;
 624         size_t offset;
 625         unsigned int npages = 0;
 626
 627         bytes = iov_iter_get_pages(iter, pages, LONG_MAX,
 628                                 VHOST_SCSI_PREALLOC_UPAGES, &offset);
 629         /* No pages were pinned */
 630         if (bytes <= 0)
 631                 return bytes < 0 ? bytes : -EFAULT;
 632
 633         iov_iter_advance(iter, bytes);
 634
 635         while (bytes) {
 636                 unsigned n = min_t(unsigned, PAGE_SIZE - offset, bytes);
 637                 sg_set_page(sg++, pages[npages++], n, offset);
 638                 bytes -= n;
 639                 offset = 0;
 640         }
 641         return npages;
 642 }
 643
 644 static int
 645 vhost_scsi_calc_sgls(struct iov_iter *iter, size_t bytes, int max_sgls)
 646 {
 647         int sgl_count = 0;
 648
 649         if (!iter || !iter->iov) {
 650                 pr_err("%s: iter->iov is NULL, but expected bytes: %zu"
 651                        " present\n", __func__, bytes);
 652                 return -EINVAL;
 653         }
 654
 655         sgl_count = iov_iter_npages(iter, 0xffff);
 656         if (sgl_count > max_sgls) {
 657                 pr_err("%s: requested sgl_count: %d exceeds pre-allocated"
 658                        " max_sgls: %d\n", __func__, sgl_count, max_sgls);
 659                 return -EINVAL;
 660         }
 661         return sgl_count;
 662 }
 663
 664 static int
 665 vhost_scsi_iov_to_sgl(struct vhost_scsi_cmd *cmd, bool write,
 666                       struct iov_iter *iter,
 667                       struct scatterlist *sg, int sg_count)
 668 {
 669         struct scatterlist *p = sg;
 670         int ret;
 671
 672         while (iov_iter_count(iter)) {
 673                 ret = vhost_scsi_map_to_sgl(cmd, iter, sg, write);
 674                 if (ret < 0) {
 675                         while (p < sg) {
 676                                 struct page *page = sg_page(p++);
 677                                 if (page)
 678                                         put_page(page);
 679                         }
 680                         return ret;
 681                 }
 682                 sg += ret;
 683         }
 684         return 0;
 685 }
 686
 687 static int
 688 vhost_scsi_mapal(struct vhost_scsi_cmd *cmd,
 689                  size_t prot_bytes, struct iov_iter *prot_iter,
 690                  size_t data_bytes, struct iov_iter *data_iter)
 691 {
 692         int sgl_count, ret;
 693         bool write = (cmd->tvc_data_direction == DMA_FROM_DEVICE);
 694
 695         if (prot_bytes) {
 696                 sgl_count = vhost_scsi_calc_sgls(prot_iter, prot_bytes,
 697                                                  VHOST_SCSI_PREALLOC_PROT_SGLS);
 698                 if (sgl_count < 0)
 699                         return sgl_count;
 700
 701                 sg_init_table(cmd->tvc_prot_sgl, sgl_count);
 702                 cmd->tvc_prot_sgl_count = sgl_count;
 703                 pr_debug("%s prot_sg %p prot_sgl_count %u\n", __func__,
 704                          cmd->tvc_prot_sgl, cmd->tvc_prot_sgl_count);
 705
 706                 ret = vhost_scsi_iov_to_sgl(cmd, write, prot_iter,
 707                                             cmd->tvc_prot_sgl,
 708                                             cmd->tvc_prot_sgl_count);
 709                 if (ret < 0) {
 710                         cmd->tvc_prot_sgl_count = 0;
 711                         return ret;
 712                 }
 713         }
 714         sgl_count = vhost_scsi_calc_sgls(data_iter, data_bytes,
 715                                          VHOST_SCSI_PREALLOC_SGLS);
 716         if (sgl_count < 0)
 717                 return sgl_count;
 718
 719         sg_init_table(cmd->tvc_sgl, sgl_count);
 720         cmd->tvc_sgl_count = sgl_count;
 721         pr_debug("%s data_sg %p data_sgl_count %u\n", __func__,
 722                   cmd->tvc_sgl, cmd->tvc_sgl_count);
 723
 724         ret = vhost_scsi_iov_to_sgl(cmd, write, data_iter,
 725                                     cmd->tvc_sgl, cmd->tvc_sgl_count);
 726         if (ret < 0) {
 727                 cmd->tvc_sgl_count = 0;
 728                 return ret;
 729         }
 730         return 0;
 731 }
 732
 733 static int vhost_scsi_to_tcm_attr(int attr)
 734 {
 735         switch (attr) {
 736         case VIRTIO_SCSI_S_SIMPLE:
 737                 return TCM_SIMPLE_TAG;
 738         case VIRTIO_SCSI_S_ORDERED:
 739                 return TCM_ORDERED_TAG;
 740         case VIRTIO_SCSI_S_HEAD:
 741                 return TCM_HEAD_TAG;
 742         case VIRTIO_SCSI_S_ACA:
 743                 return TCM_ACA_TAG;
 744         default:
 745                 break;
 746         }
 747         return TCM_SIMPLE_TAG;
 748 }
 749
 750 static void vhost_scsi_submission_work(struct work_struct *work)
 751 {
 752         struct vhost_scsi_cmd *cmd =
 753                 container_of(work, struct vhost_scsi_cmd, work);
 754         struct vhost_scsi_nexus *tv_nexus;
 755         struct se_cmd *se_cmd = &cmd->tvc_se_cmd;
 756         struct scatterlist *sg_ptr, *sg_prot_ptr = NULL;
 757         int rc;
 758
 759         /* FIXME: BIDI operation */
 760         if (cmd->tvc_sgl_count) {
 761                 sg_ptr = cmd->tvc_sgl;
 762
 763                 if (cmd->tvc_prot_sgl_count)
 764                         sg_prot_ptr = cmd->tvc_prot_sgl;
 765                 else
 766                         se_cmd->prot_pto = true;
 767         } else {
 768                 sg_ptr = NULL;
 769         }
 770         tv_nexus = cmd->tvc_nexus;
 771
 772         se_cmd->tag = 0;
 773         rc = target_submit_cmd_map_sgls(se_cmd, tv_nexus->tvn_se_sess,
 774                         cmd->tvc_cdb, &cmd->tvc_sense_buf[0],
 775                         cmd->tvc_lun, cmd->tvc_exp_data_len,
 776                         vhost_scsi_to_tcm_attr(cmd->tvc_task_attr),
 777                         cmd->tvc_data_direction, TARGET_SCF_ACK_KREF,
 778                         sg_ptr, cmd->tvc_sgl_count, NULL, 0, sg_prot_ptr,
 779                         cmd->tvc_prot_sgl_count);
 780         if (rc < 0) {
 781                 transport_send_check_condition_and_sense(se_cmd,
 782                                 TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE, 0);
 783                 transport_generic_free_cmd(se_cmd, 0);
 784         }
 785 }
 786
 787 static void
 788 vhost_scsi_send_bad_target(struct vhost_scsi *vs,
 789                            struct vhost_virtqueue *vq,
 790                            int head, unsigned out)
 791 {
 792         struct virtio_scsi_cmd_resp __user *resp;
 793         struct virtio_scsi_cmd_resp rsp;
 794         int ret;
 795
 796         memset(&rsp, 0, sizeof(rsp));
 797         rsp.response = VIRTIO_SCSI_S_BAD_TARGET;
 798         resp = vq->iov[out].iov_base;
 799         ret = __copy_to_user(resp, &rsp, sizeof(rsp));
 800         if (!ret)
 801                 vhost_add_used_and_signal(&vs->dev, vq, head, 0);
 802         else
 803                 pr_err("Faulted on virtio_scsi_cmd_resp\n");
 804 }
 805
 806 static int
 807 vhost_scsi_get_desc(struct vhost_scsi *vs, struct vhost_virtqueue *vq,
 808                     struct vhost_scsi_ctx *vc)
 809 {
 810         int ret = -ENXIO;
 811
 812         vc->head = vhost_get_vq_desc(vq, vq->iov,
 813                                      ARRAY_SIZE(vq->iov), &vc->out, &vc->in,
 814                                      NULL, NULL);
 815
 816         pr_debug("vhost_get_vq_desc: head: %d, out: %u in: %u\n",
 817                  vc->head, vc->out, vc->in);
 818
 819         /* On error, stop handling until the next kick. */
 820         if (unlikely(vc->head < 0))
 821                 goto done;
 822
 823         /* Nothing new?  Wait for eventfd to tell us they refilled. */
 824         if (vc->head == vq->num) {
 825                 if (unlikely(vhost_enable_notify(&vs->dev, vq))) {
 826                         vhost_disable_notify(&vs->dev, vq);
 827                         ret = -EAGAIN;
 828                 }
 829                 goto done;
 830         }
 831
 832         /*
 833          * Get the size of request and response buffers.
 834          * FIXME: Not correct for BIDI operation
 835          */
 836         vc->out_size = iov_length(vq->iov, vc->out);
 837         vc->in_size = iov_length(&vq->iov[vc->out], vc->in);
 838
 839         /*
 840          * Copy over the virtio-scsi request header, which for a
 841          * ANY_LAYOUT enabled guest may span multiple iovecs, or a
 842          * single iovec may contain both the header + outgoing
 843          * WRITE payloads.
 844          *
 845          * copy_from_iter() will advance out_iter, so that it will
 846          * point at the start of the outgoing WRITE payload, if
 847          * DMA_TO_DEVICE is set.
 848          */
 849         iov_iter_init(&vc->out_iter, WRITE, vq->iov, vc->out, vc->out_size);
 850         ret = 0;
 851
 852 done:
 853         return ret;
 854 }
 855
 856 static int
 857 vhost_scsi_chk_size(struct vhost_virtqueue *vq, struct vhost_scsi_ctx *vc)
 858 {
 859         if (unlikely(vc->in_size < vc->rsp_size)) {
 860                 vq_err(vq,
 861                        "Response buf too small, need min %zu bytes got %zu",
 862                        vc->rsp_size, vc->in_size);
 863                 return -EINVAL;
 864         } else if (unlikely(vc->out_size < vc->req_size)) {
 865                 vq_err(vq,
 866                        "Request buf too small, need min %zu bytes got %zu",
 867                        vc->req_size, vc->out_size);
 868                 return -EIO;
 869         }
 870
 871         return 0;
 872 }
 873
 874 static int
 875 vhost_scsi_get_req(struct vhost_virtqueue *vq, struct vhost_scsi_ctx *vc,
 876                    struct vhost_scsi_tpg **tpgp)
 877 {
 878         int ret = -EIO;
 879
 880         if (unlikely(!copy_from_iter_full(vc->req, vc->req_size,
 881                                           &vc->out_iter))) {
 882                 vq_err(vq, "Faulted on copy_from_iter_full\n");
 883         } else if (unlikely(*vc->lunp != 1)) {
 884                 /* virtio-scsi spec requires byte 0 of the lun to be 1 */
 885                 vq_err(vq, "Illegal virtio-scsi lun: %u\n", *vc->lunp);
 886         } else {
 887                 struct vhost_scsi_tpg **vs_tpg, *tpg;
 888
 889                 vs_tpg = vq->private_data;      /* validated at handler entry */
 890
 891                 tpg = READ_ONCE(vs_tpg[*vc->target]);
 892                 if (unlikely(!tpg)) {
 893                         vq_err(vq, "Target 0x%x does not exist\n", *vc->target);
 894                 } else {
 895                         if (tpgp)
 896                                 *tpgp = tpg;
 897                         ret = 0;
 898                 }
 899         }
 900
 901         return ret;
 902 }
 903
 904 static void
 905 vhost_scsi_handle_vq(struct vhost_scsi *vs, struct vhost_virtqueue *vq)
 906 {
 907         struct vhost_scsi_tpg **vs_tpg, *tpg;
 908         struct virtio_scsi_cmd_req v_req;
 909         struct virtio_scsi_cmd_req_pi v_req_pi;
 910         struct vhost_scsi_ctx vc;
 911         struct vhost_scsi_cmd *cmd;
 912         struct iov_iter in_iter, prot_iter, data_iter;
 913         u64 tag;
 914         u32 exp_data_len, data_direction;
 915         int ret, prot_bytes;
 916         u16 lun;
 917         u8 task_attr;
 918         bool t10_pi = vhost_has_feature(vq, VIRTIO_SCSI_F_T10_PI);
 919         void *cdb;
 920
 921         mutex_lock(&vq->mutex);
 922         /*
 923          * We can handle the vq only after the endpoint is setup by calling the
 924          * VHOST_SCSI_SET_ENDPOINT ioctl.
 925          */
 926         vs_tpg = vq->private_data;
 927         if (!vs_tpg)
 928                 goto out;
 929
 930         memset(&vc, 0, sizeof(vc));
 931         vc.rsp_size = sizeof(struct virtio_scsi_cmd_resp);
 932
 933         vhost_disable_notify(&vs->dev, vq);
 934
 935         for (;;) {
 936                 ret = vhost_scsi_get_desc(vs, vq, &vc);
 937                 if (ret)
 938                         goto err;
 939
 940                 /*
 941                  * Setup pointers and values based upon different virtio-scsi
 942                  * request header if T10_PI is enabled in KVM guest.
 943                  */
 944                 if (t10_pi) {
 945                         vc.req = &v_req_pi;
 946                         vc.req_size = sizeof(v_req_pi);
 947                         vc.lunp = &v_req_pi.lun[0];
 948                         vc.target = &v_req_pi.lun[1];
 949                 } else {
 950                         vc.req = &v_req;
 951                         vc.req_size = sizeof(v_req);
 952                         vc.lunp = &v_req.lun[0];
 953                         vc.target = &v_req.lun[1];
 954                 }
 955
 956                 /*
 957                  * Validate the size of request and response buffers.
 958                  * Check for a sane response buffer so we can report
 959                  * early errors back to the guest.
 960                  */
 961                 ret = vhost_scsi_chk_size(vq, &vc);
 962                 if (ret)
 963                         goto err;
 964
 965                 ret = vhost_scsi_get_req(vq, &vc, &tpg);
 966                 if (ret)
 967                         goto err;
 968
 969                 ret = -EIO;     /* bad target on any error from here on */
 970
 971                 /*
 972                  * Determine data_direction by calculating the total outgoing
 973                  * iovec sizes + incoming iovec sizes vs. virtio-scsi request +
 974                  * response headers respectively.
 975                  *
 976                  * For DMA_TO_DEVICE this is out_iter, which is already pointing
 977                  * to the right place.
 978                  *
 979                  * For DMA_FROM_DEVICE, the iovec will be just past the end
 980                  * of the virtio-scsi response header in either the same
 981                  * or immediately following iovec.
 982                  *
 983                  * Any associated T10_PI bytes for the outgoing / incoming
 984                  * payloads are included in calculation of exp_data_len here.
 985                  */
 986                 prot_bytes = 0;
 987
 988                 if (vc.out_size > vc.req_size) {
 989                         data_direction = DMA_TO_DEVICE;
 990                         exp_data_len = vc.out_size - vc.req_size;
 991                         data_iter = vc.out_iter;
 992                 } else if (vc.in_size > vc.rsp_size) {
 993                         data_direction = DMA_FROM_DEVICE;
 994                         exp_data_len = vc.in_size - vc.rsp_size;
 995
 996                         iov_iter_init(&in_iter, READ, &vq->iov[vc.out], vc.in,
 997                                       vc.rsp_size + exp_data_len);
 998                         iov_iter_advance(&in_iter, vc.rsp_size);
 999                         data_iter = in_iter;
1000                 } else {
1001                         data_direction = DMA_NONE;
1002                         exp_data_len = 0;
1003                 }
1004                 /*
1005                  * If T10_PI header + payload is present, setup prot_iter values
1006                  * and recalculate data_iter for vhost_scsi_mapal() mapping to
1007                  * host scatterlists via get_user_pages_fast().
1008                  */
1009                 if (t10_pi) {
1010                         if (v_req_pi.pi_bytesout) {
1011                                 if (data_direction != DMA_TO_DEVICE) {
1012                                         vq_err(vq, "Received non zero pi_bytesout,"
1013                                                 " but wrong data_direction\n");
1014                                         goto err;
1015                                 }
1016                                 prot_bytes = vhost32_to_cpu(vq, v_req_pi.pi_bytesout);
1017                         } else if (v_req_pi.pi_bytesin) {
1018                                 if (data_direction != DMA_FROM_DEVICE) {
1019                                         vq_err(vq, "Received non zero pi_bytesin,"
1020                                                 " but wrong data_direction\n");
1021                                         goto err;
1022                                 }
1023                                 prot_bytes = vhost32_to_cpu(vq, v_req_pi.pi_bytesin);
1024                         }
1025                         /*
1026                          * Set prot_iter to data_iter and truncate it to
1027                          * prot_bytes, and advance data_iter past any
1028                          * preceeding prot_bytes that may be present.
1029                          *
1030                          * Also fix up the exp_data_len to reflect only the
1031                          * actual data payload length.
1032                          */
1033                         if (prot_bytes) {
1034                                 exp_data_len -= prot_bytes;
1035                                 prot_iter = data_iter;
1036                                 iov_iter_truncate(&prot_iter, prot_bytes);
1037                                 iov_iter_advance(&data_iter, prot_bytes);
1038                         }
1039                         tag = vhost64_to_cpu(vq, v_req_pi.tag);
1040                         task_attr = v_req_pi.task_attr;
1041                         cdb = &v_req_pi.cdb[0];
1042                         lun = ((v_req_pi.lun[2] << 8) | v_req_pi.lun[3]) & 0x3FFF;
1043                 } else {
1044                         tag = vhost64_to_cpu(vq, v_req.tag);
1045                         task_attr = v_req.task_attr;
1046                         cdb = &v_req.cdb[0];
1047                         lun = ((v_req.lun[2] << 8) | v_req.lun[3]) & 0x3FFF;
1048                 }
1049                 /*
1050                  * Check that the received CDB size does not exceeded our
1051                  * hardcoded max for vhost-scsi, then get a pre-allocated
1052                  * cmd descriptor for the new virtio-scsi tag.
1053                  *
1054                  * TODO what if cdb was too small for varlen cdb header?
1055                  */
1056                 if (unlikely(scsi_command_size(cdb) > VHOST_SCSI_MAX_CDB_SIZE)) {
1057                         vq_err(vq, "Received SCSI CDB with command_size: %d that"
1058                                 " exceeds SCSI_MAX_VARLEN_CDB_SIZE: %d\n",
1059                                 scsi_command_size(cdb), VHOST_SCSI_MAX_CDB_SIZE);
1060                                 goto err;
1061                 }
1062                 cmd = vhost_scsi_get_tag(vq, tpg, cdb, tag, lun, task_attr,
1063                                          exp_data_len + prot_bytes,
1064                                          data_direction);
1065                 if (IS_ERR(cmd)) {
1066                         vq_err(vq, "vhost_scsi_get_tag failed %ld\n",
1067                                PTR_ERR(cmd));
1068                         goto err;
1069                 }
1070                 cmd->tvc_vhost = vs;
1071                 cmd->tvc_vq = vq;
1072                 cmd->tvc_resp_iov = vq->iov[vc.out];
1073                 cmd->tvc_in_iovs = vc.in;
1074
1075                 pr_debug("vhost_scsi got command opcode: %#02x, lun: %d\n",
1076                          cmd->tvc_cdb[0], cmd->tvc_lun);
1077                 pr_debug("cmd: %p exp_data_len: %d, prot_bytes: %d data_direction:"
1078                          " %d\n", cmd, exp_data_len, prot_bytes, data_direction);
1079
1080                 if (data_direction != DMA_NONE) {
1081                         if (unlikely(vhost_scsi_mapal(cmd, prot_bytes,
1082                                                       &prot_iter, exp_data_len,
1083                                                       &data_iter))) {
1084                                 vq_err(vq, "Failed to map iov to sgl\n");
1085                                 vhost_scsi_release_cmd(&cmd->tvc_se_cmd);
1086                                 goto err;
1087                         }
1088                 }
1089                 /*
1090                  * Save the descriptor from vhost_get_vq_desc() to be used to
1091                  * complete the virtio-scsi request in TCM callback context via
1092                  * vhost_scsi_queue_data_in() and vhost_scsi_queue_status()
1093                  */
1094                 cmd->tvc_vq_desc = vc.head;
1095                 /*
1096                  * Dispatch cmd descriptor for cmwq execution in process
1097                  * context provided by vhost_scsi_workqueue.  This also ensures
1098                  * cmd is executed on the same kworker CPU as this vhost
1099                  * thread to gain positive L2 cache locality effects.
1100                  */
1101                 INIT_WORK(&cmd->work, vhost_scsi_submission_work);
1102                 queue_work(vhost_scsi_workqueue, &cmd->work);
1103                 ret = 0;
1104 err:
1105                 /*
1106                  * ENXIO:  No more requests, or read error, wait for next kick
1107                  * EINVAL: Invalid response buffer, drop the request
1108                  * EIO:    Respond with bad target
1109                  * EAGAIN: Pending request
1110                  */
1111                 if (ret == -ENXIO)
1112                         break;
1113                 else if (ret == -EIO)
1114                         vhost_scsi_send_bad_target(vs, vq, vc.head, vc.out);
1115         }
1116 out:
1117         mutex_unlock(&vq->mutex);
1118 }
1119
1120 static void
1121 vhost_scsi_send_tmf_reject(struct vhost_scsi *vs,
1122                            struct vhost_virtqueue *vq,
1123                            struct vhost_scsi_ctx *vc)
1124 {
1125         struct virtio_scsi_ctrl_tmf_resp rsp;
1126         struct iov_iter iov_iter;
1127         int ret;
1128
1129         pr_debug("%s\n", __func__);
1130         memset(&rsp, 0, sizeof(rsp));
1131         rsp.response = VIRTIO_SCSI_S_FUNCTION_REJECTED;
1132
1133         iov_iter_init(&iov_iter, READ, &vq->iov[vc->out], vc->in, sizeof(rsp));
1134
1135         ret = copy_to_iter(&rsp, sizeof(rsp), &iov_iter);
1136         if (likely(ret == sizeof(rsp)))
1137                 vhost_add_used_and_signal(&vs->dev, vq, vc->head, 0);
1138         else
1139                 pr_err("Faulted on virtio_scsi_ctrl_tmf_resp\n");
1140 }
1141
1142 static void
1143 vhost_scsi_send_an_resp(struct vhost_scsi *vs,
1144                         struct vhost_virtqueue *vq,
1145                         struct vhost_scsi_ctx *vc)
1146 {
1147         struct virtio_scsi_ctrl_an_resp rsp;
1148         struct iov_iter iov_iter;
1149         int ret;
1150
1151         pr_debug("%s\n", __func__);
1152         memset(&rsp, 0, sizeof(rsp));   /* event_actual = 0 */
1153         rsp.response = VIRTIO_SCSI_S_OK;
1154
1155         iov_iter_init(&iov_iter, READ, &vq->iov[vc->out], vc->in, sizeof(rsp));
1156
1157         ret = copy_to_iter(&rsp, sizeof(rsp), &iov_iter);
1158         if (likely(ret == sizeof(rsp)))
1159                 vhost_add_used_and_signal(&vs->dev, vq, vc->head, 0);
1160         else
1161                 pr_err("Faulted on virtio_scsi_ctrl_an_resp\n");
1162 }
1163
1164 static void
1165 vhost_scsi_ctl_handle_vq(struct vhost_scsi *vs, struct vhost_virtqueue *vq)
1166 {
1167         union {
1168                 __virtio32 type;
1169                 struct virtio_scsi_ctrl_an_req an;
1170                 struct virtio_scsi_ctrl_tmf_req tmf;
1171         } v_req;
1172         struct vhost_scsi_ctx vc;
1173         size_t typ_size;
1174         int ret;
1175
1176         mutex_lock(&vq->mutex);
1177         /*
1178          * We can handle the vq only after the endpoint is setup by calling the
1179          * VHOST_SCSI_SET_ENDPOINT ioctl.
1180          */
1181         if (!vq->private_data)
1182                 goto out;
1183
1184         memset(&vc, 0, sizeof(vc));
1185
1186         vhost_disable_notify(&vs->dev, vq);
1187
1188         for (;;) {
1189                 ret = vhost_scsi_get_desc(vs, vq, &vc);
1190                 if (ret)
1191                         goto err;
1192
1193                 /*
1194                  * Get the request type first in order to setup
1195                  * other parameters dependent on the type.
1196                  */
1197                 vc.req = &v_req.type;
1198                 typ_size = sizeof(v_req.type);
1199
1200                 if (unlikely(!copy_from_iter_full(vc.req, typ_size,
1201                                                   &vc.out_iter))) {
1202                         vq_err(vq, "Faulted on copy_from_iter tmf type\n");
1203                         /*
1204                          * The size of the response buffer depends on the
1205                          * request type and must be validated against it.
1206                          * Since the request type is not known, don't send
1207                          * a response.
1208                          */
1209                         continue;
1210                 }
1211
1212                 switch (v_req.type) {
1213                 case VIRTIO_SCSI_T_TMF:
1214                         vc.req = &v_req.tmf;
1215                         vc.req_size = sizeof(struct virtio_scsi_ctrl_tmf_req);
1216                         vc.rsp_size = sizeof(struct virtio_scsi_ctrl_tmf_resp);
1217                         vc.lunp = &v_req.tmf.lun[0];
1218                         vc.target = &v_req.tmf.lun[1];
1219                         break;
1220                 case VIRTIO_SCSI_T_AN_QUERY:
1221                 case VIRTIO_SCSI_T_AN_SUBSCRIBE:
1222                         vc.req = &v_req.an;
1223                         vc.req_size = sizeof(struct virtio_scsi_ctrl_an_req);
1224                         vc.rsp_size = sizeof(struct virtio_scsi_ctrl_an_resp);
1225                         vc.lunp = &v_req.an.lun[0];
1226                         vc.target = NULL;
1227                         break;
1228                 default:
1229                         vq_err(vq, "Unknown control request %d", v_req.type);
1230                         continue;
1231                 }
1232
1233                 /*
1234                  * Validate the size of request and response buffers.
1235                  * Check for a sane response buffer so we can report
1236                  * early errors back to the guest.
1237                  */
1238                 ret = vhost_scsi_chk_size(vq, &vc);
1239                 if (ret)
1240                         goto err;
1241
1242                 /*
1243                  * Get the rest of the request now that its size is known.
1244                  */
1245                 vc.req += typ_size;
1246                 vc.req_size -= typ_size;
1247
1248                 ret = vhost_scsi_get_req(vq, &vc, NULL);
1249                 if (ret)
1250                         goto err;
1251
1252                 if (v_req.type == VIRTIO_SCSI_T_TMF)
1253                         vhost_scsi_send_tmf_reject(vs, vq, &vc);
1254                 else
1255                         vhost_scsi_send_an_resp(vs, vq, &vc);
1256 err:
1257                 /*
1258                  * ENXIO:  No more requests, or read error, wait for next kick
1259                  * EINVAL: Invalid response buffer, drop the request
1260                  * EIO:    Respond with bad target
1261                  * EAGAIN: Pending request
1262                  */
1263                 if (ret == -ENXIO)
1264                         break;
1265                 else if (ret == -EIO)
1266                         vhost_scsi_send_bad_target(vs, vq, vc.head, vc.out);
1267         }
1268 out:
1269         mutex_unlock(&vq->mutex);
1270 }
1271
1272 static void vhost_scsi_ctl_handle_kick(struct vhost_work *work)
1273 {
1274         struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,
1275                                                 poll.work);
1276         struct vhost_scsi *vs = container_of(vq->dev, struct vhost_scsi, dev);
1277
1278         pr_debug("%s: The handling func for control queue.\n", __func__);
1279         vhost_scsi_ctl_handle_vq(vs, vq);
1280 }
1281
1282 static void
1283 vhost_scsi_send_evt(struct vhost_scsi *vs,
1284                    struct vhost_scsi_tpg *tpg,
1285                    struct se_lun *lun,
1286                    u32 event,
1287                    u32 reason)
1288 {
1289         struct vhost_scsi_evt *evt;
1290
1291         evt = vhost_scsi_allocate_evt(vs, event, reason);
1292         if (!evt)
1293                 return;
1294
1295         if (tpg && lun) {
1296                 /* TODO: share lun setup code with virtio-scsi.ko */
1297                 /*
1298                  * Note: evt->event is zeroed when we allocate it and
1299                  * lun[4-7] need to be zero according to virtio-scsi spec.
1300                  */
1301                 evt->event.lun[0] = 0x01;
1302                 evt->event.lun[1] = tpg->tport_tpgt;
1303                 if (lun->unpacked_lun >= 256)
1304                         evt->event.lun[2] = lun->unpacked_lun >> 8 | 0x40 ;
1305                 evt->event.lun[3] = lun->unpacked_lun & 0xFF;
1306         }
1307
1308         llist_add(&evt->list, &vs->vs_event_list);
1309         vhost_work_queue(&vs->dev, &vs->vs_event_work);
1310 }
1311
1312 static void vhost_scsi_evt_handle_kick(struct vhost_work *work)
1313 {
1314         struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,
1315                                                 poll.work);
1316         struct vhost_scsi *vs = container_of(vq->dev, struct vhost_scsi, dev);
1317
1318         mutex_lock(&vq->mutex);
1319         if (!vq->private_data)
1320                 goto out;
1321
1322         if (vs->vs_events_missed)
1323                 vhost_scsi_send_evt(vs, NULL, NULL, VIRTIO_SCSI_T_NO_EVENT, 0);
1324 out:
1325         mutex_unlock(&vq->mutex);
1326 }
1327
1328 static void vhost_scsi_handle_kick(struct vhost_work *work)
1329 {
1330         struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,
1331                                                 poll.work);
1332         struct vhost_scsi *vs = container_of(vq->dev, struct vhost_scsi, dev);
1333
1334         vhost_scsi_handle_vq(vs, vq);
1335 }
1336
1337 static void vhost_scsi_flush_vq(struct vhost_scsi *vs, int index)
1338 {
1339         vhost_poll_flush(&vs->vqs[index].vq.poll);
1340 }
1341
1342 /* Callers must hold dev mutex */
1343 static void vhost_scsi_flush(struct vhost_scsi *vs)
1344 {
1345         struct vhost_scsi_inflight *old_inflight[VHOST_SCSI_MAX_VQ];
1346         int i;
1347
1348         /* Init new inflight and remember the old inflight */
1349         vhost_scsi_init_inflight(vs, old_inflight);
1350
1351         /*
1352          * The inflight->kref was initialized to 1. We decrement it here to
1353          * indicate the start of the flush operation so that it will reach 0
1354          * when all the reqs are finished.
1355          */
1356         for (i = 0; i < VHOST_SCSI_MAX_VQ; i++)
1357                 kref_put(&old_inflight[i]->kref, vhost_scsi_done_inflight);
1358
1359         /* Flush both the vhost poll and vhost work */
1360         for (i = 0; i < VHOST_SCSI_MAX_VQ; i++)
1361                 vhost_scsi_flush_vq(vs, i);
1362         vhost_work_flush(&vs->dev, &vs->vs_completion_work);
1363         vhost_work_flush(&vs->dev, &vs->vs_event_work);
1364
1365         /* Wait for all reqs issued before the flush to be finished */
1366         for (i = 0; i < VHOST_SCSI_MAX_VQ; i++)
1367                 wait_for_completion(&old_inflight[i]->comp);
1368 }
1369
1370 /*
1371  * Called from vhost_scsi_ioctl() context to walk the list of available
1372  * vhost_scsi_tpg with an active struct vhost_scsi_nexus
1373  *
1374  *  The lock nesting rule is:
1375  *    vhost_scsi_mutex -> vs->dev.mutex -> tpg->tv_tpg_mutex -> vq->mutex
1376  */
1377 static int
1378 vhost_scsi_set_endpoint(struct vhost_scsi *vs,
1379                         struct vhost_scsi_target *t)
1380 {
1381         struct se_portal_group *se_tpg;
1382         struct vhost_scsi_tport *tv_tport;
1383         struct vhost_scsi_tpg *tpg;
1384         struct vhost_scsi_tpg **vs_tpg;
1385         struct vhost_virtqueue *vq;
1386         int index, ret, i, len;
1387         bool match = false;
1388
1389         mutex_lock(&vhost_scsi_mutex);
1390         mutex_lock(&vs->dev.mutex);
1391
1392         /* Verify that ring has been setup correctly. */
1393         for (index = 0; index < vs->dev.nvqs; ++index) {
1394                 /* Verify that ring has been setup correctly. */
1395                 if (!vhost_vq_access_ok(&vs->vqs[index].vq)) {
1396                         ret = -EFAULT;
1397                         goto out;
1398                 }
1399         }
1400
1401         len = sizeof(vs_tpg[0]) * VHOST_SCSI_MAX_TARGET;
1402         vs_tpg = kzalloc(len, GFP_KERNEL);
1403         if (!vs_tpg) {
1404                 ret = -ENOMEM;
1405                 goto out;
1406         }
1407         if (vs->vs_tpg)
1408                 memcpy(vs_tpg, vs->vs_tpg, len);
1409
1410         list_for_each_entry(tpg, &vhost_scsi_list, tv_tpg_list) {
1411                 mutex_lock(&tpg->tv_tpg_mutex);
1412                 if (!tpg->tpg_nexus) {
1413                         mutex_unlock(&tpg->tv_tpg_mutex);
1414                         continue;
1415                 }
1416                 if (tpg->tv_tpg_vhost_count != 0) {
1417                         mutex_unlock(&tpg->tv_tpg_mutex);
1418                         continue;
1419                 }
1420                 tv_tport = tpg->tport;
1421
1422                 if (!strcmp(tv_tport->tport_name, t->vhost_wwpn)) {
1423                         if (vs->vs_tpg && vs->vs_tpg[tpg->tport_tpgt]) {
1424                                 kfree(vs_tpg);
1425                                 mutex_unlock(&tpg->tv_tpg_mutex);
1426                                 ret = -EEXIST;
1427                                 goto out;
1428                         }
1429                         /*
1430                          * In order to ensure individual vhost-scsi configfs
1431                          * groups cannot be removed while in use by vhost ioctl,
1432                          * go ahead and take an explicit se_tpg->tpg_group.cg_item
1433                          * dependency now.
1434                          */
1435                         se_tpg = &tpg->se_tpg;
1436                         ret = target_depend_item(&se_tpg->tpg_group.cg_item);
1437                         if (ret) {
1438                                 pr_warn("target_depend_item() failed: %d\n", ret);
1439                                 kfree(vs_tpg);
1440                                 mutex_unlock(&tpg->tv_tpg_mutex);
1441                                 goto out;
1442                         }
1443                         tpg->tv_tpg_vhost_count++;
1444                         tpg->vhost_scsi = vs;
1445                         vs_tpg[tpg->tport_tpgt] = tpg;
1446                         match = true;
1447                 }
1448                 mutex_unlock(&tpg->tv_tpg_mutex);
1449         }
1450
1451         if (match) {
1452                 memcpy(vs->vs_vhost_wwpn, t->vhost_wwpn,
1453                        sizeof(vs->vs_vhost_wwpn));
1454                 for (i = 0; i < VHOST_SCSI_MAX_VQ; i++) {
1455                         vq = &vs->vqs[i].vq;
1456                         mutex_lock(&vq->mutex);
1457                         vq->private_data = vs_tpg;
1458                         vhost_vq_init_access(vq);
1459                         mutex_unlock(&vq->mutex);
1460                 }
1461                 ret = 0;
1462         } else {
1463                 ret = -EEXIST;
1464         }
1465
1466         /*
1467          * Act as synchronize_rcu to make sure access to
1468          * old vs->vs_tpg is finished.
1469          */
1470         vhost_scsi_flush(vs);
1471         kfree(vs->vs_tpg);
1472         vs->vs_tpg = vs_tpg;
1473
1474 out:
1475         mutex_unlock(&vs->dev.mutex);
1476         mutex_unlock(&vhost_scsi_mutex);
1477         return ret;
1478 }
1479
1480 static int
1481 vhost_scsi_clear_endpoint(struct vhost_scsi *vs,
1482                           struct vhost_scsi_target *t)
1483 {
1484         struct se_portal_group *se_tpg;
1485         struct vhost_scsi_tport *tv_tport;
1486         struct vhost_scsi_tpg *tpg;
1487         struct vhost_virtqueue *vq;
1488         bool match = false;
1489         int index, ret, i;
1490         u8 target;
1491
1492         mutex_lock(&vhost_scsi_mutex);
1493         mutex_lock(&vs->dev.mutex);
1494         /* Verify that ring has been setup correctly. */
1495         for (index = 0; index < vs->dev.nvqs; ++index) {
1496                 if (!vhost_vq_access_ok(&vs->vqs[index].vq)) {
1497                         ret = -EFAULT;
1498                         goto err_dev;
1499                 }
1500         }
1501
1502         if (!vs->vs_tpg) {
1503                 ret = 0;
1504                 goto err_dev;
1505         }
1506
1507         for (i = 0; i < VHOST_SCSI_MAX_TARGET; i++) {
1508                 target = i;
1509                 tpg = vs->vs_tpg[target];
1510                 if (!tpg)
1511                         continue;
1512
1513                 mutex_lock(&tpg->tv_tpg_mutex);
1514                 tv_tport = tpg->tport;
1515                 if (!tv_tport) {
1516                         ret = -ENODEV;
1517                         goto err_tpg;
1518                 }
1519
1520                 if (strcmp(tv_tport->tport_name, t->vhost_wwpn)) {
1521                         pr_warn("tv_tport->tport_name: %s, tpg->tport_tpgt: %hu"
1522                                 " does not match t->vhost_wwpn: %s, t->vhost_tpgt: %hu\n",
1523                                 tv_tport->tport_name, tpg->tport_tpgt,
1524                                 t->vhost_wwpn, t->vhost_tpgt);
1525                         ret = -EINVAL;
1526                         goto err_tpg;
1527                 }
1528                 tpg->tv_tpg_vhost_count--;
1529                 tpg->vhost_scsi = NULL;
1530                 vs->vs_tpg[target] = NULL;
1531                 match = true;
1532                 mutex_unlock(&tpg->tv_tpg_mutex);
1533                 /*
1534                  * Release se_tpg->tpg_group.cg_item configfs dependency now
1535                  * to allow vhost-scsi WWPN se_tpg->tpg_group shutdown to occur.
1536                  */
1537                 se_tpg = &tpg->se_tpg;
1538                 target_undepend_item(&se_tpg->tpg_group.cg_item);
1539         }
1540         if (match) {
1541                 for (i = 0; i < VHOST_SCSI_MAX_VQ; i++) {
1542                         vq = &vs->vqs[i].vq;
1543                         mutex_lock(&vq->mutex);
1544                         vq->private_data = NULL;
1545                         mutex_unlock(&vq->mutex);
1546                 }
1547         }
1548         /*
1549          * Act as synchronize_rcu to make sure access to
1550          * old vs->vs_tpg is finished.
1551          */
1552         vhost_scsi_flush(vs);
1553         kfree(vs->vs_tpg);
1554         vs->vs_tpg = NULL;
1555         WARN_ON(vs->vs_events_nr);
1556         mutex_unlock(&vs->dev.mutex);
1557         mutex_unlock(&vhost_scsi_mutex);
1558         return 0;
1559
1560 err_tpg:
1561         mutex_unlock(&tpg->tv_tpg_mutex);
1562 err_dev:
1563         mutex_unlock(&vs->dev.mutex);
1564         mutex_unlock(&vhost_scsi_mutex);
1565         return ret;
1566 }
1567
1568 static int vhost_scsi_set_features(struct vhost_scsi *vs, u64 features)
1569 {
1570         struct vhost_virtqueue *vq;
1571         int i;
1572
1573         if (features & ~VHOST_SCSI_FEATURES)
1574                 return -EOPNOTSUPP;
1575
1576         mutex_lock(&vs->dev.mutex);
1577         if ((features & (1 << VHOST_F_LOG_ALL)) &&
1578             !vhost_log_access_ok(&vs->dev)) {
1579                 mutex_unlock(&vs->dev.mutex);
1580                 return -EFAULT;
1581         }
1582
1583         for (i = 0; i < VHOST_SCSI_MAX_VQ; i++) {
1584                 vq = &vs->vqs[i].vq;
1585                 mutex_lock(&vq->mutex);
1586                 vq->acked_features = features;
1587                 mutex_unlock(&vq->mutex);
1588         }
1589         mutex_unlock(&vs->dev.mutex);
1590         return 0;
1591 }
1592
1593 static int vhost_scsi_open(struct inode *inode, struct file *f)
1594 {
1595         struct vhost_scsi *vs;
1596         struct vhost_virtqueue **vqs;
1597         int r = -ENOMEM, i;
1598
1599         vs = kzalloc(sizeof(*vs), GFP_KERNEL | __GFP_NOWARN | __GFP_RETRY_MAYFAIL);
1600         if (!vs) {
1601                 vs = vzalloc(sizeof(*vs));
1602                 if (!vs)
1603                         goto err_vs;
1604         }
1605
1606         vqs = kmalloc_array(VHOST_SCSI_MAX_VQ, sizeof(*vqs), GFP_KERNEL);
1607         if (!vqs)
1608                 goto err_vqs;
1609
1610         vhost_work_init(&vs->vs_completion_work, vhost_scsi_complete_cmd_work);
1611         vhost_work_init(&vs->vs_event_work, vhost_scsi_evt_work);
1612
1613         vs->vs_events_nr = 0;
1614         vs->vs_events_missed = false;
1615
1616         vqs[VHOST_SCSI_VQ_CTL] = &vs->vqs[VHOST_SCSI_VQ_CTL].vq;
1617         vqs[VHOST_SCSI_VQ_EVT] = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;
1618         vs->vqs[VHOST_SCSI_VQ_CTL].vq.handle_kick = vhost_scsi_ctl_handle_kick;
1619         vs->vqs[VHOST_SCSI_VQ_EVT].vq.handle_kick = vhost_scsi_evt_handle_kick;
1620         for (i = VHOST_SCSI_VQ_IO; i < VHOST_SCSI_MAX_VQ; i++) {
1621                 vqs[i] = &vs->vqs[i].vq;
1622                 vs->vqs[i].vq.handle_kick = vhost_scsi_handle_kick;
1623         }
1624         vhost_dev_init(&vs->dev, vqs, VHOST_SCSI_MAX_VQ, UIO_MAXIOV);
1625
1626         vhost_scsi_init_inflight(vs, NULL);
1627
1628         f->private_data = vs;
1629         return 0;
1630
1631 err_vqs:
1632         kvfree(vs);
1633 err_vs:
1634         return r;
1635 }
1636
1637 static int vhost_scsi_release(struct inode *inode, struct file *f)
1638 {
1639         struct vhost_scsi *vs = f->private_data;
1640         struct vhost_scsi_target t;
1641
1642         mutex_lock(&vs->dev.mutex);
1643         memcpy(t.vhost_wwpn, vs->vs_vhost_wwpn, sizeof(t.vhost_wwpn));
1644         mutex_unlock(&vs->dev.mutex);
1645         vhost_scsi_clear_endpoint(vs, &t);
1646         vhost_dev_stop(&vs->dev);
1647         vhost_dev_cleanup(&vs->dev);
1648         /* Jobs can re-queue themselves in evt kick handler. Do extra flush. */
1649         vhost_scsi_flush(vs);
1650         kfree(vs->dev.vqs);
1651         kvfree(vs);
1652         return 0;
1653 }
1654
1655 static long
1656 vhost_scsi_ioctl(struct file *f,
1657                  unsigned int ioctl,
1658                  unsigned long arg)
1659 {
1660         struct vhost_scsi *vs = f->private_data;
1661         struct vhost_scsi_target backend;
1662         void __user *argp = (void __user *)arg;
1663         u64 __user *featurep = argp;
1664         u32 __user *eventsp = argp;
1665         u32 events_missed;
1666         u64 features;
1667         int r, abi_version = VHOST_SCSI_ABI_VERSION;
1668         struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;
1669
1670         switch (ioctl) {
1671         case VHOST_SCSI_SET_ENDPOINT:
1672                 if (copy_from_user(&backend, argp, sizeof backend))
1673                         return -EFAULT;
1674                 if (backend.reserved != 0)
1675                         return -EOPNOTSUPP;
1676
1677                 return vhost_scsi_set_endpoint(vs, &backend);
1678         case VHOST_SCSI_CLEAR_ENDPOINT:
1679                 if (copy_from_user(&backend, argp, sizeof backend))
1680                         return -EFAULT;
1681                 if (backend.reserved != 0)
1682                         return -EOPNOTSUPP;
1683
1684                 return vhost_scsi_clear_endpoint(vs, &backend);
1685         case VHOST_SCSI_GET_ABI_VERSION:
1686                 if (copy_to_user(argp, &abi_version, sizeof abi_version))
1687                         return -EFAULT;
1688                 return 0;
1689         case VHOST_SCSI_SET_EVENTS_MISSED:
1690                 if (get_user(events_missed, eventsp))
1691                         return -EFAULT;
1692                 mutex_lock(&vq->mutex);
1693                 vs->vs_events_missed = events_missed;
1694                 mutex_unlock(&vq->mutex);
1695                 return 0;
1696         case VHOST_SCSI_GET_EVENTS_MISSED:
1697                 mutex_lock(&vq->mutex);
1698                 events_missed = vs->vs_events_missed;
1699                 mutex_unlock(&vq->mutex);
1700                 if (put_user(events_missed, eventsp))
1701                         return -EFAULT;
1702                 return 0;
1703         case VHOST_GET_FEATURES:
1704                 features = VHOST_SCSI_FEATURES;
1705                 if (copy_to_user(featurep, &features, sizeof features))
1706                         return -EFAULT;
1707                 return 0;
1708         case VHOST_SET_FEATURES:
1709                 if (copy_from_user(&features, featurep, sizeof features))
1710                         return -EFAULT;
1711                 return vhost_scsi_set_features(vs, features);
1712         default:
1713                 mutex_lock(&vs->dev.mutex);
1714                 r = vhost_dev_ioctl(&vs->dev, ioctl, argp);
1715                 /* TODO: flush backend after dev ioctl. */
1716                 if (r == -ENOIOCTLCMD)
1717                         r = vhost_vring_ioctl(&vs->dev, ioctl, argp);
1718                 mutex_unlock(&vs->dev.mutex);
1719                 return r;
1720         }
1721 }
1722
1723 #ifdef CONFIG_COMPAT
1724 static long vhost_scsi_compat_ioctl(struct file *f, unsigned int ioctl,
1725                                 unsigned long arg)
1726 {
1727         return vhost_scsi_ioctl(f, ioctl, (unsigned long)compat_ptr(arg));
1728 }
1729 #endif
1730
1731 static const struct file_operations vhost_scsi_fops = {
1732         .owner          = THIS_MODULE,
1733         .release        = vhost_scsi_release,
1734         .unlocked_ioctl = vhost_scsi_ioctl,
1735 #ifdef CONFIG_COMPAT
1736         .compat_ioctl   = vhost_scsi_compat_ioctl,
1737 #endif
1738         .open           = vhost_scsi_open,
1739         .llseek         = noop_llseek,
1740 };
1741
1742 static struct miscdevice vhost_scsi_misc = {
1743         MISC_DYNAMIC_MINOR,
1744         "vhost-scsi",
1745         &vhost_scsi_fops,
1746 };
1747
1748 static int __init vhost_scsi_register(void)
1749 {
1750         return misc_register(&vhost_scsi_misc);
1751 }
1752
1753 static void vhost_scsi_deregister(void)
1754 {
1755         misc_deregister(&vhost_scsi_misc);
1756 }
1757
1758 static char *vhost_scsi_dump_proto_id(struct vhost_scsi_tport *tport)
1759 {
1760         switch (tport->tport_proto_id) {
1761         case SCSI_PROTOCOL_SAS:
1762                 return "SAS";
1763         case SCSI_PROTOCOL_FCP:
1764                 return "FCP";
1765         case SCSI_PROTOCOL_ISCSI:
1766                 return "iSCSI";
1767         default:
1768                 break;
1769         }
1770
1771         return "Unknown";
1772 }
1773
1774 static void
1775 vhost_scsi_do_plug(struct vhost_scsi_tpg *tpg,
1776                   struct se_lun *lun, bool plug)
1777 {
1778
1779         struct vhost_scsi *vs = tpg->vhost_scsi;
1780         struct vhost_virtqueue *vq;
1781         u32 reason;
1782
1783         if (!vs)
1784                 return;
1785
1786         mutex_lock(&vs->dev.mutex);
1787
1788         if (plug)
1789                 reason = VIRTIO_SCSI_EVT_RESET_RESCAN;
1790         else
1791                 reason = VIRTIO_SCSI_EVT_RESET_REMOVED;
1792
1793         vq = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;
1794         mutex_lock(&vq->mutex);
1795         if (vhost_has_feature(vq, VIRTIO_SCSI_F_HOTPLUG))
1796                 vhost_scsi_send_evt(vs, tpg, lun,
1797                                    VIRTIO_SCSI_T_TRANSPORT_RESET, reason);
1798         mutex_unlock(&vq->mutex);
1799         mutex_unlock(&vs->dev.mutex);
1800 }
1801
1802 static void vhost_scsi_hotplug(struct vhost_scsi_tpg *tpg, struct se_lun *lun)
1803 {
1804         vhost_scsi_do_plug(tpg, lun, true);
1805 }
1806
1807 static void vhost_scsi_hotunplug(struct vhost_scsi_tpg *tpg, struct se_lun *lun)
1808 {
1809         vhost_scsi_do_plug(tpg, lun, false);
1810 }
1811
1812 static int vhost_scsi_port_link(struct se_portal_group *se_tpg,
1813                                struct se_lun *lun)
1814 {
1815         struct vhost_scsi_tpg *tpg = container_of(se_tpg,
1816                                 struct vhost_scsi_tpg, se_tpg);
1817
1818         mutex_lock(&vhost_scsi_mutex);
1819
1820         mutex_lock(&tpg->tv_tpg_mutex);
1821         tpg->tv_tpg_port_count++;
1822         mutex_unlock(&tpg->tv_tpg_mutex);
1823
1824         vhost_scsi_hotplug(tpg, lun);
1825
1826         mutex_unlock(&vhost_scsi_mutex);
1827
1828         return 0;
1829 }
1830
1831 static void vhost_scsi_port_unlink(struct se_portal_group *se_tpg,
1832                                   struct se_lun *lun)
1833 {
1834         struct vhost_scsi_tpg *tpg = container_of(se_tpg,
1835                                 struct vhost_scsi_tpg, se_tpg);
1836
1837         mutex_lock(&vhost_scsi_mutex);
1838
1839         mutex_lock(&tpg->tv_tpg_mutex);
1840         tpg->tv_tpg_port_count--;
1841         mutex_unlock(&tpg->tv_tpg_mutex);
1842
1843         vhost_scsi_hotunplug(tpg, lun);
1844
1845         mutex_unlock(&vhost_scsi_mutex);
1846 }
1847
1848 static void vhost_scsi_free_cmd_map_res(struct se_session *se_sess)
1849 {
1850         struct vhost_scsi_cmd *tv_cmd;
1851         unsigned int i;
1852
1853         if (!se_sess->sess_cmd_map)
1854                 return;
1855
1856         for (i = 0; i < VHOST_SCSI_DEFAULT_TAGS; i++) {
1857                 tv_cmd = &((struct vhost_scsi_cmd *)se_sess->sess_cmd_map)[i];
1858
1859                 kfree(tv_cmd->tvc_sgl);
1860                 kfree(tv_cmd->tvc_prot_sgl);
1861                 kfree(tv_cmd->tvc_upages);
1862         }
1863 }
1864
1865 static ssize_t vhost_scsi_tpg_attrib_fabric_prot_type_store(
1866                 struct config_item *item, const char *page, size_t count)
1867 {
1868         struct se_portal_group *se_tpg = attrib_to_tpg(item);
1869         struct vhost_scsi_tpg *tpg = container_of(se_tpg,
1870                                 struct vhost_scsi_tpg, se_tpg);
1871         unsigned long val;
1872         int ret = kstrtoul(page, 0, &val);
1873
1874         if (ret) {
1875                 pr_err("kstrtoul() returned %d for fabric_prot_type\n", ret);
1876                 return ret;
1877         }
1878         if (val != 0 && val != 1 && val != 3) {
1879                 pr_err("Invalid vhost_scsi fabric_prot_type: %lu\n", val);
1880                 return -EINVAL;
1881         }
1882         tpg->tv_fabric_prot_type = val;
1883
1884         return count;
1885 }
1886
1887 static ssize_t vhost_scsi_tpg_attrib_fabric_prot_type_show(
1888                 struct config_item *item, char *page)
1889 {
1890         struct se_portal_group *se_tpg = attrib_to_tpg(item);
1891         struct vhost_scsi_tpg *tpg = container_of(se_tpg,
1892                                 struct vhost_scsi_tpg, se_tpg);
1893
1894         return sprintf(page, "%d\n", tpg->tv_fabric_prot_type);
1895 }
1896
1897 CONFIGFS_ATTR(vhost_scsi_tpg_attrib_, fabric_prot_type);
1898
1899 static struct configfs_attribute *vhost_scsi_tpg_attrib_attrs[] = {
1900         &vhost_scsi_tpg_attrib_attr_fabric_prot_type,
1901         NULL,
1902 };
1903
1904 static int vhost_scsi_nexus_cb(struct se_portal_group *se_tpg,
1905                                struct se_session *se_sess, void *p)
1906 {
1907         struct vhost_scsi_cmd *tv_cmd;
1908         unsigned int i;
1909
1910         for (i = 0; i < VHOST_SCSI_DEFAULT_TAGS; i++) {
1911                 tv_cmd = &((struct vhost_scsi_cmd *)se_sess->sess_cmd_map)[i];
1912
1913                 tv_cmd->tvc_sgl = kcalloc(VHOST_SCSI_PREALLOC_SGLS,
1914                                           sizeof(struct scatterlist),
1915                                           GFP_KERNEL);
1916                 if (!tv_cmd->tvc_sgl) {
1917                         pr_err("Unable to allocate tv_cmd->tvc_sgl\n");
1918                         goto out;
1919                 }
1920
1921                 tv_cmd->tvc_upages = kcalloc(VHOST_SCSI_PREALLOC_UPAGES,
1922                                              sizeof(struct page *),
1923                                              GFP_KERNEL);
1924                 if (!tv_cmd->tvc_upages) {
1925                         pr_err("Unable to allocate tv_cmd->tvc_upages\n");
1926                         goto out;
1927                 }
1928
1929                 tv_cmd->tvc_prot_sgl = kcalloc(VHOST_SCSI_PREALLOC_PROT_SGLS,
1930                                                sizeof(struct scatterlist),
1931                                                GFP_KERNEL);
1932                 if (!tv_cmd->tvc_prot_sgl) {
1933                         pr_err("Unable to allocate tv_cmd->tvc_prot_sgl\n");
1934                         goto out;
1935                 }
1936         }
1937         return 0;
1938 out:
1939         vhost_scsi_free_cmd_map_res(se_sess);
1940         return -ENOMEM;
1941 }
1942
1943 static int vhost_scsi_make_nexus(struct vhost_scsi_tpg *tpg,
1944                                 const char *name)
1945 {
1946         struct vhost_scsi_nexus *tv_nexus;
1947
1948         mutex_lock(&tpg->tv_tpg_mutex);
1949         if (tpg->tpg_nexus) {
1950                 mutex_unlock(&tpg->tv_tpg_mutex);
1951                 pr_debug("tpg->tpg_nexus already exists\n");
1952                 return -EEXIST;
1953         }
1954
1955         tv_nexus = kzalloc(sizeof(*tv_nexus), GFP_KERNEL);
1956         if (!tv_nexus) {
1957                 mutex_unlock(&tpg->tv_tpg_mutex);
1958                 pr_err("Unable to allocate struct vhost_scsi_nexus\n");
1959                 return -ENOMEM;
1960         }
1961         /*
1962          * Since we are running in 'demo mode' this call with generate a
1963          * struct se_node_acl for the vhost_scsi struct se_portal_group with
1964          * the SCSI Initiator port name of the passed configfs group 'name'.
1965          */
1966         tv_nexus->tvn_se_sess = target_setup_session(&tpg->se_tpg,
1967                                         VHOST_SCSI_DEFAULT_TAGS,
1968                                         sizeof(struct vhost_scsi_cmd),
1969                                         TARGET_PROT_DIN_PASS | TARGET_PROT_DOUT_PASS,
1970                                         (unsigned char *)name, tv_nexus,
1971                                         vhost_scsi_nexus_cb);
1972         if (IS_ERR(tv_nexus->tvn_se_sess)) {
1973                 mutex_unlock(&tpg->tv_tpg_mutex);
1974                 kfree(tv_nexus);
1975                 return -ENOMEM;
1976         }
1977         tpg->tpg_nexus = tv_nexus;
1978
1979         mutex_unlock(&tpg->tv_tpg_mutex);
1980         return 0;
1981 }
1982
1983 static int vhost_scsi_drop_nexus(struct vhost_scsi_tpg *tpg)
1984 {
1985         struct se_session *se_sess;
1986         struct vhost_scsi_nexus *tv_nexus;
1987
1988         mutex_lock(&tpg->tv_tpg_mutex);
1989         tv_nexus = tpg->tpg_nexus;
1990         if (!tv_nexus) {
1991                 mutex_unlock(&tpg->tv_tpg_mutex);
1992                 return -ENODEV;
1993         }
1994
1995         se_sess = tv_nexus->tvn_se_sess;
1996         if (!se_sess) {
1997                 mutex_unlock(&tpg->tv_tpg_mutex);
1998                 return -ENODEV;
1999         }
2000
2001         if (tpg->tv_tpg_port_count != 0) {
2002                 mutex_unlock(&tpg->tv_tpg_mutex);
2003                 pr_err("Unable to remove TCM_vhost I_T Nexus with"
2004                         " active TPG port count: %d\n",
2005                         tpg->tv_tpg_port_count);
2006                 return -EBUSY;
2007         }
2008
2009         if (tpg->tv_tpg_vhost_count != 0) {
2010                 mutex_unlock(&tpg->tv_tpg_mutex);
2011                 pr_err("Unable to remove TCM_vhost I_T Nexus with"
2012                         " active TPG vhost count: %d\n",
2013                         tpg->tv_tpg_vhost_count);
2014                 return -EBUSY;
2015         }
2016
2017         pr_debug("TCM_vhost_ConfigFS: Removing I_T Nexus to emulated"
2018                 " %s Initiator Port: %s\n", vhost_scsi_dump_proto_id(tpg->tport),
2019                 tv_nexus->tvn_se_sess->se_node_acl->initiatorname);
2020
2021         vhost_scsi_free_cmd_map_res(se_sess);
2022         /*
2023          * Release the SCSI I_T Nexus to the emulated vhost Target Port
2024          */
2025         target_remove_session(se_sess);
2026         tpg->tpg_nexus = NULL;
2027         mutex_unlock(&tpg->tv_tpg_mutex);
2028
2029         kfree(tv_nexus);
2030         return 0;
2031 }
2032
2033 static ssize_t vhost_scsi_tpg_nexus_show(struct config_item *item, char *page)
2034 {
2035         struct se_portal_group *se_tpg = to_tpg(item);
2036         struct vhost_scsi_tpg *tpg = container_of(se_tpg,
2037                                 struct vhost_scsi_tpg, se_tpg);
2038         struct vhost_scsi_nexus *tv_nexus;
2039         ssize_t ret;
2040
2041         mutex_lock(&tpg->tv_tpg_mutex);
2042         tv_nexus = tpg->tpg_nexus;
2043         if (!tv_nexus) {
2044                 mutex_unlock(&tpg->tv_tpg_mutex);
2045                 return -ENODEV;
2046         }
2047         ret = snprintf(page, PAGE_SIZE, "%s\n",
2048                         tv_nexus->tvn_se_sess->se_node_acl->initiatorname);
2049         mutex_unlock(&tpg->tv_tpg_mutex);
2050
2051         return ret;
2052 }
2053
2054 static ssize_t vhost_scsi_tpg_nexus_store(struct config_item *item,
2055                 const char *page, size_t count)
2056 {
2057         struct se_portal_group *se_tpg = to_tpg(item);
2058         struct vhost_scsi_tpg *tpg = container_of(se_tpg,
2059                                 struct vhost_scsi_tpg, se_tpg);
2060         struct vhost_scsi_tport *tport_wwn = tpg->tport;
2061         unsigned char i_port[VHOST_SCSI_NAMELEN], *ptr, *port_ptr;
2062         int ret;
2063         /*
2064          * Shutdown the active I_T nexus if 'NULL' is passed..
2065          */
2066         if (!strncmp(page, "NULL", 4)) {
2067                 ret = vhost_scsi_drop_nexus(tpg);
2068                 return (!ret) ? count : ret;
2069         }
2070         /*
2071          * Otherwise make sure the passed virtual Initiator port WWN matches
2072          * the fabric protocol_id set in vhost_scsi_make_tport(), and call
2073          * vhost_scsi_make_nexus().
2074          */
2075         if (strlen(page) >= VHOST_SCSI_NAMELEN) {
2076                 pr_err("Emulated NAA Sas Address: %s, exceeds"
2077                                 " max: %d\n", page, VHOST_SCSI_NAMELEN);
2078                 return -EINVAL;
2079         }
2080         snprintf(&i_port[0], VHOST_SCSI_NAMELEN, "%s", page);
2081
2082         ptr = strstr(i_port, "naa.");
2083         if (ptr) {
2084                 if (tport_wwn->tport_proto_id != SCSI_PROTOCOL_SAS) {
2085                         pr_err("Passed SAS Initiator Port %s does not"
2086                                 " match target port protoid: %s\n", i_port,
2087                                 vhost_scsi_dump_proto_id(tport_wwn));
2088                         return -EINVAL;
2089                 }
2090                 port_ptr = &i_port[0];
2091                 goto check_newline;
2092         }
2093         ptr = strstr(i_port, "fc.");
2094         if (ptr) {
2095                 if (tport_wwn->tport_proto_id != SCSI_PROTOCOL_FCP) {
2096                         pr_err("Passed FCP Initiator Port %s does not"
2097                                 " match target port protoid: %s\n", i_port,
2098                                 vhost_scsi_dump_proto_id(tport_wwn));
2099                         return -EINVAL;
2100                 }
2101                 port_ptr = &i_port[3]; /* Skip over "fc." */
2102                 goto check_newline;
2103         }
2104         ptr = strstr(i_port, "iqn.");
2105         if (ptr) {
2106                 if (tport_wwn->tport_proto_id != SCSI_PROTOCOL_ISCSI) {
2107                         pr_err("Passed iSCSI Initiator Port %s does not"
2108                                 " match target port protoid: %s\n", i_port,
2109                                 vhost_scsi_dump_proto_id(tport_wwn));
2110                         return -EINVAL;
2111                 }
2112                 port_ptr = &i_port[0];
2113                 goto check_newline;
2114         }
2115         pr_err("Unable to locate prefix for emulated Initiator Port:"
2116                         " %s\n", i_port);
2117         return -EINVAL;
2118         /*
2119          * Clear any trailing newline for the NAA WWN
2120          */
2121 check_newline:
2122         if (i_port[strlen(i_port)-1] == '\n')
2123                 i_port[strlen(i_port)-1] = '\0';
2124
2125         ret = vhost_scsi_make_nexus(tpg, port_ptr);
2126         if (ret < 0)
2127                 return ret;
2128
2129         return count;
2130 }
2131
2132 CONFIGFS_ATTR(vhost_scsi_tpg_, nexus);
2133
2134 static struct configfs_attribute *vhost_scsi_tpg_attrs[] = {
2135         &vhost_scsi_tpg_attr_nexus,
2136         NULL,
2137 };
2138
2139 static struct se_portal_group *
2140 vhost_scsi_make_tpg(struct se_wwn *wwn, const char *name)
2141 {
2142         struct vhost_scsi_tport *tport = container_of(wwn,
2143                         struct vhost_scsi_tport, tport_wwn);
2144
2145         struct vhost_scsi_tpg *tpg;
2146         u16 tpgt;
2147         int ret;
2148
2149         if (strstr(name, "tpgt_") != name)
2150                 return ERR_PTR(-EINVAL);
2151         if (kstrtou16(name + 5, 10, &tpgt) || tpgt >= VHOST_SCSI_MAX_TARGET)
2152                 return ERR_PTR(-EINVAL);
2153
2154         tpg = kzalloc(sizeof(*tpg), GFP_KERNEL);
2155         if (!tpg) {
2156                 pr_err("Unable to allocate struct vhost_scsi_tpg");
2157                 return ERR_PTR(-ENOMEM);
2158         }
2159         mutex_init(&tpg->tv_tpg_mutex);
2160         INIT_LIST_HEAD(&tpg->tv_tpg_list);
2161         tpg->tport = tport;
2162         tpg->tport_tpgt = tpgt;
2163
2164         ret = core_tpg_register(wwn, &tpg->se_tpg, tport->tport_proto_id);
2165         if (ret < 0) {
2166                 kfree(tpg);
2167                 return NULL;
2168         }
2169         mutex_lock(&vhost_scsi_mutex);
2170         list_add_tail(&tpg->tv_tpg_list, &vhost_scsi_list);
2171         mutex_unlock(&vhost_scsi_mutex);
2172
2173         return &tpg->se_tpg;
2174 }
2175
2176 static void vhost_scsi_drop_tpg(struct se_portal_group *se_tpg)
2177 {
2178         struct vhost_scsi_tpg *tpg = container_of(se_tpg,
2179                                 struct vhost_scsi_tpg, se_tpg);
2180
2181         mutex_lock(&vhost_scsi_mutex);
2182         list_del(&tpg->tv_tpg_list);
2183         mutex_unlock(&vhost_scsi_mutex);
2184         /*
2185          * Release the virtual I_T Nexus for this vhost TPG
2186          */
2187         vhost_scsi_drop_nexus(tpg);
2188         /*
2189          * Deregister the se_tpg from TCM..
2190          */
2191         core_tpg_deregister(se_tpg);
2192         kfree(tpg);
2193 }
2194
2195 static struct se_wwn *
2196 vhost_scsi_make_tport(struct target_fabric_configfs *tf,
2197                      struct config_group *group,
2198                      const char *name)
2199 {
2200         struct vhost_scsi_tport *tport;
2201         char *ptr;
2202         u64 wwpn = 0;
2203         int off = 0;
2204
2205         /* if (vhost_scsi_parse_wwn(name, &wwpn, 1) < 0)
2206                 return ERR_PTR(-EINVAL); */
2207
2208         tport = kzalloc(sizeof(*tport), GFP_KERNEL);
2209         if (!tport) {
2210                 pr_err("Unable to allocate struct vhost_scsi_tport");
2211                 return ERR_PTR(-ENOMEM);
2212         }
2213         tport->tport_wwpn = wwpn;
2214         /*
2215          * Determine the emulated Protocol Identifier and Target Port Name
2216          * based on the incoming configfs directory name.
2217          */
2218         ptr = strstr(name, "naa.");
2219         if (ptr) {
2220                 tport->tport_proto_id = SCSI_PROTOCOL_SAS;
2221                 goto check_len;
2222         }
2223         ptr = strstr(name, "fc.");
2224         if (ptr) {
2225                 tport->tport_proto_id = SCSI_PROTOCOL_FCP;
2226                 off = 3; /* Skip over "fc." */
2227                 goto check_len;
2228         }
2229         ptr = strstr(name, "iqn.");
2230         if (ptr) {
2231                 tport->tport_proto_id = SCSI_PROTOCOL_ISCSI;
2232                 goto check_len;
2233         }
2234
2235         pr_err("Unable to locate prefix for emulated Target Port:"
2236                         " %s\n", name);
2237         kfree(tport);
2238         return ERR_PTR(-EINVAL);
2239
2240 check_len:
2241         if (strlen(name) >= VHOST_SCSI_NAMELEN) {
2242                 pr_err("Emulated %s Address: %s, exceeds"
2243                         " max: %d\n", name, vhost_scsi_dump_proto_id(tport),
2244                         VHOST_SCSI_NAMELEN);
2245                 kfree(tport);
2246                 return ERR_PTR(-EINVAL);
2247         }
2248         snprintf(&tport->tport_name[0], VHOST_SCSI_NAMELEN, "%s", &name[off]);
2249
2250         pr_debug("TCM_VHost_ConfigFS: Allocated emulated Target"
2251                 " %s Address: %s\n", vhost_scsi_dump_proto_id(tport), name);
2252
2253         return &tport->tport_wwn;
2254 }
2255
2256 static void vhost_scsi_drop_tport(struct se_wwn *wwn)
2257 {
2258         struct vhost_scsi_tport *tport = container_of(wwn,
2259                                 struct vhost_scsi_tport, tport_wwn);
2260
2261         pr_debug("TCM_VHost_ConfigFS: Deallocating emulated Target"
2262                 " %s Address: %s\n", vhost_scsi_dump_proto_id(tport),
2263                 tport->tport_name);
2264
2265         kfree(tport);
2266 }
2267
2268 static ssize_t
2269 vhost_scsi_wwn_version_show(struct config_item *item, char *page)
2270 {
2271         return sprintf(page, "TCM_VHOST fabric module %s on %s/%s"
2272                 "on "UTS_RELEASE"\n", VHOST_SCSI_VERSION, utsname()->sysname,
2273                 utsname()->machine);
2274 }
2275
2276 CONFIGFS_ATTR_RO(vhost_scsi_wwn_, version);
2277
2278 static struct configfs_attribute *vhost_scsi_wwn_attrs[] = {
2279         &vhost_scsi_wwn_attr_version,
2280         NULL,
2281 };
2282
2283 static const struct target_core_fabric_ops vhost_scsi_ops = {
2284         .module                         = THIS_MODULE,
2285         .fabric_name                    = "vhost",
2286         .tpg_get_wwn                    = vhost_scsi_get_fabric_wwn,
2287         .tpg_get_tag                    = vhost_scsi_get_tpgt,
2288         .tpg_check_demo_mode            = vhost_scsi_check_true,
2289         .tpg_check_demo_mode_cache      = vhost_scsi_check_true,
2290         .tpg_check_demo_mode_write_protect = vhost_scsi_check_false,
2291         .tpg_check_prod_mode_write_protect = vhost_scsi_check_false,
2292         .tpg_check_prot_fabric_only     = vhost_scsi_check_prot_fabric_only,
2293         .tpg_get_inst_index             = vhost_scsi_tpg_get_inst_index,
2294         .release_cmd                    = vhost_scsi_release_cmd,
2295         .check_stop_free                = vhost_scsi_check_stop_free,
2296         .sess_get_index                 = vhost_scsi_sess_get_index,
2297         .sess_get_initiator_sid         = NULL,
2298         .write_pending                  = vhost_scsi_write_pending,
2299         .set_default_node_attributes    = vhost_scsi_set_default_node_attrs,
2300         .get_cmd_state                  = vhost_scsi_get_cmd_state,
2301         .queue_data_in                  = vhost_scsi_queue_data_in,
2302         .queue_status                   = vhost_scsi_queue_status,
2303         .queue_tm_rsp                   = vhost_scsi_queue_tm_rsp,
2304         .aborted_task                   = vhost_scsi_aborted_task,
2305         /*
2306          * Setup callers for generic logic in target_core_fabric_configfs.c
2307          */
2308         .fabric_make_wwn                = vhost_scsi_make_tport,
2309         .fabric_drop_wwn                = vhost_scsi_drop_tport,
2310         .fabric_make_tpg                = vhost_scsi_make_tpg,
2311         .fabric_drop_tpg                = vhost_scsi_drop_tpg,
2312         .fabric_post_link               = vhost_scsi_port_link,
2313         .fabric_pre_unlink              = vhost_scsi_port_unlink,
2314
2315         .tfc_wwn_attrs                  = vhost_scsi_wwn_attrs,
2316         .tfc_tpg_base_attrs             = vhost_scsi_tpg_attrs,
2317         .tfc_tpg_attrib_attrs           = vhost_scsi_tpg_attrib_attrs,
2318 };
2319
2320 static int __init vhost_scsi_init(void)
2321 {
2322         int ret = -ENOMEM;
2323
2324         pr_debug("TCM_VHOST fabric module %s on %s/%s"
2325                 " on "UTS_RELEASE"\n", VHOST_SCSI_VERSION, utsname()->sysname,
2326                 utsname()->machine);
2327
2328         /*
2329          * Use our own dedicated workqueue for submitting I/O into
2330          * target core to avoid contention within system_wq.
2331          */
2332         vhost_scsi_workqueue = alloc_workqueue("vhost_scsi", 0, 0);
2333         if (!vhost_scsi_workqueue)
2334                 goto out;
2335
2336         ret = vhost_scsi_register();
2337         if (ret < 0)
2338                 goto out_destroy_workqueue;
2339
2340         ret = target_register_template(&vhost_scsi_ops);
2341         if (ret < 0)
2342                 goto out_vhost_scsi_deregister;
2343
2344         return 0;
2345
2346 out_vhost_scsi_deregister:
2347         vhost_scsi_deregister();
2348 out_destroy_workqueue:
2349         destroy_workqueue(vhost_scsi_workqueue);
2350 out:
2351         return ret;
2352 };
2353
2354 static void vhost_scsi_exit(void)
2355 {
2356         target_unregister_template(&vhost_scsi_ops);
2357         vhost_scsi_deregister();
2358         destroy_workqueue(vhost_scsi_workqueue);
2359 };
2360
2361 MODULE_DESCRIPTION("VHOST_SCSI series fabric driver");
2362 MODULE_ALIAS("tcm_vhost");
2363 MODULE_LICENSE("GPL");
2364 module_init(vhost_scsi_init);
2365 module_exit(vhost_scsi_exit);