Merge tag 'icc-6.7-rc5' of git://git.kernel.org/pub/scm/linux/kernel/git/djakov/icc...
[linux-2.6-microblaze.git] / drivers / nvme / host / fc.c
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (c) 2016 Avago Technologies.  All rights reserved.
4  */
5 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
6 #include <linux/module.h>
7 #include <linux/parser.h>
8 #include <uapi/scsi/fc/fc_fs.h>
9 #include <uapi/scsi/fc/fc_els.h>
10 #include <linux/delay.h>
11 #include <linux/overflow.h>
12 #include <linux/blk-cgroup.h>
13 #include "nvme.h"
14 #include "fabrics.h"
15 #include <linux/nvme-fc-driver.h>
16 #include <linux/nvme-fc.h>
17 #include "fc.h"
18 #include <scsi/scsi_transport_fc.h>
19 #include <linux/blk-mq-pci.h>
20
21 /* *************************** Data Structures/Defines ****************** */
22
23
24 enum nvme_fc_queue_flags {
25         NVME_FC_Q_CONNECTED = 0,
26         NVME_FC_Q_LIVE,
27 };
28
29 #define NVME_FC_DEFAULT_DEV_LOSS_TMO    60      /* seconds */
30 #define NVME_FC_DEFAULT_RECONNECT_TMO   2       /* delay between reconnects
31                                                  * when connected and a
32                                                  * connection failure.
33                                                  */
34
35 struct nvme_fc_queue {
36         struct nvme_fc_ctrl     *ctrl;
37         struct device           *dev;
38         struct blk_mq_hw_ctx    *hctx;
39         void                    *lldd_handle;
40         size_t                  cmnd_capsule_len;
41         u32                     qnum;
42         u32                     rqcnt;
43         u32                     seqno;
44
45         u64                     connection_id;
46         atomic_t                csn;
47
48         unsigned long           flags;
49 } __aligned(sizeof(u64));       /* alignment for other things alloc'd with */
50
51 enum nvme_fcop_flags {
52         FCOP_FLAGS_TERMIO       = (1 << 0),
53         FCOP_FLAGS_AEN          = (1 << 1),
54 };
55
56 struct nvmefc_ls_req_op {
57         struct nvmefc_ls_req    ls_req;
58
59         struct nvme_fc_rport    *rport;
60         struct nvme_fc_queue    *queue;
61         struct request          *rq;
62         u32                     flags;
63
64         int                     ls_error;
65         struct completion       ls_done;
66         struct list_head        lsreq_list;     /* rport->ls_req_list */
67         bool                    req_queued;
68 };
69
70 struct nvmefc_ls_rcv_op {
71         struct nvme_fc_rport            *rport;
72         struct nvmefc_ls_rsp            *lsrsp;
73         union nvmefc_ls_requests        *rqstbuf;
74         union nvmefc_ls_responses       *rspbuf;
75         u16                             rqstdatalen;
76         bool                            handled;
77         dma_addr_t                      rspdma;
78         struct list_head                lsrcv_list;     /* rport->ls_rcv_list */
79 } __aligned(sizeof(u64));       /* alignment for other things alloc'd with */
80
81 enum nvme_fcpop_state {
82         FCPOP_STATE_UNINIT      = 0,
83         FCPOP_STATE_IDLE        = 1,
84         FCPOP_STATE_ACTIVE      = 2,
85         FCPOP_STATE_ABORTED     = 3,
86         FCPOP_STATE_COMPLETE    = 4,
87 };
88
89 struct nvme_fc_fcp_op {
90         struct nvme_request     nreq;           /*
91                                                  * nvme/host/core.c
92                                                  * requires this to be
93                                                  * the 1st element in the
94                                                  * private structure
95                                                  * associated with the
96                                                  * request.
97                                                  */
98         struct nvmefc_fcp_req   fcp_req;
99
100         struct nvme_fc_ctrl     *ctrl;
101         struct nvme_fc_queue    *queue;
102         struct request          *rq;
103
104         atomic_t                state;
105         u32                     flags;
106         u32                     rqno;
107         u32                     nents;
108
109         struct nvme_fc_cmd_iu   cmd_iu;
110         struct nvme_fc_ersp_iu  rsp_iu;
111 };
112
113 struct nvme_fcp_op_w_sgl {
114         struct nvme_fc_fcp_op   op;
115         struct scatterlist      sgl[NVME_INLINE_SG_CNT];
116         uint8_t                 priv[];
117 };
118
119 struct nvme_fc_lport {
120         struct nvme_fc_local_port       localport;
121
122         struct ida                      endp_cnt;
123         struct list_head                port_list;      /* nvme_fc_port_list */
124         struct list_head                endp_list;
125         struct device                   *dev;   /* physical device for dma */
126         struct nvme_fc_port_template    *ops;
127         struct kref                     ref;
128         atomic_t                        act_rport_cnt;
129 } __aligned(sizeof(u64));       /* alignment for other things alloc'd with */
130
131 struct nvme_fc_rport {
132         struct nvme_fc_remote_port      remoteport;
133
134         struct list_head                endp_list; /* for lport->endp_list */
135         struct list_head                ctrl_list;
136         struct list_head                ls_req_list;
137         struct list_head                ls_rcv_list;
138         struct list_head                disc_list;
139         struct device                   *dev;   /* physical device for dma */
140         struct nvme_fc_lport            *lport;
141         spinlock_t                      lock;
142         struct kref                     ref;
143         atomic_t                        act_ctrl_cnt;
144         unsigned long                   dev_loss_end;
145         struct work_struct              lsrcv_work;
146 } __aligned(sizeof(u64));       /* alignment for other things alloc'd with */
147
148 /* fc_ctrl flags values - specified as bit positions */
149 #define ASSOC_ACTIVE            0
150 #define ASSOC_FAILED            1
151 #define FCCTRL_TERMIO           2
152
153 struct nvme_fc_ctrl {
154         spinlock_t              lock;
155         struct nvme_fc_queue    *queues;
156         struct device           *dev;
157         struct nvme_fc_lport    *lport;
158         struct nvme_fc_rport    *rport;
159         u32                     cnum;
160
161         bool                    ioq_live;
162         u64                     association_id;
163         struct nvmefc_ls_rcv_op *rcv_disconn;
164
165         struct list_head        ctrl_list;      /* rport->ctrl_list */
166
167         struct blk_mq_tag_set   admin_tag_set;
168         struct blk_mq_tag_set   tag_set;
169
170         struct work_struct      ioerr_work;
171         struct delayed_work     connect_work;
172
173         struct kref             ref;
174         unsigned long           flags;
175         u32                     iocnt;
176         wait_queue_head_t       ioabort_wait;
177
178         struct nvme_fc_fcp_op   aen_ops[NVME_NR_AEN_COMMANDS];
179
180         struct nvme_ctrl        ctrl;
181 };
182
183 static inline struct nvme_fc_ctrl *
184 to_fc_ctrl(struct nvme_ctrl *ctrl)
185 {
186         return container_of(ctrl, struct nvme_fc_ctrl, ctrl);
187 }
188
189 static inline struct nvme_fc_lport *
190 localport_to_lport(struct nvme_fc_local_port *portptr)
191 {
192         return container_of(portptr, struct nvme_fc_lport, localport);
193 }
194
195 static inline struct nvme_fc_rport *
196 remoteport_to_rport(struct nvme_fc_remote_port *portptr)
197 {
198         return container_of(portptr, struct nvme_fc_rport, remoteport);
199 }
200
201 static inline struct nvmefc_ls_req_op *
202 ls_req_to_lsop(struct nvmefc_ls_req *lsreq)
203 {
204         return container_of(lsreq, struct nvmefc_ls_req_op, ls_req);
205 }
206
207 static inline struct nvme_fc_fcp_op *
208 fcp_req_to_fcp_op(struct nvmefc_fcp_req *fcpreq)
209 {
210         return container_of(fcpreq, struct nvme_fc_fcp_op, fcp_req);
211 }
212
213
214
215 /* *************************** Globals **************************** */
216
217
218 static DEFINE_SPINLOCK(nvme_fc_lock);
219
220 static LIST_HEAD(nvme_fc_lport_list);
221 static DEFINE_IDA(nvme_fc_local_port_cnt);
222 static DEFINE_IDA(nvme_fc_ctrl_cnt);
223
224 static struct workqueue_struct *nvme_fc_wq;
225
226 static bool nvme_fc_waiting_to_unload;
227 static DECLARE_COMPLETION(nvme_fc_unload_proceed);
228
229 /*
230  * These items are short-term. They will eventually be moved into
231  * a generic FC class. See comments in module init.
232  */
233 static struct device *fc_udev_device;
234
235 static void nvme_fc_complete_rq(struct request *rq);
236
237 /* *********************** FC-NVME Port Management ************************ */
238
239 static void __nvme_fc_delete_hw_queue(struct nvme_fc_ctrl *,
240                         struct nvme_fc_queue *, unsigned int);
241
242 static void nvme_fc_handle_ls_rqst_work(struct work_struct *work);
243
244
245 static void
246 nvme_fc_free_lport(struct kref *ref)
247 {
248         struct nvme_fc_lport *lport =
249                 container_of(ref, struct nvme_fc_lport, ref);
250         unsigned long flags;
251
252         WARN_ON(lport->localport.port_state != FC_OBJSTATE_DELETED);
253         WARN_ON(!list_empty(&lport->endp_list));
254
255         /* remove from transport list */
256         spin_lock_irqsave(&nvme_fc_lock, flags);
257         list_del(&lport->port_list);
258         if (nvme_fc_waiting_to_unload && list_empty(&nvme_fc_lport_list))
259                 complete(&nvme_fc_unload_proceed);
260         spin_unlock_irqrestore(&nvme_fc_lock, flags);
261
262         ida_free(&nvme_fc_local_port_cnt, lport->localport.port_num);
263         ida_destroy(&lport->endp_cnt);
264
265         put_device(lport->dev);
266
267         kfree(lport);
268 }
269
270 static void
271 nvme_fc_lport_put(struct nvme_fc_lport *lport)
272 {
273         kref_put(&lport->ref, nvme_fc_free_lport);
274 }
275
276 static int
277 nvme_fc_lport_get(struct nvme_fc_lport *lport)
278 {
279         return kref_get_unless_zero(&lport->ref);
280 }
281
282
283 static struct nvme_fc_lport *
284 nvme_fc_attach_to_unreg_lport(struct nvme_fc_port_info *pinfo,
285                         struct nvme_fc_port_template *ops,
286                         struct device *dev)
287 {
288         struct nvme_fc_lport *lport;
289         unsigned long flags;
290
291         spin_lock_irqsave(&nvme_fc_lock, flags);
292
293         list_for_each_entry(lport, &nvme_fc_lport_list, port_list) {
294                 if (lport->localport.node_name != pinfo->node_name ||
295                     lport->localport.port_name != pinfo->port_name)
296                         continue;
297
298                 if (lport->dev != dev) {
299                         lport = ERR_PTR(-EXDEV);
300                         goto out_done;
301                 }
302
303                 if (lport->localport.port_state != FC_OBJSTATE_DELETED) {
304                         lport = ERR_PTR(-EEXIST);
305                         goto out_done;
306                 }
307
308                 if (!nvme_fc_lport_get(lport)) {
309                         /*
310                          * fails if ref cnt already 0. If so,
311                          * act as if lport already deleted
312                          */
313                         lport = NULL;
314                         goto out_done;
315                 }
316
317                 /* resume the lport */
318
319                 lport->ops = ops;
320                 lport->localport.port_role = pinfo->port_role;
321                 lport->localport.port_id = pinfo->port_id;
322                 lport->localport.port_state = FC_OBJSTATE_ONLINE;
323
324                 spin_unlock_irqrestore(&nvme_fc_lock, flags);
325
326                 return lport;
327         }
328
329         lport = NULL;
330
331 out_done:
332         spin_unlock_irqrestore(&nvme_fc_lock, flags);
333
334         return lport;
335 }
336
337 /**
338  * nvme_fc_register_localport - transport entry point called by an
339  *                              LLDD to register the existence of a NVME
340  *                              host FC port.
341  * @pinfo:     pointer to information about the port to be registered
342  * @template:  LLDD entrypoints and operational parameters for the port
343  * @dev:       physical hardware device node port corresponds to. Will be
344  *             used for DMA mappings
345  * @portptr:   pointer to a local port pointer. Upon success, the routine
346  *             will allocate a nvme_fc_local_port structure and place its
347  *             address in the local port pointer. Upon failure, local port
348  *             pointer will be set to 0.
349  *
350  * Returns:
351  * a completion status. Must be 0 upon success; a negative errno
352  * (ex: -ENXIO) upon failure.
353  */
354 int
355 nvme_fc_register_localport(struct nvme_fc_port_info *pinfo,
356                         struct nvme_fc_port_template *template,
357                         struct device *dev,
358                         struct nvme_fc_local_port **portptr)
359 {
360         struct nvme_fc_lport *newrec;
361         unsigned long flags;
362         int ret, idx;
363
364         if (!template->localport_delete || !template->remoteport_delete ||
365             !template->ls_req || !template->fcp_io ||
366             !template->ls_abort || !template->fcp_abort ||
367             !template->max_hw_queues || !template->max_sgl_segments ||
368             !template->max_dif_sgl_segments || !template->dma_boundary) {
369                 ret = -EINVAL;
370                 goto out_reghost_failed;
371         }
372
373         /*
374          * look to see if there is already a localport that had been
375          * deregistered and in the process of waiting for all the
376          * references to fully be removed.  If the references haven't
377          * expired, we can simply re-enable the localport. Remoteports
378          * and controller reconnections should resume naturally.
379          */
380         newrec = nvme_fc_attach_to_unreg_lport(pinfo, template, dev);
381
382         /* found an lport, but something about its state is bad */
383         if (IS_ERR(newrec)) {
384                 ret = PTR_ERR(newrec);
385                 goto out_reghost_failed;
386
387         /* found existing lport, which was resumed */
388         } else if (newrec) {
389                 *portptr = &newrec->localport;
390                 return 0;
391         }
392
393         /* nothing found - allocate a new localport struct */
394
395         newrec = kmalloc((sizeof(*newrec) + template->local_priv_sz),
396                          GFP_KERNEL);
397         if (!newrec) {
398                 ret = -ENOMEM;
399                 goto out_reghost_failed;
400         }
401
402         idx = ida_alloc(&nvme_fc_local_port_cnt, GFP_KERNEL);
403         if (idx < 0) {
404                 ret = -ENOSPC;
405                 goto out_fail_kfree;
406         }
407
408         if (!get_device(dev) && dev) {
409                 ret = -ENODEV;
410                 goto out_ida_put;
411         }
412
413         INIT_LIST_HEAD(&newrec->port_list);
414         INIT_LIST_HEAD(&newrec->endp_list);
415         kref_init(&newrec->ref);
416         atomic_set(&newrec->act_rport_cnt, 0);
417         newrec->ops = template;
418         newrec->dev = dev;
419         ida_init(&newrec->endp_cnt);
420         if (template->local_priv_sz)
421                 newrec->localport.private = &newrec[1];
422         else
423                 newrec->localport.private = NULL;
424         newrec->localport.node_name = pinfo->node_name;
425         newrec->localport.port_name = pinfo->port_name;
426         newrec->localport.port_role = pinfo->port_role;
427         newrec->localport.port_id = pinfo->port_id;
428         newrec->localport.port_state = FC_OBJSTATE_ONLINE;
429         newrec->localport.port_num = idx;
430
431         spin_lock_irqsave(&nvme_fc_lock, flags);
432         list_add_tail(&newrec->port_list, &nvme_fc_lport_list);
433         spin_unlock_irqrestore(&nvme_fc_lock, flags);
434
435         if (dev)
436                 dma_set_seg_boundary(dev, template->dma_boundary);
437
438         *portptr = &newrec->localport;
439         return 0;
440
441 out_ida_put:
442         ida_free(&nvme_fc_local_port_cnt, idx);
443 out_fail_kfree:
444         kfree(newrec);
445 out_reghost_failed:
446         *portptr = NULL;
447
448         return ret;
449 }
450 EXPORT_SYMBOL_GPL(nvme_fc_register_localport);
451
452 /**
453  * nvme_fc_unregister_localport - transport entry point called by an
454  *                              LLDD to deregister/remove a previously
455  *                              registered a NVME host FC port.
456  * @portptr: pointer to the (registered) local port that is to be deregistered.
457  *
458  * Returns:
459  * a completion status. Must be 0 upon success; a negative errno
460  * (ex: -ENXIO) upon failure.
461  */
462 int
463 nvme_fc_unregister_localport(struct nvme_fc_local_port *portptr)
464 {
465         struct nvme_fc_lport *lport = localport_to_lport(portptr);
466         unsigned long flags;
467
468         if (!portptr)
469                 return -EINVAL;
470
471         spin_lock_irqsave(&nvme_fc_lock, flags);
472
473         if (portptr->port_state != FC_OBJSTATE_ONLINE) {
474                 spin_unlock_irqrestore(&nvme_fc_lock, flags);
475                 return -EINVAL;
476         }
477         portptr->port_state = FC_OBJSTATE_DELETED;
478
479         spin_unlock_irqrestore(&nvme_fc_lock, flags);
480
481         if (atomic_read(&lport->act_rport_cnt) == 0)
482                 lport->ops->localport_delete(&lport->localport);
483
484         nvme_fc_lport_put(lport);
485
486         return 0;
487 }
488 EXPORT_SYMBOL_GPL(nvme_fc_unregister_localport);
489
490 /*
491  * TRADDR strings, per FC-NVME are fixed format:
492  *   "nn-0x<16hexdigits>:pn-0x<16hexdigits>" - 43 characters
493  * udev event will only differ by prefix of what field is
494  * being specified:
495  *    "NVMEFC_HOST_TRADDR=" or "NVMEFC_TRADDR=" - 19 max characters
496  *  19 + 43 + null_fudge = 64 characters
497  */
498 #define FCNVME_TRADDR_LENGTH            64
499
500 static void
501 nvme_fc_signal_discovery_scan(struct nvme_fc_lport *lport,
502                 struct nvme_fc_rport *rport)
503 {
504         char hostaddr[FCNVME_TRADDR_LENGTH];    /* NVMEFC_HOST_TRADDR=...*/
505         char tgtaddr[FCNVME_TRADDR_LENGTH];     /* NVMEFC_TRADDR=...*/
506         char *envp[4] = { "FC_EVENT=nvmediscovery", hostaddr, tgtaddr, NULL };
507
508         if (!(rport->remoteport.port_role & FC_PORT_ROLE_NVME_DISCOVERY))
509                 return;
510
511         snprintf(hostaddr, sizeof(hostaddr),
512                 "NVMEFC_HOST_TRADDR=nn-0x%016llx:pn-0x%016llx",
513                 lport->localport.node_name, lport->localport.port_name);
514         snprintf(tgtaddr, sizeof(tgtaddr),
515                 "NVMEFC_TRADDR=nn-0x%016llx:pn-0x%016llx",
516                 rport->remoteport.node_name, rport->remoteport.port_name);
517         kobject_uevent_env(&fc_udev_device->kobj, KOBJ_CHANGE, envp);
518 }
519
520 static void
521 nvme_fc_free_rport(struct kref *ref)
522 {
523         struct nvme_fc_rport *rport =
524                 container_of(ref, struct nvme_fc_rport, ref);
525         struct nvme_fc_lport *lport =
526                         localport_to_lport(rport->remoteport.localport);
527         unsigned long flags;
528
529         WARN_ON(rport->remoteport.port_state != FC_OBJSTATE_DELETED);
530         WARN_ON(!list_empty(&rport->ctrl_list));
531
532         /* remove from lport list */
533         spin_lock_irqsave(&nvme_fc_lock, flags);
534         list_del(&rport->endp_list);
535         spin_unlock_irqrestore(&nvme_fc_lock, flags);
536
537         WARN_ON(!list_empty(&rport->disc_list));
538         ida_free(&lport->endp_cnt, rport->remoteport.port_num);
539
540         kfree(rport);
541
542         nvme_fc_lport_put(lport);
543 }
544
545 static void
546 nvme_fc_rport_put(struct nvme_fc_rport *rport)
547 {
548         kref_put(&rport->ref, nvme_fc_free_rport);
549 }
550
551 static int
552 nvme_fc_rport_get(struct nvme_fc_rport *rport)
553 {
554         return kref_get_unless_zero(&rport->ref);
555 }
556
557 static void
558 nvme_fc_resume_controller(struct nvme_fc_ctrl *ctrl)
559 {
560         switch (nvme_ctrl_state(&ctrl->ctrl)) {
561         case NVME_CTRL_NEW:
562         case NVME_CTRL_CONNECTING:
563                 /*
564                  * As all reconnects were suppressed, schedule a
565                  * connect.
566                  */
567                 dev_info(ctrl->ctrl.device,
568                         "NVME-FC{%d}: connectivity re-established. "
569                         "Attempting reconnect\n", ctrl->cnum);
570
571                 queue_delayed_work(nvme_wq, &ctrl->connect_work, 0);
572                 break;
573
574         case NVME_CTRL_RESETTING:
575                 /*
576                  * Controller is already in the process of terminating the
577                  * association. No need to do anything further. The reconnect
578                  * step will naturally occur after the reset completes.
579                  */
580                 break;
581
582         default:
583                 /* no action to take - let it delete */
584                 break;
585         }
586 }
587
588 static struct nvme_fc_rport *
589 nvme_fc_attach_to_suspended_rport(struct nvme_fc_lport *lport,
590                                 struct nvme_fc_port_info *pinfo)
591 {
592         struct nvme_fc_rport *rport;
593         struct nvme_fc_ctrl *ctrl;
594         unsigned long flags;
595
596         spin_lock_irqsave(&nvme_fc_lock, flags);
597
598         list_for_each_entry(rport, &lport->endp_list, endp_list) {
599                 if (rport->remoteport.node_name != pinfo->node_name ||
600                     rport->remoteport.port_name != pinfo->port_name)
601                         continue;
602
603                 if (!nvme_fc_rport_get(rport)) {
604                         rport = ERR_PTR(-ENOLCK);
605                         goto out_done;
606                 }
607
608                 spin_unlock_irqrestore(&nvme_fc_lock, flags);
609
610                 spin_lock_irqsave(&rport->lock, flags);
611
612                 /* has it been unregistered */
613                 if (rport->remoteport.port_state != FC_OBJSTATE_DELETED) {
614                         /* means lldd called us twice */
615                         spin_unlock_irqrestore(&rport->lock, flags);
616                         nvme_fc_rport_put(rport);
617                         return ERR_PTR(-ESTALE);
618                 }
619
620                 rport->remoteport.port_role = pinfo->port_role;
621                 rport->remoteport.port_id = pinfo->port_id;
622                 rport->remoteport.port_state = FC_OBJSTATE_ONLINE;
623                 rport->dev_loss_end = 0;
624
625                 /*
626                  * kick off a reconnect attempt on all associations to the
627                  * remote port. A successful reconnects will resume i/o.
628                  */
629                 list_for_each_entry(ctrl, &rport->ctrl_list, ctrl_list)
630                         nvme_fc_resume_controller(ctrl);
631
632                 spin_unlock_irqrestore(&rport->lock, flags);
633
634                 return rport;
635         }
636
637         rport = NULL;
638
639 out_done:
640         spin_unlock_irqrestore(&nvme_fc_lock, flags);
641
642         return rport;
643 }
644
645 static inline void
646 __nvme_fc_set_dev_loss_tmo(struct nvme_fc_rport *rport,
647                         struct nvme_fc_port_info *pinfo)
648 {
649         if (pinfo->dev_loss_tmo)
650                 rport->remoteport.dev_loss_tmo = pinfo->dev_loss_tmo;
651         else
652                 rport->remoteport.dev_loss_tmo = NVME_FC_DEFAULT_DEV_LOSS_TMO;
653 }
654
655 /**
656  * nvme_fc_register_remoteport - transport entry point called by an
657  *                              LLDD to register the existence of a NVME
658  *                              subsystem FC port on its fabric.
659  * @localport: pointer to the (registered) local port that the remote
660  *             subsystem port is connected to.
661  * @pinfo:     pointer to information about the port to be registered
662  * @portptr:   pointer to a remote port pointer. Upon success, the routine
663  *             will allocate a nvme_fc_remote_port structure and place its
664  *             address in the remote port pointer. Upon failure, remote port
665  *             pointer will be set to 0.
666  *
667  * Returns:
668  * a completion status. Must be 0 upon success; a negative errno
669  * (ex: -ENXIO) upon failure.
670  */
671 int
672 nvme_fc_register_remoteport(struct nvme_fc_local_port *localport,
673                                 struct nvme_fc_port_info *pinfo,
674                                 struct nvme_fc_remote_port **portptr)
675 {
676         struct nvme_fc_lport *lport = localport_to_lport(localport);
677         struct nvme_fc_rport *newrec;
678         unsigned long flags;
679         int ret, idx;
680
681         if (!nvme_fc_lport_get(lport)) {
682                 ret = -ESHUTDOWN;
683                 goto out_reghost_failed;
684         }
685
686         /*
687          * look to see if there is already a remoteport that is waiting
688          * for a reconnect (within dev_loss_tmo) with the same WWN's.
689          * If so, transition to it and reconnect.
690          */
691         newrec = nvme_fc_attach_to_suspended_rport(lport, pinfo);
692
693         /* found an rport, but something about its state is bad */
694         if (IS_ERR(newrec)) {
695                 ret = PTR_ERR(newrec);
696                 goto out_lport_put;
697
698         /* found existing rport, which was resumed */
699         } else if (newrec) {
700                 nvme_fc_lport_put(lport);
701                 __nvme_fc_set_dev_loss_tmo(newrec, pinfo);
702                 nvme_fc_signal_discovery_scan(lport, newrec);
703                 *portptr = &newrec->remoteport;
704                 return 0;
705         }
706
707         /* nothing found - allocate a new remoteport struct */
708
709         newrec = kmalloc((sizeof(*newrec) + lport->ops->remote_priv_sz),
710                          GFP_KERNEL);
711         if (!newrec) {
712                 ret = -ENOMEM;
713                 goto out_lport_put;
714         }
715
716         idx = ida_alloc(&lport->endp_cnt, GFP_KERNEL);
717         if (idx < 0) {
718                 ret = -ENOSPC;
719                 goto out_kfree_rport;
720         }
721
722         INIT_LIST_HEAD(&newrec->endp_list);
723         INIT_LIST_HEAD(&newrec->ctrl_list);
724         INIT_LIST_HEAD(&newrec->ls_req_list);
725         INIT_LIST_HEAD(&newrec->disc_list);
726         kref_init(&newrec->ref);
727         atomic_set(&newrec->act_ctrl_cnt, 0);
728         spin_lock_init(&newrec->lock);
729         newrec->remoteport.localport = &lport->localport;
730         INIT_LIST_HEAD(&newrec->ls_rcv_list);
731         newrec->dev = lport->dev;
732         newrec->lport = lport;
733         if (lport->ops->remote_priv_sz)
734                 newrec->remoteport.private = &newrec[1];
735         else
736                 newrec->remoteport.private = NULL;
737         newrec->remoteport.port_role = pinfo->port_role;
738         newrec->remoteport.node_name = pinfo->node_name;
739         newrec->remoteport.port_name = pinfo->port_name;
740         newrec->remoteport.port_id = pinfo->port_id;
741         newrec->remoteport.port_state = FC_OBJSTATE_ONLINE;
742         newrec->remoteport.port_num = idx;
743         __nvme_fc_set_dev_loss_tmo(newrec, pinfo);
744         INIT_WORK(&newrec->lsrcv_work, nvme_fc_handle_ls_rqst_work);
745
746         spin_lock_irqsave(&nvme_fc_lock, flags);
747         list_add_tail(&newrec->endp_list, &lport->endp_list);
748         spin_unlock_irqrestore(&nvme_fc_lock, flags);
749
750         nvme_fc_signal_discovery_scan(lport, newrec);
751
752         *portptr = &newrec->remoteport;
753         return 0;
754
755 out_kfree_rport:
756         kfree(newrec);
757 out_lport_put:
758         nvme_fc_lport_put(lport);
759 out_reghost_failed:
760         *portptr = NULL;
761         return ret;
762 }
763 EXPORT_SYMBOL_GPL(nvme_fc_register_remoteport);
764
765 static int
766 nvme_fc_abort_lsops(struct nvme_fc_rport *rport)
767 {
768         struct nvmefc_ls_req_op *lsop;
769         unsigned long flags;
770
771 restart:
772         spin_lock_irqsave(&rport->lock, flags);
773
774         list_for_each_entry(lsop, &rport->ls_req_list, lsreq_list) {
775                 if (!(lsop->flags & FCOP_FLAGS_TERMIO)) {
776                         lsop->flags |= FCOP_FLAGS_TERMIO;
777                         spin_unlock_irqrestore(&rport->lock, flags);
778                         rport->lport->ops->ls_abort(&rport->lport->localport,
779                                                 &rport->remoteport,
780                                                 &lsop->ls_req);
781                         goto restart;
782                 }
783         }
784         spin_unlock_irqrestore(&rport->lock, flags);
785
786         return 0;
787 }
788
789 static void
790 nvme_fc_ctrl_connectivity_loss(struct nvme_fc_ctrl *ctrl)
791 {
792         dev_info(ctrl->ctrl.device,
793                 "NVME-FC{%d}: controller connectivity lost. Awaiting "
794                 "Reconnect", ctrl->cnum);
795
796         switch (nvme_ctrl_state(&ctrl->ctrl)) {
797         case NVME_CTRL_NEW:
798         case NVME_CTRL_LIVE:
799                 /*
800                  * Schedule a controller reset. The reset will terminate the
801                  * association and schedule the reconnect timer.  Reconnects
802                  * will be attempted until either the ctlr_loss_tmo
803                  * (max_retries * connect_delay) expires or the remoteport's
804                  * dev_loss_tmo expires.
805                  */
806                 if (nvme_reset_ctrl(&ctrl->ctrl)) {
807                         dev_warn(ctrl->ctrl.device,
808                                 "NVME-FC{%d}: Couldn't schedule reset.\n",
809                                 ctrl->cnum);
810                         nvme_delete_ctrl(&ctrl->ctrl);
811                 }
812                 break;
813
814         case NVME_CTRL_CONNECTING:
815                 /*
816                  * The association has already been terminated and the
817                  * controller is attempting reconnects.  No need to do anything
818                  * futher.  Reconnects will be attempted until either the
819                  * ctlr_loss_tmo (max_retries * connect_delay) expires or the
820                  * remoteport's dev_loss_tmo expires.
821                  */
822                 break;
823
824         case NVME_CTRL_RESETTING:
825                 /*
826                  * Controller is already in the process of terminating the
827                  * association.  No need to do anything further. The reconnect
828                  * step will kick in naturally after the association is
829                  * terminated.
830                  */
831                 break;
832
833         case NVME_CTRL_DELETING:
834         case NVME_CTRL_DELETING_NOIO:
835         default:
836                 /* no action to take - let it delete */
837                 break;
838         }
839 }
840
841 /**
842  * nvme_fc_unregister_remoteport - transport entry point called by an
843  *                              LLDD to deregister/remove a previously
844  *                              registered a NVME subsystem FC port.
845  * @portptr: pointer to the (registered) remote port that is to be
846  *           deregistered.
847  *
848  * Returns:
849  * a completion status. Must be 0 upon success; a negative errno
850  * (ex: -ENXIO) upon failure.
851  */
852 int
853 nvme_fc_unregister_remoteport(struct nvme_fc_remote_port *portptr)
854 {
855         struct nvme_fc_rport *rport = remoteport_to_rport(portptr);
856         struct nvme_fc_ctrl *ctrl;
857         unsigned long flags;
858
859         if (!portptr)
860                 return -EINVAL;
861
862         spin_lock_irqsave(&rport->lock, flags);
863
864         if (portptr->port_state != FC_OBJSTATE_ONLINE) {
865                 spin_unlock_irqrestore(&rport->lock, flags);
866                 return -EINVAL;
867         }
868         portptr->port_state = FC_OBJSTATE_DELETED;
869
870         rport->dev_loss_end = jiffies + (portptr->dev_loss_tmo * HZ);
871
872         list_for_each_entry(ctrl, &rport->ctrl_list, ctrl_list) {
873                 /* if dev_loss_tmo==0, dev loss is immediate */
874                 if (!portptr->dev_loss_tmo) {
875                         dev_warn(ctrl->ctrl.device,
876                                 "NVME-FC{%d}: controller connectivity lost.\n",
877                                 ctrl->cnum);
878                         nvme_delete_ctrl(&ctrl->ctrl);
879                 } else
880                         nvme_fc_ctrl_connectivity_loss(ctrl);
881         }
882
883         spin_unlock_irqrestore(&rport->lock, flags);
884
885         nvme_fc_abort_lsops(rport);
886
887         if (atomic_read(&rport->act_ctrl_cnt) == 0)
888                 rport->lport->ops->remoteport_delete(portptr);
889
890         /*
891          * release the reference, which will allow, if all controllers
892          * go away, which should only occur after dev_loss_tmo occurs,
893          * for the rport to be torn down.
894          */
895         nvme_fc_rport_put(rport);
896
897         return 0;
898 }
899 EXPORT_SYMBOL_GPL(nvme_fc_unregister_remoteport);
900
901 /**
902  * nvme_fc_rescan_remoteport - transport entry point called by an
903  *                              LLDD to request a nvme device rescan.
904  * @remoteport: pointer to the (registered) remote port that is to be
905  *              rescanned.
906  *
907  * Returns: N/A
908  */
909 void
910 nvme_fc_rescan_remoteport(struct nvme_fc_remote_port *remoteport)
911 {
912         struct nvme_fc_rport *rport = remoteport_to_rport(remoteport);
913
914         nvme_fc_signal_discovery_scan(rport->lport, rport);
915 }
916 EXPORT_SYMBOL_GPL(nvme_fc_rescan_remoteport);
917
918 int
919 nvme_fc_set_remoteport_devloss(struct nvme_fc_remote_port *portptr,
920                         u32 dev_loss_tmo)
921 {
922         struct nvme_fc_rport *rport = remoteport_to_rport(portptr);
923         unsigned long flags;
924
925         spin_lock_irqsave(&rport->lock, flags);
926
927         if (portptr->port_state != FC_OBJSTATE_ONLINE) {
928                 spin_unlock_irqrestore(&rport->lock, flags);
929                 return -EINVAL;
930         }
931
932         /* a dev_loss_tmo of 0 (immediate) is allowed to be set */
933         rport->remoteport.dev_loss_tmo = dev_loss_tmo;
934
935         spin_unlock_irqrestore(&rport->lock, flags);
936
937         return 0;
938 }
939 EXPORT_SYMBOL_GPL(nvme_fc_set_remoteport_devloss);
940
941
942 /* *********************** FC-NVME DMA Handling **************************** */
943
944 /*
945  * The fcloop device passes in a NULL device pointer. Real LLD's will
946  * pass in a valid device pointer. If NULL is passed to the dma mapping
947  * routines, depending on the platform, it may or may not succeed, and
948  * may crash.
949  *
950  * As such:
951  * Wrapper all the dma routines and check the dev pointer.
952  *
953  * If simple mappings (return just a dma address, we'll noop them,
954  * returning a dma address of 0.
955  *
956  * On more complex mappings (dma_map_sg), a pseudo routine fills
957  * in the scatter list, setting all dma addresses to 0.
958  */
959
960 static inline dma_addr_t
961 fc_dma_map_single(struct device *dev, void *ptr, size_t size,
962                 enum dma_data_direction dir)
963 {
964         return dev ? dma_map_single(dev, ptr, size, dir) : (dma_addr_t)0L;
965 }
966
967 static inline int
968 fc_dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
969 {
970         return dev ? dma_mapping_error(dev, dma_addr) : 0;
971 }
972
973 static inline void
974 fc_dma_unmap_single(struct device *dev, dma_addr_t addr, size_t size,
975         enum dma_data_direction dir)
976 {
977         if (dev)
978                 dma_unmap_single(dev, addr, size, dir);
979 }
980
981 static inline void
982 fc_dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, size_t size,
983                 enum dma_data_direction dir)
984 {
985         if (dev)
986                 dma_sync_single_for_cpu(dev, addr, size, dir);
987 }
988
989 static inline void
990 fc_dma_sync_single_for_device(struct device *dev, dma_addr_t addr, size_t size,
991                 enum dma_data_direction dir)
992 {
993         if (dev)
994                 dma_sync_single_for_device(dev, addr, size, dir);
995 }
996
997 /* pseudo dma_map_sg call */
998 static int
999 fc_map_sg(struct scatterlist *sg, int nents)
1000 {
1001         struct scatterlist *s;
1002         int i;
1003
1004         WARN_ON(nents == 0 || sg[0].length == 0);
1005
1006         for_each_sg(sg, s, nents, i) {
1007                 s->dma_address = 0L;
1008 #ifdef CONFIG_NEED_SG_DMA_LENGTH
1009                 s->dma_length = s->length;
1010 #endif
1011         }
1012         return nents;
1013 }
1014
1015 static inline int
1016 fc_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents,
1017                 enum dma_data_direction dir)
1018 {
1019         return dev ? dma_map_sg(dev, sg, nents, dir) : fc_map_sg(sg, nents);
1020 }
1021
1022 static inline void
1023 fc_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
1024                 enum dma_data_direction dir)
1025 {
1026         if (dev)
1027                 dma_unmap_sg(dev, sg, nents, dir);
1028 }
1029
1030 /* *********************** FC-NVME LS Handling **************************** */
1031
1032 static void nvme_fc_ctrl_put(struct nvme_fc_ctrl *);
1033 static int nvme_fc_ctrl_get(struct nvme_fc_ctrl *);
1034
1035 static void nvme_fc_error_recovery(struct nvme_fc_ctrl *ctrl, char *errmsg);
1036
1037 static void
1038 __nvme_fc_finish_ls_req(struct nvmefc_ls_req_op *lsop)
1039 {
1040         struct nvme_fc_rport *rport = lsop->rport;
1041         struct nvmefc_ls_req *lsreq = &lsop->ls_req;
1042         unsigned long flags;
1043
1044         spin_lock_irqsave(&rport->lock, flags);
1045
1046         if (!lsop->req_queued) {
1047                 spin_unlock_irqrestore(&rport->lock, flags);
1048                 return;
1049         }
1050
1051         list_del(&lsop->lsreq_list);
1052
1053         lsop->req_queued = false;
1054
1055         spin_unlock_irqrestore(&rport->lock, flags);
1056
1057         fc_dma_unmap_single(rport->dev, lsreq->rqstdma,
1058                                   (lsreq->rqstlen + lsreq->rsplen),
1059                                   DMA_BIDIRECTIONAL);
1060
1061         nvme_fc_rport_put(rport);
1062 }
1063
1064 static int
1065 __nvme_fc_send_ls_req(struct nvme_fc_rport *rport,
1066                 struct nvmefc_ls_req_op *lsop,
1067                 void (*done)(struct nvmefc_ls_req *req, int status))
1068 {
1069         struct nvmefc_ls_req *lsreq = &lsop->ls_req;
1070         unsigned long flags;
1071         int ret = 0;
1072
1073         if (rport->remoteport.port_state != FC_OBJSTATE_ONLINE)
1074                 return -ECONNREFUSED;
1075
1076         if (!nvme_fc_rport_get(rport))
1077                 return -ESHUTDOWN;
1078
1079         lsreq->done = done;
1080         lsop->rport = rport;
1081         lsop->req_queued = false;
1082         INIT_LIST_HEAD(&lsop->lsreq_list);
1083         init_completion(&lsop->ls_done);
1084
1085         lsreq->rqstdma = fc_dma_map_single(rport->dev, lsreq->rqstaddr,
1086                                   lsreq->rqstlen + lsreq->rsplen,
1087                                   DMA_BIDIRECTIONAL);
1088         if (fc_dma_mapping_error(rport->dev, lsreq->rqstdma)) {
1089                 ret = -EFAULT;
1090                 goto out_putrport;
1091         }
1092         lsreq->rspdma = lsreq->rqstdma + lsreq->rqstlen;
1093
1094         spin_lock_irqsave(&rport->lock, flags);
1095
1096         list_add_tail(&lsop->lsreq_list, &rport->ls_req_list);
1097
1098         lsop->req_queued = true;
1099
1100         spin_unlock_irqrestore(&rport->lock, flags);
1101
1102         ret = rport->lport->ops->ls_req(&rport->lport->localport,
1103                                         &rport->remoteport, lsreq);
1104         if (ret)
1105                 goto out_unlink;
1106
1107         return 0;
1108
1109 out_unlink:
1110         lsop->ls_error = ret;
1111         spin_lock_irqsave(&rport->lock, flags);
1112         lsop->req_queued = false;
1113         list_del(&lsop->lsreq_list);
1114         spin_unlock_irqrestore(&rport->lock, flags);
1115         fc_dma_unmap_single(rport->dev, lsreq->rqstdma,
1116                                   (lsreq->rqstlen + lsreq->rsplen),
1117                                   DMA_BIDIRECTIONAL);
1118 out_putrport:
1119         nvme_fc_rport_put(rport);
1120
1121         return ret;
1122 }
1123
1124 static void
1125 nvme_fc_send_ls_req_done(struct nvmefc_ls_req *lsreq, int status)
1126 {
1127         struct nvmefc_ls_req_op *lsop = ls_req_to_lsop(lsreq);
1128
1129         lsop->ls_error = status;
1130         complete(&lsop->ls_done);
1131 }
1132
1133 static int
1134 nvme_fc_send_ls_req(struct nvme_fc_rport *rport, struct nvmefc_ls_req_op *lsop)
1135 {
1136         struct nvmefc_ls_req *lsreq = &lsop->ls_req;
1137         struct fcnvme_ls_rjt *rjt = lsreq->rspaddr;
1138         int ret;
1139
1140         ret = __nvme_fc_send_ls_req(rport, lsop, nvme_fc_send_ls_req_done);
1141
1142         if (!ret) {
1143                 /*
1144                  * No timeout/not interruptible as we need the struct
1145                  * to exist until the lldd calls us back. Thus mandate
1146                  * wait until driver calls back. lldd responsible for
1147                  * the timeout action
1148                  */
1149                 wait_for_completion(&lsop->ls_done);
1150
1151                 __nvme_fc_finish_ls_req(lsop);
1152
1153                 ret = lsop->ls_error;
1154         }
1155
1156         if (ret)
1157                 return ret;
1158
1159         /* ACC or RJT payload ? */
1160         if (rjt->w0.ls_cmd == FCNVME_LS_RJT)
1161                 return -ENXIO;
1162
1163         return 0;
1164 }
1165
1166 static int
1167 nvme_fc_send_ls_req_async(struct nvme_fc_rport *rport,
1168                 struct nvmefc_ls_req_op *lsop,
1169                 void (*done)(struct nvmefc_ls_req *req, int status))
1170 {
1171         /* don't wait for completion */
1172
1173         return __nvme_fc_send_ls_req(rport, lsop, done);
1174 }
1175
1176 static int
1177 nvme_fc_connect_admin_queue(struct nvme_fc_ctrl *ctrl,
1178         struct nvme_fc_queue *queue, u16 qsize, u16 ersp_ratio)
1179 {
1180         struct nvmefc_ls_req_op *lsop;
1181         struct nvmefc_ls_req *lsreq;
1182         struct fcnvme_ls_cr_assoc_rqst *assoc_rqst;
1183         struct fcnvme_ls_cr_assoc_acc *assoc_acc;
1184         unsigned long flags;
1185         int ret, fcret = 0;
1186
1187         lsop = kzalloc((sizeof(*lsop) +
1188                          sizeof(*assoc_rqst) + sizeof(*assoc_acc) +
1189                          ctrl->lport->ops->lsrqst_priv_sz), GFP_KERNEL);
1190         if (!lsop) {
1191                 dev_info(ctrl->ctrl.device,
1192                         "NVME-FC{%d}: send Create Association failed: ENOMEM\n",
1193                         ctrl->cnum);
1194                 ret = -ENOMEM;
1195                 goto out_no_memory;
1196         }
1197
1198         assoc_rqst = (struct fcnvme_ls_cr_assoc_rqst *)&lsop[1];
1199         assoc_acc = (struct fcnvme_ls_cr_assoc_acc *)&assoc_rqst[1];
1200         lsreq = &lsop->ls_req;
1201         if (ctrl->lport->ops->lsrqst_priv_sz)
1202                 lsreq->private = &assoc_acc[1];
1203         else
1204                 lsreq->private = NULL;
1205
1206         assoc_rqst->w0.ls_cmd = FCNVME_LS_CREATE_ASSOCIATION;
1207         assoc_rqst->desc_list_len =
1208                         cpu_to_be32(sizeof(struct fcnvme_lsdesc_cr_assoc_cmd));
1209
1210         assoc_rqst->assoc_cmd.desc_tag =
1211                         cpu_to_be32(FCNVME_LSDESC_CREATE_ASSOC_CMD);
1212         assoc_rqst->assoc_cmd.desc_len =
1213                         fcnvme_lsdesc_len(
1214                                 sizeof(struct fcnvme_lsdesc_cr_assoc_cmd));
1215
1216         assoc_rqst->assoc_cmd.ersp_ratio = cpu_to_be16(ersp_ratio);
1217         assoc_rqst->assoc_cmd.sqsize = cpu_to_be16(qsize - 1);
1218         /* Linux supports only Dynamic controllers */
1219         assoc_rqst->assoc_cmd.cntlid = cpu_to_be16(0xffff);
1220         uuid_copy(&assoc_rqst->assoc_cmd.hostid, &ctrl->ctrl.opts->host->id);
1221         strncpy(assoc_rqst->assoc_cmd.hostnqn, ctrl->ctrl.opts->host->nqn,
1222                 min(FCNVME_ASSOC_HOSTNQN_LEN, NVMF_NQN_SIZE));
1223         strncpy(assoc_rqst->assoc_cmd.subnqn, ctrl->ctrl.opts->subsysnqn,
1224                 min(FCNVME_ASSOC_SUBNQN_LEN, NVMF_NQN_SIZE));
1225
1226         lsop->queue = queue;
1227         lsreq->rqstaddr = assoc_rqst;
1228         lsreq->rqstlen = sizeof(*assoc_rqst);
1229         lsreq->rspaddr = assoc_acc;
1230         lsreq->rsplen = sizeof(*assoc_acc);
1231         lsreq->timeout = NVME_FC_LS_TIMEOUT_SEC;
1232
1233         ret = nvme_fc_send_ls_req(ctrl->rport, lsop);
1234         if (ret)
1235                 goto out_free_buffer;
1236
1237         /* process connect LS completion */
1238
1239         /* validate the ACC response */
1240         if (assoc_acc->hdr.w0.ls_cmd != FCNVME_LS_ACC)
1241                 fcret = VERR_LSACC;
1242         else if (assoc_acc->hdr.desc_list_len !=
1243                         fcnvme_lsdesc_len(
1244                                 sizeof(struct fcnvme_ls_cr_assoc_acc)))
1245                 fcret = VERR_CR_ASSOC_ACC_LEN;
1246         else if (assoc_acc->hdr.rqst.desc_tag !=
1247                         cpu_to_be32(FCNVME_LSDESC_RQST))
1248                 fcret = VERR_LSDESC_RQST;
1249         else if (assoc_acc->hdr.rqst.desc_len !=
1250                         fcnvme_lsdesc_len(sizeof(struct fcnvme_lsdesc_rqst)))
1251                 fcret = VERR_LSDESC_RQST_LEN;
1252         else if (assoc_acc->hdr.rqst.w0.ls_cmd != FCNVME_LS_CREATE_ASSOCIATION)
1253                 fcret = VERR_CR_ASSOC;
1254         else if (assoc_acc->associd.desc_tag !=
1255                         cpu_to_be32(FCNVME_LSDESC_ASSOC_ID))
1256                 fcret = VERR_ASSOC_ID;
1257         else if (assoc_acc->associd.desc_len !=
1258                         fcnvme_lsdesc_len(
1259                                 sizeof(struct fcnvme_lsdesc_assoc_id)))
1260                 fcret = VERR_ASSOC_ID_LEN;
1261         else if (assoc_acc->connectid.desc_tag !=
1262                         cpu_to_be32(FCNVME_LSDESC_CONN_ID))
1263                 fcret = VERR_CONN_ID;
1264         else if (assoc_acc->connectid.desc_len !=
1265                         fcnvme_lsdesc_len(sizeof(struct fcnvme_lsdesc_conn_id)))
1266                 fcret = VERR_CONN_ID_LEN;
1267
1268         if (fcret) {
1269                 ret = -EBADF;
1270                 dev_err(ctrl->dev,
1271                         "q %d Create Association LS failed: %s\n",
1272                         queue->qnum, validation_errors[fcret]);
1273         } else {
1274                 spin_lock_irqsave(&ctrl->lock, flags);
1275                 ctrl->association_id =
1276                         be64_to_cpu(assoc_acc->associd.association_id);
1277                 queue->connection_id =
1278                         be64_to_cpu(assoc_acc->connectid.connection_id);
1279                 set_bit(NVME_FC_Q_CONNECTED, &queue->flags);
1280                 spin_unlock_irqrestore(&ctrl->lock, flags);
1281         }
1282
1283 out_free_buffer:
1284         kfree(lsop);
1285 out_no_memory:
1286         if (ret)
1287                 dev_err(ctrl->dev,
1288                         "queue %d connect admin queue failed (%d).\n",
1289                         queue->qnum, ret);
1290         return ret;
1291 }
1292
1293 static int
1294 nvme_fc_connect_queue(struct nvme_fc_ctrl *ctrl, struct nvme_fc_queue *queue,
1295                         u16 qsize, u16 ersp_ratio)
1296 {
1297         struct nvmefc_ls_req_op *lsop;
1298         struct nvmefc_ls_req *lsreq;
1299         struct fcnvme_ls_cr_conn_rqst *conn_rqst;
1300         struct fcnvme_ls_cr_conn_acc *conn_acc;
1301         int ret, fcret = 0;
1302
1303         lsop = kzalloc((sizeof(*lsop) +
1304                          sizeof(*conn_rqst) + sizeof(*conn_acc) +
1305                          ctrl->lport->ops->lsrqst_priv_sz), GFP_KERNEL);
1306         if (!lsop) {
1307                 dev_info(ctrl->ctrl.device,
1308                         "NVME-FC{%d}: send Create Connection failed: ENOMEM\n",
1309                         ctrl->cnum);
1310                 ret = -ENOMEM;
1311                 goto out_no_memory;
1312         }
1313
1314         conn_rqst = (struct fcnvme_ls_cr_conn_rqst *)&lsop[1];
1315         conn_acc = (struct fcnvme_ls_cr_conn_acc *)&conn_rqst[1];
1316         lsreq = &lsop->ls_req;
1317         if (ctrl->lport->ops->lsrqst_priv_sz)
1318                 lsreq->private = (void *)&conn_acc[1];
1319         else
1320                 lsreq->private = NULL;
1321
1322         conn_rqst->w0.ls_cmd = FCNVME_LS_CREATE_CONNECTION;
1323         conn_rqst->desc_list_len = cpu_to_be32(
1324                                 sizeof(struct fcnvme_lsdesc_assoc_id) +
1325                                 sizeof(struct fcnvme_lsdesc_cr_conn_cmd));
1326
1327         conn_rqst->associd.desc_tag = cpu_to_be32(FCNVME_LSDESC_ASSOC_ID);
1328         conn_rqst->associd.desc_len =
1329                         fcnvme_lsdesc_len(
1330                                 sizeof(struct fcnvme_lsdesc_assoc_id));
1331         conn_rqst->associd.association_id = cpu_to_be64(ctrl->association_id);
1332         conn_rqst->connect_cmd.desc_tag =
1333                         cpu_to_be32(FCNVME_LSDESC_CREATE_CONN_CMD);
1334         conn_rqst->connect_cmd.desc_len =
1335                         fcnvme_lsdesc_len(
1336                                 sizeof(struct fcnvme_lsdesc_cr_conn_cmd));
1337         conn_rqst->connect_cmd.ersp_ratio = cpu_to_be16(ersp_ratio);
1338         conn_rqst->connect_cmd.qid  = cpu_to_be16(queue->qnum);
1339         conn_rqst->connect_cmd.sqsize = cpu_to_be16(qsize - 1);
1340
1341         lsop->queue = queue;
1342         lsreq->rqstaddr = conn_rqst;
1343         lsreq->rqstlen = sizeof(*conn_rqst);
1344         lsreq->rspaddr = conn_acc;
1345         lsreq->rsplen = sizeof(*conn_acc);
1346         lsreq->timeout = NVME_FC_LS_TIMEOUT_SEC;
1347
1348         ret = nvme_fc_send_ls_req(ctrl->rport, lsop);
1349         if (ret)
1350                 goto out_free_buffer;
1351
1352         /* process connect LS completion */
1353
1354         /* validate the ACC response */
1355         if (conn_acc->hdr.w0.ls_cmd != FCNVME_LS_ACC)
1356                 fcret = VERR_LSACC;
1357         else if (conn_acc->hdr.desc_list_len !=
1358                         fcnvme_lsdesc_len(sizeof(struct fcnvme_ls_cr_conn_acc)))
1359                 fcret = VERR_CR_CONN_ACC_LEN;
1360         else if (conn_acc->hdr.rqst.desc_tag != cpu_to_be32(FCNVME_LSDESC_RQST))
1361                 fcret = VERR_LSDESC_RQST;
1362         else if (conn_acc->hdr.rqst.desc_len !=
1363                         fcnvme_lsdesc_len(sizeof(struct fcnvme_lsdesc_rqst)))
1364                 fcret = VERR_LSDESC_RQST_LEN;
1365         else if (conn_acc->hdr.rqst.w0.ls_cmd != FCNVME_LS_CREATE_CONNECTION)
1366                 fcret = VERR_CR_CONN;
1367         else if (conn_acc->connectid.desc_tag !=
1368                         cpu_to_be32(FCNVME_LSDESC_CONN_ID))
1369                 fcret = VERR_CONN_ID;
1370         else if (conn_acc->connectid.desc_len !=
1371                         fcnvme_lsdesc_len(sizeof(struct fcnvme_lsdesc_conn_id)))
1372                 fcret = VERR_CONN_ID_LEN;
1373
1374         if (fcret) {
1375                 ret = -EBADF;
1376                 dev_err(ctrl->dev,
1377                         "q %d Create I/O Connection LS failed: %s\n",
1378                         queue->qnum, validation_errors[fcret]);
1379         } else {
1380                 queue->connection_id =
1381                         be64_to_cpu(conn_acc->connectid.connection_id);
1382                 set_bit(NVME_FC_Q_CONNECTED, &queue->flags);
1383         }
1384
1385 out_free_buffer:
1386         kfree(lsop);
1387 out_no_memory:
1388         if (ret)
1389                 dev_err(ctrl->dev,
1390                         "queue %d connect I/O queue failed (%d).\n",
1391                         queue->qnum, ret);
1392         return ret;
1393 }
1394
1395 static void
1396 nvme_fc_disconnect_assoc_done(struct nvmefc_ls_req *lsreq, int status)
1397 {
1398         struct nvmefc_ls_req_op *lsop = ls_req_to_lsop(lsreq);
1399
1400         __nvme_fc_finish_ls_req(lsop);
1401
1402         /* fc-nvme initiator doesn't care about success or failure of cmd */
1403
1404         kfree(lsop);
1405 }
1406
1407 /*
1408  * This routine sends a FC-NVME LS to disconnect (aka terminate)
1409  * the FC-NVME Association.  Terminating the association also
1410  * terminates the FC-NVME connections (per queue, both admin and io
1411  * queues) that are part of the association. E.g. things are torn
1412  * down, and the related FC-NVME Association ID and Connection IDs
1413  * become invalid.
1414  *
1415  * The behavior of the fc-nvme initiator is such that it's
1416  * understanding of the association and connections will implicitly
1417  * be torn down. The action is implicit as it may be due to a loss of
1418  * connectivity with the fc-nvme target, so you may never get a
1419  * response even if you tried.  As such, the action of this routine
1420  * is to asynchronously send the LS, ignore any results of the LS, and
1421  * continue on with terminating the association. If the fc-nvme target
1422  * is present and receives the LS, it too can tear down.
1423  */
1424 static void
1425 nvme_fc_xmt_disconnect_assoc(struct nvme_fc_ctrl *ctrl)
1426 {
1427         struct fcnvme_ls_disconnect_assoc_rqst *discon_rqst;
1428         struct fcnvme_ls_disconnect_assoc_acc *discon_acc;
1429         struct nvmefc_ls_req_op *lsop;
1430         struct nvmefc_ls_req *lsreq;
1431         int ret;
1432
1433         lsop = kzalloc((sizeof(*lsop) +
1434                         sizeof(*discon_rqst) + sizeof(*discon_acc) +
1435                         ctrl->lport->ops->lsrqst_priv_sz), GFP_KERNEL);
1436         if (!lsop) {
1437                 dev_info(ctrl->ctrl.device,
1438                         "NVME-FC{%d}: send Disconnect Association "
1439                         "failed: ENOMEM\n",
1440                         ctrl->cnum);
1441                 return;
1442         }
1443
1444         discon_rqst = (struct fcnvme_ls_disconnect_assoc_rqst *)&lsop[1];
1445         discon_acc = (struct fcnvme_ls_disconnect_assoc_acc *)&discon_rqst[1];
1446         lsreq = &lsop->ls_req;
1447         if (ctrl->lport->ops->lsrqst_priv_sz)
1448                 lsreq->private = (void *)&discon_acc[1];
1449         else
1450                 lsreq->private = NULL;
1451
1452         nvmefc_fmt_lsreq_discon_assoc(lsreq, discon_rqst, discon_acc,
1453                                 ctrl->association_id);
1454
1455         ret = nvme_fc_send_ls_req_async(ctrl->rport, lsop,
1456                                 nvme_fc_disconnect_assoc_done);
1457         if (ret)
1458                 kfree(lsop);
1459 }
1460
1461 static void
1462 nvme_fc_xmt_ls_rsp_done(struct nvmefc_ls_rsp *lsrsp)
1463 {
1464         struct nvmefc_ls_rcv_op *lsop = lsrsp->nvme_fc_private;
1465         struct nvme_fc_rport *rport = lsop->rport;
1466         struct nvme_fc_lport *lport = rport->lport;
1467         unsigned long flags;
1468
1469         spin_lock_irqsave(&rport->lock, flags);
1470         list_del(&lsop->lsrcv_list);
1471         spin_unlock_irqrestore(&rport->lock, flags);
1472
1473         fc_dma_sync_single_for_cpu(lport->dev, lsop->rspdma,
1474                                 sizeof(*lsop->rspbuf), DMA_TO_DEVICE);
1475         fc_dma_unmap_single(lport->dev, lsop->rspdma,
1476                         sizeof(*lsop->rspbuf), DMA_TO_DEVICE);
1477
1478         kfree(lsop->rspbuf);
1479         kfree(lsop->rqstbuf);
1480         kfree(lsop);
1481
1482         nvme_fc_rport_put(rport);
1483 }
1484
1485 static void
1486 nvme_fc_xmt_ls_rsp(struct nvmefc_ls_rcv_op *lsop)
1487 {
1488         struct nvme_fc_rport *rport = lsop->rport;
1489         struct nvme_fc_lport *lport = rport->lport;
1490         struct fcnvme_ls_rqst_w0 *w0 = &lsop->rqstbuf->w0;
1491         int ret;
1492
1493         fc_dma_sync_single_for_device(lport->dev, lsop->rspdma,
1494                                   sizeof(*lsop->rspbuf), DMA_TO_DEVICE);
1495
1496         ret = lport->ops->xmt_ls_rsp(&lport->localport, &rport->remoteport,
1497                                      lsop->lsrsp);
1498         if (ret) {
1499                 dev_warn(lport->dev,
1500                         "LLDD rejected LS RSP xmt: LS %d status %d\n",
1501                         w0->ls_cmd, ret);
1502                 nvme_fc_xmt_ls_rsp_done(lsop->lsrsp);
1503                 return;
1504         }
1505 }
1506
1507 static struct nvme_fc_ctrl *
1508 nvme_fc_match_disconn_ls(struct nvme_fc_rport *rport,
1509                       struct nvmefc_ls_rcv_op *lsop)
1510 {
1511         struct fcnvme_ls_disconnect_assoc_rqst *rqst =
1512                                         &lsop->rqstbuf->rq_dis_assoc;
1513         struct nvme_fc_ctrl *ctrl, *ret = NULL;
1514         struct nvmefc_ls_rcv_op *oldls = NULL;
1515         u64 association_id = be64_to_cpu(rqst->associd.association_id);
1516         unsigned long flags;
1517
1518         spin_lock_irqsave(&rport->lock, flags);
1519
1520         list_for_each_entry(ctrl, &rport->ctrl_list, ctrl_list) {
1521                 if (!nvme_fc_ctrl_get(ctrl))
1522                         continue;
1523                 spin_lock(&ctrl->lock);
1524                 if (association_id == ctrl->association_id) {
1525                         oldls = ctrl->rcv_disconn;
1526                         ctrl->rcv_disconn = lsop;
1527                         ret = ctrl;
1528                 }
1529                 spin_unlock(&ctrl->lock);
1530                 if (ret)
1531                         /* leave the ctrl get reference */
1532                         break;
1533                 nvme_fc_ctrl_put(ctrl);
1534         }
1535
1536         spin_unlock_irqrestore(&rport->lock, flags);
1537
1538         /* transmit a response for anything that was pending */
1539         if (oldls) {
1540                 dev_info(rport->lport->dev,
1541                         "NVME-FC{%d}: Multiple Disconnect Association "
1542                         "LS's received\n", ctrl->cnum);
1543                 /* overwrite good response with bogus failure */
1544                 oldls->lsrsp->rsplen = nvme_fc_format_rjt(oldls->rspbuf,
1545                                                 sizeof(*oldls->rspbuf),
1546                                                 rqst->w0.ls_cmd,
1547                                                 FCNVME_RJT_RC_UNAB,
1548                                                 FCNVME_RJT_EXP_NONE, 0);
1549                 nvme_fc_xmt_ls_rsp(oldls);
1550         }
1551
1552         return ret;
1553 }
1554
1555 /*
1556  * returns true to mean LS handled and ls_rsp can be sent
1557  * returns false to defer ls_rsp xmt (will be done as part of
1558  *     association termination)
1559  */
1560 static bool
1561 nvme_fc_ls_disconnect_assoc(struct nvmefc_ls_rcv_op *lsop)
1562 {
1563         struct nvme_fc_rport *rport = lsop->rport;
1564         struct fcnvme_ls_disconnect_assoc_rqst *rqst =
1565                                         &lsop->rqstbuf->rq_dis_assoc;
1566         struct fcnvme_ls_disconnect_assoc_acc *acc =
1567                                         &lsop->rspbuf->rsp_dis_assoc;
1568         struct nvme_fc_ctrl *ctrl = NULL;
1569         int ret = 0;
1570
1571         memset(acc, 0, sizeof(*acc));
1572
1573         ret = nvmefc_vldt_lsreq_discon_assoc(lsop->rqstdatalen, rqst);
1574         if (!ret) {
1575                 /* match an active association */
1576                 ctrl = nvme_fc_match_disconn_ls(rport, lsop);
1577                 if (!ctrl)
1578                         ret = VERR_NO_ASSOC;
1579         }
1580
1581         if (ret) {
1582                 dev_info(rport->lport->dev,
1583                         "Disconnect LS failed: %s\n",
1584                         validation_errors[ret]);
1585                 lsop->lsrsp->rsplen = nvme_fc_format_rjt(acc,
1586                                         sizeof(*acc), rqst->w0.ls_cmd,
1587                                         (ret == VERR_NO_ASSOC) ?
1588                                                 FCNVME_RJT_RC_INV_ASSOC :
1589                                                 FCNVME_RJT_RC_LOGIC,
1590                                         FCNVME_RJT_EXP_NONE, 0);
1591                 return true;
1592         }
1593
1594         /* format an ACCept response */
1595
1596         lsop->lsrsp->rsplen = sizeof(*acc);
1597
1598         nvme_fc_format_rsp_hdr(acc, FCNVME_LS_ACC,
1599                         fcnvme_lsdesc_len(
1600                                 sizeof(struct fcnvme_ls_disconnect_assoc_acc)),
1601                         FCNVME_LS_DISCONNECT_ASSOC);
1602
1603         /*
1604          * the transmit of the response will occur after the exchanges
1605          * for the association have been ABTS'd by
1606          * nvme_fc_delete_association().
1607          */
1608
1609         /* fail the association */
1610         nvme_fc_error_recovery(ctrl, "Disconnect Association LS received");
1611
1612         /* release the reference taken by nvme_fc_match_disconn_ls() */
1613         nvme_fc_ctrl_put(ctrl);
1614
1615         return false;
1616 }
1617
1618 /*
1619  * Actual Processing routine for received FC-NVME LS Requests from the LLD
1620  * returns true if a response should be sent afterward, false if rsp will
1621  * be sent asynchronously.
1622  */
1623 static bool
1624 nvme_fc_handle_ls_rqst(struct nvmefc_ls_rcv_op *lsop)
1625 {
1626         struct fcnvme_ls_rqst_w0 *w0 = &lsop->rqstbuf->w0;
1627         bool ret = true;
1628
1629         lsop->lsrsp->nvme_fc_private = lsop;
1630         lsop->lsrsp->rspbuf = lsop->rspbuf;
1631         lsop->lsrsp->rspdma = lsop->rspdma;
1632         lsop->lsrsp->done = nvme_fc_xmt_ls_rsp_done;
1633         /* Be preventative. handlers will later set to valid length */
1634         lsop->lsrsp->rsplen = 0;
1635
1636         /*
1637          * handlers:
1638          *   parse request input, execute the request, and format the
1639          *   LS response
1640          */
1641         switch (w0->ls_cmd) {
1642         case FCNVME_LS_DISCONNECT_ASSOC:
1643                 ret = nvme_fc_ls_disconnect_assoc(lsop);
1644                 break;
1645         case FCNVME_LS_DISCONNECT_CONN:
1646                 lsop->lsrsp->rsplen = nvme_fc_format_rjt(lsop->rspbuf,
1647                                 sizeof(*lsop->rspbuf), w0->ls_cmd,
1648                                 FCNVME_RJT_RC_UNSUP, FCNVME_RJT_EXP_NONE, 0);
1649                 break;
1650         case FCNVME_LS_CREATE_ASSOCIATION:
1651         case FCNVME_LS_CREATE_CONNECTION:
1652                 lsop->lsrsp->rsplen = nvme_fc_format_rjt(lsop->rspbuf,
1653                                 sizeof(*lsop->rspbuf), w0->ls_cmd,
1654                                 FCNVME_RJT_RC_LOGIC, FCNVME_RJT_EXP_NONE, 0);
1655                 break;
1656         default:
1657                 lsop->lsrsp->rsplen = nvme_fc_format_rjt(lsop->rspbuf,
1658                                 sizeof(*lsop->rspbuf), w0->ls_cmd,
1659                                 FCNVME_RJT_RC_INVAL, FCNVME_RJT_EXP_NONE, 0);
1660                 break;
1661         }
1662
1663         return(ret);
1664 }
1665
1666 static void
1667 nvme_fc_handle_ls_rqst_work(struct work_struct *work)
1668 {
1669         struct nvme_fc_rport *rport =
1670                 container_of(work, struct nvme_fc_rport, lsrcv_work);
1671         struct fcnvme_ls_rqst_w0 *w0;
1672         struct nvmefc_ls_rcv_op *lsop;
1673         unsigned long flags;
1674         bool sendrsp;
1675
1676 restart:
1677         sendrsp = true;
1678         spin_lock_irqsave(&rport->lock, flags);
1679         list_for_each_entry(lsop, &rport->ls_rcv_list, lsrcv_list) {
1680                 if (lsop->handled)
1681                         continue;
1682
1683                 lsop->handled = true;
1684                 if (rport->remoteport.port_state == FC_OBJSTATE_ONLINE) {
1685                         spin_unlock_irqrestore(&rport->lock, flags);
1686                         sendrsp = nvme_fc_handle_ls_rqst(lsop);
1687                 } else {
1688                         spin_unlock_irqrestore(&rport->lock, flags);
1689                         w0 = &lsop->rqstbuf->w0;
1690                         lsop->lsrsp->rsplen = nvme_fc_format_rjt(
1691                                                 lsop->rspbuf,
1692                                                 sizeof(*lsop->rspbuf),
1693                                                 w0->ls_cmd,
1694                                                 FCNVME_RJT_RC_UNAB,
1695                                                 FCNVME_RJT_EXP_NONE, 0);
1696                 }
1697                 if (sendrsp)
1698                         nvme_fc_xmt_ls_rsp(lsop);
1699                 goto restart;
1700         }
1701         spin_unlock_irqrestore(&rport->lock, flags);
1702 }
1703
1704 static
1705 void nvme_fc_rcv_ls_req_err_msg(struct nvme_fc_lport *lport,
1706                                 struct fcnvme_ls_rqst_w0 *w0)
1707 {
1708         dev_info(lport->dev, "RCV %s LS failed: No memory\n",
1709                 (w0->ls_cmd <= NVME_FC_LAST_LS_CMD_VALUE) ?
1710                         nvmefc_ls_names[w0->ls_cmd] : "");
1711 }
1712
1713 /**
1714  * nvme_fc_rcv_ls_req - transport entry point called by an LLDD
1715  *                       upon the reception of a NVME LS request.
1716  *
1717  * The nvme-fc layer will copy payload to an internal structure for
1718  * processing.  As such, upon completion of the routine, the LLDD may
1719  * immediately free/reuse the LS request buffer passed in the call.
1720  *
1721  * If this routine returns error, the LLDD should abort the exchange.
1722  *
1723  * @portptr:    pointer to the (registered) remote port that the LS
1724  *              was received from. The remoteport is associated with
1725  *              a specific localport.
1726  * @lsrsp:      pointer to a nvmefc_ls_rsp response structure to be
1727  *              used to reference the exchange corresponding to the LS
1728  *              when issuing an ls response.
1729  * @lsreqbuf:   pointer to the buffer containing the LS Request
1730  * @lsreqbuf_len: length, in bytes, of the received LS request
1731  */
1732 int
1733 nvme_fc_rcv_ls_req(struct nvme_fc_remote_port *portptr,
1734                         struct nvmefc_ls_rsp *lsrsp,
1735                         void *lsreqbuf, u32 lsreqbuf_len)
1736 {
1737         struct nvme_fc_rport *rport = remoteport_to_rport(portptr);
1738         struct nvme_fc_lport *lport = rport->lport;
1739         struct fcnvme_ls_rqst_w0 *w0 = (struct fcnvme_ls_rqst_w0 *)lsreqbuf;
1740         struct nvmefc_ls_rcv_op *lsop;
1741         unsigned long flags;
1742         int ret;
1743
1744         nvme_fc_rport_get(rport);
1745
1746         /* validate there's a routine to transmit a response */
1747         if (!lport->ops->xmt_ls_rsp) {
1748                 dev_info(lport->dev,
1749                         "RCV %s LS failed: no LLDD xmt_ls_rsp\n",
1750                         (w0->ls_cmd <= NVME_FC_LAST_LS_CMD_VALUE) ?
1751                                 nvmefc_ls_names[w0->ls_cmd] : "");
1752                 ret = -EINVAL;
1753                 goto out_put;
1754         }
1755
1756         if (lsreqbuf_len > sizeof(union nvmefc_ls_requests)) {
1757                 dev_info(lport->dev,
1758                         "RCV %s LS failed: payload too large\n",
1759                         (w0->ls_cmd <= NVME_FC_LAST_LS_CMD_VALUE) ?
1760                                 nvmefc_ls_names[w0->ls_cmd] : "");
1761                 ret = -E2BIG;
1762                 goto out_put;
1763         }
1764
1765         lsop = kzalloc(sizeof(*lsop), GFP_KERNEL);
1766         if (!lsop) {
1767                 nvme_fc_rcv_ls_req_err_msg(lport, w0);
1768                 ret = -ENOMEM;
1769                 goto out_put;
1770         }
1771
1772         lsop->rqstbuf = kzalloc(sizeof(*lsop->rqstbuf), GFP_KERNEL);
1773         lsop->rspbuf = kzalloc(sizeof(*lsop->rspbuf), GFP_KERNEL);
1774         if (!lsop->rqstbuf || !lsop->rspbuf) {
1775                 nvme_fc_rcv_ls_req_err_msg(lport, w0);
1776                 ret = -ENOMEM;
1777                 goto out_free;
1778         }
1779
1780         lsop->rspdma = fc_dma_map_single(lport->dev, lsop->rspbuf,
1781                                         sizeof(*lsop->rspbuf),
1782                                         DMA_TO_DEVICE);
1783         if (fc_dma_mapping_error(lport->dev, lsop->rspdma)) {
1784                 dev_info(lport->dev,
1785                         "RCV %s LS failed: DMA mapping failure\n",
1786                         (w0->ls_cmd <= NVME_FC_LAST_LS_CMD_VALUE) ?
1787                                 nvmefc_ls_names[w0->ls_cmd] : "");
1788                 ret = -EFAULT;
1789                 goto out_free;
1790         }
1791
1792         lsop->rport = rport;
1793         lsop->lsrsp = lsrsp;
1794
1795         memcpy(lsop->rqstbuf, lsreqbuf, lsreqbuf_len);
1796         lsop->rqstdatalen = lsreqbuf_len;
1797
1798         spin_lock_irqsave(&rport->lock, flags);
1799         if (rport->remoteport.port_state != FC_OBJSTATE_ONLINE) {
1800                 spin_unlock_irqrestore(&rport->lock, flags);
1801                 ret = -ENOTCONN;
1802                 goto out_unmap;
1803         }
1804         list_add_tail(&lsop->lsrcv_list, &rport->ls_rcv_list);
1805         spin_unlock_irqrestore(&rport->lock, flags);
1806
1807         schedule_work(&rport->lsrcv_work);
1808
1809         return 0;
1810
1811 out_unmap:
1812         fc_dma_unmap_single(lport->dev, lsop->rspdma,
1813                         sizeof(*lsop->rspbuf), DMA_TO_DEVICE);
1814 out_free:
1815         kfree(lsop->rspbuf);
1816         kfree(lsop->rqstbuf);
1817         kfree(lsop);
1818 out_put:
1819         nvme_fc_rport_put(rport);
1820         return ret;
1821 }
1822 EXPORT_SYMBOL_GPL(nvme_fc_rcv_ls_req);
1823
1824
1825 /* *********************** NVME Ctrl Routines **************************** */
1826
1827 static void
1828 __nvme_fc_exit_request(struct nvme_fc_ctrl *ctrl,
1829                 struct nvme_fc_fcp_op *op)
1830 {
1831         fc_dma_unmap_single(ctrl->lport->dev, op->fcp_req.rspdma,
1832                                 sizeof(op->rsp_iu), DMA_FROM_DEVICE);
1833         fc_dma_unmap_single(ctrl->lport->dev, op->fcp_req.cmddma,
1834                                 sizeof(op->cmd_iu), DMA_TO_DEVICE);
1835
1836         atomic_set(&op->state, FCPOP_STATE_UNINIT);
1837 }
1838
1839 static void
1840 nvme_fc_exit_request(struct blk_mq_tag_set *set, struct request *rq,
1841                 unsigned int hctx_idx)
1842 {
1843         struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(rq);
1844
1845         return __nvme_fc_exit_request(to_fc_ctrl(set->driver_data), op);
1846 }
1847
1848 static int
1849 __nvme_fc_abort_op(struct nvme_fc_ctrl *ctrl, struct nvme_fc_fcp_op *op)
1850 {
1851         unsigned long flags;
1852         int opstate;
1853
1854         spin_lock_irqsave(&ctrl->lock, flags);
1855         opstate = atomic_xchg(&op->state, FCPOP_STATE_ABORTED);
1856         if (opstate != FCPOP_STATE_ACTIVE)
1857                 atomic_set(&op->state, opstate);
1858         else if (test_bit(FCCTRL_TERMIO, &ctrl->flags)) {
1859                 op->flags |= FCOP_FLAGS_TERMIO;
1860                 ctrl->iocnt++;
1861         }
1862         spin_unlock_irqrestore(&ctrl->lock, flags);
1863
1864         if (opstate != FCPOP_STATE_ACTIVE)
1865                 return -ECANCELED;
1866
1867         ctrl->lport->ops->fcp_abort(&ctrl->lport->localport,
1868                                         &ctrl->rport->remoteport,
1869                                         op->queue->lldd_handle,
1870                                         &op->fcp_req);
1871
1872         return 0;
1873 }
1874
1875 static void
1876 nvme_fc_abort_aen_ops(struct nvme_fc_ctrl *ctrl)
1877 {
1878         struct nvme_fc_fcp_op *aen_op = ctrl->aen_ops;
1879         int i;
1880
1881         /* ensure we've initialized the ops once */
1882         if (!(aen_op->flags & FCOP_FLAGS_AEN))
1883                 return;
1884
1885         for (i = 0; i < NVME_NR_AEN_COMMANDS; i++, aen_op++)
1886                 __nvme_fc_abort_op(ctrl, aen_op);
1887 }
1888
1889 static inline void
1890 __nvme_fc_fcpop_chk_teardowns(struct nvme_fc_ctrl *ctrl,
1891                 struct nvme_fc_fcp_op *op, int opstate)
1892 {
1893         unsigned long flags;
1894
1895         if (opstate == FCPOP_STATE_ABORTED) {
1896                 spin_lock_irqsave(&ctrl->lock, flags);
1897                 if (test_bit(FCCTRL_TERMIO, &ctrl->flags) &&
1898                     op->flags & FCOP_FLAGS_TERMIO) {
1899                         if (!--ctrl->iocnt)
1900                                 wake_up(&ctrl->ioabort_wait);
1901                 }
1902                 spin_unlock_irqrestore(&ctrl->lock, flags);
1903         }
1904 }
1905
1906 static void
1907 nvme_fc_ctrl_ioerr_work(struct work_struct *work)
1908 {
1909         struct nvme_fc_ctrl *ctrl =
1910                         container_of(work, struct nvme_fc_ctrl, ioerr_work);
1911
1912         nvme_fc_error_recovery(ctrl, "transport detected io error");
1913 }
1914
1915 /*
1916  * nvme_fc_io_getuuid - Routine called to get the appid field
1917  * associated with request by the lldd
1918  * @req:IO request from nvme fc to driver
1919  * Returns: UUID if there is an appid associated with VM or
1920  * NULL if the user/libvirt has not set the appid to VM
1921  */
1922 char *nvme_fc_io_getuuid(struct nvmefc_fcp_req *req)
1923 {
1924         struct nvme_fc_fcp_op *op = fcp_req_to_fcp_op(req);
1925         struct request *rq = op->rq;
1926
1927         if (!IS_ENABLED(CONFIG_BLK_CGROUP_FC_APPID) || !rq || !rq->bio)
1928                 return NULL;
1929         return blkcg_get_fc_appid(rq->bio);
1930 }
1931 EXPORT_SYMBOL_GPL(nvme_fc_io_getuuid);
1932
1933 static void
1934 nvme_fc_fcpio_done(struct nvmefc_fcp_req *req)
1935 {
1936         struct nvme_fc_fcp_op *op = fcp_req_to_fcp_op(req);
1937         struct request *rq = op->rq;
1938         struct nvmefc_fcp_req *freq = &op->fcp_req;
1939         struct nvme_fc_ctrl *ctrl = op->ctrl;
1940         struct nvme_fc_queue *queue = op->queue;
1941         struct nvme_completion *cqe = &op->rsp_iu.cqe;
1942         struct nvme_command *sqe = &op->cmd_iu.sqe;
1943         __le16 status = cpu_to_le16(NVME_SC_SUCCESS << 1);
1944         union nvme_result result;
1945         bool terminate_assoc = true;
1946         int opstate;
1947
1948         /*
1949          * WARNING:
1950          * The current linux implementation of a nvme controller
1951          * allocates a single tag set for all io queues and sizes
1952          * the io queues to fully hold all possible tags. Thus, the
1953          * implementation does not reference or care about the sqhd
1954          * value as it never needs to use the sqhd/sqtail pointers
1955          * for submission pacing.
1956          *
1957          * This affects the FC-NVME implementation in two ways:
1958          * 1) As the value doesn't matter, we don't need to waste
1959          *    cycles extracting it from ERSPs and stamping it in the
1960          *    cases where the transport fabricates CQEs on successful
1961          *    completions.
1962          * 2) The FC-NVME implementation requires that delivery of
1963          *    ERSP completions are to go back to the nvme layer in order
1964          *    relative to the rsn, such that the sqhd value will always
1965          *    be "in order" for the nvme layer. As the nvme layer in
1966          *    linux doesn't care about sqhd, there's no need to return
1967          *    them in order.
1968          *
1969          * Additionally:
1970          * As the core nvme layer in linux currently does not look at
1971          * every field in the cqe - in cases where the FC transport must
1972          * fabricate a CQE, the following fields will not be set as they
1973          * are not referenced:
1974          *      cqe.sqid,  cqe.sqhd,  cqe.command_id
1975          *
1976          * Failure or error of an individual i/o, in a transport
1977          * detected fashion unrelated to the nvme completion status,
1978          * potentially cause the initiator and target sides to get out
1979          * of sync on SQ head/tail (aka outstanding io count allowed).
1980          * Per FC-NVME spec, failure of an individual command requires
1981          * the connection to be terminated, which in turn requires the
1982          * association to be terminated.
1983          */
1984
1985         opstate = atomic_xchg(&op->state, FCPOP_STATE_COMPLETE);
1986
1987         fc_dma_sync_single_for_cpu(ctrl->lport->dev, op->fcp_req.rspdma,
1988                                 sizeof(op->rsp_iu), DMA_FROM_DEVICE);
1989
1990         if (opstate == FCPOP_STATE_ABORTED)
1991                 status = cpu_to_le16(NVME_SC_HOST_ABORTED_CMD << 1);
1992         else if (freq->status) {
1993                 status = cpu_to_le16(NVME_SC_HOST_PATH_ERROR << 1);
1994                 dev_info(ctrl->ctrl.device,
1995                         "NVME-FC{%d}: io failed due to lldd error %d\n",
1996                         ctrl->cnum, freq->status);
1997         }
1998
1999         /*
2000          * For the linux implementation, if we have an unsuccesful
2001          * status, they blk-mq layer can typically be called with the
2002          * non-zero status and the content of the cqe isn't important.
2003          */
2004         if (status)
2005                 goto done;
2006
2007         /*
2008          * command completed successfully relative to the wire
2009          * protocol. However, validate anything received and
2010          * extract the status and result from the cqe (create it
2011          * where necessary).
2012          */
2013
2014         switch (freq->rcv_rsplen) {
2015
2016         case 0:
2017         case NVME_FC_SIZEOF_ZEROS_RSP:
2018                 /*
2019                  * No response payload or 12 bytes of payload (which
2020                  * should all be zeros) are considered successful and
2021                  * no payload in the CQE by the transport.
2022                  */
2023                 if (freq->transferred_length !=
2024                     be32_to_cpu(op->cmd_iu.data_len)) {
2025                         status = cpu_to_le16(NVME_SC_HOST_PATH_ERROR << 1);
2026                         dev_info(ctrl->ctrl.device,
2027                                 "NVME-FC{%d}: io failed due to bad transfer "
2028                                 "length: %d vs expected %d\n",
2029                                 ctrl->cnum, freq->transferred_length,
2030                                 be32_to_cpu(op->cmd_iu.data_len));
2031                         goto done;
2032                 }
2033                 result.u64 = 0;
2034                 break;
2035
2036         case sizeof(struct nvme_fc_ersp_iu):
2037                 /*
2038                  * The ERSP IU contains a full completion with CQE.
2039                  * Validate ERSP IU and look at cqe.
2040                  */
2041                 if (unlikely(be16_to_cpu(op->rsp_iu.iu_len) !=
2042                                         (freq->rcv_rsplen / 4) ||
2043                              be32_to_cpu(op->rsp_iu.xfrd_len) !=
2044                                         freq->transferred_length ||
2045                              op->rsp_iu.ersp_result ||
2046                              sqe->common.command_id != cqe->command_id)) {
2047                         status = cpu_to_le16(NVME_SC_HOST_PATH_ERROR << 1);
2048                         dev_info(ctrl->ctrl.device,
2049                                 "NVME-FC{%d}: io failed due to bad NVMe_ERSP: "
2050                                 "iu len %d, xfr len %d vs %d, status code "
2051                                 "%d, cmdid %d vs %d\n",
2052                                 ctrl->cnum, be16_to_cpu(op->rsp_iu.iu_len),
2053                                 be32_to_cpu(op->rsp_iu.xfrd_len),
2054                                 freq->transferred_length,
2055                                 op->rsp_iu.ersp_result,
2056                                 sqe->common.command_id,
2057                                 cqe->command_id);
2058                         goto done;
2059                 }
2060                 result = cqe->result;
2061                 status = cqe->status;
2062                 break;
2063
2064         default:
2065                 status = cpu_to_le16(NVME_SC_HOST_PATH_ERROR << 1);
2066                 dev_info(ctrl->ctrl.device,
2067                         "NVME-FC{%d}: io failed due to odd NVMe_xRSP iu "
2068                         "len %d\n",
2069                         ctrl->cnum, freq->rcv_rsplen);
2070                 goto done;
2071         }
2072
2073         terminate_assoc = false;
2074
2075 done:
2076         if (op->flags & FCOP_FLAGS_AEN) {
2077                 nvme_complete_async_event(&queue->ctrl->ctrl, status, &result);
2078                 __nvme_fc_fcpop_chk_teardowns(ctrl, op, opstate);
2079                 atomic_set(&op->state, FCPOP_STATE_IDLE);
2080                 op->flags = FCOP_FLAGS_AEN;     /* clear other flags */
2081                 nvme_fc_ctrl_put(ctrl);
2082                 goto check_error;
2083         }
2084
2085         __nvme_fc_fcpop_chk_teardowns(ctrl, op, opstate);
2086         if (!nvme_try_complete_req(rq, status, result))
2087                 nvme_fc_complete_rq(rq);
2088
2089 check_error:
2090         if (terminate_assoc && ctrl->ctrl.state != NVME_CTRL_RESETTING)
2091                 queue_work(nvme_reset_wq, &ctrl->ioerr_work);
2092 }
2093
2094 static int
2095 __nvme_fc_init_request(struct nvme_fc_ctrl *ctrl,
2096                 struct nvme_fc_queue *queue, struct nvme_fc_fcp_op *op,
2097                 struct request *rq, u32 rqno)
2098 {
2099         struct nvme_fcp_op_w_sgl *op_w_sgl =
2100                 container_of(op, typeof(*op_w_sgl), op);
2101         struct nvme_fc_cmd_iu *cmdiu = &op->cmd_iu;
2102         int ret = 0;
2103
2104         memset(op, 0, sizeof(*op));
2105         op->fcp_req.cmdaddr = &op->cmd_iu;
2106         op->fcp_req.cmdlen = sizeof(op->cmd_iu);
2107         op->fcp_req.rspaddr = &op->rsp_iu;
2108         op->fcp_req.rsplen = sizeof(op->rsp_iu);
2109         op->fcp_req.done = nvme_fc_fcpio_done;
2110         op->ctrl = ctrl;
2111         op->queue = queue;
2112         op->rq = rq;
2113         op->rqno = rqno;
2114
2115         cmdiu->format_id = NVME_CMD_FORMAT_ID;
2116         cmdiu->fc_id = NVME_CMD_FC_ID;
2117         cmdiu->iu_len = cpu_to_be16(sizeof(*cmdiu) / sizeof(u32));
2118         if (queue->qnum)
2119                 cmdiu->rsv_cat = fccmnd_set_cat_css(0,
2120                                         (NVME_CC_CSS_NVM >> NVME_CC_CSS_SHIFT));
2121         else
2122                 cmdiu->rsv_cat = fccmnd_set_cat_admin(0);
2123
2124         op->fcp_req.cmddma = fc_dma_map_single(ctrl->lport->dev,
2125                                 &op->cmd_iu, sizeof(op->cmd_iu), DMA_TO_DEVICE);
2126         if (fc_dma_mapping_error(ctrl->lport->dev, op->fcp_req.cmddma)) {
2127                 dev_err(ctrl->dev,
2128                         "FCP Op failed - cmdiu dma mapping failed.\n");
2129                 ret = -EFAULT;
2130                 goto out_on_error;
2131         }
2132
2133         op->fcp_req.rspdma = fc_dma_map_single(ctrl->lport->dev,
2134                                 &op->rsp_iu, sizeof(op->rsp_iu),
2135                                 DMA_FROM_DEVICE);
2136         if (fc_dma_mapping_error(ctrl->lport->dev, op->fcp_req.rspdma)) {
2137                 dev_err(ctrl->dev,
2138                         "FCP Op failed - rspiu dma mapping failed.\n");
2139                 ret = -EFAULT;
2140         }
2141
2142         atomic_set(&op->state, FCPOP_STATE_IDLE);
2143 out_on_error:
2144         return ret;
2145 }
2146
2147 static int
2148 nvme_fc_init_request(struct blk_mq_tag_set *set, struct request *rq,
2149                 unsigned int hctx_idx, unsigned int numa_node)
2150 {
2151         struct nvme_fc_ctrl *ctrl = to_fc_ctrl(set->driver_data);
2152         struct nvme_fcp_op_w_sgl *op = blk_mq_rq_to_pdu(rq);
2153         int queue_idx = (set == &ctrl->tag_set) ? hctx_idx + 1 : 0;
2154         struct nvme_fc_queue *queue = &ctrl->queues[queue_idx];
2155         int res;
2156
2157         res = __nvme_fc_init_request(ctrl, queue, &op->op, rq, queue->rqcnt++);
2158         if (res)
2159                 return res;
2160         op->op.fcp_req.first_sgl = op->sgl;
2161         op->op.fcp_req.private = &op->priv[0];
2162         nvme_req(rq)->ctrl = &ctrl->ctrl;
2163         nvme_req(rq)->cmd = &op->op.cmd_iu.sqe;
2164         return res;
2165 }
2166
2167 static int
2168 nvme_fc_init_aen_ops(struct nvme_fc_ctrl *ctrl)
2169 {
2170         struct nvme_fc_fcp_op *aen_op;
2171         struct nvme_fc_cmd_iu *cmdiu;
2172         struct nvme_command *sqe;
2173         void *private = NULL;
2174         int i, ret;
2175
2176         aen_op = ctrl->aen_ops;
2177         for (i = 0; i < NVME_NR_AEN_COMMANDS; i++, aen_op++) {
2178                 if (ctrl->lport->ops->fcprqst_priv_sz) {
2179                         private = kzalloc(ctrl->lport->ops->fcprqst_priv_sz,
2180                                                 GFP_KERNEL);
2181                         if (!private)
2182                                 return -ENOMEM;
2183                 }
2184
2185                 cmdiu = &aen_op->cmd_iu;
2186                 sqe = &cmdiu->sqe;
2187                 ret = __nvme_fc_init_request(ctrl, &ctrl->queues[0],
2188                                 aen_op, (struct request *)NULL,
2189                                 (NVME_AQ_BLK_MQ_DEPTH + i));
2190                 if (ret) {
2191                         kfree(private);
2192                         return ret;
2193                 }
2194
2195                 aen_op->flags = FCOP_FLAGS_AEN;
2196                 aen_op->fcp_req.private = private;
2197
2198                 memset(sqe, 0, sizeof(*sqe));
2199                 sqe->common.opcode = nvme_admin_async_event;
2200                 /* Note: core layer may overwrite the sqe.command_id value */
2201                 sqe->common.command_id = NVME_AQ_BLK_MQ_DEPTH + i;
2202         }
2203         return 0;
2204 }
2205
2206 static void
2207 nvme_fc_term_aen_ops(struct nvme_fc_ctrl *ctrl)
2208 {
2209         struct nvme_fc_fcp_op *aen_op;
2210         int i;
2211
2212         cancel_work_sync(&ctrl->ctrl.async_event_work);
2213         aen_op = ctrl->aen_ops;
2214         for (i = 0; i < NVME_NR_AEN_COMMANDS; i++, aen_op++) {
2215                 __nvme_fc_exit_request(ctrl, aen_op);
2216
2217                 kfree(aen_op->fcp_req.private);
2218                 aen_op->fcp_req.private = NULL;
2219         }
2220 }
2221
2222 static inline int
2223 __nvme_fc_init_hctx(struct blk_mq_hw_ctx *hctx, void *data, unsigned int qidx)
2224 {
2225         struct nvme_fc_ctrl *ctrl = to_fc_ctrl(data);
2226         struct nvme_fc_queue *queue = &ctrl->queues[qidx];
2227
2228         hctx->driver_data = queue;
2229         queue->hctx = hctx;
2230         return 0;
2231 }
2232
2233 static int
2234 nvme_fc_init_hctx(struct blk_mq_hw_ctx *hctx, void *data, unsigned int hctx_idx)
2235 {
2236         return __nvme_fc_init_hctx(hctx, data, hctx_idx + 1);
2237 }
2238
2239 static int
2240 nvme_fc_init_admin_hctx(struct blk_mq_hw_ctx *hctx, void *data,
2241                 unsigned int hctx_idx)
2242 {
2243         return __nvme_fc_init_hctx(hctx, data, hctx_idx);
2244 }
2245
2246 static void
2247 nvme_fc_init_queue(struct nvme_fc_ctrl *ctrl, int idx)
2248 {
2249         struct nvme_fc_queue *queue;
2250
2251         queue = &ctrl->queues[idx];
2252         memset(queue, 0, sizeof(*queue));
2253         queue->ctrl = ctrl;
2254         queue->qnum = idx;
2255         atomic_set(&queue->csn, 0);
2256         queue->dev = ctrl->dev;
2257
2258         if (idx > 0)
2259                 queue->cmnd_capsule_len = ctrl->ctrl.ioccsz * 16;
2260         else
2261                 queue->cmnd_capsule_len = sizeof(struct nvme_command);
2262
2263         /*
2264          * Considered whether we should allocate buffers for all SQEs
2265          * and CQEs and dma map them - mapping their respective entries
2266          * into the request structures (kernel vm addr and dma address)
2267          * thus the driver could use the buffers/mappings directly.
2268          * It only makes sense if the LLDD would use them for its
2269          * messaging api. It's very unlikely most adapter api's would use
2270          * a native NVME sqe/cqe. More reasonable if FC-NVME IU payload
2271          * structures were used instead.
2272          */
2273 }
2274
2275 /*
2276  * This routine terminates a queue at the transport level.
2277  * The transport has already ensured that all outstanding ios on
2278  * the queue have been terminated.
2279  * The transport will send a Disconnect LS request to terminate
2280  * the queue's connection. Termination of the admin queue will also
2281  * terminate the association at the target.
2282  */
2283 static void
2284 nvme_fc_free_queue(struct nvme_fc_queue *queue)
2285 {
2286         if (!test_and_clear_bit(NVME_FC_Q_CONNECTED, &queue->flags))
2287                 return;
2288
2289         clear_bit(NVME_FC_Q_LIVE, &queue->flags);
2290         /*
2291          * Current implementation never disconnects a single queue.
2292          * It always terminates a whole association. So there is never
2293          * a disconnect(queue) LS sent to the target.
2294          */
2295
2296         queue->connection_id = 0;
2297         atomic_set(&queue->csn, 0);
2298 }
2299
2300 static void
2301 __nvme_fc_delete_hw_queue(struct nvme_fc_ctrl *ctrl,
2302         struct nvme_fc_queue *queue, unsigned int qidx)
2303 {
2304         if (ctrl->lport->ops->delete_queue)
2305                 ctrl->lport->ops->delete_queue(&ctrl->lport->localport, qidx,
2306                                 queue->lldd_handle);
2307         queue->lldd_handle = NULL;
2308 }
2309
2310 static void
2311 nvme_fc_free_io_queues(struct nvme_fc_ctrl *ctrl)
2312 {
2313         int i;
2314
2315         for (i = 1; i < ctrl->ctrl.queue_count; i++)
2316                 nvme_fc_free_queue(&ctrl->queues[i]);
2317 }
2318
2319 static int
2320 __nvme_fc_create_hw_queue(struct nvme_fc_ctrl *ctrl,
2321         struct nvme_fc_queue *queue, unsigned int qidx, u16 qsize)
2322 {
2323         int ret = 0;
2324
2325         queue->lldd_handle = NULL;
2326         if (ctrl->lport->ops->create_queue)
2327                 ret = ctrl->lport->ops->create_queue(&ctrl->lport->localport,
2328                                 qidx, qsize, &queue->lldd_handle);
2329
2330         return ret;
2331 }
2332
2333 static void
2334 nvme_fc_delete_hw_io_queues(struct nvme_fc_ctrl *ctrl)
2335 {
2336         struct nvme_fc_queue *queue = &ctrl->queues[ctrl->ctrl.queue_count - 1];
2337         int i;
2338
2339         for (i = ctrl->ctrl.queue_count - 1; i >= 1; i--, queue--)
2340                 __nvme_fc_delete_hw_queue(ctrl, queue, i);
2341 }
2342
2343 static int
2344 nvme_fc_create_hw_io_queues(struct nvme_fc_ctrl *ctrl, u16 qsize)
2345 {
2346         struct nvme_fc_queue *queue = &ctrl->queues[1];
2347         int i, ret;
2348
2349         for (i = 1; i < ctrl->ctrl.queue_count; i++, queue++) {
2350                 ret = __nvme_fc_create_hw_queue(ctrl, queue, i, qsize);
2351                 if (ret)
2352                         goto delete_queues;
2353         }
2354
2355         return 0;
2356
2357 delete_queues:
2358         for (; i > 0; i--)
2359                 __nvme_fc_delete_hw_queue(ctrl, &ctrl->queues[i], i);
2360         return ret;
2361 }
2362
2363 static int
2364 nvme_fc_connect_io_queues(struct nvme_fc_ctrl *ctrl, u16 qsize)
2365 {
2366         int i, ret = 0;
2367
2368         for (i = 1; i < ctrl->ctrl.queue_count; i++) {
2369                 ret = nvme_fc_connect_queue(ctrl, &ctrl->queues[i], qsize,
2370                                         (qsize / 5));
2371                 if (ret)
2372                         break;
2373                 ret = nvmf_connect_io_queue(&ctrl->ctrl, i);
2374                 if (ret)
2375                         break;
2376
2377                 set_bit(NVME_FC_Q_LIVE, &ctrl->queues[i].flags);
2378         }
2379
2380         return ret;
2381 }
2382
2383 static void
2384 nvme_fc_init_io_queues(struct nvme_fc_ctrl *ctrl)
2385 {
2386         int i;
2387
2388         for (i = 1; i < ctrl->ctrl.queue_count; i++)
2389                 nvme_fc_init_queue(ctrl, i);
2390 }
2391
2392 static void
2393 nvme_fc_ctrl_free(struct kref *ref)
2394 {
2395         struct nvme_fc_ctrl *ctrl =
2396                 container_of(ref, struct nvme_fc_ctrl, ref);
2397         unsigned long flags;
2398
2399         if (ctrl->ctrl.tagset)
2400                 nvme_remove_io_tag_set(&ctrl->ctrl);
2401
2402         /* remove from rport list */
2403         spin_lock_irqsave(&ctrl->rport->lock, flags);
2404         list_del(&ctrl->ctrl_list);
2405         spin_unlock_irqrestore(&ctrl->rport->lock, flags);
2406
2407         nvme_unquiesce_admin_queue(&ctrl->ctrl);
2408         nvme_remove_admin_tag_set(&ctrl->ctrl);
2409
2410         kfree(ctrl->queues);
2411
2412         put_device(ctrl->dev);
2413         nvme_fc_rport_put(ctrl->rport);
2414
2415         ida_free(&nvme_fc_ctrl_cnt, ctrl->cnum);
2416         if (ctrl->ctrl.opts)
2417                 nvmf_free_options(ctrl->ctrl.opts);
2418         kfree(ctrl);
2419 }
2420
2421 static void
2422 nvme_fc_ctrl_put(struct nvme_fc_ctrl *ctrl)
2423 {
2424         kref_put(&ctrl->ref, nvme_fc_ctrl_free);
2425 }
2426
2427 static int
2428 nvme_fc_ctrl_get(struct nvme_fc_ctrl *ctrl)
2429 {
2430         return kref_get_unless_zero(&ctrl->ref);
2431 }
2432
2433 /*
2434  * All accesses from nvme core layer done - can now free the
2435  * controller. Called after last nvme_put_ctrl() call
2436  */
2437 static void
2438 nvme_fc_nvme_ctrl_freed(struct nvme_ctrl *nctrl)
2439 {
2440         struct nvme_fc_ctrl *ctrl = to_fc_ctrl(nctrl);
2441
2442         WARN_ON(nctrl != &ctrl->ctrl);
2443
2444         nvme_fc_ctrl_put(ctrl);
2445 }
2446
2447 /*
2448  * This routine is used by the transport when it needs to find active
2449  * io on a queue that is to be terminated. The transport uses
2450  * blk_mq_tagset_busy_itr() to find the busy requests, which then invoke
2451  * this routine to kill them on a 1 by 1 basis.
2452  *
2453  * As FC allocates FC exchange for each io, the transport must contact
2454  * the LLDD to terminate the exchange, thus releasing the FC exchange.
2455  * After terminating the exchange the LLDD will call the transport's
2456  * normal io done path for the request, but it will have an aborted
2457  * status. The done path will return the io request back to the block
2458  * layer with an error status.
2459  */
2460 static bool nvme_fc_terminate_exchange(struct request *req, void *data)
2461 {
2462         struct nvme_ctrl *nctrl = data;
2463         struct nvme_fc_ctrl *ctrl = to_fc_ctrl(nctrl);
2464         struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(req);
2465
2466         op->nreq.flags |= NVME_REQ_CANCELLED;
2467         __nvme_fc_abort_op(ctrl, op);
2468         return true;
2469 }
2470
2471 /*
2472  * This routine runs through all outstanding commands on the association
2473  * and aborts them.  This routine is typically be called by the
2474  * delete_association routine. It is also called due to an error during
2475  * reconnect. In that scenario, it is most likely a command that initializes
2476  * the controller, including fabric Connect commands on io queues, that
2477  * may have timed out or failed thus the io must be killed for the connect
2478  * thread to see the error.
2479  */
2480 static void
2481 __nvme_fc_abort_outstanding_ios(struct nvme_fc_ctrl *ctrl, bool start_queues)
2482 {
2483         int q;
2484
2485         /*
2486          * if aborting io, the queues are no longer good, mark them
2487          * all as not live.
2488          */
2489         if (ctrl->ctrl.queue_count > 1) {
2490                 for (q = 1; q < ctrl->ctrl.queue_count; q++)
2491                         clear_bit(NVME_FC_Q_LIVE, &ctrl->queues[q].flags);
2492         }
2493         clear_bit(NVME_FC_Q_LIVE, &ctrl->queues[0].flags);
2494
2495         /*
2496          * If io queues are present, stop them and terminate all outstanding
2497          * ios on them. As FC allocates FC exchange for each io, the
2498          * transport must contact the LLDD to terminate the exchange,
2499          * thus releasing the FC exchange. We use blk_mq_tagset_busy_itr()
2500          * to tell us what io's are busy and invoke a transport routine
2501          * to kill them with the LLDD.  After terminating the exchange
2502          * the LLDD will call the transport's normal io done path, but it
2503          * will have an aborted status. The done path will return the
2504          * io requests back to the block layer as part of normal completions
2505          * (but with error status).
2506          */
2507         if (ctrl->ctrl.queue_count > 1) {
2508                 nvme_quiesce_io_queues(&ctrl->ctrl);
2509                 nvme_sync_io_queues(&ctrl->ctrl);
2510                 blk_mq_tagset_busy_iter(&ctrl->tag_set,
2511                                 nvme_fc_terminate_exchange, &ctrl->ctrl);
2512                 blk_mq_tagset_wait_completed_request(&ctrl->tag_set);
2513                 if (start_queues)
2514                         nvme_unquiesce_io_queues(&ctrl->ctrl);
2515         }
2516
2517         /*
2518          * Other transports, which don't have link-level contexts bound
2519          * to sqe's, would try to gracefully shutdown the controller by
2520          * writing the registers for shutdown and polling (call
2521          * nvme_disable_ctrl()). Given a bunch of i/o was potentially
2522          * just aborted and we will wait on those contexts, and given
2523          * there was no indication of how live the controlelr is on the
2524          * link, don't send more io to create more contexts for the
2525          * shutdown. Let the controller fail via keepalive failure if
2526          * its still present.
2527          */
2528
2529         /*
2530          * clean up the admin queue. Same thing as above.
2531          */
2532         nvme_quiesce_admin_queue(&ctrl->ctrl);
2533         blk_sync_queue(ctrl->ctrl.admin_q);
2534         blk_mq_tagset_busy_iter(&ctrl->admin_tag_set,
2535                                 nvme_fc_terminate_exchange, &ctrl->ctrl);
2536         blk_mq_tagset_wait_completed_request(&ctrl->admin_tag_set);
2537         if (start_queues)
2538                 nvme_unquiesce_admin_queue(&ctrl->ctrl);
2539 }
2540
2541 static void
2542 nvme_fc_error_recovery(struct nvme_fc_ctrl *ctrl, char *errmsg)
2543 {
2544         /*
2545          * if an error (io timeout, etc) while (re)connecting, the remote
2546          * port requested terminating of the association (disconnect_ls)
2547          * or an error (timeout or abort) occurred on an io while creating
2548          * the controller.  Abort any ios on the association and let the
2549          * create_association error path resolve things.
2550          */
2551         enum nvme_ctrl_state state;
2552         unsigned long flags;
2553
2554         spin_lock_irqsave(&ctrl->lock, flags);
2555         state = ctrl->ctrl.state;
2556         if (state == NVME_CTRL_CONNECTING) {
2557                 set_bit(ASSOC_FAILED, &ctrl->flags);
2558                 spin_unlock_irqrestore(&ctrl->lock, flags);
2559                 __nvme_fc_abort_outstanding_ios(ctrl, true);
2560                 dev_warn(ctrl->ctrl.device,
2561                         "NVME-FC{%d}: transport error during (re)connect\n",
2562                         ctrl->cnum);
2563                 return;
2564         }
2565         spin_unlock_irqrestore(&ctrl->lock, flags);
2566
2567         /* Otherwise, only proceed if in LIVE state - e.g. on first error */
2568         if (state != NVME_CTRL_LIVE)
2569                 return;
2570
2571         dev_warn(ctrl->ctrl.device,
2572                 "NVME-FC{%d}: transport association event: %s\n",
2573                 ctrl->cnum, errmsg);
2574         dev_warn(ctrl->ctrl.device,
2575                 "NVME-FC{%d}: resetting controller\n", ctrl->cnum);
2576
2577         nvme_reset_ctrl(&ctrl->ctrl);
2578 }
2579
2580 static enum blk_eh_timer_return nvme_fc_timeout(struct request *rq)
2581 {
2582         struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(rq);
2583         struct nvme_fc_ctrl *ctrl = op->ctrl;
2584         struct nvme_fc_cmd_iu *cmdiu = &op->cmd_iu;
2585         struct nvme_command *sqe = &cmdiu->sqe;
2586
2587         /*
2588          * Attempt to abort the offending command. Command completion
2589          * will detect the aborted io and will fail the connection.
2590          */
2591         dev_info(ctrl->ctrl.device,
2592                 "NVME-FC{%d.%d}: io timeout: opcode %d fctype %d w10/11: "
2593                 "x%08x/x%08x\n",
2594                 ctrl->cnum, op->queue->qnum, sqe->common.opcode,
2595                 sqe->connect.fctype, sqe->common.cdw10, sqe->common.cdw11);
2596         if (__nvme_fc_abort_op(ctrl, op))
2597                 nvme_fc_error_recovery(ctrl, "io timeout abort failed");
2598
2599         /*
2600          * the io abort has been initiated. Have the reset timer
2601          * restarted and the abort completion will complete the io
2602          * shortly. Avoids a synchronous wait while the abort finishes.
2603          */
2604         return BLK_EH_RESET_TIMER;
2605 }
2606
2607 static int
2608 nvme_fc_map_data(struct nvme_fc_ctrl *ctrl, struct request *rq,
2609                 struct nvme_fc_fcp_op *op)
2610 {
2611         struct nvmefc_fcp_req *freq = &op->fcp_req;
2612         int ret;
2613
2614         freq->sg_cnt = 0;
2615
2616         if (!blk_rq_nr_phys_segments(rq))
2617                 return 0;
2618
2619         freq->sg_table.sgl = freq->first_sgl;
2620         ret = sg_alloc_table_chained(&freq->sg_table,
2621                         blk_rq_nr_phys_segments(rq), freq->sg_table.sgl,
2622                         NVME_INLINE_SG_CNT);
2623         if (ret)
2624                 return -ENOMEM;
2625
2626         op->nents = blk_rq_map_sg(rq->q, rq, freq->sg_table.sgl);
2627         WARN_ON(op->nents > blk_rq_nr_phys_segments(rq));
2628         freq->sg_cnt = fc_dma_map_sg(ctrl->lport->dev, freq->sg_table.sgl,
2629                                 op->nents, rq_dma_dir(rq));
2630         if (unlikely(freq->sg_cnt <= 0)) {
2631                 sg_free_table_chained(&freq->sg_table, NVME_INLINE_SG_CNT);
2632                 freq->sg_cnt = 0;
2633                 return -EFAULT;
2634         }
2635
2636         /*
2637          * TODO: blk_integrity_rq(rq)  for DIF
2638          */
2639         return 0;
2640 }
2641
2642 static void
2643 nvme_fc_unmap_data(struct nvme_fc_ctrl *ctrl, struct request *rq,
2644                 struct nvme_fc_fcp_op *op)
2645 {
2646         struct nvmefc_fcp_req *freq = &op->fcp_req;
2647
2648         if (!freq->sg_cnt)
2649                 return;
2650
2651         fc_dma_unmap_sg(ctrl->lport->dev, freq->sg_table.sgl, op->nents,
2652                         rq_dma_dir(rq));
2653
2654         sg_free_table_chained(&freq->sg_table, NVME_INLINE_SG_CNT);
2655
2656         freq->sg_cnt = 0;
2657 }
2658
2659 /*
2660  * In FC, the queue is a logical thing. At transport connect, the target
2661  * creates its "queue" and returns a handle that is to be given to the
2662  * target whenever it posts something to the corresponding SQ.  When an
2663  * SQE is sent on a SQ, FC effectively considers the SQE, or rather the
2664  * command contained within the SQE, an io, and assigns a FC exchange
2665  * to it. The SQE and the associated SQ handle are sent in the initial
2666  * CMD IU sents on the exchange. All transfers relative to the io occur
2667  * as part of the exchange.  The CQE is the last thing for the io,
2668  * which is transferred (explicitly or implicitly) with the RSP IU
2669  * sent on the exchange. After the CQE is received, the FC exchange is
2670  * terminaed and the Exchange may be used on a different io.
2671  *
2672  * The transport to LLDD api has the transport making a request for a
2673  * new fcp io request to the LLDD. The LLDD then allocates a FC exchange
2674  * resource and transfers the command. The LLDD will then process all
2675  * steps to complete the io. Upon completion, the transport done routine
2676  * is called.
2677  *
2678  * So - while the operation is outstanding to the LLDD, there is a link
2679  * level FC exchange resource that is also outstanding. This must be
2680  * considered in all cleanup operations.
2681  */
2682 static blk_status_t
2683 nvme_fc_start_fcp_op(struct nvme_fc_ctrl *ctrl, struct nvme_fc_queue *queue,
2684         struct nvme_fc_fcp_op *op, u32 data_len,
2685         enum nvmefc_fcp_datadir io_dir)
2686 {
2687         struct nvme_fc_cmd_iu *cmdiu = &op->cmd_iu;
2688         struct nvme_command *sqe = &cmdiu->sqe;
2689         int ret, opstate;
2690
2691         /*
2692          * before attempting to send the io, check to see if we believe
2693          * the target device is present
2694          */
2695         if (ctrl->rport->remoteport.port_state != FC_OBJSTATE_ONLINE)
2696                 return BLK_STS_RESOURCE;
2697
2698         if (!nvme_fc_ctrl_get(ctrl))
2699                 return BLK_STS_IOERR;
2700
2701         /* format the FC-NVME CMD IU and fcp_req */
2702         cmdiu->connection_id = cpu_to_be64(queue->connection_id);
2703         cmdiu->data_len = cpu_to_be32(data_len);
2704         switch (io_dir) {
2705         case NVMEFC_FCP_WRITE:
2706                 cmdiu->flags = FCNVME_CMD_FLAGS_WRITE;
2707                 break;
2708         case NVMEFC_FCP_READ:
2709                 cmdiu->flags = FCNVME_CMD_FLAGS_READ;
2710                 break;
2711         case NVMEFC_FCP_NODATA:
2712                 cmdiu->flags = 0;
2713                 break;
2714         }
2715         op->fcp_req.payload_length = data_len;
2716         op->fcp_req.io_dir = io_dir;
2717         op->fcp_req.transferred_length = 0;
2718         op->fcp_req.rcv_rsplen = 0;
2719         op->fcp_req.status = NVME_SC_SUCCESS;
2720         op->fcp_req.sqid = cpu_to_le16(queue->qnum);
2721
2722         /*
2723          * validate per fabric rules, set fields mandated by fabric spec
2724          * as well as those by FC-NVME spec.
2725          */
2726         WARN_ON_ONCE(sqe->common.metadata);
2727         sqe->common.flags |= NVME_CMD_SGL_METABUF;
2728
2729         /*
2730          * format SQE DPTR field per FC-NVME rules:
2731          *    type=0x5     Transport SGL Data Block Descriptor
2732          *    subtype=0xA  Transport-specific value
2733          *    address=0
2734          *    length=length of the data series
2735          */
2736         sqe->rw.dptr.sgl.type = (NVME_TRANSPORT_SGL_DATA_DESC << 4) |
2737                                         NVME_SGL_FMT_TRANSPORT_A;
2738         sqe->rw.dptr.sgl.length = cpu_to_le32(data_len);
2739         sqe->rw.dptr.sgl.addr = 0;
2740
2741         if (!(op->flags & FCOP_FLAGS_AEN)) {
2742                 ret = nvme_fc_map_data(ctrl, op->rq, op);
2743                 if (ret < 0) {
2744                         nvme_cleanup_cmd(op->rq);
2745                         nvme_fc_ctrl_put(ctrl);
2746                         if (ret == -ENOMEM || ret == -EAGAIN)
2747                                 return BLK_STS_RESOURCE;
2748                         return BLK_STS_IOERR;
2749                 }
2750         }
2751
2752         fc_dma_sync_single_for_device(ctrl->lport->dev, op->fcp_req.cmddma,
2753                                   sizeof(op->cmd_iu), DMA_TO_DEVICE);
2754
2755         atomic_set(&op->state, FCPOP_STATE_ACTIVE);
2756
2757         if (!(op->flags & FCOP_FLAGS_AEN))
2758                 nvme_start_request(op->rq);
2759
2760         cmdiu->csn = cpu_to_be32(atomic_inc_return(&queue->csn));
2761         ret = ctrl->lport->ops->fcp_io(&ctrl->lport->localport,
2762                                         &ctrl->rport->remoteport,
2763                                         queue->lldd_handle, &op->fcp_req);
2764
2765         if (ret) {
2766                 /*
2767                  * If the lld fails to send the command is there an issue with
2768                  * the csn value?  If the command that fails is the Connect,
2769                  * no - as the connection won't be live.  If it is a command
2770                  * post-connect, it's possible a gap in csn may be created.
2771                  * Does this matter?  As Linux initiators don't send fused
2772                  * commands, no.  The gap would exist, but as there's nothing
2773                  * that depends on csn order to be delivered on the target
2774                  * side, it shouldn't hurt.  It would be difficult for a
2775                  * target to even detect the csn gap as it has no idea when the
2776                  * cmd with the csn was supposed to arrive.
2777                  */
2778                 opstate = atomic_xchg(&op->state, FCPOP_STATE_COMPLETE);
2779                 __nvme_fc_fcpop_chk_teardowns(ctrl, op, opstate);
2780
2781                 if (!(op->flags & FCOP_FLAGS_AEN)) {
2782                         nvme_fc_unmap_data(ctrl, op->rq, op);
2783                         nvme_cleanup_cmd(op->rq);
2784                 }
2785
2786                 nvme_fc_ctrl_put(ctrl);
2787
2788                 if (ctrl->rport->remoteport.port_state == FC_OBJSTATE_ONLINE &&
2789                                 ret != -EBUSY)
2790                         return BLK_STS_IOERR;
2791
2792                 return BLK_STS_RESOURCE;
2793         }
2794
2795         return BLK_STS_OK;
2796 }
2797
2798 static blk_status_t
2799 nvme_fc_queue_rq(struct blk_mq_hw_ctx *hctx,
2800                         const struct blk_mq_queue_data *bd)
2801 {
2802         struct nvme_ns *ns = hctx->queue->queuedata;
2803         struct nvme_fc_queue *queue = hctx->driver_data;
2804         struct nvme_fc_ctrl *ctrl = queue->ctrl;
2805         struct request *rq = bd->rq;
2806         struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(rq);
2807         enum nvmefc_fcp_datadir io_dir;
2808         bool queue_ready = test_bit(NVME_FC_Q_LIVE, &queue->flags);
2809         u32 data_len;
2810         blk_status_t ret;
2811
2812         if (ctrl->rport->remoteport.port_state != FC_OBJSTATE_ONLINE ||
2813             !nvme_check_ready(&queue->ctrl->ctrl, rq, queue_ready))
2814                 return nvme_fail_nonready_command(&queue->ctrl->ctrl, rq);
2815
2816         ret = nvme_setup_cmd(ns, rq);
2817         if (ret)
2818                 return ret;
2819
2820         /*
2821          * nvme core doesn't quite treat the rq opaquely. Commands such
2822          * as WRITE ZEROES will return a non-zero rq payload_bytes yet
2823          * there is no actual payload to be transferred.
2824          * To get it right, key data transmission on there being 1 or
2825          * more physical segments in the sg list. If there is no
2826          * physical segments, there is no payload.
2827          */
2828         if (blk_rq_nr_phys_segments(rq)) {
2829                 data_len = blk_rq_payload_bytes(rq);
2830                 io_dir = ((rq_data_dir(rq) == WRITE) ?
2831                                         NVMEFC_FCP_WRITE : NVMEFC_FCP_READ);
2832         } else {
2833                 data_len = 0;
2834                 io_dir = NVMEFC_FCP_NODATA;
2835         }
2836
2837
2838         return nvme_fc_start_fcp_op(ctrl, queue, op, data_len, io_dir);
2839 }
2840
2841 static void
2842 nvme_fc_submit_async_event(struct nvme_ctrl *arg)
2843 {
2844         struct nvme_fc_ctrl *ctrl = to_fc_ctrl(arg);
2845         struct nvme_fc_fcp_op *aen_op;
2846         blk_status_t ret;
2847
2848         if (test_bit(FCCTRL_TERMIO, &ctrl->flags))
2849                 return;
2850
2851         aen_op = &ctrl->aen_ops[0];
2852
2853         ret = nvme_fc_start_fcp_op(ctrl, aen_op->queue, aen_op, 0,
2854                                         NVMEFC_FCP_NODATA);
2855         if (ret)
2856                 dev_err(ctrl->ctrl.device,
2857                         "failed async event work\n");
2858 }
2859
2860 static void
2861 nvme_fc_complete_rq(struct request *rq)
2862 {
2863         struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(rq);
2864         struct nvme_fc_ctrl *ctrl = op->ctrl;
2865
2866         atomic_set(&op->state, FCPOP_STATE_IDLE);
2867         op->flags &= ~FCOP_FLAGS_TERMIO;
2868
2869         nvme_fc_unmap_data(ctrl, rq, op);
2870         nvme_complete_rq(rq);
2871         nvme_fc_ctrl_put(ctrl);
2872 }
2873
2874 static void nvme_fc_map_queues(struct blk_mq_tag_set *set)
2875 {
2876         struct nvme_fc_ctrl *ctrl = to_fc_ctrl(set->driver_data);
2877         int i;
2878
2879         for (i = 0; i < set->nr_maps; i++) {
2880                 struct blk_mq_queue_map *map = &set->map[i];
2881
2882                 if (!map->nr_queues) {
2883                         WARN_ON(i == HCTX_TYPE_DEFAULT);
2884                         continue;
2885                 }
2886
2887                 /* Call LLDD map queue functionality if defined */
2888                 if (ctrl->lport->ops->map_queues)
2889                         ctrl->lport->ops->map_queues(&ctrl->lport->localport,
2890                                                      map);
2891                 else
2892                         blk_mq_map_queues(map);
2893         }
2894 }
2895
2896 static const struct blk_mq_ops nvme_fc_mq_ops = {
2897         .queue_rq       = nvme_fc_queue_rq,
2898         .complete       = nvme_fc_complete_rq,
2899         .init_request   = nvme_fc_init_request,
2900         .exit_request   = nvme_fc_exit_request,
2901         .init_hctx      = nvme_fc_init_hctx,
2902         .timeout        = nvme_fc_timeout,
2903         .map_queues     = nvme_fc_map_queues,
2904 };
2905
2906 static int
2907 nvme_fc_create_io_queues(struct nvme_fc_ctrl *ctrl)
2908 {
2909         struct nvmf_ctrl_options *opts = ctrl->ctrl.opts;
2910         unsigned int nr_io_queues;
2911         int ret;
2912
2913         nr_io_queues = min(min(opts->nr_io_queues, num_online_cpus()),
2914                                 ctrl->lport->ops->max_hw_queues);
2915         ret = nvme_set_queue_count(&ctrl->ctrl, &nr_io_queues);
2916         if (ret) {
2917                 dev_info(ctrl->ctrl.device,
2918                         "set_queue_count failed: %d\n", ret);
2919                 return ret;
2920         }
2921
2922         ctrl->ctrl.queue_count = nr_io_queues + 1;
2923         if (!nr_io_queues)
2924                 return 0;
2925
2926         nvme_fc_init_io_queues(ctrl);
2927
2928         ret = nvme_alloc_io_tag_set(&ctrl->ctrl, &ctrl->tag_set,
2929                         &nvme_fc_mq_ops, 1,
2930                         struct_size_t(struct nvme_fcp_op_w_sgl, priv,
2931                                       ctrl->lport->ops->fcprqst_priv_sz));
2932         if (ret)
2933                 return ret;
2934
2935         ret = nvme_fc_create_hw_io_queues(ctrl, ctrl->ctrl.sqsize + 1);
2936         if (ret)
2937                 goto out_cleanup_tagset;
2938
2939         ret = nvme_fc_connect_io_queues(ctrl, ctrl->ctrl.sqsize + 1);
2940         if (ret)
2941                 goto out_delete_hw_queues;
2942
2943         ctrl->ioq_live = true;
2944
2945         return 0;
2946
2947 out_delete_hw_queues:
2948         nvme_fc_delete_hw_io_queues(ctrl);
2949 out_cleanup_tagset:
2950         nvme_remove_io_tag_set(&ctrl->ctrl);
2951         nvme_fc_free_io_queues(ctrl);
2952
2953         /* force put free routine to ignore io queues */
2954         ctrl->ctrl.tagset = NULL;
2955
2956         return ret;
2957 }
2958
2959 static int
2960 nvme_fc_recreate_io_queues(struct nvme_fc_ctrl *ctrl)
2961 {
2962         struct nvmf_ctrl_options *opts = ctrl->ctrl.opts;
2963         u32 prior_ioq_cnt = ctrl->ctrl.queue_count - 1;
2964         unsigned int nr_io_queues;
2965         int ret;
2966
2967         nr_io_queues = min(min(opts->nr_io_queues, num_online_cpus()),
2968                                 ctrl->lport->ops->max_hw_queues);
2969         ret = nvme_set_queue_count(&ctrl->ctrl, &nr_io_queues);
2970         if (ret) {
2971                 dev_info(ctrl->ctrl.device,
2972                         "set_queue_count failed: %d\n", ret);
2973                 return ret;
2974         }
2975
2976         if (!nr_io_queues && prior_ioq_cnt) {
2977                 dev_info(ctrl->ctrl.device,
2978                         "Fail Reconnect: At least 1 io queue "
2979                         "required (was %d)\n", prior_ioq_cnt);
2980                 return -ENOSPC;
2981         }
2982
2983         ctrl->ctrl.queue_count = nr_io_queues + 1;
2984         /* check for io queues existing */
2985         if (ctrl->ctrl.queue_count == 1)
2986                 return 0;
2987
2988         if (prior_ioq_cnt != nr_io_queues) {
2989                 dev_info(ctrl->ctrl.device,
2990                         "reconnect: revising io queue count from %d to %d\n",
2991                         prior_ioq_cnt, nr_io_queues);
2992                 blk_mq_update_nr_hw_queues(&ctrl->tag_set, nr_io_queues);
2993         }
2994
2995         ret = nvme_fc_create_hw_io_queues(ctrl, ctrl->ctrl.sqsize + 1);
2996         if (ret)
2997                 goto out_free_io_queues;
2998
2999         ret = nvme_fc_connect_io_queues(ctrl, ctrl->ctrl.sqsize + 1);
3000         if (ret)
3001                 goto out_delete_hw_queues;
3002
3003         return 0;
3004
3005 out_delete_hw_queues:
3006         nvme_fc_delete_hw_io_queues(ctrl);
3007 out_free_io_queues:
3008         nvme_fc_free_io_queues(ctrl);
3009         return ret;
3010 }
3011
3012 static void
3013 nvme_fc_rport_active_on_lport(struct nvme_fc_rport *rport)
3014 {
3015         struct nvme_fc_lport *lport = rport->lport;
3016
3017         atomic_inc(&lport->act_rport_cnt);
3018 }
3019
3020 static void
3021 nvme_fc_rport_inactive_on_lport(struct nvme_fc_rport *rport)
3022 {
3023         struct nvme_fc_lport *lport = rport->lport;
3024         u32 cnt;
3025
3026         cnt = atomic_dec_return(&lport->act_rport_cnt);
3027         if (cnt == 0 && lport->localport.port_state == FC_OBJSTATE_DELETED)
3028                 lport->ops->localport_delete(&lport->localport);
3029 }
3030
3031 static int
3032 nvme_fc_ctlr_active_on_rport(struct nvme_fc_ctrl *ctrl)
3033 {
3034         struct nvme_fc_rport *rport = ctrl->rport;
3035         u32 cnt;
3036
3037         if (test_and_set_bit(ASSOC_ACTIVE, &ctrl->flags))
3038                 return 1;
3039
3040         cnt = atomic_inc_return(&rport->act_ctrl_cnt);
3041         if (cnt == 1)
3042                 nvme_fc_rport_active_on_lport(rport);
3043
3044         return 0;
3045 }
3046
3047 static int
3048 nvme_fc_ctlr_inactive_on_rport(struct nvme_fc_ctrl *ctrl)
3049 {
3050         struct nvme_fc_rport *rport = ctrl->rport;
3051         struct nvme_fc_lport *lport = rport->lport;
3052         u32 cnt;
3053
3054         /* clearing of ctrl->flags ASSOC_ACTIVE bit is in association delete */
3055
3056         cnt = atomic_dec_return(&rport->act_ctrl_cnt);
3057         if (cnt == 0) {
3058                 if (rport->remoteport.port_state == FC_OBJSTATE_DELETED)
3059                         lport->ops->remoteport_delete(&rport->remoteport);
3060                 nvme_fc_rport_inactive_on_lport(rport);
3061         }
3062
3063         return 0;
3064 }
3065
3066 /*
3067  * This routine restarts the controller on the host side, and
3068  * on the link side, recreates the controller association.
3069  */
3070 static int
3071 nvme_fc_create_association(struct nvme_fc_ctrl *ctrl)
3072 {
3073         struct nvmf_ctrl_options *opts = ctrl->ctrl.opts;
3074         struct nvmefc_ls_rcv_op *disls = NULL;
3075         unsigned long flags;
3076         int ret;
3077         bool changed;
3078
3079         ++ctrl->ctrl.nr_reconnects;
3080
3081         if (ctrl->rport->remoteport.port_state != FC_OBJSTATE_ONLINE)
3082                 return -ENODEV;
3083
3084         if (nvme_fc_ctlr_active_on_rport(ctrl))
3085                 return -ENOTUNIQ;
3086
3087         dev_info(ctrl->ctrl.device,
3088                 "NVME-FC{%d}: create association : host wwpn 0x%016llx "
3089                 " rport wwpn 0x%016llx: NQN \"%s\"\n",
3090                 ctrl->cnum, ctrl->lport->localport.port_name,
3091                 ctrl->rport->remoteport.port_name, ctrl->ctrl.opts->subsysnqn);
3092
3093         clear_bit(ASSOC_FAILED, &ctrl->flags);
3094
3095         /*
3096          * Create the admin queue
3097          */
3098
3099         ret = __nvme_fc_create_hw_queue(ctrl, &ctrl->queues[0], 0,
3100                                 NVME_AQ_DEPTH);
3101         if (ret)
3102                 goto out_free_queue;
3103
3104         ret = nvme_fc_connect_admin_queue(ctrl, &ctrl->queues[0],
3105                                 NVME_AQ_DEPTH, (NVME_AQ_DEPTH / 4));
3106         if (ret)
3107                 goto out_delete_hw_queue;
3108
3109         ret = nvmf_connect_admin_queue(&ctrl->ctrl);
3110         if (ret)
3111                 goto out_disconnect_admin_queue;
3112
3113         set_bit(NVME_FC_Q_LIVE, &ctrl->queues[0].flags);
3114
3115         /*
3116          * Check controller capabilities
3117          *
3118          * todo:- add code to check if ctrl attributes changed from
3119          * prior connection values
3120          */
3121
3122         ret = nvme_enable_ctrl(&ctrl->ctrl);
3123         if (!ret && test_bit(ASSOC_FAILED, &ctrl->flags))
3124                 ret = -EIO;
3125         if (ret)
3126                 goto out_disconnect_admin_queue;
3127
3128         ctrl->ctrl.max_segments = ctrl->lport->ops->max_sgl_segments;
3129         ctrl->ctrl.max_hw_sectors = ctrl->ctrl.max_segments <<
3130                                                 (ilog2(SZ_4K) - 9);
3131
3132         nvme_unquiesce_admin_queue(&ctrl->ctrl);
3133
3134         ret = nvme_init_ctrl_finish(&ctrl->ctrl, false);
3135         if (ret)
3136                 goto out_disconnect_admin_queue;
3137         if (test_bit(ASSOC_FAILED, &ctrl->flags)) {
3138                 ret = -EIO;
3139                 goto out_stop_keep_alive;
3140         }
3141         /* sanity checks */
3142
3143         /* FC-NVME does not have other data in the capsule */
3144         if (ctrl->ctrl.icdoff) {
3145                 dev_err(ctrl->ctrl.device, "icdoff %d is not supported!\n",
3146                                 ctrl->ctrl.icdoff);
3147                 ret = NVME_SC_INVALID_FIELD | NVME_SC_DNR;
3148                 goto out_stop_keep_alive;
3149         }
3150
3151         /* FC-NVME supports normal SGL Data Block Descriptors */
3152         if (!nvme_ctrl_sgl_supported(&ctrl->ctrl)) {
3153                 dev_err(ctrl->ctrl.device,
3154                         "Mandatory sgls are not supported!\n");
3155                 ret = NVME_SC_INVALID_FIELD | NVME_SC_DNR;
3156                 goto out_stop_keep_alive;
3157         }
3158
3159         if (opts->queue_size > ctrl->ctrl.maxcmd) {
3160                 /* warn if maxcmd is lower than queue_size */
3161                 dev_warn(ctrl->ctrl.device,
3162                         "queue_size %zu > ctrl maxcmd %u, reducing "
3163                         "to maxcmd\n",
3164                         opts->queue_size, ctrl->ctrl.maxcmd);
3165                 opts->queue_size = ctrl->ctrl.maxcmd;
3166                 ctrl->ctrl.sqsize = opts->queue_size - 1;
3167         }
3168
3169         ret = nvme_fc_init_aen_ops(ctrl);
3170         if (ret)
3171                 goto out_term_aen_ops;
3172
3173         /*
3174          * Create the io queues
3175          */
3176
3177         if (ctrl->ctrl.queue_count > 1) {
3178                 if (!ctrl->ioq_live)
3179                         ret = nvme_fc_create_io_queues(ctrl);
3180                 else
3181                         ret = nvme_fc_recreate_io_queues(ctrl);
3182         }
3183
3184         spin_lock_irqsave(&ctrl->lock, flags);
3185         if (!ret && test_bit(ASSOC_FAILED, &ctrl->flags))
3186                 ret = -EIO;
3187         if (ret) {
3188                 spin_unlock_irqrestore(&ctrl->lock, flags);
3189                 goto out_term_aen_ops;
3190         }
3191         changed = nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_LIVE);
3192         spin_unlock_irqrestore(&ctrl->lock, flags);
3193
3194         ctrl->ctrl.nr_reconnects = 0;
3195
3196         if (changed)
3197                 nvme_start_ctrl(&ctrl->ctrl);
3198
3199         return 0;       /* Success */
3200
3201 out_term_aen_ops:
3202         nvme_fc_term_aen_ops(ctrl);
3203 out_stop_keep_alive:
3204         nvme_stop_keep_alive(&ctrl->ctrl);
3205 out_disconnect_admin_queue:
3206         dev_warn(ctrl->ctrl.device,
3207                 "NVME-FC{%d}: create_assoc failed, assoc_id %llx ret %d\n",
3208                 ctrl->cnum, ctrl->association_id, ret);
3209         /* send a Disconnect(association) LS to fc-nvme target */
3210         nvme_fc_xmt_disconnect_assoc(ctrl);
3211         spin_lock_irqsave(&ctrl->lock, flags);
3212         ctrl->association_id = 0;
3213         disls = ctrl->rcv_disconn;
3214         ctrl->rcv_disconn = NULL;
3215         spin_unlock_irqrestore(&ctrl->lock, flags);
3216         if (disls)
3217                 nvme_fc_xmt_ls_rsp(disls);
3218 out_delete_hw_queue:
3219         __nvme_fc_delete_hw_queue(ctrl, &ctrl->queues[0], 0);
3220 out_free_queue:
3221         nvme_fc_free_queue(&ctrl->queues[0]);
3222         clear_bit(ASSOC_ACTIVE, &ctrl->flags);
3223         nvme_fc_ctlr_inactive_on_rport(ctrl);
3224
3225         return ret;
3226 }
3227
3228
3229 /*
3230  * This routine stops operation of the controller on the host side.
3231  * On the host os stack side: Admin and IO queues are stopped,
3232  *   outstanding ios on them terminated via FC ABTS.
3233  * On the link side: the association is terminated.
3234  */
3235 static void
3236 nvme_fc_delete_association(struct nvme_fc_ctrl *ctrl)
3237 {
3238         struct nvmefc_ls_rcv_op *disls = NULL;
3239         unsigned long flags;
3240
3241         if (!test_and_clear_bit(ASSOC_ACTIVE, &ctrl->flags))
3242                 return;
3243
3244         spin_lock_irqsave(&ctrl->lock, flags);
3245         set_bit(FCCTRL_TERMIO, &ctrl->flags);
3246         ctrl->iocnt = 0;
3247         spin_unlock_irqrestore(&ctrl->lock, flags);
3248
3249         __nvme_fc_abort_outstanding_ios(ctrl, false);
3250
3251         /* kill the aens as they are a separate path */
3252         nvme_fc_abort_aen_ops(ctrl);
3253
3254         /* wait for all io that had to be aborted */
3255         spin_lock_irq(&ctrl->lock);
3256         wait_event_lock_irq(ctrl->ioabort_wait, ctrl->iocnt == 0, ctrl->lock);
3257         clear_bit(FCCTRL_TERMIO, &ctrl->flags);
3258         spin_unlock_irq(&ctrl->lock);
3259
3260         nvme_fc_term_aen_ops(ctrl);
3261
3262         /*
3263          * send a Disconnect(association) LS to fc-nvme target
3264          * Note: could have been sent at top of process, but
3265          * cleaner on link traffic if after the aborts complete.
3266          * Note: if association doesn't exist, association_id will be 0
3267          */
3268         if (ctrl->association_id)
3269                 nvme_fc_xmt_disconnect_assoc(ctrl);
3270
3271         spin_lock_irqsave(&ctrl->lock, flags);
3272         ctrl->association_id = 0;
3273         disls = ctrl->rcv_disconn;
3274         ctrl->rcv_disconn = NULL;
3275         spin_unlock_irqrestore(&ctrl->lock, flags);
3276         if (disls)
3277                 /*
3278                  * if a Disconnect Request was waiting for a response, send
3279                  * now that all ABTS's have been issued (and are complete).
3280                  */
3281                 nvme_fc_xmt_ls_rsp(disls);
3282
3283         if (ctrl->ctrl.tagset) {
3284                 nvme_fc_delete_hw_io_queues(ctrl);
3285                 nvme_fc_free_io_queues(ctrl);
3286         }
3287
3288         __nvme_fc_delete_hw_queue(ctrl, &ctrl->queues[0], 0);
3289         nvme_fc_free_queue(&ctrl->queues[0]);
3290
3291         /* re-enable the admin_q so anything new can fast fail */
3292         nvme_unquiesce_admin_queue(&ctrl->ctrl);
3293
3294         /* resume the io queues so that things will fast fail */
3295         nvme_unquiesce_io_queues(&ctrl->ctrl);
3296
3297         nvme_fc_ctlr_inactive_on_rport(ctrl);
3298 }
3299
3300 static void
3301 nvme_fc_delete_ctrl(struct nvme_ctrl *nctrl)
3302 {
3303         struct nvme_fc_ctrl *ctrl = to_fc_ctrl(nctrl);
3304
3305         cancel_work_sync(&ctrl->ioerr_work);
3306         cancel_delayed_work_sync(&ctrl->connect_work);
3307         /*
3308          * kill the association on the link side.  this will block
3309          * waiting for io to terminate
3310          */
3311         nvme_fc_delete_association(ctrl);
3312 }
3313
3314 static void
3315 nvme_fc_reconnect_or_delete(struct nvme_fc_ctrl *ctrl, int status)
3316 {
3317         struct nvme_fc_rport *rport = ctrl->rport;
3318         struct nvme_fc_remote_port *portptr = &rport->remoteport;
3319         unsigned long recon_delay = ctrl->ctrl.opts->reconnect_delay * HZ;
3320         bool recon = true;
3321
3322         if (nvme_ctrl_state(&ctrl->ctrl) != NVME_CTRL_CONNECTING)
3323                 return;
3324
3325         if (portptr->port_state == FC_OBJSTATE_ONLINE) {
3326                 dev_info(ctrl->ctrl.device,
3327                         "NVME-FC{%d}: reset: Reconnect attempt failed (%d)\n",
3328                         ctrl->cnum, status);
3329                 if (status > 0 && (status & NVME_SC_DNR))
3330                         recon = false;
3331         } else if (time_after_eq(jiffies, rport->dev_loss_end))
3332                 recon = false;
3333
3334         if (recon && nvmf_should_reconnect(&ctrl->ctrl)) {
3335                 if (portptr->port_state == FC_OBJSTATE_ONLINE)
3336                         dev_info(ctrl->ctrl.device,
3337                                 "NVME-FC{%d}: Reconnect attempt in %ld "
3338                                 "seconds\n",
3339                                 ctrl->cnum, recon_delay / HZ);
3340                 else if (time_after(jiffies + recon_delay, rport->dev_loss_end))
3341                         recon_delay = rport->dev_loss_end - jiffies;
3342
3343                 queue_delayed_work(nvme_wq, &ctrl->connect_work, recon_delay);
3344         } else {
3345                 if (portptr->port_state == FC_OBJSTATE_ONLINE) {
3346                         if (status > 0 && (status & NVME_SC_DNR))
3347                                 dev_warn(ctrl->ctrl.device,
3348                                          "NVME-FC{%d}: reconnect failure\n",
3349                                          ctrl->cnum);
3350                         else
3351                                 dev_warn(ctrl->ctrl.device,
3352                                          "NVME-FC{%d}: Max reconnect attempts "
3353                                          "(%d) reached.\n",
3354                                          ctrl->cnum, ctrl->ctrl.nr_reconnects);
3355                 } else
3356                         dev_warn(ctrl->ctrl.device,
3357                                 "NVME-FC{%d}: dev_loss_tmo (%d) expired "
3358                                 "while waiting for remoteport connectivity.\n",
3359                                 ctrl->cnum, min_t(int, portptr->dev_loss_tmo,
3360                                         (ctrl->ctrl.opts->max_reconnects *
3361                                          ctrl->ctrl.opts->reconnect_delay)));
3362                 WARN_ON(nvme_delete_ctrl(&ctrl->ctrl));
3363         }
3364 }
3365
3366 static void
3367 nvme_fc_reset_ctrl_work(struct work_struct *work)
3368 {
3369         struct nvme_fc_ctrl *ctrl =
3370                 container_of(work, struct nvme_fc_ctrl, ctrl.reset_work);
3371
3372         nvme_stop_ctrl(&ctrl->ctrl);
3373
3374         /* will block will waiting for io to terminate */
3375         nvme_fc_delete_association(ctrl);
3376
3377         if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_CONNECTING))
3378                 dev_err(ctrl->ctrl.device,
3379                         "NVME-FC{%d}: error_recovery: Couldn't change state "
3380                         "to CONNECTING\n", ctrl->cnum);
3381
3382         if (ctrl->rport->remoteport.port_state == FC_OBJSTATE_ONLINE) {
3383                 if (!queue_delayed_work(nvme_wq, &ctrl->connect_work, 0)) {
3384                         dev_err(ctrl->ctrl.device,
3385                                 "NVME-FC{%d}: failed to schedule connect "
3386                                 "after reset\n", ctrl->cnum);
3387                 } else {
3388                         flush_delayed_work(&ctrl->connect_work);
3389                 }
3390         } else {
3391                 nvme_fc_reconnect_or_delete(ctrl, -ENOTCONN);
3392         }
3393 }
3394
3395
3396 static const struct nvme_ctrl_ops nvme_fc_ctrl_ops = {
3397         .name                   = "fc",
3398         .module                 = THIS_MODULE,
3399         .flags                  = NVME_F_FABRICS,
3400         .reg_read32             = nvmf_reg_read32,
3401         .reg_read64             = nvmf_reg_read64,
3402         .reg_write32            = nvmf_reg_write32,
3403         .free_ctrl              = nvme_fc_nvme_ctrl_freed,
3404         .submit_async_event     = nvme_fc_submit_async_event,
3405         .delete_ctrl            = nvme_fc_delete_ctrl,
3406         .get_address            = nvmf_get_address,
3407 };
3408
3409 static void
3410 nvme_fc_connect_ctrl_work(struct work_struct *work)
3411 {
3412         int ret;
3413
3414         struct nvme_fc_ctrl *ctrl =
3415                         container_of(to_delayed_work(work),
3416                                 struct nvme_fc_ctrl, connect_work);
3417
3418         ret = nvme_fc_create_association(ctrl);
3419         if (ret)
3420                 nvme_fc_reconnect_or_delete(ctrl, ret);
3421         else
3422                 dev_info(ctrl->ctrl.device,
3423                         "NVME-FC{%d}: controller connect complete\n",
3424                         ctrl->cnum);
3425 }
3426
3427
3428 static const struct blk_mq_ops nvme_fc_admin_mq_ops = {
3429         .queue_rq       = nvme_fc_queue_rq,
3430         .complete       = nvme_fc_complete_rq,
3431         .init_request   = nvme_fc_init_request,
3432         .exit_request   = nvme_fc_exit_request,
3433         .init_hctx      = nvme_fc_init_admin_hctx,
3434         .timeout        = nvme_fc_timeout,
3435 };
3436
3437
3438 /*
3439  * Fails a controller request if it matches an existing controller
3440  * (association) with the same tuple:
3441  * <Host NQN, Host ID, local FC port, remote FC port, SUBSYS NQN>
3442  *
3443  * The ports don't need to be compared as they are intrinsically
3444  * already matched by the port pointers supplied.
3445  */
3446 static bool
3447 nvme_fc_existing_controller(struct nvme_fc_rport *rport,
3448                 struct nvmf_ctrl_options *opts)
3449 {
3450         struct nvme_fc_ctrl *ctrl;
3451         unsigned long flags;
3452         bool found = false;
3453
3454         spin_lock_irqsave(&rport->lock, flags);
3455         list_for_each_entry(ctrl, &rport->ctrl_list, ctrl_list) {
3456                 found = nvmf_ctlr_matches_baseopts(&ctrl->ctrl, opts);
3457                 if (found)
3458                         break;
3459         }
3460         spin_unlock_irqrestore(&rport->lock, flags);
3461
3462         return found;
3463 }
3464
3465 static struct nvme_ctrl *
3466 nvme_fc_init_ctrl(struct device *dev, struct nvmf_ctrl_options *opts,
3467         struct nvme_fc_lport *lport, struct nvme_fc_rport *rport)
3468 {
3469         struct nvme_fc_ctrl *ctrl;
3470         unsigned long flags;
3471         int ret, idx, ctrl_loss_tmo;
3472
3473         if (!(rport->remoteport.port_role &
3474             (FC_PORT_ROLE_NVME_DISCOVERY | FC_PORT_ROLE_NVME_TARGET))) {
3475                 ret = -EBADR;
3476                 goto out_fail;
3477         }
3478
3479         if (!opts->duplicate_connect &&
3480             nvme_fc_existing_controller(rport, opts)) {
3481                 ret = -EALREADY;
3482                 goto out_fail;
3483         }
3484
3485         ctrl = kzalloc(sizeof(*ctrl), GFP_KERNEL);
3486         if (!ctrl) {
3487                 ret = -ENOMEM;
3488                 goto out_fail;
3489         }
3490
3491         idx = ida_alloc(&nvme_fc_ctrl_cnt, GFP_KERNEL);
3492         if (idx < 0) {
3493                 ret = -ENOSPC;
3494                 goto out_free_ctrl;
3495         }
3496
3497         /*
3498          * if ctrl_loss_tmo is being enforced and the default reconnect delay
3499          * is being used, change to a shorter reconnect delay for FC.
3500          */
3501         if (opts->max_reconnects != -1 &&
3502             opts->reconnect_delay == NVMF_DEF_RECONNECT_DELAY &&
3503             opts->reconnect_delay > NVME_FC_DEFAULT_RECONNECT_TMO) {
3504                 ctrl_loss_tmo = opts->max_reconnects * opts->reconnect_delay;
3505                 opts->reconnect_delay = NVME_FC_DEFAULT_RECONNECT_TMO;
3506                 opts->max_reconnects = DIV_ROUND_UP(ctrl_loss_tmo,
3507                                                 opts->reconnect_delay);
3508         }
3509
3510         ctrl->ctrl.opts = opts;
3511         ctrl->ctrl.nr_reconnects = 0;
3512         if (lport->dev)
3513                 ctrl->ctrl.numa_node = dev_to_node(lport->dev);
3514         else
3515                 ctrl->ctrl.numa_node = NUMA_NO_NODE;
3516         INIT_LIST_HEAD(&ctrl->ctrl_list);
3517         ctrl->lport = lport;
3518         ctrl->rport = rport;
3519         ctrl->dev = lport->dev;
3520         ctrl->cnum = idx;
3521         ctrl->ioq_live = false;
3522         init_waitqueue_head(&ctrl->ioabort_wait);
3523
3524         get_device(ctrl->dev);
3525         kref_init(&ctrl->ref);
3526
3527         INIT_WORK(&ctrl->ctrl.reset_work, nvme_fc_reset_ctrl_work);
3528         INIT_DELAYED_WORK(&ctrl->connect_work, nvme_fc_connect_ctrl_work);
3529         INIT_WORK(&ctrl->ioerr_work, nvme_fc_ctrl_ioerr_work);
3530         spin_lock_init(&ctrl->lock);
3531
3532         /* io queue count */
3533         ctrl->ctrl.queue_count = min_t(unsigned int,
3534                                 opts->nr_io_queues,
3535                                 lport->ops->max_hw_queues);
3536         ctrl->ctrl.queue_count++;       /* +1 for admin queue */
3537
3538         ctrl->ctrl.sqsize = opts->queue_size - 1;
3539         ctrl->ctrl.kato = opts->kato;
3540         ctrl->ctrl.cntlid = 0xffff;
3541
3542         ret = -ENOMEM;
3543         ctrl->queues = kcalloc(ctrl->ctrl.queue_count,
3544                                 sizeof(struct nvme_fc_queue), GFP_KERNEL);
3545         if (!ctrl->queues)
3546                 goto out_free_ida;
3547
3548         nvme_fc_init_queue(ctrl, 0);
3549
3550         /*
3551          * Would have been nice to init io queues tag set as well.
3552          * However, we require interaction from the controller
3553          * for max io queue count before we can do so.
3554          * Defer this to the connect path.
3555          */
3556
3557         ret = nvme_init_ctrl(&ctrl->ctrl, dev, &nvme_fc_ctrl_ops, 0);
3558         if (ret)
3559                 goto out_free_queues;
3560
3561         /* at this point, teardown path changes to ref counting on nvme ctrl */
3562
3563         ret = nvme_alloc_admin_tag_set(&ctrl->ctrl, &ctrl->admin_tag_set,
3564                         &nvme_fc_admin_mq_ops,
3565                         struct_size_t(struct nvme_fcp_op_w_sgl, priv,
3566                                       ctrl->lport->ops->fcprqst_priv_sz));
3567         if (ret)
3568                 goto fail_ctrl;
3569
3570         spin_lock_irqsave(&rport->lock, flags);
3571         list_add_tail(&ctrl->ctrl_list, &rport->ctrl_list);
3572         spin_unlock_irqrestore(&rport->lock, flags);
3573
3574         if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_RESETTING) ||
3575             !nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_CONNECTING)) {
3576                 dev_err(ctrl->ctrl.device,
3577                         "NVME-FC{%d}: failed to init ctrl state\n", ctrl->cnum);
3578                 goto fail_ctrl;
3579         }
3580
3581         if (!queue_delayed_work(nvme_wq, &ctrl->connect_work, 0)) {
3582                 dev_err(ctrl->ctrl.device,
3583                         "NVME-FC{%d}: failed to schedule initial connect\n",
3584                         ctrl->cnum);
3585                 goto fail_ctrl;
3586         }
3587
3588         flush_delayed_work(&ctrl->connect_work);
3589
3590         dev_info(ctrl->ctrl.device,
3591                 "NVME-FC{%d}: new ctrl: NQN \"%s\"\n",
3592                 ctrl->cnum, nvmf_ctrl_subsysnqn(&ctrl->ctrl));
3593
3594         return &ctrl->ctrl;
3595
3596 fail_ctrl:
3597         nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_DELETING);
3598         cancel_work_sync(&ctrl->ioerr_work);
3599         cancel_work_sync(&ctrl->ctrl.reset_work);
3600         cancel_delayed_work_sync(&ctrl->connect_work);
3601
3602         ctrl->ctrl.opts = NULL;
3603
3604         /* initiate nvme ctrl ref counting teardown */
3605         nvme_uninit_ctrl(&ctrl->ctrl);
3606
3607         /* Remove core ctrl ref. */
3608         nvme_put_ctrl(&ctrl->ctrl);
3609
3610         /* as we're past the point where we transition to the ref
3611          * counting teardown path, if we return a bad pointer here,
3612          * the calling routine, thinking it's prior to the
3613          * transition, will do an rport put. Since the teardown
3614          * path also does a rport put, we do an extra get here to
3615          * so proper order/teardown happens.
3616          */
3617         nvme_fc_rport_get(rport);
3618
3619         return ERR_PTR(-EIO);
3620
3621 out_free_queues:
3622         kfree(ctrl->queues);
3623 out_free_ida:
3624         put_device(ctrl->dev);
3625         ida_free(&nvme_fc_ctrl_cnt, ctrl->cnum);
3626 out_free_ctrl:
3627         kfree(ctrl);
3628 out_fail:
3629         /* exit via here doesn't follow ctlr ref points */
3630         return ERR_PTR(ret);
3631 }
3632
3633
3634 struct nvmet_fc_traddr {
3635         u64     nn;
3636         u64     pn;
3637 };
3638
3639 static int
3640 __nvme_fc_parse_u64(substring_t *sstr, u64 *val)
3641 {
3642         u64 token64;
3643
3644         if (match_u64(sstr, &token64))
3645                 return -EINVAL;
3646         *val = token64;
3647
3648         return 0;
3649 }
3650
3651 /*
3652  * This routine validates and extracts the WWN's from the TRADDR string.
3653  * As kernel parsers need the 0x to determine number base, universally
3654  * build string to parse with 0x prefix before parsing name strings.
3655  */
3656 static int
3657 nvme_fc_parse_traddr(struct nvmet_fc_traddr *traddr, char *buf, size_t blen)
3658 {
3659         char name[2 + NVME_FC_TRADDR_HEXNAMELEN + 1];
3660         substring_t wwn = { name, &name[sizeof(name)-1] };
3661         int nnoffset, pnoffset;
3662
3663         /* validate if string is one of the 2 allowed formats */
3664         if (strnlen(buf, blen) == NVME_FC_TRADDR_MAXLENGTH &&
3665                         !strncmp(buf, "nn-0x", NVME_FC_TRADDR_OXNNLEN) &&
3666                         !strncmp(&buf[NVME_FC_TRADDR_MAX_PN_OFFSET],
3667                                 "pn-0x", NVME_FC_TRADDR_OXNNLEN)) {
3668                 nnoffset = NVME_FC_TRADDR_OXNNLEN;
3669                 pnoffset = NVME_FC_TRADDR_MAX_PN_OFFSET +
3670                                                 NVME_FC_TRADDR_OXNNLEN;
3671         } else if ((strnlen(buf, blen) == NVME_FC_TRADDR_MINLENGTH &&
3672                         !strncmp(buf, "nn-", NVME_FC_TRADDR_NNLEN) &&
3673                         !strncmp(&buf[NVME_FC_TRADDR_MIN_PN_OFFSET],
3674                                 "pn-", NVME_FC_TRADDR_NNLEN))) {
3675                 nnoffset = NVME_FC_TRADDR_NNLEN;
3676                 pnoffset = NVME_FC_TRADDR_MIN_PN_OFFSET + NVME_FC_TRADDR_NNLEN;
3677         } else
3678                 goto out_einval;
3679
3680         name[0] = '0';
3681         name[1] = 'x';
3682         name[2 + NVME_FC_TRADDR_HEXNAMELEN] = 0;
3683
3684         memcpy(&name[2], &buf[nnoffset], NVME_FC_TRADDR_HEXNAMELEN);
3685         if (__nvme_fc_parse_u64(&wwn, &traddr->nn))
3686                 goto out_einval;
3687
3688         memcpy(&name[2], &buf[pnoffset], NVME_FC_TRADDR_HEXNAMELEN);
3689         if (__nvme_fc_parse_u64(&wwn, &traddr->pn))
3690                 goto out_einval;
3691
3692         return 0;
3693
3694 out_einval:
3695         pr_warn("%s: bad traddr string\n", __func__);
3696         return -EINVAL;
3697 }
3698
3699 static struct nvme_ctrl *
3700 nvme_fc_create_ctrl(struct device *dev, struct nvmf_ctrl_options *opts)
3701 {
3702         struct nvme_fc_lport *lport;
3703         struct nvme_fc_rport *rport;
3704         struct nvme_ctrl *ctrl;
3705         struct nvmet_fc_traddr laddr = { 0L, 0L };
3706         struct nvmet_fc_traddr raddr = { 0L, 0L };
3707         unsigned long flags;
3708         int ret;
3709
3710         ret = nvme_fc_parse_traddr(&raddr, opts->traddr, NVMF_TRADDR_SIZE);
3711         if (ret || !raddr.nn || !raddr.pn)
3712                 return ERR_PTR(-EINVAL);
3713
3714         ret = nvme_fc_parse_traddr(&laddr, opts->host_traddr, NVMF_TRADDR_SIZE);
3715         if (ret || !laddr.nn || !laddr.pn)
3716                 return ERR_PTR(-EINVAL);
3717
3718         /* find the host and remote ports to connect together */
3719         spin_lock_irqsave(&nvme_fc_lock, flags);
3720         list_for_each_entry(lport, &nvme_fc_lport_list, port_list) {
3721                 if (lport->localport.node_name != laddr.nn ||
3722                     lport->localport.port_name != laddr.pn ||
3723                     lport->localport.port_state != FC_OBJSTATE_ONLINE)
3724                         continue;
3725
3726                 list_for_each_entry(rport, &lport->endp_list, endp_list) {
3727                         if (rport->remoteport.node_name != raddr.nn ||
3728                             rport->remoteport.port_name != raddr.pn ||
3729                             rport->remoteport.port_state != FC_OBJSTATE_ONLINE)
3730                                 continue;
3731
3732                         /* if fail to get reference fall through. Will error */
3733                         if (!nvme_fc_rport_get(rport))
3734                                 break;
3735
3736                         spin_unlock_irqrestore(&nvme_fc_lock, flags);
3737
3738                         ctrl = nvme_fc_init_ctrl(dev, opts, lport, rport);
3739                         if (IS_ERR(ctrl))
3740                                 nvme_fc_rport_put(rport);
3741                         return ctrl;
3742                 }
3743         }
3744         spin_unlock_irqrestore(&nvme_fc_lock, flags);
3745
3746         pr_warn("%s: %s - %s combination not found\n",
3747                 __func__, opts->traddr, opts->host_traddr);
3748         return ERR_PTR(-ENOENT);
3749 }
3750
3751
3752 static struct nvmf_transport_ops nvme_fc_transport = {
3753         .name           = "fc",
3754         .module         = THIS_MODULE,
3755         .required_opts  = NVMF_OPT_TRADDR | NVMF_OPT_HOST_TRADDR,
3756         .allowed_opts   = NVMF_OPT_RECONNECT_DELAY | NVMF_OPT_CTRL_LOSS_TMO,
3757         .create_ctrl    = nvme_fc_create_ctrl,
3758 };
3759
3760 /* Arbitrary successive failures max. With lots of subsystems could be high */
3761 #define DISCOVERY_MAX_FAIL      20
3762
3763 static ssize_t nvme_fc_nvme_discovery_store(struct device *dev,
3764                 struct device_attribute *attr, const char *buf, size_t count)
3765 {
3766         unsigned long flags;
3767         LIST_HEAD(local_disc_list);
3768         struct nvme_fc_lport *lport;
3769         struct nvme_fc_rport *rport;
3770         int failcnt = 0;
3771
3772         spin_lock_irqsave(&nvme_fc_lock, flags);
3773 restart:
3774         list_for_each_entry(lport, &nvme_fc_lport_list, port_list) {
3775                 list_for_each_entry(rport, &lport->endp_list, endp_list) {
3776                         if (!nvme_fc_lport_get(lport))
3777                                 continue;
3778                         if (!nvme_fc_rport_get(rport)) {
3779                                 /*
3780                                  * This is a temporary condition. Upon restart
3781                                  * this rport will be gone from the list.
3782                                  *
3783                                  * Revert the lport put and retry.  Anything
3784                                  * added to the list already will be skipped (as
3785                                  * they are no longer list_empty).  Loops should
3786                                  * resume at rports that were not yet seen.
3787                                  */
3788                                 nvme_fc_lport_put(lport);
3789
3790                                 if (failcnt++ < DISCOVERY_MAX_FAIL)
3791                                         goto restart;
3792
3793                                 pr_err("nvme_discovery: too many reference "
3794                                        "failures\n");
3795                                 goto process_local_list;
3796                         }
3797                         if (list_empty(&rport->disc_list))
3798                                 list_add_tail(&rport->disc_list,
3799                                               &local_disc_list);
3800                 }
3801         }
3802
3803 process_local_list:
3804         while (!list_empty(&local_disc_list)) {
3805                 rport = list_first_entry(&local_disc_list,
3806                                          struct nvme_fc_rport, disc_list);
3807                 list_del_init(&rport->disc_list);
3808                 spin_unlock_irqrestore(&nvme_fc_lock, flags);
3809
3810                 lport = rport->lport;
3811                 /* signal discovery. Won't hurt if it repeats */
3812                 nvme_fc_signal_discovery_scan(lport, rport);
3813                 nvme_fc_rport_put(rport);
3814                 nvme_fc_lport_put(lport);
3815
3816                 spin_lock_irqsave(&nvme_fc_lock, flags);
3817         }
3818         spin_unlock_irqrestore(&nvme_fc_lock, flags);
3819
3820         return count;
3821 }
3822
3823 static DEVICE_ATTR(nvme_discovery, 0200, NULL, nvme_fc_nvme_discovery_store);
3824
3825 #ifdef CONFIG_BLK_CGROUP_FC_APPID
3826 /* Parse the cgroup id from a buf and return the length of cgrpid */
3827 static int fc_parse_cgrpid(const char *buf, u64 *id)
3828 {
3829         char cgrp_id[16+1];
3830         int cgrpid_len, j;
3831
3832         memset(cgrp_id, 0x0, sizeof(cgrp_id));
3833         for (cgrpid_len = 0, j = 0; cgrpid_len < 17; cgrpid_len++) {
3834                 if (buf[cgrpid_len] != ':')
3835                         cgrp_id[cgrpid_len] = buf[cgrpid_len];
3836                 else {
3837                         j = 1;
3838                         break;
3839                 }
3840         }
3841         if (!j)
3842                 return -EINVAL;
3843         if (kstrtou64(cgrp_id, 16, id) < 0)
3844                 return -EINVAL;
3845         return cgrpid_len;
3846 }
3847
3848 /*
3849  * Parse and update the appid in the blkcg associated with the cgroupid.
3850  */
3851 static ssize_t fc_appid_store(struct device *dev,
3852                 struct device_attribute *attr, const char *buf, size_t count)
3853 {
3854         size_t orig_count = count;
3855         u64 cgrp_id;
3856         int appid_len = 0;
3857         int cgrpid_len = 0;
3858         char app_id[FC_APPID_LEN];
3859         int ret = 0;
3860
3861         if (buf[count-1] == '\n')
3862                 count--;
3863
3864         if ((count > (16+1+FC_APPID_LEN)) || (!strchr(buf, ':')))
3865                 return -EINVAL;
3866
3867         cgrpid_len = fc_parse_cgrpid(buf, &cgrp_id);
3868         if (cgrpid_len < 0)
3869                 return -EINVAL;
3870         appid_len = count - cgrpid_len - 1;
3871         if (appid_len > FC_APPID_LEN)
3872                 return -EINVAL;
3873
3874         memset(app_id, 0x0, sizeof(app_id));
3875         memcpy(app_id, &buf[cgrpid_len+1], appid_len);
3876         ret = blkcg_set_fc_appid(app_id, cgrp_id, sizeof(app_id));
3877         if (ret < 0)
3878                 return ret;
3879         return orig_count;
3880 }
3881 static DEVICE_ATTR(appid_store, 0200, NULL, fc_appid_store);
3882 #endif /* CONFIG_BLK_CGROUP_FC_APPID */
3883
3884 static struct attribute *nvme_fc_attrs[] = {
3885         &dev_attr_nvme_discovery.attr,
3886 #ifdef CONFIG_BLK_CGROUP_FC_APPID
3887         &dev_attr_appid_store.attr,
3888 #endif
3889         NULL
3890 };
3891
3892 static const struct attribute_group nvme_fc_attr_group = {
3893         .attrs = nvme_fc_attrs,
3894 };
3895
3896 static const struct attribute_group *nvme_fc_attr_groups[] = {
3897         &nvme_fc_attr_group,
3898         NULL
3899 };
3900
3901 static struct class fc_class = {
3902         .name = "fc",
3903         .dev_groups = nvme_fc_attr_groups,
3904 };
3905
3906 static int __init nvme_fc_init_module(void)
3907 {
3908         int ret;
3909
3910         nvme_fc_wq = alloc_workqueue("nvme_fc_wq", WQ_MEM_RECLAIM, 0);
3911         if (!nvme_fc_wq)
3912                 return -ENOMEM;
3913
3914         /*
3915          * NOTE:
3916          * It is expected that in the future the kernel will combine
3917          * the FC-isms that are currently under scsi and now being
3918          * added to by NVME into a new standalone FC class. The SCSI
3919          * and NVME protocols and their devices would be under this
3920          * new FC class.
3921          *
3922          * As we need something to post FC-specific udev events to,
3923          * specifically for nvme probe events, start by creating the
3924          * new device class.  When the new standalone FC class is
3925          * put in place, this code will move to a more generic
3926          * location for the class.
3927          */
3928         ret = class_register(&fc_class);
3929         if (ret) {
3930                 pr_err("couldn't register class fc\n");
3931                 goto out_destroy_wq;
3932         }
3933
3934         /*
3935          * Create a device for the FC-centric udev events
3936          */
3937         fc_udev_device = device_create(&fc_class, NULL, MKDEV(0, 0), NULL,
3938                                 "fc_udev_device");
3939         if (IS_ERR(fc_udev_device)) {
3940                 pr_err("couldn't create fc_udev device!\n");
3941                 ret = PTR_ERR(fc_udev_device);
3942                 goto out_destroy_class;
3943         }
3944
3945         ret = nvmf_register_transport(&nvme_fc_transport);
3946         if (ret)
3947                 goto out_destroy_device;
3948
3949         return 0;
3950
3951 out_destroy_device:
3952         device_destroy(&fc_class, MKDEV(0, 0));
3953 out_destroy_class:
3954         class_unregister(&fc_class);
3955 out_destroy_wq:
3956         destroy_workqueue(nvme_fc_wq);
3957
3958         return ret;
3959 }
3960
3961 static void
3962 nvme_fc_delete_controllers(struct nvme_fc_rport *rport)
3963 {
3964         struct nvme_fc_ctrl *ctrl;
3965
3966         spin_lock(&rport->lock);
3967         list_for_each_entry(ctrl, &rport->ctrl_list, ctrl_list) {
3968                 dev_warn(ctrl->ctrl.device,
3969                         "NVME-FC{%d}: transport unloading: deleting ctrl\n",
3970                         ctrl->cnum);
3971                 nvme_delete_ctrl(&ctrl->ctrl);
3972         }
3973         spin_unlock(&rport->lock);
3974 }
3975
3976 static void
3977 nvme_fc_cleanup_for_unload(void)
3978 {
3979         struct nvme_fc_lport *lport;
3980         struct nvme_fc_rport *rport;
3981
3982         list_for_each_entry(lport, &nvme_fc_lport_list, port_list) {
3983                 list_for_each_entry(rport, &lport->endp_list, endp_list) {
3984                         nvme_fc_delete_controllers(rport);
3985                 }
3986         }
3987 }
3988
3989 static void __exit nvme_fc_exit_module(void)
3990 {
3991         unsigned long flags;
3992         bool need_cleanup = false;
3993
3994         spin_lock_irqsave(&nvme_fc_lock, flags);
3995         nvme_fc_waiting_to_unload = true;
3996         if (!list_empty(&nvme_fc_lport_list)) {
3997                 need_cleanup = true;
3998                 nvme_fc_cleanup_for_unload();
3999         }
4000         spin_unlock_irqrestore(&nvme_fc_lock, flags);
4001         if (need_cleanup) {
4002                 pr_info("%s: waiting for ctlr deletes\n", __func__);
4003                 wait_for_completion(&nvme_fc_unload_proceed);
4004                 pr_info("%s: ctrl deletes complete\n", __func__);
4005         }
4006
4007         nvmf_unregister_transport(&nvme_fc_transport);
4008
4009         ida_destroy(&nvme_fc_local_port_cnt);
4010         ida_destroy(&nvme_fc_ctrl_cnt);
4011
4012         device_destroy(&fc_class, MKDEV(0, 0));
4013         class_unregister(&fc_class);
4014         destroy_workqueue(nvme_fc_wq);
4015 }
4016
4017 module_init(nvme_fc_init_module);
4018 module_exit(nvme_fc_exit_module);
4019
4020 MODULE_LICENSE("GPL v2");