drivers/s390/scsi/zfcp_qdio.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * zfcp device driver
   4  *
   5  * Setup and helper functions to access QDIO.
   6  *
   7  * Copyright IBM Corp. 2002, 2020
   8  */
   9
  10 #define KMSG_COMPONENT "zfcp"
  11 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
  12
  13 #include <linux/lockdep.h>
  14 #include <linux/slab.h>
  15 #include <linux/module.h>
  16 #include "zfcp_ext.h"
  17 #include "zfcp_qdio.h"
  18
  19 static bool enable_multibuffer = true;
  20 module_param_named(datarouter, enable_multibuffer, bool, 0400);
  21 MODULE_PARM_DESC(datarouter, "Enable hardware data router support (default on)");
  22
  23 static void zfcp_qdio_handler_error(struct zfcp_qdio *qdio, char *dbftag,
  24                                     unsigned int qdio_err)
  25 {
  26         struct zfcp_adapter *adapter = qdio->adapter;
  27
  28         dev_warn(&adapter->ccw_device->dev, "A QDIO problem occurred\n");
  29
  30         if (qdio_err & QDIO_ERROR_SLSB_STATE) {
  31                 zfcp_qdio_siosl(adapter);
  32                 zfcp_erp_adapter_shutdown(adapter, 0, dbftag);
  33                 return;
  34         }
  35         zfcp_erp_adapter_reopen(adapter,
  36                                 ZFCP_STATUS_ADAPTER_LINK_UNPLUGGED |
  37                                 ZFCP_STATUS_COMMON_ERP_FAILED, dbftag);
  38 }
  39
  40 static void zfcp_qdio_zero_sbals(struct qdio_buffer *sbal[], int first, int cnt)
  41 {
  42         int i, sbal_idx;
  43
  44         for (i = first; i < first + cnt; i++) {
  45                 sbal_idx = i % QDIO_MAX_BUFFERS_PER_Q;
  46                 memset(sbal[sbal_idx], 0, sizeof(struct qdio_buffer));
  47         }
  48 }
  49
  50 /* this needs to be called prior to updating the queue fill level */
  51 static inline void zfcp_qdio_account(struct zfcp_qdio *qdio)
  52 {
  53         unsigned long long now, span;
  54         int used;
  55
  56         now = get_tod_clock_monotonic();
  57         span = (now - qdio->req_q_time) >> 12;
  58         used = QDIO_MAX_BUFFERS_PER_Q - atomic_read(&qdio->req_q_free);
  59         qdio->req_q_util += used * span;
  60         qdio->req_q_time = now;
  61 }
  62
  63 static void zfcp_qdio_int_req(struct ccw_device *cdev, unsigned int qdio_err,
  64                               int queue_no, int idx, int count,
  65                               unsigned long parm)
  66 {
  67         struct zfcp_qdio *qdio = (struct zfcp_qdio *) parm;
  68
  69         if (unlikely(qdio_err)) {
  70                 zfcp_qdio_handler_error(qdio, "qdireq1", qdio_err);
  71                 return;
  72         }
  73
  74         /* cleanup all SBALs being program-owned now */
  75         zfcp_qdio_zero_sbals(qdio->req_q, idx, count);
  76
  77         spin_lock_irq(&qdio->stat_lock);
  78         zfcp_qdio_account(qdio);
  79         spin_unlock_irq(&qdio->stat_lock);
  80         atomic_add(count, &qdio->req_q_free);
  81         wake_up(&qdio->req_q_wq);
  82 }
  83
  84 static void zfcp_qdio_int_resp(struct ccw_device *cdev, unsigned int qdio_err,
  85                                int queue_no, int idx, int count,
  86                                unsigned long parm)
  87 {
  88         struct zfcp_qdio *qdio = (struct zfcp_qdio *) parm;
  89         struct zfcp_adapter *adapter = qdio->adapter;
  90         int sbal_no, sbal_idx;
  91
  92         if (unlikely(qdio_err)) {
  93                 if (zfcp_adapter_multi_buffer_active(adapter)) {
  94                         void *pl[ZFCP_QDIO_MAX_SBALS_PER_REQ + 1];
  95                         struct qdio_buffer_element *sbale;
  96                         u64 req_id;
  97                         u8 scount;
  98
  99                         memset(pl, 0,
 100                                ZFCP_QDIO_MAX_SBALS_PER_REQ * sizeof(void *));
 101                         sbale = qdio->res_q[idx]->element;
 102                         req_id = sbale->addr;
 103                         scount = min(sbale->scount + 1,
 104                                      ZFCP_QDIO_MAX_SBALS_PER_REQ + 1);
 105                                      /* incl. signaling SBAL */
 106
 107                         for (sbal_no = 0; sbal_no < scount; sbal_no++) {
 108                                 sbal_idx = (idx + sbal_no) %
 109                                         QDIO_MAX_BUFFERS_PER_Q;
 110                                 pl[sbal_no] = qdio->res_q[sbal_idx];
 111                         }
 112                         zfcp_dbf_hba_def_err(adapter, req_id, scount, pl);
 113                 }
 114                 zfcp_qdio_handler_error(qdio, "qdires1", qdio_err);
 115                 return;
 116         }
 117
 118         /*
 119          * go through all SBALs from input queue currently
 120          * returned by QDIO layer
 121          */
 122         for (sbal_no = 0; sbal_no < count; sbal_no++) {
 123                 sbal_idx = (idx + sbal_no) % QDIO_MAX_BUFFERS_PER_Q;
 124                 /* go through all SBALEs of SBAL */
 125                 zfcp_fsf_reqid_check(qdio, sbal_idx);
 126         }
 127
 128         /*
 129          * put SBALs back to response queue
 130          */
 131         if (do_QDIO(cdev, QDIO_FLAG_SYNC_INPUT, 0, idx, count))
 132                 zfcp_erp_adapter_reopen(qdio->adapter, 0, "qdires2");
 133 }
 134
 135 static void zfcp_qdio_irq_tasklet(struct tasklet_struct *tasklet)
 136 {
 137         struct zfcp_qdio *qdio = from_tasklet(qdio, tasklet, irq_tasklet);
 138         struct ccw_device *cdev = qdio->adapter->ccw_device;
 139         unsigned int start, error;
 140         int completed;
 141
 142         /* Check the Response Queue, and kick off the Request Queue tasklet: */
 143         completed = qdio_get_next_buffers(cdev, 0, &start, &error);
 144         if (completed < 0)
 145                 return;
 146         if (completed > 0)
 147                 zfcp_qdio_int_resp(cdev, error, 0, start, completed,
 148                                    (unsigned long) qdio);
 149
 150         if (qdio_start_irq(cdev))
 151                 /* More work pending: */
 152                 tasklet_schedule(&qdio->irq_tasklet);
 153 }
 154
 155 static void zfcp_qdio_poll(struct ccw_device *cdev, unsigned long data)
 156 {
 157         struct zfcp_qdio *qdio = (struct zfcp_qdio *) data;
 158
 159         tasklet_schedule(&qdio->irq_tasklet);
 160 }
 161
 162 static struct qdio_buffer_element *
 163 zfcp_qdio_sbal_chain(struct zfcp_qdio *qdio, struct zfcp_qdio_req *q_req)
 164 {
 165         struct qdio_buffer_element *sbale;
 166
 167         /* set last entry flag in current SBALE of current SBAL */
 168         sbale = zfcp_qdio_sbale_curr(qdio, q_req);
 169         sbale->eflags |= SBAL_EFLAGS_LAST_ENTRY;
 170
 171         /* don't exceed last allowed SBAL */
 172         if (q_req->sbal_last == q_req->sbal_limit)
 173                 return NULL;
 174
 175         /* set chaining flag in first SBALE of current SBAL */
 176         sbale = zfcp_qdio_sbale_req(qdio, q_req);
 177         sbale->sflags |= SBAL_SFLAGS0_MORE_SBALS;
 178
 179         /* calculate index of next SBAL */
 180         q_req->sbal_last++;
 181         q_req->sbal_last %= QDIO_MAX_BUFFERS_PER_Q;
 182
 183         /* keep this requests number of SBALs up-to-date */
 184         q_req->sbal_number++;
 185         BUG_ON(q_req->sbal_number > ZFCP_QDIO_MAX_SBALS_PER_REQ);
 186
 187         /* start at first SBALE of new SBAL */
 188         q_req->sbale_curr = 0;
 189
 190         /* set storage-block type for new SBAL */
 191         sbale = zfcp_qdio_sbale_curr(qdio, q_req);
 192         sbale->sflags |= q_req->sbtype;
 193
 194         return sbale;
 195 }
 196
 197 static struct qdio_buffer_element *
 198 zfcp_qdio_sbale_next(struct zfcp_qdio *qdio, struct zfcp_qdio_req *q_req)
 199 {
 200         if (q_req->sbale_curr == qdio->max_sbale_per_sbal - 1)
 201                 return zfcp_qdio_sbal_chain(qdio, q_req);
 202         q_req->sbale_curr++;
 203         return zfcp_qdio_sbale_curr(qdio, q_req);
 204 }
 205
 206 /**
 207  * zfcp_qdio_sbals_from_sg - fill SBALs from scatter-gather list
 208  * @qdio: pointer to struct zfcp_qdio
 209  * @q_req: pointer to struct zfcp_qdio_req
 210  * @sg: scatter-gather list
 211  * Returns: zero or -EINVAL on error
 212  */
 213 int zfcp_qdio_sbals_from_sg(struct zfcp_qdio *qdio, struct zfcp_qdio_req *q_req,
 214                             struct scatterlist *sg)
 215 {
 216         struct qdio_buffer_element *sbale;
 217
 218         /* set storage-block type for this request */
 219         sbale = zfcp_qdio_sbale_req(qdio, q_req);
 220         sbale->sflags |= q_req->sbtype;
 221
 222         for (; sg; sg = sg_next(sg)) {
 223                 sbale = zfcp_qdio_sbale_next(qdio, q_req);
 224                 if (!sbale) {
 225                         atomic_inc(&qdio->req_q_full);
 226                         zfcp_qdio_zero_sbals(qdio->req_q, q_req->sbal_first,
 227                                              q_req->sbal_number);
 228                         return -EINVAL;
 229                 }
 230                 sbale->addr = sg_phys(sg);
 231                 sbale->length = sg->length;
 232         }
 233         return 0;
 234 }
 235
 236 static int zfcp_qdio_sbal_check(struct zfcp_qdio *qdio)
 237 {
 238         if (atomic_read(&qdio->req_q_free) ||
 239             !(atomic_read(&qdio->adapter->status) & ZFCP_STATUS_ADAPTER_QDIOUP))
 240                 return 1;
 241         return 0;
 242 }
 243
 244 /**
 245  * zfcp_qdio_sbal_get - get free sbal in request queue, wait if necessary
 246  * @qdio: pointer to struct zfcp_qdio
 247  *
 248  * The req_q_lock must be held by the caller of this function, and
 249  * this function may only be called from process context; it will
 250  * sleep when waiting for a free sbal.
 251  *
 252  * Returns: 0 on success, -EIO if there is no free sbal after waiting.
 253  */
 254 int zfcp_qdio_sbal_get(struct zfcp_qdio *qdio)
 255 {
 256         long ret;
 257
 258         ret = wait_event_interruptible_lock_irq_timeout(qdio->req_q_wq,
 259                        zfcp_qdio_sbal_check(qdio), qdio->req_q_lock, 5 * HZ);
 260
 261         if (!(atomic_read(&qdio->adapter->status) & ZFCP_STATUS_ADAPTER_QDIOUP))
 262                 return -EIO;
 263
 264         if (ret > 0)
 265                 return 0;
 266
 267         if (!ret) {
 268                 atomic_inc(&qdio->req_q_full);
 269                 /* assume hanging outbound queue, try queue recovery */
 270                 zfcp_erp_adapter_reopen(qdio->adapter, 0, "qdsbg_1");
 271         }
 272
 273         return -EIO;
 274 }
 275
 276 /**
 277  * zfcp_qdio_send - send req to QDIO
 278  * @qdio: pointer to struct zfcp_qdio
 279  * @q_req: pointer to struct zfcp_qdio_req
 280  * Returns: 0 on success, error otherwise
 281  */
 282 int zfcp_qdio_send(struct zfcp_qdio *qdio, struct zfcp_qdio_req *q_req)
 283 {
 284         int retval;
 285         u8 sbal_number = q_req->sbal_number;
 286
 287         /*
 288          * This should actually be a spin_lock_bh(stat_lock), to protect against
 289          * zfcp_qdio_int_req() in tasklet context.
 290          * But we can't do so (and are safe), as we always get called with IRQs
 291          * disabled by spin_lock_irq[save](req_q_lock).
 292          */
 293         lockdep_assert_irqs_disabled();
 294         spin_lock(&qdio->stat_lock);
 295         zfcp_qdio_account(qdio);
 296         spin_unlock(&qdio->stat_lock);
 297
 298         atomic_sub(sbal_number, &qdio->req_q_free);
 299
 300         retval = do_QDIO(qdio->adapter->ccw_device, QDIO_FLAG_SYNC_OUTPUT, 0,
 301                          q_req->sbal_first, sbal_number);
 302
 303         if (unlikely(retval)) {
 304                 /* Failed to submit the IO, roll back our modifications. */
 305                 atomic_add(sbal_number, &qdio->req_q_free);
 306                 zfcp_qdio_zero_sbals(qdio->req_q, q_req->sbal_first,
 307                                      sbal_number);
 308                 return retval;
 309         }
 310
 311         /* account for transferred buffers */
 312         qdio->req_q_idx += sbal_number;
 313         qdio->req_q_idx %= QDIO_MAX_BUFFERS_PER_Q;
 314
 315         return 0;
 316 }
 317
 318 /**
 319  * zfcp_qdio_allocate - allocate queue memory and initialize QDIO data
 320  * @qdio: pointer to struct zfcp_qdio
 321  * Returns: -ENOMEM on memory allocation error or return value from
 322  *          qdio_allocate
 323  */
 324 static int zfcp_qdio_allocate(struct zfcp_qdio *qdio)
 325 {
 326         int ret;
 327
 328         ret = qdio_alloc_buffers(qdio->req_q, QDIO_MAX_BUFFERS_PER_Q);
 329         if (ret)
 330                 return -ENOMEM;
 331
 332         ret = qdio_alloc_buffers(qdio->res_q, QDIO_MAX_BUFFERS_PER_Q);
 333         if (ret)
 334                 goto free_req_q;
 335
 336         init_waitqueue_head(&qdio->req_q_wq);
 337
 338         ret = qdio_allocate(qdio->adapter->ccw_device, 1, 1);
 339         if (ret)
 340                 goto free_res_q;
 341
 342         return 0;
 343
 344 free_res_q:
 345         qdio_free_buffers(qdio->res_q, QDIO_MAX_BUFFERS_PER_Q);
 346 free_req_q:
 347         qdio_free_buffers(qdio->req_q, QDIO_MAX_BUFFERS_PER_Q);
 348         return ret;
 349 }
 350
 351 /**
 352  * zfcp_close_qdio - close qdio queues for an adapter
 353  * @qdio: pointer to structure zfcp_qdio
 354  */
 355 void zfcp_qdio_close(struct zfcp_qdio *qdio)
 356 {
 357         struct zfcp_adapter *adapter = qdio->adapter;
 358         int idx, count;
 359
 360         if (!(atomic_read(&adapter->status) & ZFCP_STATUS_ADAPTER_QDIOUP))
 361                 return;
 362
 363         /* clear QDIOUP flag, thus do_QDIO is not called during qdio_shutdown */
 364         spin_lock_irq(&qdio->req_q_lock);
 365         atomic_andnot(ZFCP_STATUS_ADAPTER_QDIOUP, &adapter->status);
 366         spin_unlock_irq(&qdio->req_q_lock);
 367
 368         wake_up(&qdio->req_q_wq);
 369
 370         tasklet_disable(&qdio->irq_tasklet);
 371         qdio_stop_irq(adapter->ccw_device);
 372         qdio_shutdown(adapter->ccw_device, QDIO_FLAG_CLEANUP_USING_CLEAR);
 373
 374         /* cleanup used outbound sbals */
 375         count = atomic_read(&qdio->req_q_free);
 376         if (count < QDIO_MAX_BUFFERS_PER_Q) {
 377                 idx = (qdio->req_q_idx + count) % QDIO_MAX_BUFFERS_PER_Q;
 378                 count = QDIO_MAX_BUFFERS_PER_Q - count;
 379                 zfcp_qdio_zero_sbals(qdio->req_q, idx, count);
 380         }
 381         qdio->req_q_idx = 0;
 382         atomic_set(&qdio->req_q_free, 0);
 383 }
 384
 385 void zfcp_qdio_shost_update(struct zfcp_adapter *const adapter,
 386                             const struct zfcp_qdio *const qdio)
 387 {
 388         struct Scsi_Host *const shost = adapter->scsi_host;
 389
 390         if (shost == NULL)
 391                 return;
 392
 393         shost->sg_tablesize = qdio->max_sbale_per_req;
 394         shost->max_sectors = qdio->max_sbale_per_req * 8;
 395 }
 396
 397 /**
 398  * zfcp_qdio_open - prepare and initialize response queue
 399  * @qdio: pointer to struct zfcp_qdio
 400  * Returns: 0 on success, otherwise -EIO
 401  */
 402 int zfcp_qdio_open(struct zfcp_qdio *qdio)
 403 {
 404         struct qdio_buffer **input_sbals[1] = {qdio->res_q};
 405         struct qdio_buffer **output_sbals[1] = {qdio->req_q};
 406         struct qdio_buffer_element *sbale;
 407         struct qdio_initialize init_data = {0};
 408         struct zfcp_adapter *adapter = qdio->adapter;
 409         struct ccw_device *cdev = adapter->ccw_device;
 410         struct qdio_ssqd_desc ssqd;
 411         int cc;
 412
 413         if (atomic_read(&adapter->status) & ZFCP_STATUS_ADAPTER_QDIOUP)
 414                 return -EIO;
 415
 416         atomic_andnot(ZFCP_STATUS_ADAPTER_SIOSL_ISSUED,
 417                           &qdio->adapter->status);
 418
 419         init_data.q_format = QDIO_ZFCP_QFMT;
 420         init_data.qib_rflags = QIB_RFLAGS_ENABLE_DATA_DIV;
 421         if (enable_multibuffer)
 422                 init_data.qdr_ac |= QDR_AC_MULTI_BUFFER_ENABLE;
 423         init_data.no_input_qs = 1;
 424         init_data.no_output_qs = 1;
 425         init_data.input_handler = zfcp_qdio_int_resp;
 426         init_data.output_handler = zfcp_qdio_int_req;
 427         init_data.irq_poll = zfcp_qdio_poll;
 428         init_data.int_parm = (unsigned long) qdio;
 429         init_data.input_sbal_addr_array = input_sbals;
 430         init_data.output_sbal_addr_array = output_sbals;
 431         init_data.scan_threshold =
 432                 QDIO_MAX_BUFFERS_PER_Q - ZFCP_QDIO_MAX_SBALS_PER_REQ * 2;
 433
 434         if (qdio_establish(cdev, &init_data))
 435                 goto failed_establish;
 436
 437         if (qdio_get_ssqd_desc(cdev, &ssqd))
 438                 goto failed_qdio;
 439
 440         if (ssqd.qdioac2 & CHSC_AC2_DATA_DIV_ENABLED)
 441                 atomic_or(ZFCP_STATUS_ADAPTER_DATA_DIV_ENABLED,
 442                                 &qdio->adapter->status);
 443
 444         if (ssqd.qdioac2 & CHSC_AC2_MULTI_BUFFER_ENABLED) {
 445                 atomic_or(ZFCP_STATUS_ADAPTER_MB_ACT, &adapter->status);
 446                 qdio->max_sbale_per_sbal = QDIO_MAX_ELEMENTS_PER_BUFFER;
 447         } else {
 448                 atomic_andnot(ZFCP_STATUS_ADAPTER_MB_ACT, &adapter->status);
 449                 qdio->max_sbale_per_sbal = QDIO_MAX_ELEMENTS_PER_BUFFER - 1;
 450         }
 451
 452         qdio->max_sbale_per_req =
 453                 ZFCP_QDIO_MAX_SBALS_PER_REQ * qdio->max_sbale_per_sbal
 454                 - 2;
 455         if (qdio_activate(cdev))
 456                 goto failed_qdio;
 457
 458         for (cc = 0; cc < QDIO_MAX_BUFFERS_PER_Q; cc++) {
 459                 sbale = &(qdio->res_q[cc]->element[0]);
 460                 sbale->length = 0;
 461                 sbale->eflags = SBAL_EFLAGS_LAST_ENTRY;
 462                 sbale->sflags = 0;
 463                 sbale->addr = 0;
 464         }
 465
 466         if (do_QDIO(cdev, QDIO_FLAG_SYNC_INPUT, 0, 0, QDIO_MAX_BUFFERS_PER_Q))
 467                 goto failed_qdio;
 468
 469         /* set index of first available SBALS / number of available SBALS */
 470         qdio->req_q_idx = 0;
 471         atomic_set(&qdio->req_q_free, QDIO_MAX_BUFFERS_PER_Q);
 472         atomic_or(ZFCP_STATUS_ADAPTER_QDIOUP, &qdio->adapter->status);
 473
 474         /* Enable processing for QDIO interrupts: */
 475         tasklet_enable(&qdio->irq_tasklet);
 476         /* This results in a qdio_start_irq(): */
 477         tasklet_schedule(&qdio->irq_tasklet);
 478
 479         zfcp_qdio_shost_update(adapter, qdio);
 480
 481         return 0;
 482
 483 failed_qdio:
 484         qdio_shutdown(cdev, QDIO_FLAG_CLEANUP_USING_CLEAR);
 485 failed_establish:
 486         dev_err(&cdev->dev,
 487                 "Setting up the QDIO connection to the FCP adapter failed\n");
 488         return -EIO;
 489 }
 490
 491 void zfcp_qdio_destroy(struct zfcp_qdio *qdio)
 492 {
 493         if (!qdio)
 494                 return;
 495
 496         tasklet_kill(&qdio->irq_tasklet);
 497
 498         if (qdio->adapter->ccw_device)
 499                 qdio_free(qdio->adapter->ccw_device);
 500
 501         qdio_free_buffers(qdio->req_q, QDIO_MAX_BUFFERS_PER_Q);
 502         qdio_free_buffers(qdio->res_q, QDIO_MAX_BUFFERS_PER_Q);
 503         kfree(qdio);
 504 }
 505
 506 int zfcp_qdio_setup(struct zfcp_adapter *adapter)
 507 {
 508         struct zfcp_qdio *qdio;
 509
 510         qdio = kzalloc(sizeof(struct zfcp_qdio), GFP_KERNEL);
 511         if (!qdio)
 512                 return -ENOMEM;
 513
 514         qdio->adapter = adapter;
 515
 516         if (zfcp_qdio_allocate(qdio)) {
 517                 kfree(qdio);
 518                 return -ENOMEM;
 519         }
 520
 521         spin_lock_init(&qdio->req_q_lock);
 522         spin_lock_init(&qdio->stat_lock);
 523         tasklet_setup(&qdio->irq_tasklet, zfcp_qdio_irq_tasklet);
 524         tasklet_disable(&qdio->irq_tasklet);
 525
 526         adapter->qdio = qdio;
 527         return 0;
 528 }
 529
 530 /**
 531  * zfcp_qdio_siosl - Trigger logging in FCP channel
 532  * @adapter: The zfcp_adapter where to trigger logging
 533  *
 534  * Call the cio siosl function to trigger hardware logging.  This
 535  * wrapper function sets a flag to ensure hardware logging is only
 536  * triggered once before going through qdio shutdown.
 537  *
 538  * The triggers are always run from qdio tasklet context, so no
 539  * additional synchronization is necessary.
 540  */
 541 void zfcp_qdio_siosl(struct zfcp_adapter *adapter)
 542 {
 543         int rc;
 544
 545         if (atomic_read(&adapter->status) & ZFCP_STATUS_ADAPTER_SIOSL_ISSUED)
 546                 return;
 547
 548         rc = ccw_device_siosl(adapter->ccw_device);
 549         if (!rc)
 550                 atomic_or(ZFCP_STATUS_ADAPTER_SIOSL_ISSUED,
 551                                 &adapter->status);
 552 }