drivers/misc/genwqe/card_ddcb.c

   1 /**
   2  * IBM Accelerator Family 'GenWQE'
   3  *
   4  * (C) Copyright IBM Corp. 2013
   5  *
   6  * Author: Frank Haverkamp <haver@linux.vnet.ibm.com>
   7  * Author: Joerg-Stephan Vogt <jsvogt@de.ibm.com>
   8  * Author: Michael Jung <mijung@gmx.net>
   9  * Author: Michael Ruettger <michael@ibmra.de>
  10  *
  11  * This program is free software; you can redistribute it and/or modify
  12  * it under the terms of the GNU General Public License (version 2 only)
  13  * as published by the Free Software Foundation.
  14  *
  15  * This program is distributed in the hope that it will be useful,
  16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  18  * GNU General Public License for more details.
  19  */
  20
  21 /*
  22  * Device Driver Control Block (DDCB) queue support. Definition of
  23  * interrupt handlers for queue support as well as triggering the
  24  * health monitor code in case of problems. The current hardware uses
  25  * an MSI interrupt which is shared between error handling and
  26  * functional code.
  27  */
  28
  29 #include <linux/types.h>
  30 #include <linux/sched.h>
  31 #include <linux/wait.h>
  32 #include <linux/pci.h>
  33 #include <linux/string.h>
  34 #include <linux/dma-mapping.h>
  35 #include <linux/delay.h>
  36 #include <linux/module.h>
  37 #include <linux/interrupt.h>
  38 #include <linux/crc-itu-t.h>
  39
  40 #include "card_base.h"
  41 #include "card_ddcb.h"
  42
  43 /*
  44  * N: next DDCB, this is where the next DDCB will be put.
  45  * A: active DDCB, this is where the code will look for the next completion.
  46  * x: DDCB is enqueued, we are waiting for its completion.
  47
  48  * Situation (1): Empty queue
  49  *  +---+---+---+---+---+---+---+---+
  50  *  | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |
  51  *  |   |   |   |   |   |   |   |   |
  52  *  +---+---+---+---+---+---+---+---+
  53  *           A/N
  54  *  enqueued_ddcbs = A - N = 2 - 2 = 0
  55  *
  56  * Situation (2): Wrapped, N > A
  57  *  +---+---+---+---+---+---+---+---+
  58  *  | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |
  59  *  |   |   | x | x |   |   |   |   |
  60  *  +---+---+---+---+---+---+---+---+
  61  *            A       N
  62  *  enqueued_ddcbs = N - A = 4 - 2 = 2
  63  *
  64  * Situation (3): Queue wrapped, A > N
  65  *  +---+---+---+---+---+---+---+---+
  66  *  | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |
  67  *  | x | x |   |   | x | x | x | x |
  68  *  +---+---+---+---+---+---+---+---+
  69  *            N       A
  70  *  enqueued_ddcbs = queue_max  - (A - N) = 8 - (4 - 2) = 6
  71  *
  72  * Situation (4a): Queue full N > A
  73  *  +---+---+---+---+---+---+---+---+
  74  *  | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |
  75  *  | x | x | x | x | x | x | x |   |
  76  *  +---+---+---+---+---+---+---+---+
  77  *    A                           N
  78  *
  79  *  enqueued_ddcbs = N - A = 7 - 0 = 7
  80  *
  81  * Situation (4a): Queue full A > N
  82  *  +---+---+---+---+---+---+---+---+
  83  *  | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 |
  84  *  | x | x | x |   | x | x | x | x |
  85  *  +---+---+---+---+---+---+---+---+
  86  *                N   A
  87  *  enqueued_ddcbs = queue_max - (A - N) = 8 - (4 - 3) = 7
  88  */
  89
  90 static int queue_empty(struct ddcb_queue *queue)
  91 {
  92         return queue->ddcb_next == queue->ddcb_act;
  93 }
  94
  95 static int queue_enqueued_ddcbs(struct ddcb_queue *queue)
  96 {
  97         if (queue->ddcb_next >= queue->ddcb_act)
  98                 return queue->ddcb_next - queue->ddcb_act;
  99
 100         return queue->ddcb_max - (queue->ddcb_act - queue->ddcb_next);
 101 }
 102
 103 static int queue_free_ddcbs(struct ddcb_queue *queue)
 104 {
 105         int free_ddcbs = queue->ddcb_max - queue_enqueued_ddcbs(queue) - 1;
 106
 107         if (WARN_ON_ONCE(free_ddcbs < 0)) { /* must never ever happen! */
 108                 return 0;
 109         }
 110         return free_ddcbs;
 111 }
 112
 113 /*
 114  * Use of the PRIV field in the DDCB for queue debugging:
 115  *
 116  * (1) Trying to get rid of a DDCB which saw a timeout:
 117  *     pddcb->priv[6] = 0xcc;   # cleared
 118  *
 119  * (2) Append a DDCB via NEXT bit:
 120  *     pddcb->priv[7] = 0xaa;   # appended
 121  *
 122  * (3) DDCB needed tapping:
 123  *     pddcb->priv[7] = 0xbb;   # tapped
 124  *
 125  * (4) DDCB marked as correctly finished:
 126  *     pddcb->priv[6] = 0xff;   # finished
 127  */
 128
 129 static inline void ddcb_mark_tapped(struct ddcb *pddcb)
 130 {
 131         pddcb->priv[7] = 0xbb;  /* tapped */
 132 }
 133
 134 static inline void ddcb_mark_appended(struct ddcb *pddcb)
 135 {
 136         pddcb->priv[7] = 0xaa;  /* appended */
 137 }
 138
 139 static inline void ddcb_mark_cleared(struct ddcb *pddcb)
 140 {
 141         pddcb->priv[6] = 0xcc; /* cleared */
 142 }
 143
 144 static inline void ddcb_mark_finished(struct ddcb *pddcb)
 145 {
 146         pddcb->priv[6] = 0xff;  /* finished */
 147 }
 148
 149 static inline void ddcb_mark_unused(struct ddcb *pddcb)
 150 {
 151         pddcb->priv_64 = cpu_to_be64(0); /* not tapped */
 152 }
 153
 154 /**
 155  * genwqe_crc16() - Generate 16-bit crc as required for DDCBs
 156  * @buff:       pointer to data buffer
 157  * @len:        length of data for calculation
 158  * @init:       initial crc (0xffff at start)
 159  *
 160  * Polynomial = x^16 + x^12 + x^5 + 1   (0x1021)
 161  * Example: 4 bytes 0x01 0x02 0x03 0x04 with init = 0xffff
 162  *          should result in a crc16 of 0x89c3
 163  *
 164  * Return: crc16 checksum in big endian format !
 165  */
 166 static inline u16 genwqe_crc16(const u8 *buff, size_t len, u16 init)
 167 {
 168         return crc_itu_t(init, buff, len);
 169 }
 170
 171 static void print_ddcb_info(struct genwqe_dev *cd, struct ddcb_queue *queue)
 172 {
 173         int i;
 174         struct ddcb *pddcb;
 175         unsigned long flags;
 176         struct pci_dev *pci_dev = cd->pci_dev;
 177
 178         spin_lock_irqsave(&cd->print_lock, flags);
 179
 180         dev_info(&pci_dev->dev,
 181                  "DDCB list for card #%d (ddcb_act=%d / ddcb_next=%d):\n",
 182                  cd->card_idx, queue->ddcb_act, queue->ddcb_next);
 183
 184         pddcb = queue->ddcb_vaddr;
 185         for (i = 0; i < queue->ddcb_max; i++) {
 186                 dev_err(&pci_dev->dev,
 187                         "  %c %-3d: RETC=%03x SEQ=%04x HSI=%02X SHI=%02x PRIV=%06llx CMD=%03x\n",
 188                         i == queue->ddcb_act ? '>' : ' ',
 189                         i,
 190                         be16_to_cpu(pddcb->retc_16),
 191                         be16_to_cpu(pddcb->seqnum_16),
 192                         pddcb->hsi,
 193                         pddcb->shi,
 194                         be64_to_cpu(pddcb->priv_64),
 195                         pddcb->cmd);
 196                 pddcb++;
 197         }
 198         spin_unlock_irqrestore(&cd->print_lock, flags);
 199 }
 200
 201 struct genwqe_ddcb_cmd *ddcb_requ_alloc(void)
 202 {
 203         struct ddcb_requ *req;
 204
 205         req = kzalloc(sizeof(*req), GFP_KERNEL);
 206         if (!req)
 207                 return NULL;
 208
 209         return &req->cmd;
 210 }
 211
 212 void ddcb_requ_free(struct genwqe_ddcb_cmd *cmd)
 213 {
 214         struct ddcb_requ *req = container_of(cmd, struct ddcb_requ, cmd);
 215
 216         kfree(req);
 217 }
 218
 219 static inline enum genwqe_requ_state ddcb_requ_get_state(struct ddcb_requ *req)
 220 {
 221         return req->req_state;
 222 }
 223
 224 static inline void ddcb_requ_set_state(struct ddcb_requ *req,
 225                                        enum genwqe_requ_state new_state)
 226 {
 227         req->req_state = new_state;
 228 }
 229
 230 static inline int ddcb_requ_collect_debug_data(struct ddcb_requ *req)
 231 {
 232         return req->cmd.ddata_addr != 0x0;
 233 }
 234
 235 /**
 236  * ddcb_requ_finished() - Returns the hardware state of the associated DDCB
 237  * @cd:          pointer to genwqe device descriptor
 238  * @req:         DDCB work request
 239  *
 240  * Status of ddcb_requ mirrors this hardware state, but is copied in
 241  * the ddcb_requ on interrupt/polling function. The lowlevel code
 242  * should check the hardware state directly, the higher level code
 243  * should check the copy.
 244  *
 245  * This function will also return true if the state of the queue is
 246  * not GENWQE_CARD_USED. This enables us to purge all DDCBs in the
 247  * shutdown case.
 248  */
 249 static int ddcb_requ_finished(struct genwqe_dev *cd, struct ddcb_requ *req)
 250 {
 251         return (ddcb_requ_get_state(req) == GENWQE_REQU_FINISHED) ||
 252                 (cd->card_state != GENWQE_CARD_USED);
 253 }
 254
 255 /**
 256  * enqueue_ddcb() - Enqueue a DDCB
 257  * @cd:         pointer to genwqe device descriptor
 258  * @queue:      queue this operation should be done on
 259  * @ddcb_no:    pointer to ddcb number being tapped
 260  *
 261  * Start execution of DDCB by tapping or append to queue via NEXT
 262  * bit. This is done by an atomic 'compare and swap' instruction and
 263  * checking SHI and HSI of the previous DDCB.
 264  *
 265  * This function must only be called with ddcb_lock held.
 266  *
 267  * Return: 1 if new DDCB is appended to previous
 268  *         2 if DDCB queue is tapped via register/simulation
 269  */
 270 #define RET_DDCB_APPENDED 1
 271 #define RET_DDCB_TAPPED   2
 272
 273 static int enqueue_ddcb(struct genwqe_dev *cd, struct ddcb_queue *queue,
 274                         struct ddcb *pddcb, int ddcb_no)
 275 {
 276         unsigned int try;
 277         int prev_no;
 278         struct ddcb *prev_ddcb;
 279         __be32 old, new, icrc_hsi_shi;
 280         u64 num;
 281
 282         /*
 283          * For performance checks a Dispatch Timestamp can be put into
 284          * DDCB It is supposed to use the SLU's free running counter,
 285          * but this requires PCIe cycles.
 286          */
 287         ddcb_mark_unused(pddcb);
 288
 289         /* check previous DDCB if already fetched */
 290         prev_no = (ddcb_no == 0) ? queue->ddcb_max - 1 : ddcb_no - 1;
 291         prev_ddcb = &queue->ddcb_vaddr[prev_no];
 292
 293         /*
 294          * It might have happened that the HSI.FETCHED bit is
 295          * set. Retry in this case. Therefore I expect maximum 2 times
 296          * trying.
 297          */
 298         ddcb_mark_appended(pddcb);
 299         for (try = 0; try < 2; try++) {
 300                 old = prev_ddcb->icrc_hsi_shi_32; /* read SHI/HSI in BE32 */
 301
 302                 /* try to append via NEXT bit if prev DDCB is not completed */
 303                 if ((old & DDCB_COMPLETED_BE32) != 0x00000000)
 304                         break;
 305
 306                 new = (old | DDCB_NEXT_BE32);
 307
 308                 wmb();          /* need to ensure write ordering */
 309                 icrc_hsi_shi = cmpxchg(&prev_ddcb->icrc_hsi_shi_32, old, new);
 310
 311                 if (icrc_hsi_shi == old)
 312                         return RET_DDCB_APPENDED; /* appended to queue */
 313         }
 314
 315         /* Queue must be re-started by updating QUEUE_OFFSET */
 316         ddcb_mark_tapped(pddcb);
 317         num = (u64)ddcb_no << 8;
 318
 319         wmb();                  /* need to ensure write ordering */
 320         __genwqe_writeq(cd, queue->IO_QUEUE_OFFSET, num); /* start queue */
 321
 322         return RET_DDCB_TAPPED;
 323 }
 324
 325 /**
 326  * copy_ddcb_results() - Copy output state from real DDCB to request
 327  *
 328  * Copy DDCB ASV to request struct. There is no endian
 329  * conversion made, since data structure in ASV is still
 330  * unknown here.
 331  *
 332  * This is needed by:
 333  *   - genwqe_purge_ddcb()
 334  *   - genwqe_check_ddcb_queue()
 335  */
 336 static void copy_ddcb_results(struct ddcb_requ *req, int ddcb_no)
 337 {
 338         struct ddcb_queue *queue = req->queue;
 339         struct ddcb *pddcb = &queue->ddcb_vaddr[req->num];
 340
 341         memcpy(&req->cmd.asv[0], &pddcb->asv[0], DDCB_ASV_LENGTH);
 342
 343         /* copy status flags of the variant part */
 344         req->cmd.vcrc     = be16_to_cpu(pddcb->vcrc_16);
 345         req->cmd.deque_ts = be64_to_cpu(pddcb->deque_ts_64);
 346         req->cmd.cmplt_ts = be64_to_cpu(pddcb->cmplt_ts_64);
 347
 348         req->cmd.attn     = be16_to_cpu(pddcb->attn_16);
 349         req->cmd.progress = be32_to_cpu(pddcb->progress_32);
 350         req->cmd.retc     = be16_to_cpu(pddcb->retc_16);
 351
 352         if (ddcb_requ_collect_debug_data(req)) {
 353                 int prev_no = (ddcb_no == 0) ?
 354                         queue->ddcb_max - 1 : ddcb_no - 1;
 355                 struct ddcb *prev_pddcb = &queue->ddcb_vaddr[prev_no];
 356
 357                 memcpy(&req->debug_data.ddcb_finished, pddcb,
 358                        sizeof(req->debug_data.ddcb_finished));
 359                 memcpy(&req->debug_data.ddcb_prev, prev_pddcb,
 360                        sizeof(req->debug_data.ddcb_prev));
 361         }
 362 }
 363
 364 /**
 365  * genwqe_check_ddcb_queue() - Checks DDCB queue for completed work equests.
 366  * @cd:         pointer to genwqe device descriptor
 367  *
 368  * Return: Number of DDCBs which were finished
 369  */
 370 static int genwqe_check_ddcb_queue(struct genwqe_dev *cd,
 371                                    struct ddcb_queue *queue)
 372 {
 373         unsigned long flags;
 374         int ddcbs_finished = 0;
 375         struct pci_dev *pci_dev = cd->pci_dev;
 376
 377         spin_lock_irqsave(&queue->ddcb_lock, flags);
 378
 379         /* FIXME avoid soft locking CPU */
 380         while (!queue_empty(queue) && (ddcbs_finished < queue->ddcb_max)) {
 381
 382                 struct ddcb *pddcb;
 383                 struct ddcb_requ *req;
 384                 u16 vcrc, vcrc_16, retc_16;
 385
 386                 pddcb = &queue->ddcb_vaddr[queue->ddcb_act];
 387
 388                 if ((pddcb->icrc_hsi_shi_32 & DDCB_COMPLETED_BE32) ==
 389                     0x00000000)
 390                         goto go_home; /* not completed, continue waiting */
 391
 392                 wmb();  /*  Add sync to decouple prev. read operations */
 393
 394                 /* Note: DDCB could be purged */
 395                 req = queue->ddcb_req[queue->ddcb_act];
 396                 if (req == NULL) {
 397                         /* this occurs if DDCB is purged, not an error */
 398                         /* Move active DDCB further; Nothing to do anymore. */
 399                         goto pick_next_one;
 400                 }
 401
 402                 /*
 403                  * HSI=0x44 (fetched and completed), but RETC is
 404                  * 0x101, or even worse 0x000.
 405                  *
 406                  * In case of seeing the queue in inconsistent state
 407                  * we read the errcnts and the queue status to provide
 408                  * a trigger for our PCIe analyzer stop capturing.
 409                  */
 410                 retc_16 = be16_to_cpu(pddcb->retc_16);
 411                 if ((pddcb->hsi == 0x44) && (retc_16 <= 0x101)) {
 412                         u64 errcnts, status;
 413                         u64 ddcb_offs = (u64)pddcb - (u64)queue->ddcb_vaddr;
 414
 415                         errcnts = __genwqe_readq(cd, queue->IO_QUEUE_ERRCNTS);
 416                         status  = __genwqe_readq(cd, queue->IO_QUEUE_STATUS);
 417
 418                         dev_err(&pci_dev->dev,
 419                                 "[%s] SEQN=%04x HSI=%02x RETC=%03x Q_ERRCNTS=%016llx Q_STATUS=%016llx DDCB_DMA_ADDR=%016llx\n",
 420                                 __func__, be16_to_cpu(pddcb->seqnum_16),
 421                                 pddcb->hsi, retc_16, errcnts, status,
 422                                 queue->ddcb_daddr + ddcb_offs);
 423                 }
 424
 425                 copy_ddcb_results(req, queue->ddcb_act);
 426                 queue->ddcb_req[queue->ddcb_act] = NULL; /* take from queue */
 427
 428                 dev_dbg(&pci_dev->dev, "FINISHED DDCB#%d\n", req->num);
 429                 genwqe_hexdump(pci_dev, pddcb, sizeof(*pddcb));
 430
 431                 ddcb_mark_finished(pddcb);
 432
 433                 /* calculate CRC_16 to see if VCRC is correct */
 434                 vcrc = genwqe_crc16(pddcb->asv,
 435                                    VCRC_LENGTH(req->cmd.asv_length),
 436                                    0xffff);
 437                 vcrc_16 = be16_to_cpu(pddcb->vcrc_16);
 438                 if (vcrc != vcrc_16) {
 439                         printk_ratelimited(KERN_ERR
 440                                 "%s %s: err: wrong VCRC pre=%02x vcrc_len=%d bytes vcrc_data=%04x is not vcrc_card=%04x\n",
 441                                 GENWQE_DEVNAME, dev_name(&pci_dev->dev),
 442                                 pddcb->pre, VCRC_LENGTH(req->cmd.asv_length),
 443                                 vcrc, vcrc_16);
 444                 }
 445
 446                 ddcb_requ_set_state(req, GENWQE_REQU_FINISHED);
 447                 queue->ddcbs_completed++;
 448                 queue->ddcbs_in_flight--;
 449
 450                 /* wake up process waiting for this DDCB, and
 451                    processes on the busy queue */
 452                 wake_up_interruptible(&queue->ddcb_waitqs[queue->ddcb_act]);
 453                 wake_up_interruptible(&queue->busy_waitq);
 454
 455 pick_next_one:
 456                 queue->ddcb_act = (queue->ddcb_act + 1) % queue->ddcb_max;
 457                 ddcbs_finished++;
 458         }
 459
 460  go_home:
 461         spin_unlock_irqrestore(&queue->ddcb_lock, flags);
 462         return ddcbs_finished;
 463 }
 464
 465 /**
 466  * __genwqe_wait_ddcb(): Waits until DDCB is completed
 467  * @cd:         pointer to genwqe device descriptor
 468  * @req:        pointer to requsted DDCB parameters
 469  *
 470  * The Service Layer will update the RETC in DDCB when processing is
 471  * pending or done.
 472  *
 473  * Return: > 0 remaining jiffies, DDCB completed
 474  *           -ETIMEDOUT when timeout
 475  *           -ERESTARTSYS when ^C
 476  *           -EINVAL when unknown error condition
 477  *
 478  * When an error is returned the called needs to ensure that
 479  * purge_ddcb() is being called to get the &req removed from the
 480  * queue.
 481  */
 482 int __genwqe_wait_ddcb(struct genwqe_dev *cd, struct ddcb_requ *req)
 483 {
 484         int rc;
 485         unsigned int ddcb_no;
 486         struct ddcb_queue *queue;
 487         struct pci_dev *pci_dev = cd->pci_dev;
 488
 489         if (req == NULL)
 490                 return -EINVAL;
 491
 492         queue = req->queue;
 493         if (queue == NULL)
 494                 return -EINVAL;
 495
 496         ddcb_no = req->num;
 497         if (ddcb_no >= queue->ddcb_max)
 498                 return -EINVAL;
 499
 500         rc = wait_event_interruptible_timeout(queue->ddcb_waitqs[ddcb_no],
 501                                 ddcb_requ_finished(cd, req),
 502                                 GENWQE_DDCB_SOFTWARE_TIMEOUT * HZ);
 503
 504         /*
 505          * We need to distinguish 3 cases here:
 506          *   1. rc == 0              timeout occured
 507          *   2. rc == -ERESTARTSYS   signal received
 508          *   3. rc > 0               remaining jiffies condition is true
 509          */
 510         if (rc == 0) {
 511                 struct ddcb_queue *queue = req->queue;
 512                 struct ddcb *pddcb;
 513
 514                 /*
 515                  * Timeout may be caused by long task switching time.
 516                  * When timeout happens, check if the request has
 517                  * meanwhile completed.
 518                  */
 519                 genwqe_check_ddcb_queue(cd, req->queue);
 520                 if (ddcb_requ_finished(cd, req))
 521                         return rc;
 522
 523                 dev_err(&pci_dev->dev,
 524                         "[%s] err: DDCB#%d timeout rc=%d state=%d req @ %p\n",
 525                         __func__, req->num, rc, ddcb_requ_get_state(req),
 526                         req);
 527                 dev_err(&pci_dev->dev,
 528                         "[%s]      IO_QUEUE_STATUS=0x%016llx\n", __func__,
 529                         __genwqe_readq(cd, queue->IO_QUEUE_STATUS));
 530
 531                 pddcb = &queue->ddcb_vaddr[req->num];
 532                 genwqe_hexdump(pci_dev, pddcb, sizeof(*pddcb));
 533
 534                 print_ddcb_info(cd, req->queue);
 535                 return -ETIMEDOUT;
 536
 537         } else if (rc == -ERESTARTSYS) {
 538                 return rc;
 539                 /*
 540                  * EINTR:       Stops the application
 541                  * ERESTARTSYS: Restartable systemcall; called again
 542                  */
 543
 544         } else if (rc < 0) {
 545                 dev_err(&pci_dev->dev,
 546                         "[%s] err: DDCB#%d unknown result (rc=%d) %d!\n",
 547                         __func__, req->num, rc, ddcb_requ_get_state(req));
 548                 return -EINVAL;
 549         }
 550
 551         /* Severe error occured. Driver is forced to stop operation */
 552         if (cd->card_state != GENWQE_CARD_USED) {
 553                 dev_err(&pci_dev->dev,
 554                         "[%s] err: DDCB#%d forced to stop (rc=%d)\n",
 555                         __func__, req->num, rc);
 556                 return -EIO;
 557         }
 558         return rc;
 559 }
 560
 561 /**
 562  * get_next_ddcb() - Get next available DDCB
 563  * @cd:         pointer to genwqe device descriptor
 564  *
 565  * DDCB's content is completely cleared but presets for PRE and
 566  * SEQNUM. This function must only be called when ddcb_lock is held.
 567  *
 568  * Return: NULL if no empty DDCB available otherwise ptr to next DDCB.
 569  */
 570 static struct ddcb *get_next_ddcb(struct genwqe_dev *cd,
 571                                   struct ddcb_queue *queue,
 572                                   int *num)
 573 {
 574         u64 *pu64;
 575         struct ddcb *pddcb;
 576
 577         if (queue_free_ddcbs(queue) == 0) /* queue is  full */
 578                 return NULL;
 579
 580         /* find new ddcb */
 581         pddcb = &queue->ddcb_vaddr[queue->ddcb_next];
 582
 583         /* if it is not completed, we are not allowed to use it */
 584         /* barrier(); */
 585         if ((pddcb->icrc_hsi_shi_32 & DDCB_COMPLETED_BE32) == 0x00000000)
 586                 return NULL;
 587
 588         *num = queue->ddcb_next;        /* internal DDCB number */
 589         queue->ddcb_next = (queue->ddcb_next + 1) % queue->ddcb_max;
 590
 591         /* clear important DDCB fields */
 592         pu64 = (u64 *)pddcb;
 593         pu64[0] = 0ULL;         /* offs 0x00 (ICRC,HSI,SHI,...) */
 594         pu64[1] = 0ULL;         /* offs 0x01 (ACFUNC,CMD...) */
 595
 596         /* destroy previous results in ASV */
 597         pu64[0x80/8] = 0ULL;    /* offs 0x80 (ASV + 0) */
 598         pu64[0x88/8] = 0ULL;    /* offs 0x88 (ASV + 0x08) */
 599         pu64[0x90/8] = 0ULL;    /* offs 0x90 (ASV + 0x10) */
 600         pu64[0x98/8] = 0ULL;    /* offs 0x98 (ASV + 0x18) */
 601         pu64[0xd0/8] = 0ULL;    /* offs 0xd0 (RETC,ATTN...) */
 602
 603         pddcb->pre = DDCB_PRESET_PRE; /* 128 */
 604         pddcb->seqnum_16 = cpu_to_be16(queue->ddcb_seq++);
 605         return pddcb;
 606 }
 607
 608 /**
 609  * __genwqe_purge_ddcb() - Remove a DDCB from the workqueue
 610  * @cd:         genwqe device descriptor
 611  * @req:        DDCB request
 612  *
 613  * This will fail when the request was already FETCHED. In this case
 614  * we need to wait until it is finished. Else the DDCB can be
 615  * reused. This function also ensures that the request data structure
 616  * is removed from ddcb_req[].
 617  *
 618  * Do not forget to call this function when genwqe_wait_ddcb() fails,
 619  * such that the request gets really removed from ddcb_req[].
 620  *
 621  * Return: 0 success
 622  */
 623 int __genwqe_purge_ddcb(struct genwqe_dev *cd, struct ddcb_requ *req)
 624 {
 625         struct ddcb *pddcb = NULL;
 626         unsigned int t;
 627         unsigned long flags;
 628         struct ddcb_queue *queue = req->queue;
 629         struct pci_dev *pci_dev = cd->pci_dev;
 630         u64 queue_status;
 631         __be32 icrc_hsi_shi = 0x0000;
 632         __be32 old, new;
 633
 634         /* unsigned long flags; */
 635         if (GENWQE_DDCB_SOFTWARE_TIMEOUT <= 0) {
 636                 dev_err(&pci_dev->dev,
 637                         "[%s] err: software timeout is not set!\n", __func__);
 638                 return -EFAULT;
 639         }
 640
 641         pddcb = &queue->ddcb_vaddr[req->num];
 642
 643         for (t = 0; t < GENWQE_DDCB_SOFTWARE_TIMEOUT * 10; t++) {
 644
 645                 spin_lock_irqsave(&queue->ddcb_lock, flags);
 646
 647                 /* Check if req was meanwhile finished */
 648                 if (ddcb_requ_get_state(req) == GENWQE_REQU_FINISHED)
 649                         goto go_home;
 650
 651                 /* try to set PURGE bit if FETCHED/COMPLETED are not set */
 652                 old = pddcb->icrc_hsi_shi_32;   /* read SHI/HSI in BE32 */
 653                 if ((old & DDCB_FETCHED_BE32) == 0x00000000) {
 654
 655                         new = (old | DDCB_PURGE_BE32);
 656                         icrc_hsi_shi = cmpxchg(&pddcb->icrc_hsi_shi_32,
 657                                                old, new);
 658                         if (icrc_hsi_shi == old)
 659                                 goto finish_ddcb;
 660                 }
 661
 662                 /* normal finish with HSI bit */
 663                 barrier();
 664                 icrc_hsi_shi = pddcb->icrc_hsi_shi_32;
 665                 if (icrc_hsi_shi & DDCB_COMPLETED_BE32)
 666                         goto finish_ddcb;
 667
 668                 spin_unlock_irqrestore(&queue->ddcb_lock, flags);
 669
 670                 /*
 671                  * Here the check_ddcb() function will most likely
 672                  * discover this DDCB to be finished some point in
 673                  * time. It will mark the req finished and free it up
 674                  * in the list.
 675                  */
 676
 677                 copy_ddcb_results(req, req->num); /* for the failing case */
 678                 msleep(100); /* sleep for 1/10 second and try again */
 679                 continue;
 680
 681 finish_ddcb:
 682                 copy_ddcb_results(req, req->num);
 683                 ddcb_requ_set_state(req, GENWQE_REQU_FINISHED);
 684                 queue->ddcbs_in_flight--;
 685                 queue->ddcb_req[req->num] = NULL; /* delete from array */
 686                 ddcb_mark_cleared(pddcb);
 687
 688                 /* Move active DDCB further; Nothing to do here anymore. */
 689
 690                 /*
 691                  * We need to ensure that there is at least one free
 692                  * DDCB in the queue. To do that, we must update
 693                  * ddcb_act only if the COMPLETED bit is set for the
 694                  * DDCB we are working on else we treat that DDCB even
 695                  * if we PURGED it as occupied (hardware is supposed
 696                  * to set the COMPLETED bit yet!).
 697                  */
 698                 icrc_hsi_shi = pddcb->icrc_hsi_shi_32;
 699                 if ((icrc_hsi_shi & DDCB_COMPLETED_BE32) &&
 700                     (queue->ddcb_act == req->num)) {
 701                         queue->ddcb_act = ((queue->ddcb_act + 1) %
 702                                            queue->ddcb_max);
 703                 }
 704 go_home:
 705                 spin_unlock_irqrestore(&queue->ddcb_lock, flags);
 706                 return 0;
 707         }
 708
 709         /*
 710          * If the card is dead and the queue is forced to stop, we
 711          * might see this in the queue status register.
 712          */
 713         queue_status = __genwqe_readq(cd, queue->IO_QUEUE_STATUS);
 714
 715         dev_dbg(&pci_dev->dev, "UN/FINISHED DDCB#%d\n", req->num);
 716         genwqe_hexdump(pci_dev, pddcb, sizeof(*pddcb));
 717
 718         dev_err(&pci_dev->dev,
 719                 "[%s] err: DDCB#%d not purged and not completed after %d seconds QSTAT=%016llx!!\n",
 720                 __func__, req->num, GENWQE_DDCB_SOFTWARE_TIMEOUT,
 721                 queue_status);
 722
 723         print_ddcb_info(cd, req->queue);
 724
 725         return -EFAULT;
 726 }
 727
 728 int genwqe_init_debug_data(struct genwqe_dev *cd, struct genwqe_debug_data *d)
 729 {
 730         int len;
 731         struct pci_dev *pci_dev = cd->pci_dev;
 732
 733         if (d == NULL) {
 734                 dev_err(&pci_dev->dev,
 735                         "[%s] err: invalid memory for debug data!\n",
 736                         __func__);
 737                 return -EFAULT;
 738         }
 739
 740         len  = sizeof(d->driver_version);
 741         snprintf(d->driver_version, len, "%s", DRV_VERSION);
 742         d->slu_unitcfg = cd->slu_unitcfg;
 743         d->app_unitcfg = cd->app_unitcfg;
 744         return 0;
 745 }
 746
 747 /**
 748  * __genwqe_enqueue_ddcb() - Enqueue a DDCB
 749  * @cd:         pointer to genwqe device descriptor
 750  * @req:        pointer to DDCB execution request
 751  * @f_flags:    file mode: blocking, non-blocking
 752  *
 753  * Return: 0 if enqueuing succeeded
 754  *         -EIO if card is unusable/PCIe problems
 755  *         -EBUSY if enqueuing failed
 756  */
 757 int __genwqe_enqueue_ddcb(struct genwqe_dev *cd, struct ddcb_requ *req,
 758                           unsigned int f_flags)
 759 {
 760         struct ddcb *pddcb;
 761         unsigned long flags;
 762         struct ddcb_queue *queue;
 763         struct pci_dev *pci_dev = cd->pci_dev;
 764         u16 icrc;
 765
 766  retry:
 767         if (cd->card_state != GENWQE_CARD_USED) {
 768                 printk_ratelimited(KERN_ERR
 769                         "%s %s: [%s] Card is unusable/PCIe problem Req#%d\n",
 770                         GENWQE_DEVNAME, dev_name(&pci_dev->dev),
 771                         __func__, req->num);
 772                 return -EIO;
 773         }
 774
 775         queue = req->queue = &cd->queue;
 776
 777         /* FIXME circumvention to improve performance when no irq is
 778          * there.
 779          */
 780         if (GENWQE_POLLING_ENABLED)
 781                 genwqe_check_ddcb_queue(cd, queue);
 782
 783         /*
 784          * It must be ensured to process all DDCBs in successive
 785          * order. Use a lock here in order to prevent nested DDCB
 786          * enqueuing.
 787          */
 788         spin_lock_irqsave(&queue->ddcb_lock, flags);
 789
 790         pddcb = get_next_ddcb(cd, queue, &req->num);    /* get ptr and num */
 791         if (pddcb == NULL) {
 792                 int rc;
 793
 794                 spin_unlock_irqrestore(&queue->ddcb_lock, flags);
 795
 796                 if (f_flags & O_NONBLOCK) {
 797                         queue->return_on_busy++;
 798                         return -EBUSY;
 799                 }
 800
 801                 queue->wait_on_busy++;
 802                 rc = wait_event_interruptible(queue->busy_waitq,
 803                                               queue_free_ddcbs(queue) != 0);
 804                 dev_dbg(&pci_dev->dev, "[%s] waiting for free DDCB: rc=%d\n",
 805                         __func__, rc);
 806                 if (rc == -ERESTARTSYS)
 807                         return rc;  /* interrupted by a signal */
 808
 809                 goto retry;
 810         }
 811
 812         if (queue->ddcb_req[req->num] != NULL) {
 813                 spin_unlock_irqrestore(&queue->ddcb_lock, flags);
 814
 815                 dev_err(&pci_dev->dev,
 816                         "[%s] picked DDCB %d with req=%p still in use!!\n",
 817                         __func__, req->num, req);
 818                 return -EFAULT;
 819         }
 820         ddcb_requ_set_state(req, GENWQE_REQU_ENQUEUED);
 821         queue->ddcb_req[req->num] = req;
 822
 823         pddcb->cmdopts_16 = cpu_to_be16(req->cmd.cmdopts);
 824         pddcb->cmd = req->cmd.cmd;
 825         pddcb->acfunc = req->cmd.acfunc;        /* functional unit */
 826
 827         /*
 828          * We know that we can get retc 0x104 with CRC error, do not
 829          * stop the queue in those cases for this command. XDIR = 1
 830          * does not work for old SLU versions.
 831          *
 832          * Last bitstream with the old XDIR behavior had SLU_ID
 833          * 0x34199.
 834          */
 835         if ((cd->slu_unitcfg & 0xFFFF0ull) > 0x34199ull)
 836                 pddcb->xdir = 0x1;
 837         else
 838                 pddcb->xdir = 0x0;
 839
 840
 841         pddcb->psp = (((req->cmd.asiv_length / 8) << 4) |
 842                       ((req->cmd.asv_length  / 8)));
 843         pddcb->disp_ts_64 = cpu_to_be64(req->cmd.disp_ts);
 844
 845         /*
 846          * If copying the whole DDCB_ASIV_LENGTH is impacting
 847          * performance we need to change it to
 848          * req->cmd.asiv_length. But simulation benefits from some
 849          * non-architectured bits behind the architectured content.
 850          *
 851          * How much data is copied depends on the availability of the
 852          * ATS field, which was introduced late. If the ATS field is
 853          * supported ASIV is 8 bytes shorter than it used to be. Since
 854          * the ATS field is copied too, the code should do exactly
 855          * what it did before, but I wanted to make copying of the ATS
 856          * field very explicit.
 857          */
 858         if (genwqe_get_slu_id(cd) <= 0x2) {
 859                 memcpy(&pddcb->__asiv[0],       /* destination */
 860                        &req->cmd.__asiv[0],     /* source */
 861                        DDCB_ASIV_LENGTH);       /* req->cmd.asiv_length */
 862         } else {
 863                 pddcb->n.ats_64 = cpu_to_be64(req->cmd.ats);
 864                 memcpy(&pddcb->n.asiv[0],       /* destination */
 865                         &req->cmd.asiv[0],      /* source */
 866                         DDCB_ASIV_LENGTH_ATS);  /* req->cmd.asiv_length */
 867         }
 868
 869         pddcb->icrc_hsi_shi_32 = cpu_to_be32(0x00000000); /* for crc */
 870
 871         /*
 872          * Calculate CRC_16 for corresponding range PSP(7:4). Include
 873          * empty 4 bytes prior to the data.
 874          */
 875         icrc = genwqe_crc16((const u8 *)pddcb,
 876                            ICRC_LENGTH(req->cmd.asiv_length), 0xffff);
 877         pddcb->icrc_hsi_shi_32 = cpu_to_be32((u32)icrc << 16);
 878
 879         /* enable DDCB completion irq */
 880         if (!GENWQE_POLLING_ENABLED)
 881                 pddcb->icrc_hsi_shi_32 |= DDCB_INTR_BE32;
 882
 883         dev_dbg(&pci_dev->dev, "INPUT DDCB#%d\n", req->num);
 884         genwqe_hexdump(pci_dev, pddcb, sizeof(*pddcb));
 885
 886         if (ddcb_requ_collect_debug_data(req)) {
 887                 /* use the kernel copy of debug data. copying back to
 888                    user buffer happens later */
 889
 890                 genwqe_init_debug_data(cd, &req->debug_data);
 891                 memcpy(&req->debug_data.ddcb_before, pddcb,
 892                        sizeof(req->debug_data.ddcb_before));
 893         }
 894
 895         enqueue_ddcb(cd, queue, pddcb, req->num);
 896         queue->ddcbs_in_flight++;
 897
 898         if (queue->ddcbs_in_flight > queue->ddcbs_max_in_flight)
 899                 queue->ddcbs_max_in_flight = queue->ddcbs_in_flight;
 900
 901         ddcb_requ_set_state(req, GENWQE_REQU_TAPPED);
 902         spin_unlock_irqrestore(&queue->ddcb_lock, flags);
 903         wake_up_interruptible(&cd->queue_waitq);
 904
 905         return 0;
 906 }
 907
 908 /**
 909  * __genwqe_execute_raw_ddcb() - Setup and execute DDCB
 910  * @cd:         pointer to genwqe device descriptor
 911  * @req:        user provided DDCB request
 912  * @f_flags:    file mode: blocking, non-blocking
 913  */
 914 int __genwqe_execute_raw_ddcb(struct genwqe_dev *cd,
 915                               struct genwqe_ddcb_cmd *cmd,
 916                               unsigned int f_flags)
 917 {
 918         int rc = 0;
 919         struct pci_dev *pci_dev = cd->pci_dev;
 920         struct ddcb_requ *req = container_of(cmd, struct ddcb_requ, cmd);
 921
 922         if (cmd->asiv_length > DDCB_ASIV_LENGTH) {
 923                 dev_err(&pci_dev->dev, "[%s] err: wrong asiv_length of %d\n",
 924                         __func__, cmd->asiv_length);
 925                 return -EINVAL;
 926         }
 927         if (cmd->asv_length > DDCB_ASV_LENGTH) {
 928                 dev_err(&pci_dev->dev, "[%s] err: wrong asv_length of %d\n",
 929                         __func__, cmd->asiv_length);
 930                 return -EINVAL;
 931         }
 932         rc = __genwqe_enqueue_ddcb(cd, req, f_flags);
 933         if (rc != 0)
 934                 return rc;
 935
 936         rc = __genwqe_wait_ddcb(cd, req);
 937         if (rc < 0)             /* error or signal interrupt */
 938                 goto err_exit;
 939
 940         if (ddcb_requ_collect_debug_data(req)) {
 941                 if (copy_to_user((struct genwqe_debug_data __user *)
 942                                  (unsigned long)cmd->ddata_addr,
 943                                  &req->debug_data,
 944                                  sizeof(struct genwqe_debug_data)))
 945                         return -EFAULT;
 946         }
 947
 948         /*
 949          * Higher values than 0x102 indicate completion with faults,
 950          * lower values than 0x102 indicate processing faults. Note
 951          * that DDCB might have been purged. E.g. Cntl+C.
 952          */
 953         if (cmd->retc != DDCB_RETC_COMPLETE) {
 954                 /* This might happen e.g. flash read, and needs to be
 955                    handled by the upper layer code. */
 956                 rc = -EBADMSG;  /* not processed/error retc */
 957         }
 958
 959         return rc;
 960
 961  err_exit:
 962         __genwqe_purge_ddcb(cd, req);
 963
 964         if (ddcb_requ_collect_debug_data(req)) {
 965                 if (copy_to_user((struct genwqe_debug_data __user *)
 966                                  (unsigned long)cmd->ddata_addr,
 967                                  &req->debug_data,
 968                                  sizeof(struct genwqe_debug_data)))
 969                         return -EFAULT;
 970         }
 971         return rc;
 972 }
 973
 974 /**
 975  * genwqe_next_ddcb_ready() - Figure out if the next DDCB is already finished
 976  *
 977  * We use this as condition for our wait-queue code.
 978  */
 979 static int genwqe_next_ddcb_ready(struct genwqe_dev *cd)
 980 {
 981         unsigned long flags;
 982         struct ddcb *pddcb;
 983         struct ddcb_queue *queue = &cd->queue;
 984
 985         spin_lock_irqsave(&queue->ddcb_lock, flags);
 986
 987         if (queue_empty(queue)) { /* emtpy queue */
 988                 spin_unlock_irqrestore(&queue->ddcb_lock, flags);
 989                 return 0;
 990         }
 991
 992         pddcb = &queue->ddcb_vaddr[queue->ddcb_act];
 993         if (pddcb->icrc_hsi_shi_32 & DDCB_COMPLETED_BE32) { /* ddcb ready */
 994                 spin_unlock_irqrestore(&queue->ddcb_lock, flags);
 995                 return 1;
 996         }
 997
 998         spin_unlock_irqrestore(&queue->ddcb_lock, flags);
 999         return 0;
1000 }
1001
1002 /**
1003  * genwqe_ddcbs_in_flight() - Check how many DDCBs are in flight
1004  *
1005  * Keep track on the number of DDCBs which ware currently in the
1006  * queue. This is needed for statistics as well as conditon if we want
1007  * to wait or better do polling in case of no interrupts available.
1008  */
1009 int genwqe_ddcbs_in_flight(struct genwqe_dev *cd)
1010 {
1011         unsigned long flags;
1012         int ddcbs_in_flight = 0;
1013         struct ddcb_queue *queue = &cd->queue;
1014
1015         spin_lock_irqsave(&queue->ddcb_lock, flags);
1016         ddcbs_in_flight += queue->ddcbs_in_flight;
1017         spin_unlock_irqrestore(&queue->ddcb_lock, flags);
1018
1019         return ddcbs_in_flight;
1020 }
1021
1022 static int setup_ddcb_queue(struct genwqe_dev *cd, struct ddcb_queue *queue)
1023 {
1024         int rc, i;
1025         struct ddcb *pddcb;
1026         u64 val64;
1027         unsigned int queue_size;
1028         struct pci_dev *pci_dev = cd->pci_dev;
1029
1030         if (GENWQE_DDCB_MAX < 2)
1031                 return -EINVAL;
1032
1033         queue_size = roundup(GENWQE_DDCB_MAX * sizeof(struct ddcb), PAGE_SIZE);
1034
1035         queue->ddcbs_in_flight = 0;  /* statistics */
1036         queue->ddcbs_max_in_flight = 0;
1037         queue->ddcbs_completed = 0;
1038         queue->return_on_busy = 0;
1039         queue->wait_on_busy = 0;
1040
1041         queue->ddcb_seq   = 0x100; /* start sequence number */
1042         queue->ddcb_max   = GENWQE_DDCB_MAX;
1043         queue->ddcb_vaddr = __genwqe_alloc_consistent(cd, queue_size,
1044                                                 &queue->ddcb_daddr);
1045         if (queue->ddcb_vaddr == NULL) {
1046                 dev_err(&pci_dev->dev,
1047                         "[%s] **err: could not allocate DDCB **\n", __func__);
1048                 return -ENOMEM;
1049         }
1050         queue->ddcb_req = kcalloc(queue->ddcb_max, sizeof(struct ddcb_requ *),
1051                                   GFP_KERNEL);
1052         if (!queue->ddcb_req) {
1053                 rc = -ENOMEM;
1054                 goto free_ddcbs;
1055         }
1056
1057         queue->ddcb_waitqs = kcalloc(queue->ddcb_max,
1058                                      sizeof(wait_queue_head_t),
1059                                      GFP_KERNEL);
1060         if (!queue->ddcb_waitqs) {
1061                 rc = -ENOMEM;
1062                 goto free_requs;
1063         }
1064
1065         for (i = 0; i < queue->ddcb_max; i++) {
1066                 pddcb = &queue->ddcb_vaddr[i];               /* DDCBs */
1067                 pddcb->icrc_hsi_shi_32 = DDCB_COMPLETED_BE32;
1068                 pddcb->retc_16 = cpu_to_be16(0xfff);
1069
1070                 queue->ddcb_req[i] = NULL;                   /* requests */
1071                 init_waitqueue_head(&queue->ddcb_waitqs[i]); /* waitqueues */
1072         }
1073
1074         queue->ddcb_act  = 0;
1075         queue->ddcb_next = 0;   /* queue is empty */
1076
1077         spin_lock_init(&queue->ddcb_lock);
1078         init_waitqueue_head(&queue->busy_waitq);
1079
1080         val64 = ((u64)(queue->ddcb_max - 1) <<  8); /* lastptr */
1081         __genwqe_writeq(cd, queue->IO_QUEUE_CONFIG,  0x07);  /* iCRC/vCRC */
1082         __genwqe_writeq(cd, queue->IO_QUEUE_SEGMENT, queue->ddcb_daddr);
1083         __genwqe_writeq(cd, queue->IO_QUEUE_INITSQN, queue->ddcb_seq);
1084         __genwqe_writeq(cd, queue->IO_QUEUE_WRAP,    val64);
1085         return 0;
1086
1087  free_requs:
1088         kfree(queue->ddcb_req);
1089         queue->ddcb_req = NULL;
1090  free_ddcbs:
1091         __genwqe_free_consistent(cd, queue_size, queue->ddcb_vaddr,
1092                                 queue->ddcb_daddr);
1093         queue->ddcb_vaddr = NULL;
1094         queue->ddcb_daddr = 0ull;
1095         return -ENODEV;
1096
1097 }
1098
1099 static int ddcb_queue_initialized(struct ddcb_queue *queue)
1100 {
1101         return queue->ddcb_vaddr != NULL;
1102 }
1103
1104 static void free_ddcb_queue(struct genwqe_dev *cd, struct ddcb_queue *queue)
1105 {
1106         unsigned int queue_size;
1107
1108         queue_size = roundup(queue->ddcb_max * sizeof(struct ddcb), PAGE_SIZE);
1109
1110         kfree(queue->ddcb_req);
1111         queue->ddcb_req = NULL;
1112
1113         if (queue->ddcb_vaddr) {
1114                 __genwqe_free_consistent(cd, queue_size, queue->ddcb_vaddr,
1115                                         queue->ddcb_daddr);
1116                 queue->ddcb_vaddr = NULL;
1117                 queue->ddcb_daddr = 0ull;
1118         }
1119 }
1120
1121 static irqreturn_t genwqe_pf_isr(int irq, void *dev_id)
1122 {
1123         u64 gfir;
1124         struct genwqe_dev *cd = (struct genwqe_dev *)dev_id;
1125         struct pci_dev *pci_dev = cd->pci_dev;
1126
1127         /*
1128          * In case of fatal FIR error the queue is stopped, such that
1129          * we can safely check it without risking anything.
1130          */
1131         cd->irqs_processed++;
1132         wake_up_interruptible(&cd->queue_waitq);
1133
1134         /*
1135          * Checking for errors before kicking the queue might be
1136          * safer, but slower for the good-case ... See above.
1137          */
1138         gfir = __genwqe_readq(cd, IO_SLC_CFGREG_GFIR);
1139         if (((gfir & GFIR_ERR_TRIGGER) != 0x0) &&
1140             !pci_channel_offline(pci_dev)) {
1141
1142                 if (cd->use_platform_recovery) {
1143                         /*
1144                          * Since we use raw accessors, EEH errors won't be
1145                          * detected by the platform until we do a non-raw
1146                          * MMIO or config space read
1147                          */
1148                         readq(cd->mmio + IO_SLC_CFGREG_GFIR);
1149
1150                         /* Don't do anything if the PCI channel is frozen */
1151                         if (pci_channel_offline(pci_dev))
1152                                 goto exit;
1153                 }
1154
1155                 wake_up_interruptible(&cd->health_waitq);
1156
1157                 /*
1158                  * By default GFIRs causes recovery actions. This
1159                  * count is just for debug when recovery is masked.
1160                  */
1161                 dev_err_ratelimited(&pci_dev->dev,
1162                                     "[%s] GFIR=%016llx\n",
1163                                     __func__, gfir);
1164         }
1165
1166  exit:
1167         return IRQ_HANDLED;
1168 }
1169
1170 static irqreturn_t genwqe_vf_isr(int irq, void *dev_id)
1171 {
1172         struct genwqe_dev *cd = (struct genwqe_dev *)dev_id;
1173
1174         cd->irqs_processed++;
1175         wake_up_interruptible(&cd->queue_waitq);
1176
1177         return IRQ_HANDLED;
1178 }
1179
1180 /**
1181  * genwqe_card_thread() - Work thread for the DDCB queue
1182  *
1183  * The idea is to check if there are DDCBs in processing. If there are
1184  * some finished DDCBs, we process them and wakeup the
1185  * requestors. Otherwise we give other processes time using
1186  * cond_resched().
1187  */
1188 static int genwqe_card_thread(void *data)
1189 {
1190         int should_stop = 0, rc = 0;
1191         struct genwqe_dev *cd = (struct genwqe_dev *)data;
1192
1193         while (!kthread_should_stop()) {
1194
1195                 genwqe_check_ddcb_queue(cd, &cd->queue);
1196
1197                 if (GENWQE_POLLING_ENABLED) {
1198                         rc = wait_event_interruptible_timeout(
1199                                 cd->queue_waitq,
1200                                 genwqe_ddcbs_in_flight(cd) ||
1201                                 (should_stop = kthread_should_stop()), 1);
1202                 } else {
1203                         rc = wait_event_interruptible_timeout(
1204                                 cd->queue_waitq,
1205                                 genwqe_next_ddcb_ready(cd) ||
1206                                 (should_stop = kthread_should_stop()), HZ);
1207                 }
1208                 if (should_stop)
1209                         break;
1210
1211                 /*
1212                  * Avoid soft lockups on heavy loads; we do not want
1213                  * to disable our interrupts.
1214                  */
1215                 cond_resched();
1216         }
1217         return 0;
1218 }
1219
1220 /**
1221  * genwqe_setup_service_layer() - Setup DDCB queue
1222  * @cd:         pointer to genwqe device descriptor
1223  *
1224  * Allocate DDCBs. Configure Service Layer Controller (SLC).
1225  *
1226  * Return: 0 success
1227  */
1228 int genwqe_setup_service_layer(struct genwqe_dev *cd)
1229 {
1230         int rc;
1231         struct ddcb_queue *queue;
1232         struct pci_dev *pci_dev = cd->pci_dev;
1233
1234         if (genwqe_is_privileged(cd)) {
1235                 rc = genwqe_card_reset(cd);
1236                 if (rc < 0) {
1237                         dev_err(&pci_dev->dev,
1238                                 "[%s] err: reset failed.\n", __func__);
1239                         return rc;
1240                 }
1241                 genwqe_read_softreset(cd);
1242         }
1243
1244         queue = &cd->queue;
1245         queue->IO_QUEUE_CONFIG  = IO_SLC_QUEUE_CONFIG;
1246         queue->IO_QUEUE_STATUS  = IO_SLC_QUEUE_STATUS;
1247         queue->IO_QUEUE_SEGMENT = IO_SLC_QUEUE_SEGMENT;
1248         queue->IO_QUEUE_INITSQN = IO_SLC_QUEUE_INITSQN;
1249         queue->IO_QUEUE_OFFSET  = IO_SLC_QUEUE_OFFSET;
1250         queue->IO_QUEUE_WRAP    = IO_SLC_QUEUE_WRAP;
1251         queue->IO_QUEUE_WTIME   = IO_SLC_QUEUE_WTIME;
1252         queue->IO_QUEUE_ERRCNTS = IO_SLC_QUEUE_ERRCNTS;
1253         queue->IO_QUEUE_LRW     = IO_SLC_QUEUE_LRW;
1254
1255         rc = setup_ddcb_queue(cd, queue);
1256         if (rc != 0) {
1257                 rc = -ENODEV;
1258                 goto err_out;
1259         }
1260
1261         init_waitqueue_head(&cd->queue_waitq);
1262         cd->card_thread = kthread_run(genwqe_card_thread, cd,
1263                                       GENWQE_DEVNAME "%d_thread",
1264                                       cd->card_idx);
1265         if (IS_ERR(cd->card_thread)) {
1266                 rc = PTR_ERR(cd->card_thread);
1267                 cd->card_thread = NULL;
1268                 goto stop_free_queue;
1269         }
1270
1271         rc = genwqe_set_interrupt_capability(cd, GENWQE_MSI_IRQS);
1272         if (rc)
1273                 goto stop_kthread;
1274
1275         /*
1276          * We must have all wait-queues initialized when we enable the
1277          * interrupts. Otherwise we might crash if we get an early
1278          * irq.
1279          */
1280         init_waitqueue_head(&cd->health_waitq);
1281
1282         if (genwqe_is_privileged(cd)) {
1283                 rc = request_irq(pci_dev->irq, genwqe_pf_isr, IRQF_SHARED,
1284                                  GENWQE_DEVNAME, cd);
1285         } else {
1286                 rc = request_irq(pci_dev->irq, genwqe_vf_isr, IRQF_SHARED,
1287                                  GENWQE_DEVNAME, cd);
1288         }
1289         if (rc < 0) {
1290                 dev_err(&pci_dev->dev, "irq %d not free.\n", pci_dev->irq);
1291                 goto stop_irq_cap;
1292         }
1293
1294         cd->card_state = GENWQE_CARD_USED;
1295         return 0;
1296
1297  stop_irq_cap:
1298         genwqe_reset_interrupt_capability(cd);
1299  stop_kthread:
1300         kthread_stop(cd->card_thread);
1301         cd->card_thread = NULL;
1302  stop_free_queue:
1303         free_ddcb_queue(cd, queue);
1304  err_out:
1305         return rc;
1306 }
1307
1308 /**
1309  * queue_wake_up_all() - Handles fatal error case
1310  *
1311  * The PCI device got unusable and we have to stop all pending
1312  * requests as fast as we can. The code after this must purge the
1313  * DDCBs in question and ensure that all mappings are freed.
1314  */
1315 static int queue_wake_up_all(struct genwqe_dev *cd)
1316 {
1317         unsigned int i;
1318         unsigned long flags;
1319         struct ddcb_queue *queue = &cd->queue;
1320
1321         spin_lock_irqsave(&queue->ddcb_lock, flags);
1322
1323         for (i = 0; i < queue->ddcb_max; i++)
1324                 wake_up_interruptible(&queue->ddcb_waitqs[queue->ddcb_act]);
1325
1326         wake_up_interruptible(&queue->busy_waitq);
1327         spin_unlock_irqrestore(&queue->ddcb_lock, flags);
1328
1329         return 0;
1330 }
1331
1332 /**
1333  * genwqe_finish_queue() - Remove any genwqe devices and user-interfaces
1334  *
1335  * Relies on the pre-condition that there are no users of the card
1336  * device anymore e.g. with open file-descriptors.
1337  *
1338  * This function must be robust enough to be called twice.
1339  */
1340 int genwqe_finish_queue(struct genwqe_dev *cd)
1341 {
1342         int i, rc = 0, in_flight;
1343         int waitmax = GENWQE_DDCB_SOFTWARE_TIMEOUT;
1344         struct pci_dev *pci_dev = cd->pci_dev;
1345         struct ddcb_queue *queue = &cd->queue;
1346
1347         if (!ddcb_queue_initialized(queue))
1348                 return 0;
1349
1350         /* Do not wipe out the error state. */
1351         if (cd->card_state == GENWQE_CARD_USED)
1352                 cd->card_state = GENWQE_CARD_UNUSED;
1353
1354         /* Wake up all requests in the DDCB queue such that they
1355            should be removed nicely. */
1356         queue_wake_up_all(cd);
1357
1358         /* We must wait to get rid of the DDCBs in flight */
1359         for (i = 0; i < waitmax; i++) {
1360                 in_flight = genwqe_ddcbs_in_flight(cd);
1361
1362                 if (in_flight == 0)
1363                         break;
1364
1365                 dev_dbg(&pci_dev->dev,
1366                         "  DEBUG [%d/%d] waiting for queue to get empty: %d requests!\n",
1367                         i, waitmax, in_flight);
1368
1369                 /*
1370                  * Severe severe error situation: The card itself has
1371                  * 16 DDCB queues, each queue has e.g. 32 entries,
1372                  * each DDBC has a hardware timeout of currently 250
1373                  * msec but the PFs have a hardware timeout of 8 sec
1374                  * ... so I take something large.
1375                  */
1376                 msleep(1000);
1377         }
1378         if (i == waitmax) {
1379                 dev_err(&pci_dev->dev, "  [%s] err: queue is not empty!!\n",
1380                         __func__);
1381                 rc = -EIO;
1382         }
1383         return rc;
1384 }
1385
1386 /**
1387  * genwqe_release_service_layer() - Shutdown DDCB queue
1388  * @cd:       genwqe device descriptor
1389  *
1390  * This function must be robust enough to be called twice.
1391  */
1392 int genwqe_release_service_layer(struct genwqe_dev *cd)
1393 {
1394         struct pci_dev *pci_dev = cd->pci_dev;
1395
1396         if (!ddcb_queue_initialized(&cd->queue))
1397                 return 1;
1398
1399         free_irq(pci_dev->irq, cd);
1400         genwqe_reset_interrupt_capability(cd);
1401
1402         if (cd->card_thread != NULL) {
1403                 kthread_stop(cd->card_thread);
1404                 cd->card_thread = NULL;
1405         }
1406
1407         free_ddcb_queue(cd, &cd->queue);
1408         return 0;
1409 }