drivers/usb/dwc3/gadget.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * gadget.c - DesignWare USB3 DRD Controller Gadget Framework Link
   4  *
   5  * Copyright (C) 2010-2011 Texas Instruments Incorporated - http://www.ti.com
   6  *
   7  * Authors: Felipe Balbi <balbi@ti.com>,
   8  *          Sebastian Andrzej Siewior <bigeasy@linutronix.de>
   9  */
  10
  11 #include <linux/kernel.h>
  12 #include <linux/delay.h>
  13 #include <linux/slab.h>
  14 #include <linux/spinlock.h>
  15 #include <linux/platform_device.h>
  16 #include <linux/pm_runtime.h>
  17 #include <linux/interrupt.h>
  18 #include <linux/io.h>
  19 #include <linux/list.h>
  20 #include <linux/dma-mapping.h>
  21
  22 #include <linux/usb/ch9.h>
  23 #include <linux/usb/gadget.h>
  24
  25 #include "debug.h"
  26 #include "core.h"
  27 #include "gadget.h"
  28 #include "io.h"
  29
  30 /**
  31  * dwc3_gadget_set_test_mode - enables usb2 test modes
  32  * @dwc: pointer to our context structure
  33  * @mode: the mode to set (J, K SE0 NAK, Force Enable)
  34  *
  35  * Caller should take care of locking. This function will return 0 on
  36  * success or -EINVAL if wrong Test Selector is passed.
  37  */
  38 int dwc3_gadget_set_test_mode(struct dwc3 *dwc, int mode)
  39 {
  40         u32             reg;
  41
  42         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
  43         reg &= ~DWC3_DCTL_TSTCTRL_MASK;
  44
  45         switch (mode) {
  46         case TEST_J:
  47         case TEST_K:
  48         case TEST_SE0_NAK:
  49         case TEST_PACKET:
  50         case TEST_FORCE_EN:
  51                 reg |= mode << 1;
  52                 break;
  53         default:
  54                 return -EINVAL;
  55         }
  56
  57         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
  58
  59         return 0;
  60 }
  61
  62 /**
  63  * dwc3_gadget_get_link_state - gets current state of usb link
  64  * @dwc: pointer to our context structure
  65  *
  66  * Caller should take care of locking. This function will
  67  * return the link state on success (>= 0) or -ETIMEDOUT.
  68  */
  69 int dwc3_gadget_get_link_state(struct dwc3 *dwc)
  70 {
  71         u32             reg;
  72
  73         reg = dwc3_readl(dwc->regs, DWC3_DSTS);
  74
  75         return DWC3_DSTS_USBLNKST(reg);
  76 }
  77
  78 /**
  79  * dwc3_gadget_set_link_state - sets usb link to a particular state
  80  * @dwc: pointer to our context structure
  81  * @state: the state to put link into
  82  *
  83  * Caller should take care of locking. This function will
  84  * return 0 on success or -ETIMEDOUT.
  85  */
  86 int dwc3_gadget_set_link_state(struct dwc3 *dwc, enum dwc3_link_state state)
  87 {
  88         int             retries = 10000;
  89         u32             reg;
  90
  91         /*
  92          * Wait until device controller is ready. Only applies to 1.94a and
  93          * later RTL.
  94          */
  95         if (dwc->revision >= DWC3_REVISION_194A) {
  96                 while (--retries) {
  97                         reg = dwc3_readl(dwc->regs, DWC3_DSTS);
  98                         if (reg & DWC3_DSTS_DCNRD)
  99                                 udelay(5);
 100                         else
 101                                 break;
 102                 }
 103
 104                 if (retries <= 0)
 105                         return -ETIMEDOUT;
 106         }
 107
 108         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
 109         reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;
 110
 111         /* set requested state */
 112         reg |= DWC3_DCTL_ULSTCHNGREQ(state);
 113         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
 114
 115         /*
 116          * The following code is racy when called from dwc3_gadget_wakeup,
 117          * and is not needed, at least on newer versions
 118          */
 119         if (dwc->revision >= DWC3_REVISION_194A)
 120                 return 0;
 121
 122         /* wait for a change in DSTS */
 123         retries = 10000;
 124         while (--retries) {
 125                 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
 126
 127                 if (DWC3_DSTS_USBLNKST(reg) == state)
 128                         return 0;
 129
 130                 udelay(5);
 131         }
 132
 133         return -ETIMEDOUT;
 134 }
 135
 136 /**
 137  * dwc3_ep_inc_trb - increment a trb index.
 138  * @index: Pointer to the TRB index to increment.
 139  *
 140  * The index should never point to the link TRB. After incrementing,
 141  * if it is point to the link TRB, wrap around to the beginning. The
 142  * link TRB is always at the last TRB entry.
 143  */
 144 static void dwc3_ep_inc_trb(u8 *index)
 145 {
 146         (*index)++;
 147         if (*index == (DWC3_TRB_NUM - 1))
 148                 *index = 0;
 149 }
 150
 151 /**
 152  * dwc3_ep_inc_enq - increment endpoint's enqueue pointer
 153  * @dep: The endpoint whose enqueue pointer we're incrementing
 154  */
 155 static void dwc3_ep_inc_enq(struct dwc3_ep *dep)
 156 {
 157         dwc3_ep_inc_trb(&dep->trb_enqueue);
 158 }
 159
 160 /**
 161  * dwc3_ep_inc_deq - increment endpoint's dequeue pointer
 162  * @dep: The endpoint whose enqueue pointer we're incrementing
 163  */
 164 static void dwc3_ep_inc_deq(struct dwc3_ep *dep)
 165 {
 166         dwc3_ep_inc_trb(&dep->trb_dequeue);
 167 }
 168
 169 void dwc3_gadget_del_and_unmap_request(struct dwc3_ep *dep,
 170                 struct dwc3_request *req, int status)
 171 {
 172         struct dwc3                     *dwc = dep->dwc;
 173
 174         req->started = false;
 175         list_del(&req->list);
 176         req->remaining = 0;
 177
 178         if (req->request.status == -EINPROGRESS)
 179                 req->request.status = status;
 180
 181         if (req->trb)
 182                 usb_gadget_unmap_request_by_dev(dwc->sysdev,
 183                                 &req->request, req->direction);
 184
 185         req->trb = NULL;
 186         trace_dwc3_gadget_giveback(req);
 187
 188         if (dep->number > 1)
 189                 pm_runtime_put(dwc->dev);
 190 }
 191
 192 /**
 193  * dwc3_gadget_giveback - call struct usb_request's ->complete callback
 194  * @dep: The endpoint to whom the request belongs to
 195  * @req: The request we're giving back
 196  * @status: completion code for the request
 197  *
 198  * Must be called with controller's lock held and interrupts disabled. This
 199  * function will unmap @req and call its ->complete() callback to notify upper
 200  * layers that it has completed.
 201  */
 202 void dwc3_gadget_giveback(struct dwc3_ep *dep, struct dwc3_request *req,
 203                 int status)
 204 {
 205         struct dwc3                     *dwc = dep->dwc;
 206
 207         dwc3_gadget_del_and_unmap_request(dep, req, status);
 208
 209         spin_unlock(&dwc->lock);
 210         usb_gadget_giveback_request(&dep->endpoint, &req->request);
 211         spin_lock(&dwc->lock);
 212 }
 213
 214 /**
 215  * dwc3_send_gadget_generic_command - issue a generic command for the controller
 216  * @dwc: pointer to the controller context
 217  * @cmd: the command to be issued
 218  * @param: command parameter
 219  *
 220  * Caller should take care of locking. Issue @cmd with a given @param to @dwc
 221  * and wait for its completion.
 222  */
 223 int dwc3_send_gadget_generic_command(struct dwc3 *dwc, unsigned cmd, u32 param)
 224 {
 225         u32             timeout = 500;
 226         int             status = 0;
 227         int             ret = 0;
 228         u32             reg;
 229
 230         dwc3_writel(dwc->regs, DWC3_DGCMDPAR, param);
 231         dwc3_writel(dwc->regs, DWC3_DGCMD, cmd | DWC3_DGCMD_CMDACT);
 232
 233         do {
 234                 reg = dwc3_readl(dwc->regs, DWC3_DGCMD);
 235                 if (!(reg & DWC3_DGCMD_CMDACT)) {
 236                         status = DWC3_DGCMD_STATUS(reg);
 237                         if (status)
 238                                 ret = -EINVAL;
 239                         break;
 240                 }
 241         } while (--timeout);
 242
 243         if (!timeout) {
 244                 ret = -ETIMEDOUT;
 245                 status = -ETIMEDOUT;
 246         }
 247
 248         trace_dwc3_gadget_generic_cmd(cmd, param, status);
 249
 250         return ret;
 251 }
 252
 253 static int __dwc3_gadget_wakeup(struct dwc3 *dwc);
 254
 255 /**
 256  * dwc3_send_gadget_ep_cmd - issue an endpoint command
 257  * @dep: the endpoint to which the command is going to be issued
 258  * @cmd: the command to be issued
 259  * @params: parameters to the command
 260  *
 261  * Caller should handle locking. This function will issue @cmd with given
 262  * @params to @dep and wait for its completion.
 263  */
 264 int dwc3_send_gadget_ep_cmd(struct dwc3_ep *dep, unsigned cmd,
 265                 struct dwc3_gadget_ep_cmd_params *params)
 266 {
 267         const struct usb_endpoint_descriptor *desc = dep->endpoint.desc;
 268         struct dwc3             *dwc = dep->dwc;
 269         u32                     timeout = 1000;
 270         u32                     reg;
 271
 272         int                     cmd_status = 0;
 273         int                     susphy = false;
 274         int                     ret = -EINVAL;
 275
 276         /*
 277          * Synopsys Databook 2.60a states, on section 6.3.2.5.[1-8], that if
 278          * we're issuing an endpoint command, we must check if
 279          * GUSB2PHYCFG.SUSPHY bit is set. If it is, then we need to clear it.
 280          *
 281          * We will also set SUSPHY bit to what it was before returning as stated
 282          * by the same section on Synopsys databook.
 283          */
 284         if (dwc->gadget.speed <= USB_SPEED_HIGH) {
 285                 reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0));
 286                 if (unlikely(reg & DWC3_GUSB2PHYCFG_SUSPHY)) {
 287                         susphy = true;
 288                         reg &= ~DWC3_GUSB2PHYCFG_SUSPHY;
 289                         dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg);
 290                 }
 291         }
 292
 293         if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_STARTTRANSFER) {
 294                 int             needs_wakeup;
 295
 296                 needs_wakeup = (dwc->link_state == DWC3_LINK_STATE_U1 ||
 297                                 dwc->link_state == DWC3_LINK_STATE_U2 ||
 298                                 dwc->link_state == DWC3_LINK_STATE_U3);
 299
 300                 if (unlikely(needs_wakeup)) {
 301                         ret = __dwc3_gadget_wakeup(dwc);
 302                         dev_WARN_ONCE(dwc->dev, ret, "wakeup failed --> %d\n",
 303                                         ret);
 304                 }
 305         }
 306
 307         dwc3_writel(dep->regs, DWC3_DEPCMDPAR0, params->param0);
 308         dwc3_writel(dep->regs, DWC3_DEPCMDPAR1, params->param1);
 309         dwc3_writel(dep->regs, DWC3_DEPCMDPAR2, params->param2);
 310
 311         /*
 312          * Synopsys Databook 2.60a states in section 6.3.2.5.6 of that if we're
 313          * not relying on XferNotReady, we can make use of a special "No
 314          * Response Update Transfer" command where we should clear both CmdAct
 315          * and CmdIOC bits.
 316          *
 317          * With this, we don't need to wait for command completion and can
 318          * straight away issue further commands to the endpoint.
 319          *
 320          * NOTICE: We're making an assumption that control endpoints will never
 321          * make use of Update Transfer command. This is a safe assumption
 322          * because we can never have more than one request at a time with
 323          * Control Endpoints. If anybody changes that assumption, this chunk
 324          * needs to be updated accordingly.
 325          */
 326         if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_UPDATETRANSFER &&
 327                         !usb_endpoint_xfer_isoc(desc))
 328                 cmd &= ~(DWC3_DEPCMD_CMDIOC | DWC3_DEPCMD_CMDACT);
 329         else
 330                 cmd |= DWC3_DEPCMD_CMDACT;
 331
 332         dwc3_writel(dep->regs, DWC3_DEPCMD, cmd);
 333         do {
 334                 reg = dwc3_readl(dep->regs, DWC3_DEPCMD);
 335                 if (!(reg & DWC3_DEPCMD_CMDACT)) {
 336                         cmd_status = DWC3_DEPCMD_STATUS(reg);
 337
 338                         switch (cmd_status) {
 339                         case 0:
 340                                 ret = 0;
 341                                 break;
 342                         case DEPEVT_TRANSFER_NO_RESOURCE:
 343                                 ret = -EINVAL;
 344                                 break;
 345                         case DEPEVT_TRANSFER_BUS_EXPIRY:
 346                                 /*
 347                                  * SW issues START TRANSFER command to
 348                                  * isochronous ep with future frame interval. If
 349                                  * future interval time has already passed when
 350                                  * core receives the command, it will respond
 351                                  * with an error status of 'Bus Expiry'.
 352                                  *
 353                                  * Instead of always returning -EINVAL, let's
 354                                  * give a hint to the gadget driver that this is
 355                                  * the case by returning -EAGAIN.
 356                                  */
 357                                 ret = -EAGAIN;
 358                                 break;
 359                         default:
 360                                 dev_WARN(dwc->dev, "UNKNOWN cmd status\n");
 361                         }
 362
 363                         break;
 364                 }
 365         } while (--timeout);
 366
 367         if (timeout == 0) {
 368                 ret = -ETIMEDOUT;
 369                 cmd_status = -ETIMEDOUT;
 370         }
 371
 372         trace_dwc3_gadget_ep_cmd(dep, cmd, params, cmd_status);
 373
 374         if (ret == 0) {
 375                 switch (DWC3_DEPCMD_CMD(cmd)) {
 376                 case DWC3_DEPCMD_STARTTRANSFER:
 377                         dep->flags |= DWC3_EP_TRANSFER_STARTED;
 378                         break;
 379                 case DWC3_DEPCMD_ENDTRANSFER:
 380                         dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
 381                         break;
 382                 default:
 383                         /* nothing */
 384                         break;
 385                 }
 386         }
 387
 388         if (unlikely(susphy)) {
 389                 reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0));
 390                 reg |= DWC3_GUSB2PHYCFG_SUSPHY;
 391                 dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg);
 392         }
 393
 394         return ret;
 395 }
 396
 397 static int dwc3_send_clear_stall_ep_cmd(struct dwc3_ep *dep)
 398 {
 399         struct dwc3 *dwc = dep->dwc;
 400         struct dwc3_gadget_ep_cmd_params params;
 401         u32 cmd = DWC3_DEPCMD_CLEARSTALL;
 402
 403         /*
 404          * As of core revision 2.60a the recommended programming model
 405          * is to set the ClearPendIN bit when issuing a Clear Stall EP
 406          * command for IN endpoints. This is to prevent an issue where
 407          * some (non-compliant) hosts may not send ACK TPs for pending
 408          * IN transfers due to a mishandled error condition. Synopsys
 409          * STAR 9000614252.
 410          */
 411         if (dep->direction && (dwc->revision >= DWC3_REVISION_260A) &&
 412             (dwc->gadget.speed >= USB_SPEED_SUPER))
 413                 cmd |= DWC3_DEPCMD_CLEARPENDIN;
 414
 415         memset(&params, 0, sizeof(params));
 416
 417         return dwc3_send_gadget_ep_cmd(dep, cmd, &params);
 418 }
 419
 420 static dma_addr_t dwc3_trb_dma_offset(struct dwc3_ep *dep,
 421                 struct dwc3_trb *trb)
 422 {
 423         u32             offset = (char *) trb - (char *) dep->trb_pool;
 424
 425         return dep->trb_pool_dma + offset;
 426 }
 427
 428 static int dwc3_alloc_trb_pool(struct dwc3_ep *dep)
 429 {
 430         struct dwc3             *dwc = dep->dwc;
 431
 432         if (dep->trb_pool)
 433                 return 0;
 434
 435         dep->trb_pool = dma_alloc_coherent(dwc->sysdev,
 436                         sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
 437                         &dep->trb_pool_dma, GFP_KERNEL);
 438         if (!dep->trb_pool) {
 439                 dev_err(dep->dwc->dev, "failed to allocate trb pool for %s\n",
 440                                 dep->name);
 441                 return -ENOMEM;
 442         }
 443
 444         return 0;
 445 }
 446
 447 static void dwc3_free_trb_pool(struct dwc3_ep *dep)
 448 {
 449         struct dwc3             *dwc = dep->dwc;
 450
 451         dma_free_coherent(dwc->sysdev, sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
 452                         dep->trb_pool, dep->trb_pool_dma);
 453
 454         dep->trb_pool = NULL;
 455         dep->trb_pool_dma = 0;
 456 }
 457
 458 static int dwc3_gadget_set_xfer_resource(struct dwc3 *dwc, struct dwc3_ep *dep);
 459
 460 /**
 461  * dwc3_gadget_start_config - configure ep resources
 462  * @dwc: pointer to our controller context structure
 463  * @dep: endpoint that is being enabled
 464  *
 465  * Issue a %DWC3_DEPCMD_DEPSTARTCFG command to @dep. After the command's
 466  * completion, it will set Transfer Resource for all available endpoints.
 467  *
 468  * The assignment of transfer resources cannot perfectly follow the data book
 469  * due to the fact that the controller driver does not have all knowledge of the
 470  * configuration in advance. It is given this information piecemeal by the
 471  * composite gadget framework after every SET_CONFIGURATION and
 472  * SET_INTERFACE. Trying to follow the databook programming model in this
 473  * scenario can cause errors. For two reasons:
 474  *
 475  * 1) The databook says to do %DWC3_DEPCMD_DEPSTARTCFG for every
 476  * %USB_REQ_SET_CONFIGURATION and %USB_REQ_SET_INTERFACE (8.1.5). This is
 477  * incorrect in the scenario of multiple interfaces.
 478  *
 479  * 2) The databook does not mention doing more %DWC3_DEPCMD_DEPXFERCFG for new
 480  * endpoint on alt setting (8.1.6).
 481  *
 482  * The following simplified method is used instead:
 483  *
 484  * All hardware endpoints can be assigned a transfer resource and this setting
 485  * will stay persistent until either a core reset or hibernation. So whenever we
 486  * do a %DWC3_DEPCMD_DEPSTARTCFG(0) we can go ahead and do
 487  * %DWC3_DEPCMD_DEPXFERCFG for every hardware endpoint as well. We are
 488  * guaranteed that there are as many transfer resources as endpoints.
 489  *
 490  * This function is called for each endpoint when it is being enabled but is
 491  * triggered only when called for EP0-out, which always happens first, and which
 492  * should only happen in one of the above conditions.
 493  */
 494 static int dwc3_gadget_start_config(struct dwc3 *dwc, struct dwc3_ep *dep)
 495 {
 496         struct dwc3_gadget_ep_cmd_params params;
 497         u32                     cmd;
 498         int                     i;
 499         int                     ret;
 500
 501         if (dep->number)
 502                 return 0;
 503
 504         memset(&params, 0x00, sizeof(params));
 505         cmd = DWC3_DEPCMD_DEPSTARTCFG;
 506
 507         ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
 508         if (ret)
 509                 return ret;
 510
 511         for (i = 0; i < DWC3_ENDPOINTS_NUM; i++) {
 512                 struct dwc3_ep *dep = dwc->eps[i];
 513
 514                 if (!dep)
 515                         continue;
 516
 517                 ret = dwc3_gadget_set_xfer_resource(dwc, dep);
 518                 if (ret)
 519                         return ret;
 520         }
 521
 522         return 0;
 523 }
 524
 525 static int dwc3_gadget_set_ep_config(struct dwc3 *dwc, struct dwc3_ep *dep,
 526                 bool modify, bool restore)
 527 {
 528         const struct usb_ss_ep_comp_descriptor *comp_desc;
 529         const struct usb_endpoint_descriptor *desc;
 530         struct dwc3_gadget_ep_cmd_params params;
 531
 532         if (dev_WARN_ONCE(dwc->dev, modify && restore,
 533                                         "Can't modify and restore\n"))
 534                 return -EINVAL;
 535
 536         comp_desc = dep->endpoint.comp_desc;
 537         desc = dep->endpoint.desc;
 538
 539         memset(&params, 0x00, sizeof(params));
 540
 541         params.param0 = DWC3_DEPCFG_EP_TYPE(usb_endpoint_type(desc))
 542                 | DWC3_DEPCFG_MAX_PACKET_SIZE(usb_endpoint_maxp(desc));
 543
 544         /* Burst size is only needed in SuperSpeed mode */
 545         if (dwc->gadget.speed >= USB_SPEED_SUPER) {
 546                 u32 burst = dep->endpoint.maxburst;
 547                 params.param0 |= DWC3_DEPCFG_BURST_SIZE(burst - 1);
 548         }
 549
 550         if (modify) {
 551                 params.param0 |= DWC3_DEPCFG_ACTION_MODIFY;
 552         } else if (restore) {
 553                 params.param0 |= DWC3_DEPCFG_ACTION_RESTORE;
 554                 params.param2 |= dep->saved_state;
 555         } else {
 556                 params.param0 |= DWC3_DEPCFG_ACTION_INIT;
 557         }
 558
 559         if (usb_endpoint_xfer_control(desc))
 560                 params.param1 = DWC3_DEPCFG_XFER_COMPLETE_EN;
 561
 562         if (dep->number <= 1 || usb_endpoint_xfer_isoc(desc))
 563                 params.param1 |= DWC3_DEPCFG_XFER_NOT_READY_EN;
 564
 565         if (usb_ss_max_streams(comp_desc) && usb_endpoint_xfer_bulk(desc)) {
 566                 params.param1 |= DWC3_DEPCFG_STREAM_CAPABLE
 567                         | DWC3_DEPCFG_STREAM_EVENT_EN;
 568                 dep->stream_capable = true;
 569         }
 570
 571         if (!usb_endpoint_xfer_control(desc))
 572                 params.param1 |= DWC3_DEPCFG_XFER_IN_PROGRESS_EN;
 573
 574         /*
 575          * We are doing 1:1 mapping for endpoints, meaning
 576          * Physical Endpoints 2 maps to Logical Endpoint 2 and
 577          * so on. We consider the direction bit as part of the physical
 578          * endpoint number. So USB endpoint 0x81 is 0x03.
 579          */
 580         params.param1 |= DWC3_DEPCFG_EP_NUMBER(dep->number);
 581
 582         /*
 583          * We must use the lower 16 TX FIFOs even though
 584          * HW might have more
 585          */
 586         if (dep->direction)
 587                 params.param0 |= DWC3_DEPCFG_FIFO_NUMBER(dep->number >> 1);
 588
 589         if (desc->bInterval) {
 590                 params.param1 |= DWC3_DEPCFG_BINTERVAL_M1(desc->bInterval - 1);
 591                 dep->interval = 1 << (desc->bInterval - 1);
 592         }
 593
 594         return dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETEPCONFIG, &params);
 595 }
 596
 597 static int dwc3_gadget_set_xfer_resource(struct dwc3 *dwc, struct dwc3_ep *dep)
 598 {
 599         struct dwc3_gadget_ep_cmd_params params;
 600
 601         memset(&params, 0x00, sizeof(params));
 602
 603         params.param0 = DWC3_DEPXFERCFG_NUM_XFER_RES(1);
 604
 605         return dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETTRANSFRESOURCE,
 606                         &params);
 607 }
 608
 609 /**
 610  * __dwc3_gadget_ep_enable - initializes a hw endpoint
 611  * @dep: endpoint to be initialized
 612  * @modify: if true, modify existing endpoint configuration
 613  * @restore: if true, restore endpoint configuration from scratch buffer
 614  *
 615  * Caller should take care of locking. Execute all necessary commands to
 616  * initialize a HW endpoint so it can be used by a gadget driver.
 617  */
 618 static int __dwc3_gadget_ep_enable(struct dwc3_ep *dep,
 619                 bool modify, bool restore)
 620 {
 621         const struct usb_endpoint_descriptor *desc = dep->endpoint.desc;
 622         struct dwc3             *dwc = dep->dwc;
 623
 624         u32                     reg;
 625         int                     ret;
 626
 627         if (!(dep->flags & DWC3_EP_ENABLED)) {
 628                 ret = dwc3_gadget_start_config(dwc, dep);
 629                 if (ret)
 630                         return ret;
 631         }
 632
 633         ret = dwc3_gadget_set_ep_config(dwc, dep, modify, restore);
 634         if (ret)
 635                 return ret;
 636
 637         if (!(dep->flags & DWC3_EP_ENABLED)) {
 638                 struct dwc3_trb *trb_st_hw;
 639                 struct dwc3_trb *trb_link;
 640
 641                 dep->type = usb_endpoint_type(desc);
 642                 dep->flags |= DWC3_EP_ENABLED;
 643                 dep->flags &= ~DWC3_EP_END_TRANSFER_PENDING;
 644
 645                 reg = dwc3_readl(dwc->regs, DWC3_DALEPENA);
 646                 reg |= DWC3_DALEPENA_EP(dep->number);
 647                 dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);
 648
 649                 init_waitqueue_head(&dep->wait_end_transfer);
 650
 651                 if (usb_endpoint_xfer_control(desc))
 652                         goto out;
 653
 654                 /* Initialize the TRB ring */
 655                 dep->trb_dequeue = 0;
 656                 dep->trb_enqueue = 0;
 657                 memset(dep->trb_pool, 0,
 658                        sizeof(struct dwc3_trb) * DWC3_TRB_NUM);
 659
 660                 /* Link TRB. The HWO bit is never reset */
 661                 trb_st_hw = &dep->trb_pool[0];
 662
 663                 trb_link = &dep->trb_pool[DWC3_TRB_NUM - 1];
 664                 trb_link->bpl = lower_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));
 665                 trb_link->bph = upper_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));
 666                 trb_link->ctrl |= DWC3_TRBCTL_LINK_TRB;
 667                 trb_link->ctrl |= DWC3_TRB_CTRL_HWO;
 668         }
 669
 670         /*
 671          * Issue StartTransfer here with no-op TRB so we can always rely on No
 672          * Response Update Transfer command.
 673          */
 674         if (usb_endpoint_xfer_bulk(desc) ||
 675                         usb_endpoint_xfer_int(desc)) {
 676                 struct dwc3_gadget_ep_cmd_params params;
 677                 struct dwc3_trb *trb;
 678                 dma_addr_t trb_dma;
 679                 u32 cmd;
 680
 681                 memset(&params, 0, sizeof(params));
 682                 trb = &dep->trb_pool[0];
 683                 trb_dma = dwc3_trb_dma_offset(dep, trb);
 684
 685                 params.param0 = upper_32_bits(trb_dma);
 686                 params.param1 = lower_32_bits(trb_dma);
 687
 688                 cmd = DWC3_DEPCMD_STARTTRANSFER;
 689
 690                 ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
 691                 if (ret < 0)
 692                         return ret;
 693
 694                 dep->flags |= DWC3_EP_BUSY;
 695
 696                 dep->resource_index = dwc3_gadget_ep_get_transfer_index(dep);
 697                 WARN_ON_ONCE(!dep->resource_index);
 698         }
 699
 700 out:
 701         trace_dwc3_gadget_ep_enable(dep);
 702
 703         return 0;
 704 }
 705
 706 static void dwc3_stop_active_transfer(struct dwc3 *dwc, u32 epnum, bool force);
 707 static void dwc3_remove_requests(struct dwc3 *dwc, struct dwc3_ep *dep)
 708 {
 709         struct dwc3_request             *req;
 710
 711         dwc3_stop_active_transfer(dwc, dep->number, true);
 712
 713         /* - giveback all requests to gadget driver */
 714         while (!list_empty(&dep->started_list)) {
 715                 req = next_request(&dep->started_list);
 716
 717                 dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
 718         }
 719
 720         while (!list_empty(&dep->pending_list)) {
 721                 req = next_request(&dep->pending_list);
 722
 723                 dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
 724         }
 725 }
 726
 727 /**
 728  * __dwc3_gadget_ep_disable - disables a hw endpoint
 729  * @dep: the endpoint to disable
 730  *
 731  * This function undoes what __dwc3_gadget_ep_enable did and also removes
 732  * requests which are currently being processed by the hardware and those which
 733  * are not yet scheduled.
 734  *
 735  * Caller should take care of locking.
 736  */
 737 static int __dwc3_gadget_ep_disable(struct dwc3_ep *dep)
 738 {
 739         struct dwc3             *dwc = dep->dwc;
 740         u32                     reg;
 741
 742         trace_dwc3_gadget_ep_disable(dep);
 743
 744         dwc3_remove_requests(dwc, dep);
 745
 746         /* make sure HW endpoint isn't stalled */
 747         if (dep->flags & DWC3_EP_STALL)
 748                 __dwc3_gadget_ep_set_halt(dep, 0, false);
 749
 750         reg = dwc3_readl(dwc->regs, DWC3_DALEPENA);
 751         reg &= ~DWC3_DALEPENA_EP(dep->number);
 752         dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);
 753
 754         dep->stream_capable = false;
 755         dep->type = 0;
 756         dep->flags &= DWC3_EP_END_TRANSFER_PENDING;
 757
 758         /* Clear out the ep descriptors for non-ep0 */
 759         if (dep->number > 1) {
 760                 dep->endpoint.comp_desc = NULL;
 761                 dep->endpoint.desc = NULL;
 762         }
 763
 764         return 0;
 765 }
 766
 767 /* -------------------------------------------------------------------------- */
 768
 769 static int dwc3_gadget_ep0_enable(struct usb_ep *ep,
 770                 const struct usb_endpoint_descriptor *desc)
 771 {
 772         return -EINVAL;
 773 }
 774
 775 static int dwc3_gadget_ep0_disable(struct usb_ep *ep)
 776 {
 777         return -EINVAL;
 778 }
 779
 780 /* -------------------------------------------------------------------------- */
 781
 782 static int dwc3_gadget_ep_enable(struct usb_ep *ep,
 783                 const struct usb_endpoint_descriptor *desc)
 784 {
 785         struct dwc3_ep                  *dep;
 786         struct dwc3                     *dwc;
 787         unsigned long                   flags;
 788         int                             ret;
 789
 790         if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT) {
 791                 pr_debug("dwc3: invalid parameters\n");
 792                 return -EINVAL;
 793         }
 794
 795         if (!desc->wMaxPacketSize) {
 796                 pr_debug("dwc3: missing wMaxPacketSize\n");
 797                 return -EINVAL;
 798         }
 799
 800         dep = to_dwc3_ep(ep);
 801         dwc = dep->dwc;
 802
 803         if (dev_WARN_ONCE(dwc->dev, dep->flags & DWC3_EP_ENABLED,
 804                                         "%s is already enabled\n",
 805                                         dep->name))
 806                 return 0;
 807
 808         spin_lock_irqsave(&dwc->lock, flags);
 809         ret = __dwc3_gadget_ep_enable(dep, false, false);
 810         spin_unlock_irqrestore(&dwc->lock, flags);
 811
 812         return ret;
 813 }
 814
 815 static int dwc3_gadget_ep_disable(struct usb_ep *ep)
 816 {
 817         struct dwc3_ep                  *dep;
 818         struct dwc3                     *dwc;
 819         unsigned long                   flags;
 820         int                             ret;
 821
 822         if (!ep) {
 823                 pr_debug("dwc3: invalid parameters\n");
 824                 return -EINVAL;
 825         }
 826
 827         dep = to_dwc3_ep(ep);
 828         dwc = dep->dwc;
 829
 830         if (dev_WARN_ONCE(dwc->dev, !(dep->flags & DWC3_EP_ENABLED),
 831                                         "%s is already disabled\n",
 832                                         dep->name))
 833                 return 0;
 834
 835         spin_lock_irqsave(&dwc->lock, flags);
 836         ret = __dwc3_gadget_ep_disable(dep);
 837         spin_unlock_irqrestore(&dwc->lock, flags);
 838
 839         return ret;
 840 }
 841
 842 static struct usb_request *dwc3_gadget_ep_alloc_request(struct usb_ep *ep,
 843                 gfp_t gfp_flags)
 844 {
 845         struct dwc3_request             *req;
 846         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
 847
 848         req = kzalloc(sizeof(*req), gfp_flags);
 849         if (!req)
 850                 return NULL;
 851
 852         req->epnum      = dep->number;
 853         req->dep        = dep;
 854
 855         trace_dwc3_alloc_request(req);
 856
 857         return &req->request;
 858 }
 859
 860 static void dwc3_gadget_ep_free_request(struct usb_ep *ep,
 861                 struct usb_request *request)
 862 {
 863         struct dwc3_request             *req = to_dwc3_request(request);
 864
 865         trace_dwc3_free_request(req);
 866         kfree(req);
 867 }
 868
 869 static u32 dwc3_calc_trbs_left(struct dwc3_ep *dep);
 870
 871 static void __dwc3_prepare_one_trb(struct dwc3_ep *dep, struct dwc3_trb *trb,
 872                 dma_addr_t dma, unsigned length, unsigned chain, unsigned node,
 873                 unsigned stream_id, unsigned short_not_ok, unsigned no_interrupt)
 874 {
 875         struct dwc3             *dwc = dep->dwc;
 876         struct usb_gadget       *gadget = &dwc->gadget;
 877         enum usb_device_speed   speed = gadget->speed;
 878
 879         dwc3_ep_inc_enq(dep);
 880
 881         trb->size = DWC3_TRB_SIZE_LENGTH(length);
 882         trb->bpl = lower_32_bits(dma);
 883         trb->bph = upper_32_bits(dma);
 884
 885         switch (usb_endpoint_type(dep->endpoint.desc)) {
 886         case USB_ENDPOINT_XFER_CONTROL:
 887                 trb->ctrl = DWC3_TRBCTL_CONTROL_SETUP;
 888                 break;
 889
 890         case USB_ENDPOINT_XFER_ISOC:
 891                 if (!node) {
 892                         trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS_FIRST;
 893
 894                         /*
 895                          * USB Specification 2.0 Section 5.9.2 states that: "If
 896                          * there is only a single transaction in the microframe,
 897                          * only a DATA0 data packet PID is used.  If there are
 898                          * two transactions per microframe, DATA1 is used for
 899                          * the first transaction data packet and DATA0 is used
 900                          * for the second transaction data packet.  If there are
 901                          * three transactions per microframe, DATA2 is used for
 902                          * the first transaction data packet, DATA1 is used for
 903                          * the second, and DATA0 is used for the third."
 904                          *
 905                          * IOW, we should satisfy the following cases:
 906                          *
 907                          * 1) length <= maxpacket
 908                          *      - DATA0
 909                          *
 910                          * 2) maxpacket < length <= (2 * maxpacket)
 911                          *      - DATA1, DATA0
 912                          *
 913                          * 3) (2 * maxpacket) < length <= (3 * maxpacket)
 914                          *      - DATA2, DATA1, DATA0
 915                          */
 916                         if (speed == USB_SPEED_HIGH) {
 917                                 struct usb_ep *ep = &dep->endpoint;
 918                                 unsigned int mult = 2;
 919                                 unsigned int maxp = usb_endpoint_maxp(ep->desc);
 920
 921                                 if (length <= (2 * maxp))
 922                                         mult--;
 923
 924                                 if (length <= maxp)
 925                                         mult--;
 926
 927                                 trb->size |= DWC3_TRB_SIZE_PCM1(mult);
 928                         }
 929                 } else {
 930                         trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS;
 931                 }
 932
 933                 /* always enable Interrupt on Missed ISOC */
 934                 trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI;
 935                 break;
 936
 937         case USB_ENDPOINT_XFER_BULK:
 938         case USB_ENDPOINT_XFER_INT:
 939                 trb->ctrl = DWC3_TRBCTL_NORMAL;
 940                 break;
 941         default:
 942                 /*
 943                  * This is only possible with faulty memory because we
 944                  * checked it already :)
 945                  */
 946                 dev_WARN(dwc->dev, "Unknown endpoint type %d\n",
 947                                 usb_endpoint_type(dep->endpoint.desc));
 948         }
 949
 950         /* always enable Continue on Short Packet */
 951         if (usb_endpoint_dir_out(dep->endpoint.desc)) {
 952                 trb->ctrl |= DWC3_TRB_CTRL_CSP;
 953
 954                 if (short_not_ok)
 955                         trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI;
 956         }
 957
 958         if ((!no_interrupt && !chain) ||
 959                         (dwc3_calc_trbs_left(dep) == 0))
 960                 trb->ctrl |= DWC3_TRB_CTRL_IOC;
 961
 962         if (chain)
 963                 trb->ctrl |= DWC3_TRB_CTRL_CHN;
 964
 965         if (usb_endpoint_xfer_bulk(dep->endpoint.desc) && dep->stream_capable)
 966                 trb->ctrl |= DWC3_TRB_CTRL_SID_SOFN(stream_id);
 967
 968         trb->ctrl |= DWC3_TRB_CTRL_HWO;
 969
 970         trace_dwc3_prepare_trb(dep, trb);
 971 }
 972
 973 /**
 974  * dwc3_prepare_one_trb - setup one TRB from one request
 975  * @dep: endpoint for which this request is prepared
 976  * @req: dwc3_request pointer
 977  * @chain: should this TRB be chained to the next?
 978  * @node: only for isochronous endpoints. First TRB needs different type.
 979  */
 980 static void dwc3_prepare_one_trb(struct dwc3_ep *dep,
 981                 struct dwc3_request *req, unsigned chain, unsigned node)
 982 {
 983         struct dwc3_trb         *trb;
 984         unsigned int            length;
 985         dma_addr_t              dma;
 986         unsigned                stream_id = req->request.stream_id;
 987         unsigned                short_not_ok = req->request.short_not_ok;
 988         unsigned                no_interrupt = req->request.no_interrupt;
 989
 990         if (req->request.num_sgs > 0) {
 991                 length = sg_dma_len(req->start_sg);
 992                 dma = sg_dma_address(req->start_sg);
 993         } else {
 994                 length = req->request.length;
 995                 dma = req->request.dma;
 996         }
 997
 998         trb = &dep->trb_pool[dep->trb_enqueue];
 999
1000         if (!req->trb) {
1001                 dwc3_gadget_move_started_request(req);
1002                 req->trb = trb;
1003                 req->trb_dma = dwc3_trb_dma_offset(dep, trb);
1004         }
1005
1006         __dwc3_prepare_one_trb(dep, trb, dma, length, chain, node,
1007                         stream_id, short_not_ok, no_interrupt);
1008 }
1009
1010 /**
1011  * dwc3_ep_prev_trb - returns the previous TRB in the ring
1012  * @dep: The endpoint with the TRB ring
1013  * @index: The index of the current TRB in the ring
1014  *
1015  * Returns the TRB prior to the one pointed to by the index. If the
1016  * index is 0, we will wrap backwards, skip the link TRB, and return
1017  * the one just before that.
1018  */
1019 static struct dwc3_trb *dwc3_ep_prev_trb(struct dwc3_ep *dep, u8 index)
1020 {
1021         u8 tmp = index;
1022
1023         if (!tmp)
1024                 tmp = DWC3_TRB_NUM - 1;
1025
1026         return &dep->trb_pool[tmp - 1];
1027 }
1028
1029 static u32 dwc3_calc_trbs_left(struct dwc3_ep *dep)
1030 {
1031         struct dwc3_trb         *tmp;
1032         u8                      trbs_left;
1033
1034         /*
1035          * If enqueue & dequeue are equal than it is either full or empty.
1036          *
1037          * One way to know for sure is if the TRB right before us has HWO bit
1038          * set or not. If it has, then we're definitely full and can't fit any
1039          * more transfers in our ring.
1040          */
1041         if (dep->trb_enqueue == dep->trb_dequeue) {
1042                 tmp = dwc3_ep_prev_trb(dep, dep->trb_enqueue);
1043                 if (tmp->ctrl & DWC3_TRB_CTRL_HWO)
1044                         return 0;
1045
1046                 return DWC3_TRB_NUM - 1;
1047         }
1048
1049         trbs_left = dep->trb_dequeue - dep->trb_enqueue;
1050         trbs_left &= (DWC3_TRB_NUM - 1);
1051
1052         if (dep->trb_dequeue < dep->trb_enqueue)
1053                 trbs_left--;
1054
1055         return trbs_left;
1056 }
1057
1058 static void dwc3_prepare_one_trb_sg(struct dwc3_ep *dep,
1059                 struct dwc3_request *req)
1060 {
1061         struct scatterlist *sg = req->start_sg;
1062         struct scatterlist *s;
1063         int             i;
1064
1065         unsigned int remaining = req->request.num_mapped_sgs
1066                 - req->num_queued_sgs;
1067
1068         for_each_sg(sg, s, remaining, i) {
1069                 unsigned int length = req->request.length;
1070                 unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc);
1071                 unsigned int rem = length % maxp;
1072                 unsigned chain = true;
1073
1074                 if (sg_is_last(s))
1075                         chain = false;
1076
1077                 if (rem && usb_endpoint_dir_out(dep->endpoint.desc) && !chain) {
1078                         struct dwc3     *dwc = dep->dwc;
1079                         struct dwc3_trb *trb;
1080
1081                         req->unaligned = true;
1082
1083                         /* prepare normal TRB */
1084                         dwc3_prepare_one_trb(dep, req, true, i);
1085
1086                         /* Now prepare one extra TRB to align transfer size */
1087                         trb = &dep->trb_pool[dep->trb_enqueue];
1088                         __dwc3_prepare_one_trb(dep, trb, dwc->bounce_addr,
1089                                         maxp - rem, false, 0,
1090                                         req->request.stream_id,
1091                                         req->request.short_not_ok,
1092                                         req->request.no_interrupt);
1093                 } else {
1094                         dwc3_prepare_one_trb(dep, req, chain, i);
1095                 }
1096
1097                 /*
1098                  * There can be a situation where all sgs in sglist are not
1099                  * queued because of insufficient trb number. To handle this
1100                  * case, update start_sg to next sg to be queued, so that
1101                  * we have free trbs we can continue queuing from where we
1102                  * previously stopped
1103                  */
1104                 if (chain)
1105                         req->start_sg = sg_next(s);
1106
1107                 req->num_queued_sgs++;
1108
1109                 if (!dwc3_calc_trbs_left(dep))
1110                         break;
1111         }
1112 }
1113
1114 static void dwc3_prepare_one_trb_linear(struct dwc3_ep *dep,
1115                 struct dwc3_request *req)
1116 {
1117         unsigned int length = req->request.length;
1118         unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc);
1119         unsigned int rem = length % maxp;
1120
1121         if (rem && usb_endpoint_dir_out(dep->endpoint.desc)) {
1122                 struct dwc3     *dwc = dep->dwc;
1123                 struct dwc3_trb *trb;
1124
1125                 req->unaligned = true;
1126
1127                 /* prepare normal TRB */
1128                 dwc3_prepare_one_trb(dep, req, true, 0);
1129
1130                 /* Now prepare one extra TRB to align transfer size */
1131                 trb = &dep->trb_pool[dep->trb_enqueue];
1132                 __dwc3_prepare_one_trb(dep, trb, dwc->bounce_addr, maxp - rem,
1133                                 false, 0, req->request.stream_id,
1134                                 req->request.short_not_ok,
1135                                 req->request.no_interrupt);
1136         } else if (req->request.zero && req->request.length &&
1137                    (IS_ALIGNED(req->request.length,dep->endpoint.maxpacket))) {
1138                 struct dwc3     *dwc = dep->dwc;
1139                 struct dwc3_trb *trb;
1140
1141                 req->zero = true;
1142
1143                 /* prepare normal TRB */
1144                 dwc3_prepare_one_trb(dep, req, true, 0);
1145
1146                 /* Now prepare one extra TRB to handle ZLP */
1147                 trb = &dep->trb_pool[dep->trb_enqueue];
1148                 __dwc3_prepare_one_trb(dep, trb, dwc->bounce_addr, 0,
1149                                 false, 0, req->request.stream_id,
1150                                 req->request.short_not_ok,
1151                                 req->request.no_interrupt);
1152         } else {
1153                 dwc3_prepare_one_trb(dep, req, false, 0);
1154         }
1155 }
1156
1157 /*
1158  * dwc3_prepare_trbs - setup TRBs from requests
1159  * @dep: endpoint for which requests are being prepared
1160  *
1161  * The function goes through the requests list and sets up TRBs for the
1162  * transfers. The function returns once there are no more TRBs available or
1163  * it runs out of requests.
1164  */
1165 static void dwc3_prepare_trbs(struct dwc3_ep *dep)
1166 {
1167         struct dwc3_request     *req, *n;
1168
1169         BUILD_BUG_ON_NOT_POWER_OF_2(DWC3_TRB_NUM);
1170
1171         /*
1172          * We can get in a situation where there's a request in the started list
1173          * but there weren't enough TRBs to fully kick it in the first time
1174          * around, so it has been waiting for more TRBs to be freed up.
1175          *
1176          * In that case, we should check if we have a request with pending_sgs
1177          * in the started list and prepare TRBs for that request first,
1178          * otherwise we will prepare TRBs completely out of order and that will
1179          * break things.
1180          */
1181         list_for_each_entry(req, &dep->started_list, list) {
1182                 if (req->num_pending_sgs > 0)
1183                         dwc3_prepare_one_trb_sg(dep, req);
1184
1185                 if (!dwc3_calc_trbs_left(dep))
1186                         return;
1187         }
1188
1189         list_for_each_entry_safe(req, n, &dep->pending_list, list) {
1190                 struct dwc3     *dwc = dep->dwc;
1191                 int             ret;
1192
1193                 ret = usb_gadget_map_request_by_dev(dwc->sysdev, &req->request,
1194                                                     dep->direction);
1195                 if (ret)
1196                         return;
1197
1198                 req->sg                 = req->request.sg;
1199                 req->start_sg           = req->sg;
1200                 req->num_queued_sgs     = 0;
1201                 req->num_pending_sgs    = req->request.num_mapped_sgs;
1202
1203                 if (req->num_pending_sgs > 0)
1204                         dwc3_prepare_one_trb_sg(dep, req);
1205                 else
1206                         dwc3_prepare_one_trb_linear(dep, req);
1207
1208                 if (!dwc3_calc_trbs_left(dep))
1209                         return;
1210         }
1211 }
1212
1213 static int __dwc3_gadget_kick_transfer(struct dwc3_ep *dep)
1214 {
1215         struct dwc3_gadget_ep_cmd_params params;
1216         struct dwc3_request             *req;
1217         int                             starting;
1218         int                             ret;
1219         u32                             cmd;
1220
1221         if (!dwc3_calc_trbs_left(dep))
1222                 return 0;
1223
1224         starting = !(dep->flags & DWC3_EP_BUSY);
1225
1226         dwc3_prepare_trbs(dep);
1227         req = next_request(&dep->started_list);
1228         if (!req) {
1229                 dep->flags |= DWC3_EP_PENDING_REQUEST;
1230                 return 0;
1231         }
1232
1233         memset(&params, 0, sizeof(params));
1234
1235         if (starting) {
1236                 params.param0 = upper_32_bits(req->trb_dma);
1237                 params.param1 = lower_32_bits(req->trb_dma);
1238                 cmd = DWC3_DEPCMD_STARTTRANSFER;
1239
1240                 if (usb_endpoint_xfer_isoc(dep->endpoint.desc))
1241                         cmd |= DWC3_DEPCMD_PARAM(dep->frame_number);
1242         } else {
1243                 cmd = DWC3_DEPCMD_UPDATETRANSFER |
1244                         DWC3_DEPCMD_PARAM(dep->resource_index);
1245         }
1246
1247         ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
1248         if (ret < 0) {
1249                 /*
1250                  * FIXME we need to iterate over the list of requests
1251                  * here and stop, unmap, free and del each of the linked
1252                  * requests instead of what we do now.
1253                  */
1254                 if (req->trb)
1255                         memset(req->trb, 0, sizeof(struct dwc3_trb));
1256                 dwc3_gadget_del_and_unmap_request(dep, req, ret);
1257                 return ret;
1258         }
1259
1260         dep->flags |= DWC3_EP_BUSY;
1261
1262         if (starting) {
1263                 dep->resource_index = dwc3_gadget_ep_get_transfer_index(dep);
1264                 WARN_ON_ONCE(!dep->resource_index);
1265         }
1266
1267         return 0;
1268 }
1269
1270 static int __dwc3_gadget_get_frame(struct dwc3 *dwc)
1271 {
1272         u32                     reg;
1273
1274         reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1275         return DWC3_DSTS_SOFFN(reg);
1276 }
1277
1278 static void __dwc3_gadget_start_isoc(struct dwc3 *dwc,
1279                 struct dwc3_ep *dep, u32 cur_uf)
1280 {
1281         if (list_empty(&dep->pending_list)) {
1282                 dev_info(dwc->dev, "%s: ran out of requests\n",
1283                                 dep->name);
1284                 dep->flags |= DWC3_EP_PENDING_REQUEST;
1285                 return;
1286         }
1287
1288         /*
1289          * Schedule the first trb for one interval in the future or at
1290          * least 4 microframes.
1291          */
1292         dep->frame_number = cur_uf + max_t(u32, 4, dep->interval);
1293         __dwc3_gadget_kick_transfer(dep);
1294 }
1295
1296 static void dwc3_gadget_endpoint_transfer_not_ready(struct dwc3 *dwc,
1297                 struct dwc3_ep *dep, const struct dwc3_event_depevt *event)
1298 {
1299         u32 cur_uf, mask;
1300
1301         mask = ~(dep->interval - 1);
1302         cur_uf = event->parameters & mask;
1303
1304         __dwc3_gadget_start_isoc(dwc, dep, cur_uf);
1305 }
1306
1307 static int __dwc3_gadget_ep_queue(struct dwc3_ep *dep, struct dwc3_request *req)
1308 {
1309         struct dwc3             *dwc = dep->dwc;
1310
1311         if (!dep->endpoint.desc) {
1312                 dev_err(dwc->dev, "%s: can't queue to disabled endpoint\n",
1313                                 dep->name);
1314                 return -ESHUTDOWN;
1315         }
1316
1317         if (WARN(req->dep != dep, "request %pK belongs to '%s'\n",
1318                                 &req->request, req->dep->name))
1319                 return -EINVAL;
1320
1321         pm_runtime_get(dwc->dev);
1322
1323         req->request.actual     = 0;
1324         req->request.status     = -EINPROGRESS;
1325         req->direction          = dep->direction;
1326         req->epnum              = dep->number;
1327
1328         trace_dwc3_ep_queue(req);
1329
1330         list_add_tail(&req->list, &dep->pending_list);
1331
1332         /*
1333          * NOTICE: Isochronous endpoints should NEVER be prestarted. We must
1334          * wait for a XferNotReady event so we will know what's the current
1335          * (micro-)frame number.
1336          *
1337          * Without this trick, we are very, very likely gonna get Bus Expiry
1338          * errors which will force us issue EndTransfer command.
1339          */
1340         if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
1341                 if ((dep->flags & DWC3_EP_PENDING_REQUEST)) {
1342                         if (dep->flags & DWC3_EP_TRANSFER_STARTED) {
1343                                 dwc3_stop_active_transfer(dwc, dep->number, true);
1344                                 dep->flags = DWC3_EP_ENABLED;
1345                         } else {
1346                                 u32 cur_uf;
1347
1348                                 cur_uf = __dwc3_gadget_get_frame(dwc);
1349                                 __dwc3_gadget_start_isoc(dwc, dep, cur_uf);
1350                                 dep->flags &= ~DWC3_EP_PENDING_REQUEST;
1351                         }
1352                         return 0;
1353                 }
1354
1355                 if ((dep->flags & DWC3_EP_BUSY) &&
1356                     !(dep->flags & DWC3_EP_MISSED_ISOC))
1357                         goto out;
1358
1359                 return 0;
1360         }
1361
1362 out:
1363         return __dwc3_gadget_kick_transfer(dep);
1364 }
1365
1366 static int dwc3_gadget_ep_queue(struct usb_ep *ep, struct usb_request *request,
1367         gfp_t gfp_flags)
1368 {
1369         struct dwc3_request             *req = to_dwc3_request(request);
1370         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
1371         struct dwc3                     *dwc = dep->dwc;
1372
1373         unsigned long                   flags;
1374
1375         int                             ret;
1376
1377         spin_lock_irqsave(&dwc->lock, flags);
1378         ret = __dwc3_gadget_ep_queue(dep, req);
1379         spin_unlock_irqrestore(&dwc->lock, flags);
1380
1381         return ret;
1382 }
1383
1384 static int dwc3_gadget_ep_dequeue(struct usb_ep *ep,
1385                 struct usb_request *request)
1386 {
1387         struct dwc3_request             *req = to_dwc3_request(request);
1388         struct dwc3_request             *r = NULL;
1389
1390         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
1391         struct dwc3                     *dwc = dep->dwc;
1392
1393         unsigned long                   flags;
1394         int                             ret = 0;
1395
1396         trace_dwc3_ep_dequeue(req);
1397
1398         spin_lock_irqsave(&dwc->lock, flags);
1399
1400         list_for_each_entry(r, &dep->pending_list, list) {
1401                 if (r == req)
1402                         break;
1403         }
1404
1405         if (r != req) {
1406                 list_for_each_entry(r, &dep->started_list, list) {
1407                         if (r == req)
1408                                 break;
1409                 }
1410                 if (r == req) {
1411                         /* wait until it is processed */
1412                         dwc3_stop_active_transfer(dwc, dep->number, true);
1413
1414                         /*
1415                          * If request was already started, this means we had to
1416                          * stop the transfer. With that we also need to ignore
1417                          * all TRBs used by the request, however TRBs can only
1418                          * be modified after completion of END_TRANSFER
1419                          * command. So what we do here is that we wait for
1420                          * END_TRANSFER completion and only after that, we jump
1421                          * over TRBs by clearing HWO and incrementing dequeue
1422                          * pointer.
1423                          *
1424                          * Note that we have 2 possible types of transfers here:
1425                          *
1426                          * i) Linear buffer request
1427                          * ii) SG-list based request
1428                          *
1429                          * SG-list based requests will have r->num_pending_sgs
1430                          * set to a valid number (> 0). Linear requests,
1431                          * normally use a single TRB.
1432                          *
1433                          * For each of these two cases, if r->unaligned flag is
1434                          * set, one extra TRB has been used to align transfer
1435                          * size to wMaxPacketSize.
1436                          *
1437                          * All of these cases need to be taken into
1438                          * consideration so we don't mess up our TRB ring
1439                          * pointers.
1440                          */
1441                         wait_event_lock_irq(dep->wait_end_transfer,
1442                                         !(dep->flags & DWC3_EP_END_TRANSFER_PENDING),
1443                                         dwc->lock);
1444
1445                         if (!r->trb)
1446                                 goto out1;
1447
1448                         if (r->num_pending_sgs) {
1449                                 struct dwc3_trb *trb;
1450                                 int i = 0;
1451
1452                                 for (i = 0; i < r->num_pending_sgs; i++) {
1453                                         trb = r->trb + i;
1454                                         trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
1455                                         dwc3_ep_inc_deq(dep);
1456                                 }
1457
1458                                 if (r->unaligned || r->zero) {
1459                                         trb = r->trb + r->num_pending_sgs + 1;
1460                                         trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
1461                                         dwc3_ep_inc_deq(dep);
1462                                 }
1463                         } else {
1464                                 struct dwc3_trb *trb = r->trb;
1465
1466                                 trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
1467                                 dwc3_ep_inc_deq(dep);
1468
1469                                 if (r->unaligned || r->zero) {
1470                                         trb = r->trb + 1;
1471                                         trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
1472                                         dwc3_ep_inc_deq(dep);
1473                                 }
1474                         }
1475                         goto out1;
1476                 }
1477                 dev_err(dwc->dev, "request %pK was not queued to %s\n",
1478                                 request, ep->name);
1479                 ret = -EINVAL;
1480                 goto out0;
1481         }
1482
1483 out1:
1484         /* giveback the request */
1485
1486         dwc3_gadget_giveback(dep, req, -ECONNRESET);
1487
1488 out0:
1489         spin_unlock_irqrestore(&dwc->lock, flags);
1490
1491         return ret;
1492 }
1493
1494 int __dwc3_gadget_ep_set_halt(struct dwc3_ep *dep, int value, int protocol)
1495 {
1496         struct dwc3_gadget_ep_cmd_params        params;
1497         struct dwc3                             *dwc = dep->dwc;
1498         int                                     ret;
1499
1500         if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
1501                 dev_err(dwc->dev, "%s is of Isochronous type\n", dep->name);
1502                 return -EINVAL;
1503         }
1504
1505         memset(&params, 0x00, sizeof(params));
1506
1507         if (value) {
1508                 struct dwc3_trb *trb;
1509
1510                 unsigned transfer_in_flight;
1511                 unsigned started;
1512
1513                 if (dep->flags & DWC3_EP_STALL)
1514                         return 0;
1515
1516                 if (dep->number > 1)
1517                         trb = dwc3_ep_prev_trb(dep, dep->trb_enqueue);
1518                 else
1519                         trb = &dwc->ep0_trb[dep->trb_enqueue];
1520
1521                 transfer_in_flight = trb->ctrl & DWC3_TRB_CTRL_HWO;
1522                 started = !list_empty(&dep->started_list);
1523
1524                 if (!protocol && ((dep->direction && transfer_in_flight) ||
1525                                 (!dep->direction && started))) {
1526                         return -EAGAIN;
1527                 }
1528
1529                 ret = dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETSTALL,
1530                                 &params);
1531                 if (ret)
1532                         dev_err(dwc->dev, "failed to set STALL on %s\n",
1533                                         dep->name);
1534                 else
1535                         dep->flags |= DWC3_EP_STALL;
1536         } else {
1537                 if (!(dep->flags & DWC3_EP_STALL))
1538                         return 0;
1539
1540                 ret = dwc3_send_clear_stall_ep_cmd(dep);
1541                 if (ret)
1542                         dev_err(dwc->dev, "failed to clear STALL on %s\n",
1543                                         dep->name);
1544                 else
1545                         dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE);
1546         }
1547
1548         return ret;
1549 }
1550
1551 static int dwc3_gadget_ep_set_halt(struct usb_ep *ep, int value)
1552 {
1553         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
1554         struct dwc3                     *dwc = dep->dwc;
1555
1556         unsigned long                   flags;
1557
1558         int                             ret;
1559
1560         spin_lock_irqsave(&dwc->lock, flags);
1561         ret = __dwc3_gadget_ep_set_halt(dep, value, false);
1562         spin_unlock_irqrestore(&dwc->lock, flags);
1563
1564         return ret;
1565 }
1566
1567 static int dwc3_gadget_ep_set_wedge(struct usb_ep *ep)
1568 {
1569         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
1570         struct dwc3                     *dwc = dep->dwc;
1571         unsigned long                   flags;
1572         int                             ret;
1573
1574         spin_lock_irqsave(&dwc->lock, flags);
1575         dep->flags |= DWC3_EP_WEDGE;
1576
1577         if (dep->number == 0 || dep->number == 1)
1578                 ret = __dwc3_gadget_ep0_set_halt(ep, 1);
1579         else
1580                 ret = __dwc3_gadget_ep_set_halt(dep, 1, false);
1581         spin_unlock_irqrestore(&dwc->lock, flags);
1582
1583         return ret;
1584 }
1585
1586 /* -------------------------------------------------------------------------- */
1587
1588 static struct usb_endpoint_descriptor dwc3_gadget_ep0_desc = {
1589         .bLength        = USB_DT_ENDPOINT_SIZE,
1590         .bDescriptorType = USB_DT_ENDPOINT,
1591         .bmAttributes   = USB_ENDPOINT_XFER_CONTROL,
1592 };
1593
1594 static const struct usb_ep_ops dwc3_gadget_ep0_ops = {
1595         .enable         = dwc3_gadget_ep0_enable,
1596         .disable        = dwc3_gadget_ep0_disable,
1597         .alloc_request  = dwc3_gadget_ep_alloc_request,
1598         .free_request   = dwc3_gadget_ep_free_request,
1599         .queue          = dwc3_gadget_ep0_queue,
1600         .dequeue        = dwc3_gadget_ep_dequeue,
1601         .set_halt       = dwc3_gadget_ep0_set_halt,
1602         .set_wedge      = dwc3_gadget_ep_set_wedge,
1603 };
1604
1605 static const struct usb_ep_ops dwc3_gadget_ep_ops = {
1606         .enable         = dwc3_gadget_ep_enable,
1607         .disable        = dwc3_gadget_ep_disable,
1608         .alloc_request  = dwc3_gadget_ep_alloc_request,
1609         .free_request   = dwc3_gadget_ep_free_request,
1610         .queue          = dwc3_gadget_ep_queue,
1611         .dequeue        = dwc3_gadget_ep_dequeue,
1612         .set_halt       = dwc3_gadget_ep_set_halt,
1613         .set_wedge      = dwc3_gadget_ep_set_wedge,
1614 };
1615
1616 /* -------------------------------------------------------------------------- */
1617
1618 static int dwc3_gadget_get_frame(struct usb_gadget *g)
1619 {
1620         struct dwc3             *dwc = gadget_to_dwc(g);
1621
1622         return __dwc3_gadget_get_frame(dwc);
1623 }
1624
1625 static int __dwc3_gadget_wakeup(struct dwc3 *dwc)
1626 {
1627         int                     retries;
1628
1629         int                     ret;
1630         u32                     reg;
1631
1632         u8                      link_state;
1633         u8                      speed;
1634
1635         /*
1636          * According to the Databook Remote wakeup request should
1637          * be issued only when the device is in early suspend state.
1638          *
1639          * We can check that via USB Link State bits in DSTS register.
1640          */
1641         reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1642
1643         speed = reg & DWC3_DSTS_CONNECTSPD;
1644         if ((speed == DWC3_DSTS_SUPERSPEED) ||
1645             (speed == DWC3_DSTS_SUPERSPEED_PLUS))
1646                 return 0;
1647
1648         link_state = DWC3_DSTS_USBLNKST(reg);
1649
1650         switch (link_state) {
1651         case DWC3_LINK_STATE_RX_DET:    /* in HS, means Early Suspend */
1652         case DWC3_LINK_STATE_U3:        /* in HS, means SUSPEND */
1653                 break;
1654         default:
1655                 return -EINVAL;
1656         }
1657
1658         ret = dwc3_gadget_set_link_state(dwc, DWC3_LINK_STATE_RECOV);
1659         if (ret < 0) {
1660                 dev_err(dwc->dev, "failed to put link in Recovery\n");
1661                 return ret;
1662         }
1663
1664         /* Recent versions do this automatically */
1665         if (dwc->revision < DWC3_REVISION_194A) {
1666                 /* write zeroes to Link Change Request */
1667                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
1668                 reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;
1669                 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
1670         }
1671
1672         /* poll until Link State changes to ON */
1673         retries = 20000;
1674
1675         while (retries--) {
1676                 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1677
1678                 /* in HS, means ON */
1679                 if (DWC3_DSTS_USBLNKST(reg) == DWC3_LINK_STATE_U0)
1680                         break;
1681         }
1682
1683         if (DWC3_DSTS_USBLNKST(reg) != DWC3_LINK_STATE_U0) {
1684                 dev_err(dwc->dev, "failed to send remote wakeup\n");
1685                 return -EINVAL;
1686         }
1687
1688         return 0;
1689 }
1690
1691 static int dwc3_gadget_wakeup(struct usb_gadget *g)
1692 {
1693         struct dwc3             *dwc = gadget_to_dwc(g);
1694         unsigned long           flags;
1695         int                     ret;
1696
1697         spin_lock_irqsave(&dwc->lock, flags);
1698         ret = __dwc3_gadget_wakeup(dwc);
1699         spin_unlock_irqrestore(&dwc->lock, flags);
1700
1701         return ret;
1702 }
1703
1704 static int dwc3_gadget_set_selfpowered(struct usb_gadget *g,
1705                 int is_selfpowered)
1706 {
1707         struct dwc3             *dwc = gadget_to_dwc(g);
1708         unsigned long           flags;
1709
1710         spin_lock_irqsave(&dwc->lock, flags);
1711         g->is_selfpowered = !!is_selfpowered;
1712         spin_unlock_irqrestore(&dwc->lock, flags);
1713
1714         return 0;
1715 }
1716
1717 static int dwc3_gadget_run_stop(struct dwc3 *dwc, int is_on, int suspend)
1718 {
1719         u32                     reg;
1720         u32                     timeout = 500;
1721
1722         if (pm_runtime_suspended(dwc->dev))
1723                 return 0;
1724
1725         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
1726         if (is_on) {
1727                 if (dwc->revision <= DWC3_REVISION_187A) {
1728                         reg &= ~DWC3_DCTL_TRGTULST_MASK;
1729                         reg |= DWC3_DCTL_TRGTULST_RX_DET;
1730                 }
1731
1732                 if (dwc->revision >= DWC3_REVISION_194A)
1733                         reg &= ~DWC3_DCTL_KEEP_CONNECT;
1734                 reg |= DWC3_DCTL_RUN_STOP;
1735
1736                 if (dwc->has_hibernation)
1737                         reg |= DWC3_DCTL_KEEP_CONNECT;
1738
1739                 dwc->pullups_connected = true;
1740         } else {
1741                 reg &= ~DWC3_DCTL_RUN_STOP;
1742
1743                 if (dwc->has_hibernation && !suspend)
1744                         reg &= ~DWC3_DCTL_KEEP_CONNECT;
1745
1746                 dwc->pullups_connected = false;
1747         }
1748
1749         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
1750
1751         do {
1752                 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1753                 reg &= DWC3_DSTS_DEVCTRLHLT;
1754         } while (--timeout && !(!is_on ^ !reg));
1755
1756         if (!timeout)
1757                 return -ETIMEDOUT;
1758
1759         return 0;
1760 }
1761
1762 static int dwc3_gadget_pullup(struct usb_gadget *g, int is_on)
1763 {
1764         struct dwc3             *dwc = gadget_to_dwc(g);
1765         unsigned long           flags;
1766         int                     ret;
1767
1768         is_on = !!is_on;
1769
1770         /*
1771          * Per databook, when we want to stop the gadget, if a control transfer
1772          * is still in process, complete it and get the core into setup phase.
1773          */
1774         if (!is_on && dwc->ep0state != EP0_SETUP_PHASE) {
1775                 reinit_completion(&dwc->ep0_in_setup);
1776
1777                 ret = wait_for_completion_timeout(&dwc->ep0_in_setup,
1778                                 msecs_to_jiffies(DWC3_PULL_UP_TIMEOUT));
1779                 if (ret == 0) {
1780                         dev_err(dwc->dev, "timed out waiting for SETUP phase\n");
1781                         return -ETIMEDOUT;
1782                 }
1783         }
1784
1785         spin_lock_irqsave(&dwc->lock, flags);
1786         ret = dwc3_gadget_run_stop(dwc, is_on, false);
1787         spin_unlock_irqrestore(&dwc->lock, flags);
1788
1789         return ret;
1790 }
1791
1792 static void dwc3_gadget_enable_irq(struct dwc3 *dwc)
1793 {
1794         u32                     reg;
1795
1796         /* Enable all but Start and End of Frame IRQs */
1797         reg = (DWC3_DEVTEN_VNDRDEVTSTRCVEDEN |
1798                         DWC3_DEVTEN_EVNTOVERFLOWEN |
1799                         DWC3_DEVTEN_CMDCMPLTEN |
1800                         DWC3_DEVTEN_ERRTICERREN |
1801                         DWC3_DEVTEN_WKUPEVTEN |
1802                         DWC3_DEVTEN_CONNECTDONEEN |
1803                         DWC3_DEVTEN_USBRSTEN |
1804                         DWC3_DEVTEN_DISCONNEVTEN);
1805
1806         if (dwc->revision < DWC3_REVISION_250A)
1807                 reg |= DWC3_DEVTEN_ULSTCNGEN;
1808
1809         dwc3_writel(dwc->regs, DWC3_DEVTEN, reg);
1810 }
1811
1812 static void dwc3_gadget_disable_irq(struct dwc3 *dwc)
1813 {
1814         /* mask all interrupts */
1815         dwc3_writel(dwc->regs, DWC3_DEVTEN, 0x00);
1816 }
1817
1818 static irqreturn_t dwc3_interrupt(int irq, void *_dwc);
1819 static irqreturn_t dwc3_thread_interrupt(int irq, void *_dwc);
1820
1821 /**
1822  * dwc3_gadget_setup_nump - calculate and initialize NUMP field of %DWC3_DCFG
1823  * @dwc: pointer to our context structure
1824  *
1825  * The following looks like complex but it's actually very simple. In order to
1826  * calculate the number of packets we can burst at once on OUT transfers, we're
1827  * gonna use RxFIFO size.
1828  *
1829  * To calculate RxFIFO size we need two numbers:
1830  * MDWIDTH = size, in bits, of the internal memory bus
1831  * RAM2_DEPTH = depth, in MDWIDTH, of internal RAM2 (where RxFIFO sits)
1832  *
1833  * Given these two numbers, the formula is simple:
1834  *
1835  * RxFIFO Size = (RAM2_DEPTH * MDWIDTH / 8) - 24 - 16;
1836  *
1837  * 24 bytes is for 3x SETUP packets
1838  * 16 bytes is a clock domain crossing tolerance
1839  *
1840  * Given RxFIFO Size, NUMP = RxFIFOSize / 1024;
1841  */
1842 static void dwc3_gadget_setup_nump(struct dwc3 *dwc)
1843 {
1844         u32 ram2_depth;
1845         u32 mdwidth;
1846         u32 nump;
1847         u32 reg;
1848
1849         ram2_depth = DWC3_GHWPARAMS7_RAM2_DEPTH(dwc->hwparams.hwparams7);
1850         mdwidth = DWC3_GHWPARAMS0_MDWIDTH(dwc->hwparams.hwparams0);
1851
1852         nump = ((ram2_depth * mdwidth / 8) - 24 - 16) / 1024;
1853         nump = min_t(u32, nump, 16);
1854
1855         /* update NumP */
1856         reg = dwc3_readl(dwc->regs, DWC3_DCFG);
1857         reg &= ~DWC3_DCFG_NUMP_MASK;
1858         reg |= nump << DWC3_DCFG_NUMP_SHIFT;
1859         dwc3_writel(dwc->regs, DWC3_DCFG, reg);
1860 }
1861
1862 static int __dwc3_gadget_start(struct dwc3 *dwc)
1863 {
1864         struct dwc3_ep          *dep;
1865         int                     ret = 0;
1866         u32                     reg;
1867
1868         /*
1869          * Use IMOD if enabled via dwc->imod_interval. Otherwise, if
1870          * the core supports IMOD, disable it.
1871          */
1872         if (dwc->imod_interval) {
1873                 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), dwc->imod_interval);
1874                 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), DWC3_GEVNTCOUNT_EHB);
1875         } else if (dwc3_has_imod(dwc)) {
1876                 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), 0);
1877         }
1878
1879         /*
1880          * We are telling dwc3 that we want to use DCFG.NUMP as ACK TP's NUMP
1881          * field instead of letting dwc3 itself calculate that automatically.
1882          *
1883          * This way, we maximize the chances that we'll be able to get several
1884          * bursts of data without going through any sort of endpoint throttling.
1885          */
1886         reg = dwc3_readl(dwc->regs, DWC3_GRXTHRCFG);
1887         if (dwc3_is_usb31(dwc))
1888                 reg &= ~DWC31_GRXTHRCFG_PKTCNTSEL;
1889         else
1890                 reg &= ~DWC3_GRXTHRCFG_PKTCNTSEL;
1891
1892         dwc3_writel(dwc->regs, DWC3_GRXTHRCFG, reg);
1893
1894         dwc3_gadget_setup_nump(dwc);
1895
1896         /* Start with SuperSpeed Default */
1897         dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
1898
1899         dep = dwc->eps[0];
1900         ret = __dwc3_gadget_ep_enable(dep, false, false);
1901         if (ret) {
1902                 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
1903                 goto err0;
1904         }
1905
1906         dep = dwc->eps[1];
1907         ret = __dwc3_gadget_ep_enable(dep, false, false);
1908         if (ret) {
1909                 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
1910                 goto err1;
1911         }
1912
1913         /* begin to receive SETUP packets */
1914         dwc->ep0state = EP0_SETUP_PHASE;
1915         dwc3_ep0_out_start(dwc);
1916
1917         dwc3_gadget_enable_irq(dwc);
1918
1919         return 0;
1920
1921 err1:
1922         __dwc3_gadget_ep_disable(dwc->eps[0]);
1923
1924 err0:
1925         return ret;
1926 }
1927
1928 static int dwc3_gadget_start(struct usb_gadget *g,
1929                 struct usb_gadget_driver *driver)
1930 {
1931         struct dwc3             *dwc = gadget_to_dwc(g);
1932         unsigned long           flags;
1933         int                     ret = 0;
1934         int                     irq;
1935
1936         irq = dwc->irq_gadget;
1937         ret = request_threaded_irq(irq, dwc3_interrupt, dwc3_thread_interrupt,
1938                         IRQF_SHARED, "dwc3", dwc->ev_buf);
1939         if (ret) {
1940                 dev_err(dwc->dev, "failed to request irq #%d --> %d\n",
1941                                 irq, ret);
1942                 goto err0;
1943         }
1944
1945         spin_lock_irqsave(&dwc->lock, flags);
1946         if (dwc->gadget_driver) {
1947                 dev_err(dwc->dev, "%s is already bound to %s\n",
1948                                 dwc->gadget.name,
1949                                 dwc->gadget_driver->driver.name);
1950                 ret = -EBUSY;
1951                 goto err1;
1952         }
1953
1954         dwc->gadget_driver      = driver;
1955
1956         if (pm_runtime_active(dwc->dev))
1957                 __dwc3_gadget_start(dwc);
1958
1959         spin_unlock_irqrestore(&dwc->lock, flags);
1960
1961         return 0;
1962
1963 err1:
1964         spin_unlock_irqrestore(&dwc->lock, flags);
1965         free_irq(irq, dwc);
1966
1967 err0:
1968         return ret;
1969 }
1970
1971 static void __dwc3_gadget_stop(struct dwc3 *dwc)
1972 {
1973         dwc3_gadget_disable_irq(dwc);
1974         __dwc3_gadget_ep_disable(dwc->eps[0]);
1975         __dwc3_gadget_ep_disable(dwc->eps[1]);
1976 }
1977
1978 static int dwc3_gadget_stop(struct usb_gadget *g)
1979 {
1980         struct dwc3             *dwc = gadget_to_dwc(g);
1981         unsigned long           flags;
1982         int                     epnum;
1983         u32                     tmo_eps = 0;
1984
1985         spin_lock_irqsave(&dwc->lock, flags);
1986
1987         if (pm_runtime_suspended(dwc->dev))
1988                 goto out;
1989
1990         __dwc3_gadget_stop(dwc);
1991
1992         for (epnum = 2; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
1993                 struct dwc3_ep  *dep = dwc->eps[epnum];
1994                 int ret;
1995
1996                 if (!dep)
1997                         continue;
1998
1999                 if (!(dep->flags & DWC3_EP_END_TRANSFER_PENDING))
2000                         continue;
2001
2002                 ret = wait_event_interruptible_lock_irq_timeout(dep->wait_end_transfer,
2003                             !(dep->flags & DWC3_EP_END_TRANSFER_PENDING),
2004                             dwc->lock, msecs_to_jiffies(5));
2005
2006                 if (ret <= 0) {
2007                         /* Timed out or interrupted! There's nothing much
2008                          * we can do so we just log here and print which
2009                          * endpoints timed out at the end.
2010                          */
2011                         tmo_eps |= 1 << epnum;
2012                         dep->flags &= DWC3_EP_END_TRANSFER_PENDING;
2013                 }
2014         }
2015
2016         if (tmo_eps) {
2017                 dev_err(dwc->dev,
2018                         "end transfer timed out on endpoints 0x%x [bitmap]\n",
2019                         tmo_eps);
2020         }
2021
2022 out:
2023         dwc->gadget_driver      = NULL;
2024         spin_unlock_irqrestore(&dwc->lock, flags);
2025
2026         free_irq(dwc->irq_gadget, dwc->ev_buf);
2027
2028         return 0;
2029 }
2030
2031 static void dwc3_gadget_set_speed(struct usb_gadget *g,
2032                                   enum usb_device_speed speed)
2033 {
2034         struct dwc3             *dwc = gadget_to_dwc(g);
2035         unsigned long           flags;
2036         u32                     reg;
2037
2038         spin_lock_irqsave(&dwc->lock, flags);
2039         reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2040         reg &= ~(DWC3_DCFG_SPEED_MASK);
2041
2042         /*
2043          * WORKAROUND: DWC3 revision < 2.20a have an issue
2044          * which would cause metastability state on Run/Stop
2045          * bit if we try to force the IP to USB2-only mode.
2046          *
2047          * Because of that, we cannot configure the IP to any
2048          * speed other than the SuperSpeed
2049          *
2050          * Refers to:
2051          *
2052          * STAR#9000525659: Clock Domain Crossing on DCTL in
2053          * USB 2.0 Mode
2054          */
2055         if (dwc->revision < DWC3_REVISION_220A &&
2056             !dwc->dis_metastability_quirk) {
2057                 reg |= DWC3_DCFG_SUPERSPEED;
2058         } else {
2059                 switch (speed) {
2060                 case USB_SPEED_LOW:
2061                         reg |= DWC3_DCFG_LOWSPEED;
2062                         break;
2063                 case USB_SPEED_FULL:
2064                         reg |= DWC3_DCFG_FULLSPEED;
2065                         break;
2066                 case USB_SPEED_HIGH:
2067                         reg |= DWC3_DCFG_HIGHSPEED;
2068                         break;
2069                 case USB_SPEED_SUPER:
2070                         reg |= DWC3_DCFG_SUPERSPEED;
2071                         break;
2072                 case USB_SPEED_SUPER_PLUS:
2073                         if (dwc3_is_usb31(dwc))
2074                                 reg |= DWC3_DCFG_SUPERSPEED_PLUS;
2075                         else
2076                                 reg |= DWC3_DCFG_SUPERSPEED;
2077                         break;
2078                 default:
2079                         dev_err(dwc->dev, "invalid speed (%d)\n", speed);
2080
2081                         if (dwc->revision & DWC3_REVISION_IS_DWC31)
2082                                 reg |= DWC3_DCFG_SUPERSPEED_PLUS;
2083                         else
2084                                 reg |= DWC3_DCFG_SUPERSPEED;
2085                 }
2086         }
2087         dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2088
2089         spin_unlock_irqrestore(&dwc->lock, flags);
2090 }
2091
2092 static const struct usb_gadget_ops dwc3_gadget_ops = {
2093         .get_frame              = dwc3_gadget_get_frame,
2094         .wakeup                 = dwc3_gadget_wakeup,
2095         .set_selfpowered        = dwc3_gadget_set_selfpowered,
2096         .pullup                 = dwc3_gadget_pullup,
2097         .udc_start              = dwc3_gadget_start,
2098         .udc_stop               = dwc3_gadget_stop,
2099         .udc_set_speed          = dwc3_gadget_set_speed,
2100 };
2101
2102 /* -------------------------------------------------------------------------- */
2103
2104 static int dwc3_gadget_init_endpoints(struct dwc3 *dwc, u8 total)
2105 {
2106         struct dwc3_ep                  *dep;
2107         u8                              epnum;
2108
2109         INIT_LIST_HEAD(&dwc->gadget.ep_list);
2110
2111         for (epnum = 0; epnum < total; epnum++) {
2112                 bool                    direction = epnum & 1;
2113                 u8                      num = epnum >> 1;
2114
2115                 dep = kzalloc(sizeof(*dep), GFP_KERNEL);
2116                 if (!dep)
2117                         return -ENOMEM;
2118
2119                 dep->dwc = dwc;
2120                 dep->number = epnum;
2121                 dep->direction = direction;
2122                 dep->regs = dwc->regs + DWC3_DEP_BASE(epnum);
2123                 dwc->eps[epnum] = dep;
2124
2125                 snprintf(dep->name, sizeof(dep->name), "ep%u%s", num,
2126                                 direction ? "in" : "out");
2127
2128                 dep->endpoint.name = dep->name;
2129
2130                 if (!(dep->number > 1)) {
2131                         dep->endpoint.desc = &dwc3_gadget_ep0_desc;
2132                         dep->endpoint.comp_desc = NULL;
2133                 }
2134
2135                 spin_lock_init(&dep->lock);
2136
2137                 if (num == 0) {
2138                         usb_ep_set_maxpacket_limit(&dep->endpoint, 512);
2139                         dep->endpoint.maxburst = 1;
2140                         dep->endpoint.ops = &dwc3_gadget_ep0_ops;
2141                         if (!direction)
2142                                 dwc->gadget.ep0 = &dep->endpoint;
2143                 } else if (direction) {
2144                         int mdwidth;
2145                         int kbytes;
2146                         int size;
2147                         int ret;
2148
2149                         mdwidth = DWC3_MDWIDTH(dwc->hwparams.hwparams0);
2150                         /* MDWIDTH is represented in bits, we need it in bytes */
2151                         mdwidth /= 8;
2152
2153                         size = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(num));
2154                         if (dwc3_is_usb31(dwc))
2155                                 size = DWC31_GTXFIFOSIZ_TXFDEF(size);
2156                         else
2157                                 size = DWC3_GTXFIFOSIZ_TXFDEF(size);
2158
2159                         /* FIFO Depth is in MDWDITH bytes. Multiply */
2160                         size *= mdwidth;
2161
2162                         kbytes = size / 1024;
2163                         if (kbytes == 0)
2164                                 kbytes = 1;
2165
2166                         /*
2167                          * FIFO sizes account an extra MDWIDTH * (kbytes + 1) bytes for
2168                          * internal overhead. We don't really know how these are used,
2169                          * but documentation say it exists.
2170                          */
2171                         size -= mdwidth * (kbytes + 1);
2172                         size /= kbytes;
2173
2174                         usb_ep_set_maxpacket_limit(&dep->endpoint, size);
2175
2176                         dep->endpoint.max_streams = 15;
2177                         dep->endpoint.ops = &dwc3_gadget_ep_ops;
2178                         list_add_tail(&dep->endpoint.ep_list,
2179                                         &dwc->gadget.ep_list);
2180
2181                         ret = dwc3_alloc_trb_pool(dep);
2182                         if (ret)
2183                                 return ret;
2184                 } else {
2185                         int             ret;
2186
2187                         usb_ep_set_maxpacket_limit(&dep->endpoint, 1024);
2188                         dep->endpoint.max_streams = 15;
2189                         dep->endpoint.ops = &dwc3_gadget_ep_ops;
2190                         list_add_tail(&dep->endpoint.ep_list,
2191                                         &dwc->gadget.ep_list);
2192
2193                         ret = dwc3_alloc_trb_pool(dep);
2194                         if (ret)
2195                                 return ret;
2196                 }
2197
2198                 if (num == 0) {
2199                         dep->endpoint.caps.type_control = true;
2200                 } else {
2201                         dep->endpoint.caps.type_iso = true;
2202                         dep->endpoint.caps.type_bulk = true;
2203                         dep->endpoint.caps.type_int = true;
2204                 }
2205
2206                 dep->endpoint.caps.dir_in = direction;
2207                 dep->endpoint.caps.dir_out = !direction;
2208
2209                 INIT_LIST_HEAD(&dep->pending_list);
2210                 INIT_LIST_HEAD(&dep->started_list);
2211         }
2212
2213         return 0;
2214 }
2215
2216 static void dwc3_gadget_free_endpoints(struct dwc3 *dwc)
2217 {
2218         struct dwc3_ep                  *dep;
2219         u8                              epnum;
2220
2221         for (epnum = 0; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
2222                 dep = dwc->eps[epnum];
2223                 if (!dep)
2224                         continue;
2225                 /*
2226                  * Physical endpoints 0 and 1 are special; they form the
2227                  * bi-directional USB endpoint 0.
2228                  *
2229                  * For those two physical endpoints, we don't allocate a TRB
2230                  * pool nor do we add them the endpoints list. Due to that, we
2231                  * shouldn't do these two operations otherwise we would end up
2232                  * with all sorts of bugs when removing dwc3.ko.
2233                  */
2234                 if (epnum != 0 && epnum != 1) {
2235                         dwc3_free_trb_pool(dep);
2236                         list_del(&dep->endpoint.ep_list);
2237                 }
2238
2239                 kfree(dep);
2240         }
2241 }
2242
2243 /* -------------------------------------------------------------------------- */
2244
2245 static int dwc3_gadget_ep_reclaim_completed_trb(struct dwc3 *dwc,
2246                 struct dwc3_ep *dep, struct dwc3_request *req,
2247                 struct dwc3_trb *trb, const struct dwc3_event_depevt *event,
2248                 int status, int chain)
2249 {
2250         unsigned int            count;
2251         unsigned int            s_pkt = 0;
2252         unsigned int            trb_status;
2253
2254         dwc3_ep_inc_deq(dep);
2255
2256         trace_dwc3_complete_trb(dep, trb);
2257
2258         /*
2259          * If we're in the middle of series of chained TRBs and we
2260          * receive a short transfer along the way, DWC3 will skip
2261          * through all TRBs including the last TRB in the chain (the
2262          * where CHN bit is zero. DWC3 will also avoid clearing HWO
2263          * bit and SW has to do it manually.
2264          *
2265          * We're going to do that here to avoid problems of HW trying
2266          * to use bogus TRBs for transfers.
2267          */
2268         if (chain && (trb->ctrl & DWC3_TRB_CTRL_HWO))
2269                 trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
2270
2271         /*
2272          * If we're dealing with unaligned size OUT transfer, we will be left
2273          * with one TRB pending in the ring. We need to manually clear HWO bit
2274          * from that TRB.
2275          */
2276         if ((req->zero || req->unaligned) && (trb->ctrl & DWC3_TRB_CTRL_HWO)) {
2277                 trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
2278                 return 1;
2279         }
2280
2281         count = trb->size & DWC3_TRB_SIZE_MASK;
2282         req->remaining += count;
2283
2284         if ((trb->ctrl & DWC3_TRB_CTRL_HWO) && status != -ESHUTDOWN)
2285                 return 1;
2286
2287         if (dep->direction) {
2288                 if (count) {
2289                         trb_status = DWC3_TRB_SIZE_TRBSTS(trb->size);
2290                         if (trb_status == DWC3_TRBSTS_MISSED_ISOC) {
2291                                 /*
2292                                  * If missed isoc occurred and there is
2293                                  * no request queued then issue END
2294                                  * TRANSFER, so that core generates
2295                                  * next xfernotready and we will issue
2296                                  * a fresh START TRANSFER.
2297                                  * If there are still queued request
2298                                  * then wait, do not issue either END
2299                                  * or UPDATE TRANSFER, just attach next
2300                                  * request in pending_list during
2301                                  * giveback.If any future queued request
2302                                  * is successfully transferred then we
2303                                  * will issue UPDATE TRANSFER for all
2304                                  * request in the pending_list.
2305                                  */
2306                                 dep->flags |= DWC3_EP_MISSED_ISOC;
2307                         } else {
2308                                 dev_err(dwc->dev, "incomplete IN transfer %s\n",
2309                                                 dep->name);
2310                                 status = -ECONNRESET;
2311                         }
2312                 } else {
2313                         dep->flags &= ~DWC3_EP_MISSED_ISOC;
2314                 }
2315         } else {
2316                 if (count && (event->status & DEPEVT_STATUS_SHORT))
2317                         s_pkt = 1;
2318         }
2319
2320         if (s_pkt && !chain)
2321                 return 1;
2322
2323         if ((event->status & DEPEVT_STATUS_IOC) &&
2324                         (trb->ctrl & DWC3_TRB_CTRL_IOC))
2325                 return 1;
2326
2327         return 0;
2328 }
2329
2330 static int dwc3_gadget_ep_cleanup_completed_requests(struct dwc3 *dwc,
2331                 struct dwc3_ep *dep, const struct dwc3_event_depevt *event,
2332                 int status)
2333 {
2334         struct dwc3_request     *req, *n;
2335         struct dwc3_trb         *trb;
2336         bool                    ioc = false;
2337         int                     ret = 0;
2338
2339         list_for_each_entry_safe(req, n, &dep->started_list, list) {
2340                 unsigned length;
2341                 int chain;
2342
2343                 length = req->request.length;
2344                 chain = req->num_pending_sgs > 0;
2345                 if (chain) {
2346                         struct scatterlist *sg = req->sg;
2347                         struct scatterlist *s;
2348                         unsigned int pending = req->num_pending_sgs;
2349                         unsigned int i;
2350
2351                         for_each_sg(sg, s, pending, i) {
2352                                 trb = &dep->trb_pool[dep->trb_dequeue];
2353
2354                                 if (trb->ctrl & DWC3_TRB_CTRL_HWO)
2355                                         break;
2356
2357                                 req->sg = sg_next(s);
2358                                 req->num_pending_sgs--;
2359
2360                                 ret = dwc3_gadget_ep_reclaim_completed_trb(dwc,
2361                                                 dep, req, trb, event, status,
2362                                                 chain);
2363                                 if (ret)
2364                                         break;
2365                         }
2366                 } else {
2367                         trb = &dep->trb_pool[dep->trb_dequeue];
2368                         ret = dwc3_gadget_ep_reclaim_completed_trb(dwc, dep,
2369                                         req, trb, event, status, chain);
2370                 }
2371
2372                 if (req->unaligned || req->zero) {
2373                         trb = &dep->trb_pool[dep->trb_dequeue];
2374                         ret = dwc3_gadget_ep_reclaim_completed_trb(dwc, dep,
2375                                         req, trb, event, status, false);
2376                         req->unaligned = false;
2377                         req->zero = false;
2378                 }
2379
2380                 req->request.actual = length - req->remaining;
2381
2382                 if (req->request.actual < length || req->num_pending_sgs) {
2383                         /*
2384                          * There could be a scenario where the whole req can't
2385                          * be mapped into available TRB's. In that case, we need
2386                          * to kick transfer again if (req->num_pending_sgs > 0)
2387                          */
2388                         if (req->num_pending_sgs) {
2389                                 dev_WARN_ONCE(dwc->dev,
2390                                               (req->request.actual == length),
2391                                               "There are some pending sg's that needs to be queued again\n");
2392                                 return __dwc3_gadget_kick_transfer(dep);
2393                         }
2394                 }
2395
2396                 dwc3_gadget_giveback(dep, req, status);
2397
2398                 if (ret) {
2399                         if ((event->status & DEPEVT_STATUS_IOC) &&
2400                             (trb->ctrl & DWC3_TRB_CTRL_IOC))
2401                                 ioc = true;
2402                         break;
2403                 }
2404         }
2405
2406         /*
2407          * Our endpoint might get disabled by another thread during
2408          * dwc3_gadget_giveback(). If that happens, we're just gonna return 1
2409          * early on so DWC3_EP_BUSY flag gets cleared
2410          */
2411         if (!dep->endpoint.desc)
2412                 return 1;
2413
2414         if (usb_endpoint_xfer_isoc(dep->endpoint.desc) &&
2415                         list_empty(&dep->started_list)) {
2416                 if (list_empty(&dep->pending_list)) {
2417                         /*
2418                          * If there is no entry in request list then do
2419                          * not issue END TRANSFER now. Just set PENDING
2420                          * flag, so that END TRANSFER is issued when an
2421                          * entry is added into request list.
2422                          */
2423                         dep->flags = DWC3_EP_PENDING_REQUEST;
2424                 } else {
2425                         dwc3_stop_active_transfer(dwc, dep->number, true);
2426                         dep->flags = DWC3_EP_ENABLED;
2427                 }
2428                 return 1;
2429         }
2430
2431         if (usb_endpoint_xfer_isoc(dep->endpoint.desc) && ioc)
2432                 return 0;
2433
2434         return 1;
2435 }
2436
2437 static void dwc3_gadget_endpoint_transfer_in_progress(struct dwc3 *dwc,
2438                 struct dwc3_ep *dep, const struct dwc3_event_depevt *event)
2439 {
2440         unsigned                status = 0;
2441         int                     clean_busy;
2442
2443         if (event->status & DEPEVT_STATUS_BUSERR)
2444                 status = -ECONNRESET;
2445
2446         clean_busy = dwc3_gadget_ep_cleanup_completed_requests(dwc, dep, event,
2447                         status);
2448         if (clean_busy && (!dep->endpoint.desc ||
2449                                 usb_endpoint_xfer_isoc(dep->endpoint.desc)))
2450                 dep->flags &= ~DWC3_EP_BUSY;
2451
2452         /*
2453          * WORKAROUND: This is the 2nd half of U1/U2 -> U0 workaround.
2454          * See dwc3_gadget_linksts_change_interrupt() for 1st half.
2455          */
2456         if (dwc->revision < DWC3_REVISION_183A) {
2457                 u32             reg;
2458                 int             i;
2459
2460                 for (i = 0; i < DWC3_ENDPOINTS_NUM; i++) {
2461                         dep = dwc->eps[i];
2462
2463                         if (!(dep->flags & DWC3_EP_ENABLED))
2464                                 continue;
2465
2466                         if (!list_empty(&dep->started_list))
2467                                 return;
2468                 }
2469
2470                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2471                 reg |= dwc->u1u2;
2472                 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2473
2474                 dwc->u1u2 = 0;
2475         }
2476 }
2477
2478 static void dwc3_endpoint_interrupt(struct dwc3 *dwc,
2479                 const struct dwc3_event_depevt *event)
2480 {
2481         struct dwc3_ep          *dep;
2482         u8                      epnum = event->endpoint_number;
2483         u8                      cmd;
2484
2485         dep = dwc->eps[epnum];
2486
2487         if (!(dep->flags & DWC3_EP_ENABLED)) {
2488                 if (!(dep->flags & DWC3_EP_END_TRANSFER_PENDING))
2489                         return;
2490
2491                 /* Handle only EPCMDCMPLT when EP disabled */
2492                 if (event->endpoint_event != DWC3_DEPEVT_EPCMDCMPLT)
2493                         return;
2494         }
2495
2496         if (epnum == 0 || epnum == 1) {
2497                 dwc3_ep0_interrupt(dwc, event);
2498                 return;
2499         }
2500
2501         switch (event->endpoint_event) {
2502         case DWC3_DEPEVT_XFERINPROGRESS:
2503                 dwc3_gadget_endpoint_transfer_in_progress(dwc, dep, event);
2504                 break;
2505         case DWC3_DEPEVT_XFERNOTREADY:
2506                 dwc3_gadget_endpoint_transfer_not_ready(dwc, dep, event);
2507                 break;
2508         case DWC3_DEPEVT_EPCMDCMPLT:
2509                 cmd = DEPEVT_PARAMETER_CMD(event->parameters);
2510
2511                 if (cmd == DWC3_DEPCMD_ENDTRANSFER) {
2512                         dep->flags &= ~DWC3_EP_END_TRANSFER_PENDING;
2513                         wake_up(&dep->wait_end_transfer);
2514                 }
2515                 break;
2516         case DWC3_DEPEVT_STREAMEVT:
2517         case DWC3_DEPEVT_XFERCOMPLETE:
2518         case DWC3_DEPEVT_RXTXFIFOEVT:
2519                 break;
2520         }
2521 }
2522
2523 static void dwc3_disconnect_gadget(struct dwc3 *dwc)
2524 {
2525         if (dwc->gadget_driver && dwc->gadget_driver->disconnect) {
2526                 spin_unlock(&dwc->lock);
2527                 dwc->gadget_driver->disconnect(&dwc->gadget);
2528                 spin_lock(&dwc->lock);
2529         }
2530 }
2531
2532 static void dwc3_suspend_gadget(struct dwc3 *dwc)
2533 {
2534         if (dwc->gadget_driver && dwc->gadget_driver->suspend) {
2535                 spin_unlock(&dwc->lock);
2536                 dwc->gadget_driver->suspend(&dwc->gadget);
2537                 spin_lock(&dwc->lock);
2538         }
2539 }
2540
2541 static void dwc3_resume_gadget(struct dwc3 *dwc)
2542 {
2543         if (dwc->gadget_driver && dwc->gadget_driver->resume) {
2544                 spin_unlock(&dwc->lock);
2545                 dwc->gadget_driver->resume(&dwc->gadget);
2546                 spin_lock(&dwc->lock);
2547         }
2548 }
2549
2550 static void dwc3_reset_gadget(struct dwc3 *dwc)
2551 {
2552         if (!dwc->gadget_driver)
2553                 return;
2554
2555         if (dwc->gadget.speed != USB_SPEED_UNKNOWN) {
2556                 spin_unlock(&dwc->lock);
2557                 usb_gadget_udc_reset(&dwc->gadget, dwc->gadget_driver);
2558                 spin_lock(&dwc->lock);
2559         }
2560 }
2561
2562 static void dwc3_stop_active_transfer(struct dwc3 *dwc, u32 epnum, bool force)
2563 {
2564         struct dwc3_ep *dep;
2565         struct dwc3_gadget_ep_cmd_params params;
2566         u32 cmd;
2567         int ret;
2568
2569         dep = dwc->eps[epnum];
2570
2571         if ((dep->flags & DWC3_EP_END_TRANSFER_PENDING) ||
2572             !dep->resource_index)
2573                 return;
2574
2575         /*
2576          * NOTICE: We are violating what the Databook says about the
2577          * EndTransfer command. Ideally we would _always_ wait for the
2578          * EndTransfer Command Completion IRQ, but that's causing too
2579          * much trouble synchronizing between us and gadget driver.
2580          *
2581          * We have discussed this with the IP Provider and it was
2582          * suggested to giveback all requests here, but give HW some
2583          * extra time to synchronize with the interconnect. We're using
2584          * an arbitrary 100us delay for that.
2585          *
2586          * Note also that a similar handling was tested by Synopsys
2587          * (thanks a lot Paul) and nothing bad has come out of it.
2588          * In short, what we're doing is:
2589          *
2590          * - Issue EndTransfer WITH CMDIOC bit set
2591          * - Wait 100us
2592          *
2593          * As of IP version 3.10a of the DWC_usb3 IP, the controller
2594          * supports a mode to work around the above limitation. The
2595          * software can poll the CMDACT bit in the DEPCMD register
2596          * after issuing a EndTransfer command. This mode is enabled
2597          * by writing GUCTL2[14]. This polling is already done in the
2598          * dwc3_send_gadget_ep_cmd() function so if the mode is
2599          * enabled, the EndTransfer command will have completed upon
2600          * returning from this function and we don't need to delay for
2601          * 100us.
2602          *
2603          * This mode is NOT available on the DWC_usb31 IP.
2604          */
2605
2606         cmd = DWC3_DEPCMD_ENDTRANSFER;
2607         cmd |= force ? DWC3_DEPCMD_HIPRI_FORCERM : 0;
2608         cmd |= DWC3_DEPCMD_CMDIOC;
2609         cmd |= DWC3_DEPCMD_PARAM(dep->resource_index);
2610         memset(&params, 0, sizeof(params));
2611         ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
2612         WARN_ON_ONCE(ret);
2613         dep->resource_index = 0;
2614         dep->flags &= ~DWC3_EP_BUSY;
2615
2616         if (dwc3_is_usb31(dwc) || dwc->revision < DWC3_REVISION_310A) {
2617                 dep->flags |= DWC3_EP_END_TRANSFER_PENDING;
2618                 udelay(100);
2619         }
2620 }
2621
2622 static void dwc3_clear_stall_all_ep(struct dwc3 *dwc)
2623 {
2624         u32 epnum;
2625
2626         for (epnum = 1; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
2627                 struct dwc3_ep *dep;
2628                 int ret;
2629
2630                 dep = dwc->eps[epnum];
2631                 if (!dep)
2632                         continue;
2633
2634                 if (!(dep->flags & DWC3_EP_STALL))
2635                         continue;
2636
2637                 dep->flags &= ~DWC3_EP_STALL;
2638
2639                 ret = dwc3_send_clear_stall_ep_cmd(dep);
2640                 WARN_ON_ONCE(ret);
2641         }
2642 }
2643
2644 static void dwc3_gadget_disconnect_interrupt(struct dwc3 *dwc)
2645 {
2646         int                     reg;
2647
2648         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2649         reg &= ~DWC3_DCTL_INITU1ENA;
2650         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2651
2652         reg &= ~DWC3_DCTL_INITU2ENA;
2653         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2654
2655         dwc3_disconnect_gadget(dwc);
2656
2657         dwc->gadget.speed = USB_SPEED_UNKNOWN;
2658         dwc->setup_packet_pending = false;
2659         usb_gadget_set_state(&dwc->gadget, USB_STATE_NOTATTACHED);
2660
2661         dwc->connected = false;
2662 }
2663
2664 static void dwc3_gadget_reset_interrupt(struct dwc3 *dwc)
2665 {
2666         u32                     reg;
2667
2668         dwc->connected = true;
2669
2670         /*
2671          * WORKAROUND: DWC3 revisions <1.88a have an issue which
2672          * would cause a missing Disconnect Event if there's a
2673          * pending Setup Packet in the FIFO.
2674          *
2675          * There's no suggested workaround on the official Bug
2676          * report, which states that "unless the driver/application
2677          * is doing any special handling of a disconnect event,
2678          * there is no functional issue".
2679          *
2680          * Unfortunately, it turns out that we _do_ some special
2681          * handling of a disconnect event, namely complete all
2682          * pending transfers, notify gadget driver of the
2683          * disconnection, and so on.
2684          *
2685          * Our suggested workaround is to follow the Disconnect
2686          * Event steps here, instead, based on a setup_packet_pending
2687          * flag. Such flag gets set whenever we have a SETUP_PENDING
2688          * status for EP0 TRBs and gets cleared on XferComplete for the
2689          * same endpoint.
2690          *
2691          * Refers to:
2692          *
2693          * STAR#9000466709: RTL: Device : Disconnect event not
2694          * generated if setup packet pending in FIFO
2695          */
2696         if (dwc->revision < DWC3_REVISION_188A) {
2697                 if (dwc->setup_packet_pending)
2698                         dwc3_gadget_disconnect_interrupt(dwc);
2699         }
2700
2701         dwc3_reset_gadget(dwc);
2702
2703         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2704         reg &= ~DWC3_DCTL_TSTCTRL_MASK;
2705         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2706         dwc->test_mode = false;
2707         dwc3_clear_stall_all_ep(dwc);
2708
2709         /* Reset device address to zero */
2710         reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2711         reg &= ~(DWC3_DCFG_DEVADDR_MASK);
2712         dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2713 }
2714
2715 static void dwc3_gadget_conndone_interrupt(struct dwc3 *dwc)
2716 {
2717         struct dwc3_ep          *dep;
2718         int                     ret;
2719         u32                     reg;
2720         u8                      speed;
2721
2722         reg = dwc3_readl(dwc->regs, DWC3_DSTS);
2723         speed = reg & DWC3_DSTS_CONNECTSPD;
2724         dwc->speed = speed;
2725
2726         /*
2727          * RAMClkSel is reset to 0 after USB reset, so it must be reprogrammed
2728          * each time on Connect Done.
2729          *
2730          * Currently we always use the reset value. If any platform
2731          * wants to set this to a different value, we need to add a
2732          * setting and update GCTL.RAMCLKSEL here.
2733          */
2734
2735         switch (speed) {
2736         case DWC3_DSTS_SUPERSPEED_PLUS:
2737                 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
2738                 dwc->gadget.ep0->maxpacket = 512;
2739                 dwc->gadget.speed = USB_SPEED_SUPER_PLUS;
2740                 break;
2741         case DWC3_DSTS_SUPERSPEED:
2742                 /*
2743                  * WORKAROUND: DWC3 revisions <1.90a have an issue which
2744                  * would cause a missing USB3 Reset event.
2745                  *
2746                  * In such situations, we should force a USB3 Reset
2747                  * event by calling our dwc3_gadget_reset_interrupt()
2748                  * routine.
2749                  *
2750                  * Refers to:
2751                  *
2752                  * STAR#9000483510: RTL: SS : USB3 reset event may
2753                  * not be generated always when the link enters poll
2754                  */
2755                 if (dwc->revision < DWC3_REVISION_190A)
2756                         dwc3_gadget_reset_interrupt(dwc);
2757
2758                 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
2759                 dwc->gadget.ep0->maxpacket = 512;
2760                 dwc->gadget.speed = USB_SPEED_SUPER;
2761                 break;
2762         case DWC3_DSTS_HIGHSPEED:
2763                 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
2764                 dwc->gadget.ep0->maxpacket = 64;
2765                 dwc->gadget.speed = USB_SPEED_HIGH;
2766                 break;
2767         case DWC3_DSTS_FULLSPEED:
2768                 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
2769                 dwc->gadget.ep0->maxpacket = 64;
2770                 dwc->gadget.speed = USB_SPEED_FULL;
2771                 break;
2772         case DWC3_DSTS_LOWSPEED:
2773                 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(8);
2774                 dwc->gadget.ep0->maxpacket = 8;
2775                 dwc->gadget.speed = USB_SPEED_LOW;
2776                 break;
2777         }
2778
2779         dwc->eps[1]->endpoint.maxpacket = dwc->gadget.ep0->maxpacket;
2780
2781         /* Enable USB2 LPM Capability */
2782
2783         if ((dwc->revision > DWC3_REVISION_194A) &&
2784             (speed != DWC3_DSTS_SUPERSPEED) &&
2785             (speed != DWC3_DSTS_SUPERSPEED_PLUS)) {
2786                 reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2787                 reg |= DWC3_DCFG_LPM_CAP;
2788                 dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2789
2790                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2791                 reg &= ~(DWC3_DCTL_HIRD_THRES_MASK | DWC3_DCTL_L1_HIBER_EN);
2792
2793                 reg |= DWC3_DCTL_HIRD_THRES(dwc->hird_threshold);
2794
2795                 /*
2796                  * When dwc3 revisions >= 2.40a, LPM Erratum is enabled and
2797                  * DCFG.LPMCap is set, core responses with an ACK and the
2798                  * BESL value in the LPM token is less than or equal to LPM
2799                  * NYET threshold.
2800                  */
2801                 WARN_ONCE(dwc->revision < DWC3_REVISION_240A
2802                                 && dwc->has_lpm_erratum,
2803                                 "LPM Erratum not available on dwc3 revisions < 2.40a\n");
2804
2805                 if (dwc->has_lpm_erratum && dwc->revision >= DWC3_REVISION_240A)
2806                         reg |= DWC3_DCTL_LPM_ERRATA(dwc->lpm_nyet_threshold);
2807
2808                 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2809         } else {
2810                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2811                 reg &= ~DWC3_DCTL_HIRD_THRES_MASK;
2812                 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2813         }
2814
2815         dep = dwc->eps[0];
2816         ret = __dwc3_gadget_ep_enable(dep, true, false);
2817         if (ret) {
2818                 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
2819                 return;
2820         }
2821
2822         dep = dwc->eps[1];
2823         ret = __dwc3_gadget_ep_enable(dep, true, false);
2824         if (ret) {
2825                 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
2826                 return;
2827         }
2828
2829         /*
2830          * Configure PHY via GUSB3PIPECTLn if required.
2831          *
2832          * Update GTXFIFOSIZn
2833          *
2834          * In both cases reset values should be sufficient.
2835          */
2836 }
2837
2838 static void dwc3_gadget_wakeup_interrupt(struct dwc3 *dwc)
2839 {
2840         /*
2841          * TODO take core out of low power mode when that's
2842          * implemented.
2843          */
2844
2845         if (dwc->gadget_driver && dwc->gadget_driver->resume) {
2846                 spin_unlock(&dwc->lock);
2847                 dwc->gadget_driver->resume(&dwc->gadget);
2848                 spin_lock(&dwc->lock);
2849         }
2850 }
2851
2852 static void dwc3_gadget_linksts_change_interrupt(struct dwc3 *dwc,
2853                 unsigned int evtinfo)
2854 {
2855         enum dwc3_link_state    next = evtinfo & DWC3_LINK_STATE_MASK;
2856         unsigned int            pwropt;
2857
2858         /*
2859          * WORKAROUND: DWC3 < 2.50a have an issue when configured without
2860          * Hibernation mode enabled which would show up when device detects
2861          * host-initiated U3 exit.
2862          *
2863          * In that case, device will generate a Link State Change Interrupt
2864          * from U3 to RESUME which is only necessary if Hibernation is
2865          * configured in.
2866          *
2867          * There are no functional changes due to such spurious event and we
2868          * just need to ignore it.
2869          *
2870          * Refers to:
2871          *
2872          * STAR#9000570034 RTL: SS Resume event generated in non-Hibernation
2873          * operational mode
2874          */
2875         pwropt = DWC3_GHWPARAMS1_EN_PWROPT(dwc->hwparams.hwparams1);
2876         if ((dwc->revision < DWC3_REVISION_250A) &&
2877                         (pwropt != DWC3_GHWPARAMS1_EN_PWROPT_HIB)) {
2878                 if ((dwc->link_state == DWC3_LINK_STATE_U3) &&
2879                                 (next == DWC3_LINK_STATE_RESUME)) {
2880                         return;
2881                 }
2882         }
2883
2884         /*
2885          * WORKAROUND: DWC3 Revisions <1.83a have an issue which, depending
2886          * on the link partner, the USB session might do multiple entry/exit
2887          * of low power states before a transfer takes place.
2888          *
2889          * Due to this problem, we might experience lower throughput. The
2890          * suggested workaround is to disable DCTL[12:9] bits if we're
2891          * transitioning from U1/U2 to U0 and enable those bits again
2892          * after a transfer completes and there are no pending transfers
2893          * on any of the enabled endpoints.
2894          *
2895          * This is the first half of that workaround.
2896          *
2897          * Refers to:
2898          *
2899          * STAR#9000446952: RTL: Device SS : if U1/U2 ->U0 takes >128us
2900          * core send LGO_Ux entering U0
2901          */
2902         if (dwc->revision < DWC3_REVISION_183A) {
2903                 if (next == DWC3_LINK_STATE_U0) {
2904                         u32     u1u2;
2905                         u32     reg;
2906
2907                         switch (dwc->link_state) {
2908                         case DWC3_LINK_STATE_U1:
2909                         case DWC3_LINK_STATE_U2:
2910                                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2911                                 u1u2 = reg & (DWC3_DCTL_INITU2ENA
2912                                                 | DWC3_DCTL_ACCEPTU2ENA
2913                                                 | DWC3_DCTL_INITU1ENA
2914                                                 | DWC3_DCTL_ACCEPTU1ENA);
2915
2916                                 if (!dwc->u1u2)
2917                                         dwc->u1u2 = reg & u1u2;
2918
2919                                 reg &= ~u1u2;
2920
2921                                 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2922                                 break;
2923                         default:
2924                                 /* do nothing */
2925                                 break;
2926                         }
2927                 }
2928         }
2929
2930         switch (next) {
2931         case DWC3_LINK_STATE_U1:
2932                 if (dwc->speed == USB_SPEED_SUPER)
2933                         dwc3_suspend_gadget(dwc);
2934                 break;
2935         case DWC3_LINK_STATE_U2:
2936         case DWC3_LINK_STATE_U3:
2937                 dwc3_suspend_gadget(dwc);
2938                 break;
2939         case DWC3_LINK_STATE_RESUME:
2940                 dwc3_resume_gadget(dwc);
2941                 break;
2942         default:
2943                 /* do nothing */
2944                 break;
2945         }
2946
2947         dwc->link_state = next;
2948 }
2949
2950 static void dwc3_gadget_suspend_interrupt(struct dwc3 *dwc,
2951                                           unsigned int evtinfo)
2952 {
2953         enum dwc3_link_state next = evtinfo & DWC3_LINK_STATE_MASK;
2954
2955         if (dwc->link_state != next && next == DWC3_LINK_STATE_U3)
2956                 dwc3_suspend_gadget(dwc);
2957
2958         dwc->link_state = next;
2959 }
2960
2961 static void dwc3_gadget_hibernation_interrupt(struct dwc3 *dwc,
2962                 unsigned int evtinfo)
2963 {
2964         unsigned int is_ss = evtinfo & BIT(4);
2965
2966         /*
2967          * WORKAROUND: DWC3 revison 2.20a with hibernation support
2968          * have a known issue which can cause USB CV TD.9.23 to fail
2969          * randomly.
2970          *
2971          * Because of this issue, core could generate bogus hibernation
2972          * events which SW needs to ignore.
2973          *
2974          * Refers to:
2975          *
2976          * STAR#9000546576: Device Mode Hibernation: Issue in USB 2.0
2977          * Device Fallback from SuperSpeed
2978          */
2979         if (is_ss ^ (dwc->speed == USB_SPEED_SUPER))
2980                 return;
2981
2982         /* enter hibernation here */
2983 }
2984
2985 static void dwc3_gadget_interrupt(struct dwc3 *dwc,
2986                 const struct dwc3_event_devt *event)
2987 {
2988         switch (event->type) {
2989         case DWC3_DEVICE_EVENT_DISCONNECT:
2990                 dwc3_gadget_disconnect_interrupt(dwc);
2991                 break;
2992         case DWC3_DEVICE_EVENT_RESET:
2993                 dwc3_gadget_reset_interrupt(dwc);
2994                 break;
2995         case DWC3_DEVICE_EVENT_CONNECT_DONE:
2996                 dwc3_gadget_conndone_interrupt(dwc);
2997                 break;
2998         case DWC3_DEVICE_EVENT_WAKEUP:
2999                 dwc3_gadget_wakeup_interrupt(dwc);
3000                 break;
3001         case DWC3_DEVICE_EVENT_HIBER_REQ:
3002                 if (dev_WARN_ONCE(dwc->dev, !dwc->has_hibernation,
3003                                         "unexpected hibernation event\n"))
3004                         break;
3005
3006                 dwc3_gadget_hibernation_interrupt(dwc, event->event_info);
3007                 break;
3008         case DWC3_DEVICE_EVENT_LINK_STATUS_CHANGE:
3009                 dwc3_gadget_linksts_change_interrupt(dwc, event->event_info);
3010                 break;
3011         case DWC3_DEVICE_EVENT_EOPF:
3012                 /* It changed to be suspend event for version 2.30a and above */
3013                 if (dwc->revision >= DWC3_REVISION_230A) {
3014                         /*
3015                          * Ignore suspend event until the gadget enters into
3016                          * USB_STATE_CONFIGURED state.
3017                          */
3018                         if (dwc->gadget.state >= USB_STATE_CONFIGURED)
3019                                 dwc3_gadget_suspend_interrupt(dwc,
3020                                                 event->event_info);
3021                 }
3022                 break;
3023         case DWC3_DEVICE_EVENT_SOF:
3024         case DWC3_DEVICE_EVENT_ERRATIC_ERROR:
3025         case DWC3_DEVICE_EVENT_CMD_CMPL:
3026         case DWC3_DEVICE_EVENT_OVERFLOW:
3027                 break;
3028         default:
3029                 dev_WARN(dwc->dev, "UNKNOWN IRQ %d\n", event->type);
3030         }
3031 }
3032
3033 static void dwc3_process_event_entry(struct dwc3 *dwc,
3034                 const union dwc3_event *event)
3035 {
3036         trace_dwc3_event(event->raw, dwc);
3037
3038         if (!event->type.is_devspec)
3039                 dwc3_endpoint_interrupt(dwc, &event->depevt);
3040         else if (event->type.type == DWC3_EVENT_TYPE_DEV)
3041                 dwc3_gadget_interrupt(dwc, &event->devt);
3042         else
3043                 dev_err(dwc->dev, "UNKNOWN IRQ type %d\n", event->raw);
3044 }
3045
3046 static irqreturn_t dwc3_process_event_buf(struct dwc3_event_buffer *evt)
3047 {
3048         struct dwc3 *dwc = evt->dwc;
3049         irqreturn_t ret = IRQ_NONE;
3050         int left;
3051         u32 reg;
3052
3053         left = evt->count;
3054
3055         if (!(evt->flags & DWC3_EVENT_PENDING))
3056                 return IRQ_NONE;
3057
3058         while (left > 0) {
3059                 union dwc3_event event;
3060
3061                 event.raw = *(u32 *) (evt->cache + evt->lpos);
3062
3063                 dwc3_process_event_entry(dwc, &event);
3064
3065                 /*
3066                  * FIXME we wrap around correctly to the next entry as
3067                  * almost all entries are 4 bytes in size. There is one
3068                  * entry which has 12 bytes which is a regular entry
3069                  * followed by 8 bytes data. ATM I don't know how
3070                  * things are organized if we get next to the a
3071                  * boundary so I worry about that once we try to handle
3072                  * that.
3073                  */
3074                 evt->lpos = (evt->lpos + 4) % evt->length;
3075                 left -= 4;
3076         }
3077
3078         evt->count = 0;
3079         evt->flags &= ~DWC3_EVENT_PENDING;
3080         ret = IRQ_HANDLED;
3081
3082         /* Unmask interrupt */
3083         reg = dwc3_readl(dwc->regs, DWC3_GEVNTSIZ(0));
3084         reg &= ~DWC3_GEVNTSIZ_INTMASK;
3085         dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0), reg);
3086
3087         if (dwc->imod_interval) {
3088                 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), DWC3_GEVNTCOUNT_EHB);
3089                 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), dwc->imod_interval);
3090         }
3091
3092         return ret;
3093 }
3094
3095 static irqreturn_t dwc3_thread_interrupt(int irq, void *_evt)
3096 {
3097         struct dwc3_event_buffer *evt = _evt;
3098         struct dwc3 *dwc = evt->dwc;
3099         unsigned long flags;
3100         irqreturn_t ret = IRQ_NONE;
3101
3102         spin_lock_irqsave(&dwc->lock, flags);
3103         ret = dwc3_process_event_buf(evt);
3104         spin_unlock_irqrestore(&dwc->lock, flags);
3105
3106         return ret;
3107 }
3108
3109 static irqreturn_t dwc3_check_event_buf(struct dwc3_event_buffer *evt)
3110 {
3111         struct dwc3 *dwc = evt->dwc;
3112         u32 amount;
3113         u32 count;
3114         u32 reg;
3115
3116         if (pm_runtime_suspended(dwc->dev)) {
3117                 pm_runtime_get(dwc->dev);
3118                 disable_irq_nosync(dwc->irq_gadget);
3119                 dwc->pending_events = true;
3120                 return IRQ_HANDLED;
3121         }
3122
3123         /*
3124          * With PCIe legacy interrupt, test shows that top-half irq handler can
3125          * be called again after HW interrupt deassertion. Check if bottom-half
3126          * irq event handler completes before caching new event to prevent
3127          * losing events.
3128          */
3129         if (evt->flags & DWC3_EVENT_PENDING)
3130                 return IRQ_HANDLED;
3131
3132         count = dwc3_readl(dwc->regs, DWC3_GEVNTCOUNT(0));
3133         count &= DWC3_GEVNTCOUNT_MASK;
3134         if (!count)
3135                 return IRQ_NONE;
3136
3137         evt->count = count;
3138         evt->flags |= DWC3_EVENT_PENDING;
3139
3140         /* Mask interrupt */
3141         reg = dwc3_readl(dwc->regs, DWC3_GEVNTSIZ(0));
3142         reg |= DWC3_GEVNTSIZ_INTMASK;
3143         dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0), reg);
3144
3145         amount = min(count, evt->length - evt->lpos);
3146         memcpy(evt->cache + evt->lpos, evt->buf + evt->lpos, amount);
3147
3148         if (amount < count)
3149                 memcpy(evt->cache, evt->buf, count - amount);
3150
3151         dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), count);
3152
3153         return IRQ_WAKE_THREAD;
3154 }
3155
3156 static irqreturn_t dwc3_interrupt(int irq, void *_evt)
3157 {
3158         struct dwc3_event_buffer        *evt = _evt;
3159
3160         return dwc3_check_event_buf(evt);
3161 }
3162
3163 static int dwc3_gadget_get_irq(struct dwc3 *dwc)
3164 {
3165         struct platform_device *dwc3_pdev = to_platform_device(dwc->dev);
3166         int irq;
3167
3168         irq = platform_get_irq_byname(dwc3_pdev, "peripheral");
3169         if (irq > 0)
3170                 goto out;
3171
3172         if (irq == -EPROBE_DEFER)
3173                 goto out;
3174
3175         irq = platform_get_irq_byname(dwc3_pdev, "dwc_usb3");
3176         if (irq > 0)
3177                 goto out;
3178
3179         if (irq == -EPROBE_DEFER)
3180                 goto out;
3181
3182         irq = platform_get_irq(dwc3_pdev, 0);
3183         if (irq > 0)
3184                 goto out;
3185
3186         if (irq != -EPROBE_DEFER)
3187                 dev_err(dwc->dev, "missing peripheral IRQ\n");
3188
3189         if (!irq)
3190                 irq = -EINVAL;
3191
3192 out:
3193         return irq;
3194 }
3195
3196 /**
3197  * dwc3_gadget_init - initializes gadget related registers
3198  * @dwc: pointer to our controller context structure
3199  *
3200  * Returns 0 on success otherwise negative errno.
3201  */
3202 int dwc3_gadget_init(struct dwc3 *dwc)
3203 {
3204         int ret;
3205         int irq;
3206
3207         irq = dwc3_gadget_get_irq(dwc);
3208         if (irq < 0) {
3209                 ret = irq;
3210                 goto err0;
3211         }
3212
3213         dwc->irq_gadget = irq;
3214
3215         dwc->ep0_trb = dma_alloc_coherent(dwc->sysdev,
3216                                           sizeof(*dwc->ep0_trb) * 2,
3217                                           &dwc->ep0_trb_addr, GFP_KERNEL);
3218         if (!dwc->ep0_trb) {
3219                 dev_err(dwc->dev, "failed to allocate ep0 trb\n");
3220                 ret = -ENOMEM;
3221                 goto err0;
3222         }
3223
3224         dwc->setup_buf = kzalloc(DWC3_EP0_SETUP_SIZE, GFP_KERNEL);
3225         if (!dwc->setup_buf) {
3226                 ret = -ENOMEM;
3227                 goto err1;
3228         }
3229
3230         dwc->bounce = dma_alloc_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE,
3231                         &dwc->bounce_addr, GFP_KERNEL);
3232         if (!dwc->bounce) {
3233                 ret = -ENOMEM;
3234                 goto err2;
3235         }
3236
3237         init_completion(&dwc->ep0_in_setup);
3238
3239         dwc->gadget.ops                 = &dwc3_gadget_ops;
3240         dwc->gadget.speed               = USB_SPEED_UNKNOWN;
3241         dwc->gadget.sg_supported        = true;
3242         dwc->gadget.name                = "dwc3-gadget";
3243         dwc->gadget.is_otg              = dwc->dr_mode == USB_DR_MODE_OTG;
3244
3245         /*
3246          * FIXME We might be setting max_speed to <SUPER, however versions
3247          * <2.20a of dwc3 have an issue with metastability (documented
3248          * elsewhere in this driver) which tells us we can't set max speed to
3249          * anything lower than SUPER.
3250          *
3251          * Because gadget.max_speed is only used by composite.c and function
3252          * drivers (i.e. it won't go into dwc3's registers) we are allowing this
3253          * to happen so we avoid sending SuperSpeed Capability descriptor
3254          * together with our BOS descriptor as that could confuse host into
3255          * thinking we can handle super speed.
3256          *
3257          * Note that, in fact, we won't even support GetBOS requests when speed
3258          * is less than super speed because we don't have means, yet, to tell
3259          * composite.c that we are USB 2.0 + LPM ECN.
3260          */
3261         if (dwc->revision < DWC3_REVISION_220A &&
3262             !dwc->dis_metastability_quirk)
3263                 dev_info(dwc->dev, "changing max_speed on rev %08x\n",
3264                                 dwc->revision);
3265
3266         dwc->gadget.max_speed           = dwc->maximum_speed;
3267
3268         /*
3269          * REVISIT: Here we should clear all pending IRQs to be
3270          * sure we're starting from a well known location.
3271          */
3272
3273         ret = dwc3_gadget_init_endpoints(dwc, dwc->num_eps);
3274         if (ret)
3275                 goto err3;
3276
3277         ret = usb_add_gadget_udc(dwc->dev, &dwc->gadget);
3278         if (ret) {
3279                 dev_err(dwc->dev, "failed to register udc\n");
3280                 goto err4;
3281         }
3282
3283         return 0;
3284
3285 err4:
3286         dwc3_gadget_free_endpoints(dwc);
3287
3288 err3:
3289         dma_free_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE, dwc->bounce,
3290                         dwc->bounce_addr);
3291
3292 err2:
3293         kfree(dwc->setup_buf);
3294
3295 err1:
3296         dma_free_coherent(dwc->sysdev, sizeof(*dwc->ep0_trb) * 2,
3297                         dwc->ep0_trb, dwc->ep0_trb_addr);
3298
3299 err0:
3300         return ret;
3301 }
3302
3303 /* -------------------------------------------------------------------------- */
3304
3305 void dwc3_gadget_exit(struct dwc3 *dwc)
3306 {
3307         usb_del_gadget_udc(&dwc->gadget);
3308         dwc3_gadget_free_endpoints(dwc);
3309         dma_free_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE, dwc->bounce,
3310                           dwc->bounce_addr);
3311         kfree(dwc->setup_buf);
3312         dma_free_coherent(dwc->sysdev, sizeof(*dwc->ep0_trb) * 2,
3313                           dwc->ep0_trb, dwc->ep0_trb_addr);
3314 }
3315
3316 int dwc3_gadget_suspend(struct dwc3 *dwc)
3317 {
3318         if (!dwc->gadget_driver)
3319                 return 0;
3320
3321         dwc3_gadget_run_stop(dwc, false, false);
3322         dwc3_disconnect_gadget(dwc);
3323         __dwc3_gadget_stop(dwc);
3324
3325         return 0;
3326 }
3327
3328 int dwc3_gadget_resume(struct dwc3 *dwc)
3329 {
3330         int                     ret;
3331
3332         if (!dwc->gadget_driver)
3333                 return 0;
3334
3335         ret = __dwc3_gadget_start(dwc);
3336         if (ret < 0)
3337                 goto err0;
3338
3339         ret = dwc3_gadget_run_stop(dwc, true, false);
3340         if (ret < 0)
3341                 goto err1;
3342
3343         return 0;
3344
3345 err1:
3346         __dwc3_gadget_stop(dwc);
3347
3348 err0:
3349         return ret;
3350 }
3351
3352 void dwc3_gadget_process_pending_events(struct dwc3 *dwc)
3353 {
3354         if (dwc->pending_events) {
3355                 dwc3_interrupt(dwc->irq_gadget, dwc->ev_buf);
3356                 dwc->pending_events = false;
3357                 enable_irq(dwc->irq_gadget);
3358         }
3359 }