drivers/usb/dwc3/gadget.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * gadget.c - DesignWare USB3 DRD Controller Gadget Framework Link
   4  *
   5  * Copyright (C) 2010-2011 Texas Instruments Incorporated - https://www.ti.com
   6  *
   7  * Authors: Felipe Balbi <balbi@ti.com>,
   8  *          Sebastian Andrzej Siewior <bigeasy@linutronix.de>
   9  */
  10
  11 #include <linux/kernel.h>
  12 #include <linux/delay.h>
  13 #include <linux/slab.h>
  14 #include <linux/spinlock.h>
  15 #include <linux/platform_device.h>
  16 #include <linux/pm_runtime.h>
  17 #include <linux/interrupt.h>
  18 #include <linux/io.h>
  19 #include <linux/list.h>
  20 #include <linux/dma-mapping.h>
  21
  22 #include <linux/usb/ch9.h>
  23 #include <linux/usb/gadget.h>
  24
  25 #include "debug.h"
  26 #include "core.h"
  27 #include "gadget.h"
  28 #include "io.h"
  29
  30 #define DWC3_ALIGN_FRAME(d, n)  (((d)->frame_number + ((d)->interval * (n))) \
  31                                         & ~((d)->interval - 1))
  32
  33 /**
  34  * dwc3_gadget_set_test_mode - enables usb2 test modes
  35  * @dwc: pointer to our context structure
  36  * @mode: the mode to set (J, K SE0 NAK, Force Enable)
  37  *
  38  * Caller should take care of locking. This function will return 0 on
  39  * success or -EINVAL if wrong Test Selector is passed.
  40  */
  41 int dwc3_gadget_set_test_mode(struct dwc3 *dwc, int mode)
  42 {
  43         u32             reg;
  44
  45         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
  46         reg &= ~DWC3_DCTL_TSTCTRL_MASK;
  47
  48         switch (mode) {
  49         case USB_TEST_J:
  50         case USB_TEST_K:
  51         case USB_TEST_SE0_NAK:
  52         case USB_TEST_PACKET:
  53         case USB_TEST_FORCE_ENABLE:
  54                 reg |= mode << 1;
  55                 break;
  56         default:
  57                 return -EINVAL;
  58         }
  59
  60         dwc3_gadget_dctl_write_safe(dwc, reg);
  61
  62         return 0;
  63 }
  64
  65 /**
  66  * dwc3_gadget_get_link_state - gets current state of usb link
  67  * @dwc: pointer to our context structure
  68  *
  69  * Caller should take care of locking. This function will
  70  * return the link state on success (>= 0) or -ETIMEDOUT.
  71  */
  72 int dwc3_gadget_get_link_state(struct dwc3 *dwc)
  73 {
  74         u32             reg;
  75
  76         reg = dwc3_readl(dwc->regs, DWC3_DSTS);
  77
  78         return DWC3_DSTS_USBLNKST(reg);
  79 }
  80
  81 /**
  82  * dwc3_gadget_set_link_state - sets usb link to a particular state
  83  * @dwc: pointer to our context structure
  84  * @state: the state to put link into
  85  *
  86  * Caller should take care of locking. This function will
  87  * return 0 on success or -ETIMEDOUT.
  88  */
  89 int dwc3_gadget_set_link_state(struct dwc3 *dwc, enum dwc3_link_state state)
  90 {
  91         int             retries = 10000;
  92         u32             reg;
  93
  94         /*
  95          * Wait until device controller is ready. Only applies to 1.94a and
  96          * later RTL.
  97          */
  98         if (!DWC3_VER_IS_PRIOR(DWC3, 194A)) {
  99                 while (--retries) {
 100                         reg = dwc3_readl(dwc->regs, DWC3_DSTS);
 101                         if (reg & DWC3_DSTS_DCNRD)
 102                                 udelay(5);
 103                         else
 104                                 break;
 105                 }
 106
 107                 if (retries <= 0)
 108                         return -ETIMEDOUT;
 109         }
 110
 111         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
 112         reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;
 113
 114         /* set no action before sending new link state change */
 115         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
 116
 117         /* set requested state */
 118         reg |= DWC3_DCTL_ULSTCHNGREQ(state);
 119         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
 120
 121         /*
 122          * The following code is racy when called from dwc3_gadget_wakeup,
 123          * and is not needed, at least on newer versions
 124          */
 125         if (!DWC3_VER_IS_PRIOR(DWC3, 194A))
 126                 return 0;
 127
 128         /* wait for a change in DSTS */
 129         retries = 10000;
 130         while (--retries) {
 131                 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
 132
 133                 if (DWC3_DSTS_USBLNKST(reg) == state)
 134                         return 0;
 135
 136                 udelay(5);
 137         }
 138
 139         return -ETIMEDOUT;
 140 }
 141
 142 /**
 143  * dwc3_ep_inc_trb - increment a trb index.
 144  * @index: Pointer to the TRB index to increment.
 145  *
 146  * The index should never point to the link TRB. After incrementing,
 147  * if it is point to the link TRB, wrap around to the beginning. The
 148  * link TRB is always at the last TRB entry.
 149  */
 150 static void dwc3_ep_inc_trb(u8 *index)
 151 {
 152         (*index)++;
 153         if (*index == (DWC3_TRB_NUM - 1))
 154                 *index = 0;
 155 }
 156
 157 /**
 158  * dwc3_ep_inc_enq - increment endpoint's enqueue pointer
 159  * @dep: The endpoint whose enqueue pointer we're incrementing
 160  */
 161 static void dwc3_ep_inc_enq(struct dwc3_ep *dep)
 162 {
 163         dwc3_ep_inc_trb(&dep->trb_enqueue);
 164 }
 165
 166 /**
 167  * dwc3_ep_inc_deq - increment endpoint's dequeue pointer
 168  * @dep: The endpoint whose enqueue pointer we're incrementing
 169  */
 170 static void dwc3_ep_inc_deq(struct dwc3_ep *dep)
 171 {
 172         dwc3_ep_inc_trb(&dep->trb_dequeue);
 173 }
 174
 175 static void dwc3_gadget_del_and_unmap_request(struct dwc3_ep *dep,
 176                 struct dwc3_request *req, int status)
 177 {
 178         struct dwc3                     *dwc = dep->dwc;
 179
 180         list_del(&req->list);
 181         req->remaining = 0;
 182         req->needs_extra_trb = false;
 183
 184         if (req->request.status == -EINPROGRESS)
 185                 req->request.status = status;
 186
 187         if (req->trb)
 188                 usb_gadget_unmap_request_by_dev(dwc->sysdev,
 189                                 &req->request, req->direction);
 190
 191         req->trb = NULL;
 192         trace_dwc3_gadget_giveback(req);
 193
 194         if (dep->number > 1)
 195                 pm_runtime_put(dwc->dev);
 196 }
 197
 198 /**
 199  * dwc3_gadget_giveback - call struct usb_request's ->complete callback
 200  * @dep: The endpoint to whom the request belongs to
 201  * @req: The request we're giving back
 202  * @status: completion code for the request
 203  *
 204  * Must be called with controller's lock held and interrupts disabled. This
 205  * function will unmap @req and call its ->complete() callback to notify upper
 206  * layers that it has completed.
 207  */
 208 void dwc3_gadget_giveback(struct dwc3_ep *dep, struct dwc3_request *req,
 209                 int status)
 210 {
 211         struct dwc3                     *dwc = dep->dwc;
 212
 213         dwc3_gadget_del_and_unmap_request(dep, req, status);
 214         req->status = DWC3_REQUEST_STATUS_COMPLETED;
 215
 216         spin_unlock(&dwc->lock);
 217         usb_gadget_giveback_request(&dep->endpoint, &req->request);
 218         spin_lock(&dwc->lock);
 219 }
 220
 221 /**
 222  * dwc3_send_gadget_generic_command - issue a generic command for the controller
 223  * @dwc: pointer to the controller context
 224  * @cmd: the command to be issued
 225  * @param: command parameter
 226  *
 227  * Caller should take care of locking. Issue @cmd with a given @param to @dwc
 228  * and wait for its completion.
 229  */
 230 int dwc3_send_gadget_generic_command(struct dwc3 *dwc, unsigned int cmd,
 231                 u32 param)
 232 {
 233         u32             timeout = 500;
 234         int             status = 0;
 235         int             ret = 0;
 236         u32             reg;
 237
 238         dwc3_writel(dwc->regs, DWC3_DGCMDPAR, param);
 239         dwc3_writel(dwc->regs, DWC3_DGCMD, cmd | DWC3_DGCMD_CMDACT);
 240
 241         do {
 242                 reg = dwc3_readl(dwc->regs, DWC3_DGCMD);
 243                 if (!(reg & DWC3_DGCMD_CMDACT)) {
 244                         status = DWC3_DGCMD_STATUS(reg);
 245                         if (status)
 246                                 ret = -EINVAL;
 247                         break;
 248                 }
 249         } while (--timeout);
 250
 251         if (!timeout) {
 252                 ret = -ETIMEDOUT;
 253                 status = -ETIMEDOUT;
 254         }
 255
 256         trace_dwc3_gadget_generic_cmd(cmd, param, status);
 257
 258         return ret;
 259 }
 260
 261 static int __dwc3_gadget_wakeup(struct dwc3 *dwc);
 262
 263 /**
 264  * dwc3_send_gadget_ep_cmd - issue an endpoint command
 265  * @dep: the endpoint to which the command is going to be issued
 266  * @cmd: the command to be issued
 267  * @params: parameters to the command
 268  *
 269  * Caller should handle locking. This function will issue @cmd with given
 270  * @params to @dep and wait for its completion.
 271  */
 272 int dwc3_send_gadget_ep_cmd(struct dwc3_ep *dep, unsigned int cmd,
 273                 struct dwc3_gadget_ep_cmd_params *params)
 274 {
 275         const struct usb_endpoint_descriptor *desc = dep->endpoint.desc;
 276         struct dwc3             *dwc = dep->dwc;
 277         u32                     timeout = 5000;
 278         u32                     saved_config = 0;
 279         u32                     reg;
 280
 281         int                     cmd_status = 0;
 282         int                     ret = -EINVAL;
 283
 284         /*
 285          * When operating in USB 2.0 speeds (HS/FS), if GUSB2PHYCFG.ENBLSLPM or
 286          * GUSB2PHYCFG.SUSPHY is set, it must be cleared before issuing an
 287          * endpoint command.
 288          *
 289          * Save and clear both GUSB2PHYCFG.ENBLSLPM and GUSB2PHYCFG.SUSPHY
 290          * settings. Restore them after the command is completed.
 291          *
 292          * DWC_usb3 3.30a and DWC_usb31 1.90a programming guide section 3.2.2
 293          */
 294         if (dwc->gadget->speed <= USB_SPEED_HIGH) {
 295                 reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0));
 296                 if (unlikely(reg & DWC3_GUSB2PHYCFG_SUSPHY)) {
 297                         saved_config |= DWC3_GUSB2PHYCFG_SUSPHY;
 298                         reg &= ~DWC3_GUSB2PHYCFG_SUSPHY;
 299                 }
 300
 301                 if (reg & DWC3_GUSB2PHYCFG_ENBLSLPM) {
 302                         saved_config |= DWC3_GUSB2PHYCFG_ENBLSLPM;
 303                         reg &= ~DWC3_GUSB2PHYCFG_ENBLSLPM;
 304                 }
 305
 306                 if (saved_config)
 307                         dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg);
 308         }
 309
 310         if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_STARTTRANSFER) {
 311                 int link_state;
 312
 313                 /*
 314                  * Initiate remote wakeup if the link state is in U3 when
 315                  * operating in SS/SSP or L1/L2 when operating in HS/FS. If the
 316                  * link state is in U1/U2, no remote wakeup is needed. The Start
 317                  * Transfer command will initiate the link recovery.
 318                  */
 319                 link_state = dwc3_gadget_get_link_state(dwc);
 320                 switch (link_state) {
 321                 case DWC3_LINK_STATE_U2:
 322                         if (dwc->gadget->speed >= USB_SPEED_SUPER)
 323                                 break;
 324
 325                         fallthrough;
 326                 case DWC3_LINK_STATE_U3:
 327                         ret = __dwc3_gadget_wakeup(dwc);
 328                         dev_WARN_ONCE(dwc->dev, ret, "wakeup failed --> %d\n",
 329                                         ret);
 330                         break;
 331                 }
 332         }
 333
 334         /*
 335          * For some commands such as Update Transfer command, DEPCMDPARn
 336          * registers are reserved. Since the driver often sends Update Transfer
 337          * command, don't write to DEPCMDPARn to avoid register write delays and
 338          * improve performance.
 339          */
 340         if (DWC3_DEPCMD_CMD(cmd) != DWC3_DEPCMD_UPDATETRANSFER) {
 341                 dwc3_writel(dep->regs, DWC3_DEPCMDPAR0, params->param0);
 342                 dwc3_writel(dep->regs, DWC3_DEPCMDPAR1, params->param1);
 343                 dwc3_writel(dep->regs, DWC3_DEPCMDPAR2, params->param2);
 344         }
 345
 346         /*
 347          * Synopsys Databook 2.60a states in section 6.3.2.5.6 of that if we're
 348          * not relying on XferNotReady, we can make use of a special "No
 349          * Response Update Transfer" command where we should clear both CmdAct
 350          * and CmdIOC bits.
 351          *
 352          * With this, we don't need to wait for command completion and can
 353          * straight away issue further commands to the endpoint.
 354          *
 355          * NOTICE: We're making an assumption that control endpoints will never
 356          * make use of Update Transfer command. This is a safe assumption
 357          * because we can never have more than one request at a time with
 358          * Control Endpoints. If anybody changes that assumption, this chunk
 359          * needs to be updated accordingly.
 360          */
 361         if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_UPDATETRANSFER &&
 362                         !usb_endpoint_xfer_isoc(desc))
 363                 cmd &= ~(DWC3_DEPCMD_CMDIOC | DWC3_DEPCMD_CMDACT);
 364         else
 365                 cmd |= DWC3_DEPCMD_CMDACT;
 366
 367         dwc3_writel(dep->regs, DWC3_DEPCMD, cmd);
 368
 369         if (!(cmd & DWC3_DEPCMD_CMDACT)) {
 370                 ret = 0;
 371                 goto skip_status;
 372         }
 373
 374         do {
 375                 reg = dwc3_readl(dep->regs, DWC3_DEPCMD);
 376                 if (!(reg & DWC3_DEPCMD_CMDACT)) {
 377                         cmd_status = DWC3_DEPCMD_STATUS(reg);
 378
 379                         switch (cmd_status) {
 380                         case 0:
 381                                 ret = 0;
 382                                 break;
 383                         case DEPEVT_TRANSFER_NO_RESOURCE:
 384                                 dev_WARN(dwc->dev, "No resource for %s\n",
 385                                          dep->name);
 386                                 ret = -EINVAL;
 387                                 break;
 388                         case DEPEVT_TRANSFER_BUS_EXPIRY:
 389                                 /*
 390                                  * SW issues START TRANSFER command to
 391                                  * isochronous ep with future frame interval. If
 392                                  * future interval time has already passed when
 393                                  * core receives the command, it will respond
 394                                  * with an error status of 'Bus Expiry'.
 395                                  *
 396                                  * Instead of always returning -EINVAL, let's
 397                                  * give a hint to the gadget driver that this is
 398                                  * the case by returning -EAGAIN.
 399                                  */
 400                                 ret = -EAGAIN;
 401                                 break;
 402                         default:
 403                                 dev_WARN(dwc->dev, "UNKNOWN cmd status\n");
 404                         }
 405
 406                         break;
 407                 }
 408         } while (--timeout);
 409
 410         if (timeout == 0) {
 411                 ret = -ETIMEDOUT;
 412                 cmd_status = -ETIMEDOUT;
 413         }
 414
 415 skip_status:
 416         trace_dwc3_gadget_ep_cmd(dep, cmd, params, cmd_status);
 417
 418         if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_STARTTRANSFER) {
 419                 if (ret == 0)
 420                         dep->flags |= DWC3_EP_TRANSFER_STARTED;
 421
 422                 if (ret != -ETIMEDOUT)
 423                         dwc3_gadget_ep_get_transfer_index(dep);
 424         }
 425
 426         if (saved_config) {
 427                 reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0));
 428                 reg |= saved_config;
 429                 dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg);
 430         }
 431
 432         return ret;
 433 }
 434
 435 static int dwc3_send_clear_stall_ep_cmd(struct dwc3_ep *dep)
 436 {
 437         struct dwc3 *dwc = dep->dwc;
 438         struct dwc3_gadget_ep_cmd_params params;
 439         u32 cmd = DWC3_DEPCMD_CLEARSTALL;
 440
 441         /*
 442          * As of core revision 2.60a the recommended programming model
 443          * is to set the ClearPendIN bit when issuing a Clear Stall EP
 444          * command for IN endpoints. This is to prevent an issue where
 445          * some (non-compliant) hosts may not send ACK TPs for pending
 446          * IN transfers due to a mishandled error condition. Synopsys
 447          * STAR 9000614252.
 448          */
 449         if (dep->direction &&
 450             !DWC3_VER_IS_PRIOR(DWC3, 260A) &&
 451             (dwc->gadget->speed >= USB_SPEED_SUPER))
 452                 cmd |= DWC3_DEPCMD_CLEARPENDIN;
 453
 454         memset(&params, 0, sizeof(params));
 455
 456         return dwc3_send_gadget_ep_cmd(dep, cmd, &params);
 457 }
 458
 459 static dma_addr_t dwc3_trb_dma_offset(struct dwc3_ep *dep,
 460                 struct dwc3_trb *trb)
 461 {
 462         u32             offset = (char *) trb - (char *) dep->trb_pool;
 463
 464         return dep->trb_pool_dma + offset;
 465 }
 466
 467 static int dwc3_alloc_trb_pool(struct dwc3_ep *dep)
 468 {
 469         struct dwc3             *dwc = dep->dwc;
 470
 471         if (dep->trb_pool)
 472                 return 0;
 473
 474         dep->trb_pool = dma_alloc_coherent(dwc->sysdev,
 475                         sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
 476                         &dep->trb_pool_dma, GFP_KERNEL);
 477         if (!dep->trb_pool) {
 478                 dev_err(dep->dwc->dev, "failed to allocate trb pool for %s\n",
 479                                 dep->name);
 480                 return -ENOMEM;
 481         }
 482
 483         return 0;
 484 }
 485
 486 static void dwc3_free_trb_pool(struct dwc3_ep *dep)
 487 {
 488         struct dwc3             *dwc = dep->dwc;
 489
 490         dma_free_coherent(dwc->sysdev, sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
 491                         dep->trb_pool, dep->trb_pool_dma);
 492
 493         dep->trb_pool = NULL;
 494         dep->trb_pool_dma = 0;
 495 }
 496
 497 static int dwc3_gadget_set_xfer_resource(struct dwc3_ep *dep)
 498 {
 499         struct dwc3_gadget_ep_cmd_params params;
 500
 501         memset(&params, 0x00, sizeof(params));
 502
 503         params.param0 = DWC3_DEPXFERCFG_NUM_XFER_RES(1);
 504
 505         return dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETTRANSFRESOURCE,
 506                         &params);
 507 }
 508
 509 /**
 510  * dwc3_gadget_start_config - configure ep resources
 511  * @dep: endpoint that is being enabled
 512  *
 513  * Issue a %DWC3_DEPCMD_DEPSTARTCFG command to @dep. After the command's
 514  * completion, it will set Transfer Resource for all available endpoints.
 515  *
 516  * The assignment of transfer resources cannot perfectly follow the data book
 517  * due to the fact that the controller driver does not have all knowledge of the
 518  * configuration in advance. It is given this information piecemeal by the
 519  * composite gadget framework after every SET_CONFIGURATION and
 520  * SET_INTERFACE. Trying to follow the databook programming model in this
 521  * scenario can cause errors. For two reasons:
 522  *
 523  * 1) The databook says to do %DWC3_DEPCMD_DEPSTARTCFG for every
 524  * %USB_REQ_SET_CONFIGURATION and %USB_REQ_SET_INTERFACE (8.1.5). This is
 525  * incorrect in the scenario of multiple interfaces.
 526  *
 527  * 2) The databook does not mention doing more %DWC3_DEPCMD_DEPXFERCFG for new
 528  * endpoint on alt setting (8.1.6).
 529  *
 530  * The following simplified method is used instead:
 531  *
 532  * All hardware endpoints can be assigned a transfer resource and this setting
 533  * will stay persistent until either a core reset or hibernation. So whenever we
 534  * do a %DWC3_DEPCMD_DEPSTARTCFG(0) we can go ahead and do
 535  * %DWC3_DEPCMD_DEPXFERCFG for every hardware endpoint as well. We are
 536  * guaranteed that there are as many transfer resources as endpoints.
 537  *
 538  * This function is called for each endpoint when it is being enabled but is
 539  * triggered only when called for EP0-out, which always happens first, and which
 540  * should only happen in one of the above conditions.
 541  */
 542 static int dwc3_gadget_start_config(struct dwc3_ep *dep)
 543 {
 544         struct dwc3_gadget_ep_cmd_params params;
 545         struct dwc3             *dwc;
 546         u32                     cmd;
 547         int                     i;
 548         int                     ret;
 549
 550         if (dep->number)
 551                 return 0;
 552
 553         memset(&params, 0x00, sizeof(params));
 554         cmd = DWC3_DEPCMD_DEPSTARTCFG;
 555         dwc = dep->dwc;
 556
 557         ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
 558         if (ret)
 559                 return ret;
 560
 561         for (i = 0; i < DWC3_ENDPOINTS_NUM; i++) {
 562                 struct dwc3_ep *dep = dwc->eps[i];
 563
 564                 if (!dep)
 565                         continue;
 566
 567                 ret = dwc3_gadget_set_xfer_resource(dep);
 568                 if (ret)
 569                         return ret;
 570         }
 571
 572         return 0;
 573 }
 574
 575 static int dwc3_gadget_set_ep_config(struct dwc3_ep *dep, unsigned int action)
 576 {
 577         const struct usb_ss_ep_comp_descriptor *comp_desc;
 578         const struct usb_endpoint_descriptor *desc;
 579         struct dwc3_gadget_ep_cmd_params params;
 580         struct dwc3 *dwc = dep->dwc;
 581
 582         comp_desc = dep->endpoint.comp_desc;
 583         desc = dep->endpoint.desc;
 584
 585         memset(&params, 0x00, sizeof(params));
 586
 587         params.param0 = DWC3_DEPCFG_EP_TYPE(usb_endpoint_type(desc))
 588                 | DWC3_DEPCFG_MAX_PACKET_SIZE(usb_endpoint_maxp(desc));
 589
 590         /* Burst size is only needed in SuperSpeed mode */
 591         if (dwc->gadget->speed >= USB_SPEED_SUPER) {
 592                 u32 burst = dep->endpoint.maxburst;
 593
 594                 params.param0 |= DWC3_DEPCFG_BURST_SIZE(burst - 1);
 595         }
 596
 597         params.param0 |= action;
 598         if (action == DWC3_DEPCFG_ACTION_RESTORE)
 599                 params.param2 |= dep->saved_state;
 600
 601         if (usb_endpoint_xfer_control(desc))
 602                 params.param1 = DWC3_DEPCFG_XFER_COMPLETE_EN;
 603
 604         if (dep->number <= 1 || usb_endpoint_xfer_isoc(desc))
 605                 params.param1 |= DWC3_DEPCFG_XFER_NOT_READY_EN;
 606
 607         if (usb_ss_max_streams(comp_desc) && usb_endpoint_xfer_bulk(desc)) {
 608                 params.param1 |= DWC3_DEPCFG_STREAM_CAPABLE
 609                         | DWC3_DEPCFG_XFER_COMPLETE_EN
 610                         | DWC3_DEPCFG_STREAM_EVENT_EN;
 611                 dep->stream_capable = true;
 612         }
 613
 614         if (!usb_endpoint_xfer_control(desc))
 615                 params.param1 |= DWC3_DEPCFG_XFER_IN_PROGRESS_EN;
 616
 617         /*
 618          * We are doing 1:1 mapping for endpoints, meaning
 619          * Physical Endpoints 2 maps to Logical Endpoint 2 and
 620          * so on. We consider the direction bit as part of the physical
 621          * endpoint number. So USB endpoint 0x81 is 0x03.
 622          */
 623         params.param1 |= DWC3_DEPCFG_EP_NUMBER(dep->number);
 624
 625         /*
 626          * We must use the lower 16 TX FIFOs even though
 627          * HW might have more
 628          */
 629         if (dep->direction)
 630                 params.param0 |= DWC3_DEPCFG_FIFO_NUMBER(dep->number >> 1);
 631
 632         if (desc->bInterval) {
 633                 u8 bInterval_m1;
 634
 635                 /*
 636                  * Valid range for DEPCFG.bInterval_m1 is from 0 to 13.
 637                  *
 638                  * NOTE: The programming guide incorrectly stated bInterval_m1
 639                  * must be set to 0 when operating in fullspeed. Internally the
 640                  * controller does not have this limitation. See DWC_usb3x
 641                  * programming guide section 3.2.2.1.
 642                  */
 643                 bInterval_m1 = min_t(u8, desc->bInterval - 1, 13);
 644
 645                 if (usb_endpoint_type(desc) == USB_ENDPOINT_XFER_INT &&
 646                     dwc->gadget->speed == USB_SPEED_FULL)
 647                         dep->interval = desc->bInterval;
 648                 else
 649                         dep->interval = 1 << (desc->bInterval - 1);
 650
 651                 params.param1 |= DWC3_DEPCFG_BINTERVAL_M1(bInterval_m1);
 652         }
 653
 654         return dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETEPCONFIG, &params);
 655 }
 656
 657 /**
 658  * dwc3_gadget_calc_tx_fifo_size - calculates the txfifo size value
 659  * @dwc: pointer to the DWC3 context
 660  *
 661  * Calculates the size value based on the equation below:
 662  *
 663  * DWC3 revision 280A and prior:
 664  * fifo_size = mult * (max_packet / mdwidth) + 1;
 665  *
 666  * DWC3 revision 290A and onwards:
 667  * fifo_size = mult * ((max_packet + mdwidth)/mdwidth + 1) + 1
 668  *
 669  * The max packet size is set to 1024, as the txfifo requirements mainly apply
 670  * to super speed USB use cases.  However, it is safe to overestimate the fifo
 671  * allocations for other scenarios, i.e. high speed USB.
 672  */
 673 static int dwc3_gadget_calc_tx_fifo_size(struct dwc3 *dwc, int mult)
 674 {
 675         int max_packet = 1024;
 676         int fifo_size;
 677         int mdwidth;
 678
 679         mdwidth = dwc3_mdwidth(dwc);
 680
 681         /* MDWIDTH is represented in bits, we need it in bytes */
 682         mdwidth >>= 3;
 683
 684         if (DWC3_VER_IS_PRIOR(DWC3, 290A))
 685                 fifo_size = mult * (max_packet / mdwidth) + 1;
 686         else
 687                 fifo_size = mult * ((max_packet + mdwidth) / mdwidth) + 1;
 688         return fifo_size;
 689 }
 690
 691 /**
 692  * dwc3_gadget_clear_tx_fifos - Clears txfifo allocation
 693  * @dwc: pointer to the DWC3 context
 694  *
 695  * Iterates through all the endpoint registers and clears the previous txfifo
 696  * allocations.
 697  */
 698 void dwc3_gadget_clear_tx_fifos(struct dwc3 *dwc)
 699 {
 700         struct dwc3_ep *dep;
 701         int fifo_depth;
 702         int size;
 703         int num;
 704
 705         if (!dwc->do_fifo_resize)
 706                 return;
 707
 708         /* Read ep0IN related TXFIFO size */
 709         dep = dwc->eps[1];
 710         size = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(0));
 711         if (DWC3_IP_IS(DWC3))
 712                 fifo_depth = DWC3_GTXFIFOSIZ_TXFDEP(size);
 713         else
 714                 fifo_depth = DWC31_GTXFIFOSIZ_TXFDEP(size);
 715
 716         dwc->last_fifo_depth = fifo_depth;
 717         /* Clear existing TXFIFO for all IN eps except ep0 */
 718         for (num = 3; num < min_t(int, dwc->num_eps, DWC3_ENDPOINTS_NUM);
 719              num += 2) {
 720                 dep = dwc->eps[num];
 721                 /* Don't change TXFRAMNUM on usb31 version */
 722                 size = DWC3_IP_IS(DWC3) ? 0 :
 723                         dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(num >> 1)) &
 724                                    DWC31_GTXFIFOSIZ_TXFRAMNUM;
 725
 726                 dwc3_writel(dwc->regs, DWC3_GTXFIFOSIZ(num >> 1), size);
 727                 dep->flags &= ~DWC3_EP_TXFIFO_RESIZED;
 728         }
 729         dwc->num_ep_resized = 0;
 730 }
 731
 732 /*
 733  * dwc3_gadget_resize_tx_fifos - reallocate fifo spaces for current use-case
 734  * @dwc: pointer to our context structure
 735  *
 736  * This function will a best effort FIFO allocation in order
 737  * to improve FIFO usage and throughput, while still allowing
 738  * us to enable as many endpoints as possible.
 739  *
 740  * Keep in mind that this operation will be highly dependent
 741  * on the configured size for RAM1 - which contains TxFifo -,
 742  * the amount of endpoints enabled on coreConsultant tool, and
 743  * the width of the Master Bus.
 744  *
 745  * In general, FIFO depths are represented with the following equation:
 746  *
 747  * fifo_size = mult * ((max_packet + mdwidth)/mdwidth + 1) + 1
 748  *
 749  * In conjunction with dwc3_gadget_check_config(), this resizing logic will
 750  * ensure that all endpoints will have enough internal memory for one max
 751  * packet per endpoint.
 752  */
 753 static int dwc3_gadget_resize_tx_fifos(struct dwc3_ep *dep)
 754 {
 755         struct dwc3 *dwc = dep->dwc;
 756         int fifo_0_start;
 757         int ram1_depth;
 758         int fifo_size;
 759         int min_depth;
 760         int num_in_ep;
 761         int remaining;
 762         int num_fifos = 1;
 763         int fifo;
 764         int tmp;
 765
 766         if (!dwc->do_fifo_resize)
 767                 return 0;
 768
 769         /* resize IN endpoints except ep0 */
 770         if (!usb_endpoint_dir_in(dep->endpoint.desc) || dep->number <= 1)
 771                 return 0;
 772
 773         /* bail if already resized */
 774         if (dep->flags & DWC3_EP_TXFIFO_RESIZED)
 775                 return 0;
 776
 777         ram1_depth = DWC3_RAM1_DEPTH(dwc->hwparams.hwparams7);
 778
 779         if ((dep->endpoint.maxburst > 1 &&
 780              usb_endpoint_xfer_bulk(dep->endpoint.desc)) ||
 781             usb_endpoint_xfer_isoc(dep->endpoint.desc))
 782                 num_fifos = 3;
 783
 784         if (dep->endpoint.maxburst > 6 &&
 785             (usb_endpoint_xfer_bulk(dep->endpoint.desc) ||
 786              usb_endpoint_xfer_isoc(dep->endpoint.desc)) && DWC3_IP_IS(DWC31))
 787                 num_fifos = dwc->tx_fifo_resize_max_num;
 788
 789         /* FIFO size for a single buffer */
 790         fifo = dwc3_gadget_calc_tx_fifo_size(dwc, 1);
 791
 792         /* Calculate the number of remaining EPs w/o any FIFO */
 793         num_in_ep = dwc->max_cfg_eps;
 794         num_in_ep -= dwc->num_ep_resized;
 795
 796         /* Reserve at least one FIFO for the number of IN EPs */
 797         min_depth = num_in_ep * (fifo + 1);
 798         remaining = ram1_depth - min_depth - dwc->last_fifo_depth;
 799         remaining = max_t(int, 0, remaining);
 800         /*
 801          * We've already reserved 1 FIFO per EP, so check what we can fit in
 802          * addition to it.  If there is not enough remaining space, allocate
 803          * all the remaining space to the EP.
 804          */
 805         fifo_size = (num_fifos - 1) * fifo;
 806         if (remaining < fifo_size)
 807                 fifo_size = remaining;
 808
 809         fifo_size += fifo;
 810         /* Last increment according to the TX FIFO size equation */
 811         fifo_size++;
 812
 813         /* Check if TXFIFOs start at non-zero addr */
 814         tmp = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(0));
 815         fifo_0_start = DWC3_GTXFIFOSIZ_TXFSTADDR(tmp);
 816
 817         fifo_size |= (fifo_0_start + (dwc->last_fifo_depth << 16));
 818         if (DWC3_IP_IS(DWC3))
 819                 dwc->last_fifo_depth += DWC3_GTXFIFOSIZ_TXFDEP(fifo_size);
 820         else
 821                 dwc->last_fifo_depth += DWC31_GTXFIFOSIZ_TXFDEP(fifo_size);
 822
 823         /* Check fifo size allocation doesn't exceed available RAM size. */
 824         if (dwc->last_fifo_depth >= ram1_depth) {
 825                 dev_err(dwc->dev, "Fifosize(%d) > RAM size(%d) %s depth:%d\n",
 826                         dwc->last_fifo_depth, ram1_depth,
 827                         dep->endpoint.name, fifo_size);
 828                 if (DWC3_IP_IS(DWC3))
 829                         fifo_size = DWC3_GTXFIFOSIZ_TXFDEP(fifo_size);
 830                 else
 831                         fifo_size = DWC31_GTXFIFOSIZ_TXFDEP(fifo_size);
 832
 833                 dwc->last_fifo_depth -= fifo_size;
 834                 return -ENOMEM;
 835         }
 836
 837         dwc3_writel(dwc->regs, DWC3_GTXFIFOSIZ(dep->number >> 1), fifo_size);
 838         dep->flags |= DWC3_EP_TXFIFO_RESIZED;
 839         dwc->num_ep_resized++;
 840
 841         return 0;
 842 }
 843
 844 /**
 845  * __dwc3_gadget_ep_enable - initializes a hw endpoint
 846  * @dep: endpoint to be initialized
 847  * @action: one of INIT, MODIFY or RESTORE
 848  *
 849  * Caller should take care of locking. Execute all necessary commands to
 850  * initialize a HW endpoint so it can be used by a gadget driver.
 851  */
 852 static int __dwc3_gadget_ep_enable(struct dwc3_ep *dep, unsigned int action)
 853 {
 854         const struct usb_endpoint_descriptor *desc = dep->endpoint.desc;
 855         struct dwc3             *dwc = dep->dwc;
 856
 857         u32                     reg;
 858         int                     ret;
 859
 860         if (!(dep->flags & DWC3_EP_ENABLED)) {
 861                 ret = dwc3_gadget_resize_tx_fifos(dep);
 862                 if (ret)
 863                         return ret;
 864
 865                 ret = dwc3_gadget_start_config(dep);
 866                 if (ret)
 867                         return ret;
 868         }
 869
 870         ret = dwc3_gadget_set_ep_config(dep, action);
 871         if (ret)
 872                 return ret;
 873
 874         if (!(dep->flags & DWC3_EP_ENABLED)) {
 875                 struct dwc3_trb *trb_st_hw;
 876                 struct dwc3_trb *trb_link;
 877
 878                 dep->type = usb_endpoint_type(desc);
 879                 dep->flags |= DWC3_EP_ENABLED;
 880
 881                 reg = dwc3_readl(dwc->regs, DWC3_DALEPENA);
 882                 reg |= DWC3_DALEPENA_EP(dep->number);
 883                 dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);
 884
 885                 if (usb_endpoint_xfer_control(desc))
 886                         goto out;
 887
 888                 /* Initialize the TRB ring */
 889                 dep->trb_dequeue = 0;
 890                 dep->trb_enqueue = 0;
 891                 memset(dep->trb_pool, 0,
 892                        sizeof(struct dwc3_trb) * DWC3_TRB_NUM);
 893
 894                 /* Link TRB. The HWO bit is never reset */
 895                 trb_st_hw = &dep->trb_pool[0];
 896
 897                 trb_link = &dep->trb_pool[DWC3_TRB_NUM - 1];
 898                 trb_link->bpl = lower_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));
 899                 trb_link->bph = upper_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));
 900                 trb_link->ctrl |= DWC3_TRBCTL_LINK_TRB;
 901                 trb_link->ctrl |= DWC3_TRB_CTRL_HWO;
 902         }
 903
 904         /*
 905          * Issue StartTransfer here with no-op TRB so we can always rely on No
 906          * Response Update Transfer command.
 907          */
 908         if (usb_endpoint_xfer_bulk(desc) ||
 909                         usb_endpoint_xfer_int(desc)) {
 910                 struct dwc3_gadget_ep_cmd_params params;
 911                 struct dwc3_trb *trb;
 912                 dma_addr_t trb_dma;
 913                 u32 cmd;
 914
 915                 memset(&params, 0, sizeof(params));
 916                 trb = &dep->trb_pool[0];
 917                 trb_dma = dwc3_trb_dma_offset(dep, trb);
 918
 919                 params.param0 = upper_32_bits(trb_dma);
 920                 params.param1 = lower_32_bits(trb_dma);
 921
 922                 cmd = DWC3_DEPCMD_STARTTRANSFER;
 923
 924                 ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
 925                 if (ret < 0)
 926                         return ret;
 927
 928                 if (dep->stream_capable) {
 929                         /*
 930                          * For streams, at start, there maybe a race where the
 931                          * host primes the endpoint before the function driver
 932                          * queues a request to initiate a stream. In that case,
 933                          * the controller will not see the prime to generate the
 934                          * ERDY and start stream. To workaround this, issue a
 935                          * no-op TRB as normal, but end it immediately. As a
 936                          * result, when the function driver queues the request,
 937                          * the next START_TRANSFER command will cause the
 938                          * controller to generate an ERDY to initiate the
 939                          * stream.
 940                          */
 941                         dwc3_stop_active_transfer(dep, true, true);
 942
 943                         /*
 944                          * All stream eps will reinitiate stream on NoStream
 945                          * rejection until we can determine that the host can
 946                          * prime after the first transfer.
 947                          *
 948                          * However, if the controller is capable of
 949                          * TXF_FLUSH_BYPASS, then IN direction endpoints will
 950                          * automatically restart the stream without the driver
 951                          * initiation.
 952                          */
 953                         if (!dep->direction ||
 954                             !(dwc->hwparams.hwparams9 &
 955                               DWC3_GHWPARAMS9_DEV_TXF_FLUSH_BYPASS))
 956                                 dep->flags |= DWC3_EP_FORCE_RESTART_STREAM;
 957                 }
 958         }
 959
 960 out:
 961         trace_dwc3_gadget_ep_enable(dep);
 962
 963         return 0;
 964 }
 965
 966 static void dwc3_remove_requests(struct dwc3 *dwc, struct dwc3_ep *dep)
 967 {
 968         struct dwc3_request             *req;
 969
 970         dwc3_stop_active_transfer(dep, true, false);
 971
 972         /* - giveback all requests to gadget driver */
 973         while (!list_empty(&dep->started_list)) {
 974                 req = next_request(&dep->started_list);
 975
 976                 dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
 977         }
 978
 979         while (!list_empty(&dep->pending_list)) {
 980                 req = next_request(&dep->pending_list);
 981
 982                 dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
 983         }
 984
 985         while (!list_empty(&dep->cancelled_list)) {
 986                 req = next_request(&dep->cancelled_list);
 987
 988                 dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
 989         }
 990 }
 991
 992 /**
 993  * __dwc3_gadget_ep_disable - disables a hw endpoint
 994  * @dep: the endpoint to disable
 995  *
 996  * This function undoes what __dwc3_gadget_ep_enable did and also removes
 997  * requests which are currently being processed by the hardware and those which
 998  * are not yet scheduled.
 999  *
1000  * Caller should take care of locking.
1001  */
1002 static int __dwc3_gadget_ep_disable(struct dwc3_ep *dep)
1003 {
1004         struct dwc3             *dwc = dep->dwc;
1005         u32                     reg;
1006
1007         trace_dwc3_gadget_ep_disable(dep);
1008
1009         /* make sure HW endpoint isn't stalled */
1010         if (dep->flags & DWC3_EP_STALL)
1011                 __dwc3_gadget_ep_set_halt(dep, 0, false);
1012
1013         reg = dwc3_readl(dwc->regs, DWC3_DALEPENA);
1014         reg &= ~DWC3_DALEPENA_EP(dep->number);
1015         dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);
1016
1017         /* Clear out the ep descriptors for non-ep0 */
1018         if (dep->number > 1) {
1019                 dep->endpoint.comp_desc = NULL;
1020                 dep->endpoint.desc = NULL;
1021         }
1022
1023         dwc3_remove_requests(dwc, dep);
1024
1025         dep->stream_capable = false;
1026         dep->type = 0;
1027         dep->flags &= DWC3_EP_TXFIFO_RESIZED;
1028
1029         return 0;
1030 }
1031
1032 /* -------------------------------------------------------------------------- */
1033
1034 static int dwc3_gadget_ep0_enable(struct usb_ep *ep,
1035                 const struct usb_endpoint_descriptor *desc)
1036 {
1037         return -EINVAL;
1038 }
1039
1040 static int dwc3_gadget_ep0_disable(struct usb_ep *ep)
1041 {
1042         return -EINVAL;
1043 }
1044
1045 /* -------------------------------------------------------------------------- */
1046
1047 static int dwc3_gadget_ep_enable(struct usb_ep *ep,
1048                 const struct usb_endpoint_descriptor *desc)
1049 {
1050         struct dwc3_ep                  *dep;
1051         struct dwc3                     *dwc;
1052         unsigned long                   flags;
1053         int                             ret;
1054
1055         if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT) {
1056                 pr_debug("dwc3: invalid parameters\n");
1057                 return -EINVAL;
1058         }
1059
1060         if (!desc->wMaxPacketSize) {
1061                 pr_debug("dwc3: missing wMaxPacketSize\n");
1062                 return -EINVAL;
1063         }
1064
1065         dep = to_dwc3_ep(ep);
1066         dwc = dep->dwc;
1067
1068         if (dev_WARN_ONCE(dwc->dev, dep->flags & DWC3_EP_ENABLED,
1069                                         "%s is already enabled\n",
1070                                         dep->name))
1071                 return 0;
1072
1073         spin_lock_irqsave(&dwc->lock, flags);
1074         ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT);
1075         spin_unlock_irqrestore(&dwc->lock, flags);
1076
1077         return ret;
1078 }
1079
1080 static int dwc3_gadget_ep_disable(struct usb_ep *ep)
1081 {
1082         struct dwc3_ep                  *dep;
1083         struct dwc3                     *dwc;
1084         unsigned long                   flags;
1085         int                             ret;
1086
1087         if (!ep) {
1088                 pr_debug("dwc3: invalid parameters\n");
1089                 return -EINVAL;
1090         }
1091
1092         dep = to_dwc3_ep(ep);
1093         dwc = dep->dwc;
1094
1095         if (dev_WARN_ONCE(dwc->dev, !(dep->flags & DWC3_EP_ENABLED),
1096                                         "%s is already disabled\n",
1097                                         dep->name))
1098                 return 0;
1099
1100         spin_lock_irqsave(&dwc->lock, flags);
1101         ret = __dwc3_gadget_ep_disable(dep);
1102         spin_unlock_irqrestore(&dwc->lock, flags);
1103
1104         return ret;
1105 }
1106
1107 static struct usb_request *dwc3_gadget_ep_alloc_request(struct usb_ep *ep,
1108                 gfp_t gfp_flags)
1109 {
1110         struct dwc3_request             *req;
1111         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
1112
1113         req = kzalloc(sizeof(*req), gfp_flags);
1114         if (!req)
1115                 return NULL;
1116
1117         req->direction  = dep->direction;
1118         req->epnum      = dep->number;
1119         req->dep        = dep;
1120         req->status     = DWC3_REQUEST_STATUS_UNKNOWN;
1121
1122         trace_dwc3_alloc_request(req);
1123
1124         return &req->request;
1125 }
1126
1127 static void dwc3_gadget_ep_free_request(struct usb_ep *ep,
1128                 struct usb_request *request)
1129 {
1130         struct dwc3_request             *req = to_dwc3_request(request);
1131
1132         trace_dwc3_free_request(req);
1133         kfree(req);
1134 }
1135
1136 /**
1137  * dwc3_ep_prev_trb - returns the previous TRB in the ring
1138  * @dep: The endpoint with the TRB ring
1139  * @index: The index of the current TRB in the ring
1140  *
1141  * Returns the TRB prior to the one pointed to by the index. If the
1142  * index is 0, we will wrap backwards, skip the link TRB, and return
1143  * the one just before that.
1144  */
1145 static struct dwc3_trb *dwc3_ep_prev_trb(struct dwc3_ep *dep, u8 index)
1146 {
1147         u8 tmp = index;
1148
1149         if (!tmp)
1150                 tmp = DWC3_TRB_NUM - 1;
1151
1152         return &dep->trb_pool[tmp - 1];
1153 }
1154
1155 static u32 dwc3_calc_trbs_left(struct dwc3_ep *dep)
1156 {
1157         u8                      trbs_left;
1158
1159         /*
1160          * If the enqueue & dequeue are equal then the TRB ring is either full
1161          * or empty. It's considered full when there are DWC3_TRB_NUM-1 of TRBs
1162          * pending to be processed by the driver.
1163          */
1164         if (dep->trb_enqueue == dep->trb_dequeue) {
1165                 /*
1166                  * If there is any request remained in the started_list at
1167                  * this point, that means there is no TRB available.
1168                  */
1169                 if (!list_empty(&dep->started_list))
1170                         return 0;
1171
1172                 return DWC3_TRB_NUM - 1;
1173         }
1174
1175         trbs_left = dep->trb_dequeue - dep->trb_enqueue;
1176         trbs_left &= (DWC3_TRB_NUM - 1);
1177
1178         if (dep->trb_dequeue < dep->trb_enqueue)
1179                 trbs_left--;
1180
1181         return trbs_left;
1182 }
1183
1184 static void __dwc3_prepare_one_trb(struct dwc3_ep *dep, struct dwc3_trb *trb,
1185                 dma_addr_t dma, unsigned int length, unsigned int chain,
1186                 unsigned int node, unsigned int stream_id,
1187                 unsigned int short_not_ok, unsigned int no_interrupt,
1188                 unsigned int is_last, bool must_interrupt)
1189 {
1190         struct dwc3             *dwc = dep->dwc;
1191         struct usb_gadget       *gadget = dwc->gadget;
1192         enum usb_device_speed   speed = gadget->speed;
1193
1194         trb->size = DWC3_TRB_SIZE_LENGTH(length);
1195         trb->bpl = lower_32_bits(dma);
1196         trb->bph = upper_32_bits(dma);
1197
1198         switch (usb_endpoint_type(dep->endpoint.desc)) {
1199         case USB_ENDPOINT_XFER_CONTROL:
1200                 trb->ctrl = DWC3_TRBCTL_CONTROL_SETUP;
1201                 break;
1202
1203         case USB_ENDPOINT_XFER_ISOC:
1204                 if (!node) {
1205                         trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS_FIRST;
1206
1207                         /*
1208                          * USB Specification 2.0 Section 5.9.2 states that: "If
1209                          * there is only a single transaction in the microframe,
1210                          * only a DATA0 data packet PID is used.  If there are
1211                          * two transactions per microframe, DATA1 is used for
1212                          * the first transaction data packet and DATA0 is used
1213                          * for the second transaction data packet.  If there are
1214                          * three transactions per microframe, DATA2 is used for
1215                          * the first transaction data packet, DATA1 is used for
1216                          * the second, and DATA0 is used for the third."
1217                          *
1218                          * IOW, we should satisfy the following cases:
1219                          *
1220                          * 1) length <= maxpacket
1221                          *      - DATA0
1222                          *
1223                          * 2) maxpacket < length <= (2 * maxpacket)
1224                          *      - DATA1, DATA0
1225                          *
1226                          * 3) (2 * maxpacket) < length <= (3 * maxpacket)
1227                          *      - DATA2, DATA1, DATA0
1228                          */
1229                         if (speed == USB_SPEED_HIGH) {
1230                                 struct usb_ep *ep = &dep->endpoint;
1231                                 unsigned int mult = 2;
1232                                 unsigned int maxp = usb_endpoint_maxp(ep->desc);
1233
1234                                 if (length <= (2 * maxp))
1235                                         mult--;
1236
1237                                 if (length <= maxp)
1238                                         mult--;
1239
1240                                 trb->size |= DWC3_TRB_SIZE_PCM1(mult);
1241                         }
1242                 } else {
1243                         trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS;
1244                 }
1245
1246                 /* always enable Interrupt on Missed ISOC */
1247                 trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI;
1248                 break;
1249
1250         case USB_ENDPOINT_XFER_BULK:
1251         case USB_ENDPOINT_XFER_INT:
1252                 trb->ctrl = DWC3_TRBCTL_NORMAL;
1253                 break;
1254         default:
1255                 /*
1256                  * This is only possible with faulty memory because we
1257                  * checked it already :)
1258                  */
1259                 dev_WARN(dwc->dev, "Unknown endpoint type %d\n",
1260                                 usb_endpoint_type(dep->endpoint.desc));
1261         }
1262
1263         /*
1264          * Enable Continue on Short Packet
1265          * when endpoint is not a stream capable
1266          */
1267         if (usb_endpoint_dir_out(dep->endpoint.desc)) {
1268                 if (!dep->stream_capable)
1269                         trb->ctrl |= DWC3_TRB_CTRL_CSP;
1270
1271                 if (short_not_ok)
1272                         trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI;
1273         }
1274
1275         /* All TRBs setup for MST must set CSP=1 when LST=0 */
1276         if (dep->stream_capable && DWC3_MST_CAPABLE(&dwc->hwparams))
1277                 trb->ctrl |= DWC3_TRB_CTRL_CSP;
1278
1279         if ((!no_interrupt && !chain) || must_interrupt)
1280                 trb->ctrl |= DWC3_TRB_CTRL_IOC;
1281
1282         if (chain)
1283                 trb->ctrl |= DWC3_TRB_CTRL_CHN;
1284         else if (dep->stream_capable && is_last &&
1285                  !DWC3_MST_CAPABLE(&dwc->hwparams))
1286                 trb->ctrl |= DWC3_TRB_CTRL_LST;
1287
1288         if (usb_endpoint_xfer_bulk(dep->endpoint.desc) && dep->stream_capable)
1289                 trb->ctrl |= DWC3_TRB_CTRL_SID_SOFN(stream_id);
1290
1291         /*
1292          * As per data book 4.2.3.2TRB Control Bit Rules section
1293          *
1294          * The controller autonomously checks the HWO field of a TRB to determine if the
1295          * entire TRB is valid. Therefore, software must ensure that the rest of the TRB
1296          * is valid before setting the HWO field to '1'. In most systems, this means that
1297          * software must update the fourth DWORD of a TRB last.
1298          *
1299          * However there is a possibility of CPU re-ordering here which can cause
1300          * controller to observe the HWO bit set prematurely.
1301          * Add a write memory barrier to prevent CPU re-ordering.
1302          */
1303         wmb();
1304         trb->ctrl |= DWC3_TRB_CTRL_HWO;
1305
1306         dwc3_ep_inc_enq(dep);
1307
1308         trace_dwc3_prepare_trb(dep, trb);
1309 }
1310
1311 /**
1312  * dwc3_prepare_one_trb - setup one TRB from one request
1313  * @dep: endpoint for which this request is prepared
1314  * @req: dwc3_request pointer
1315  * @trb_length: buffer size of the TRB
1316  * @chain: should this TRB be chained to the next?
1317  * @node: only for isochronous endpoints. First TRB needs different type.
1318  * @use_bounce_buffer: set to use bounce buffer
1319  * @must_interrupt: set to interrupt on TRB completion
1320  */
1321 static void dwc3_prepare_one_trb(struct dwc3_ep *dep,
1322                 struct dwc3_request *req, unsigned int trb_length,
1323                 unsigned int chain, unsigned int node, bool use_bounce_buffer,
1324                 bool must_interrupt)
1325 {
1326         struct dwc3_trb         *trb;
1327         dma_addr_t              dma;
1328         unsigned int            stream_id = req->request.stream_id;
1329         unsigned int            short_not_ok = req->request.short_not_ok;
1330         unsigned int            no_interrupt = req->request.no_interrupt;
1331         unsigned int            is_last = req->request.is_last;
1332
1333         if (use_bounce_buffer)
1334                 dma = dep->dwc->bounce_addr;
1335         else if (req->request.num_sgs > 0)
1336                 dma = sg_dma_address(req->start_sg);
1337         else
1338                 dma = req->request.dma;
1339
1340         trb = &dep->trb_pool[dep->trb_enqueue];
1341
1342         if (!req->trb) {
1343                 dwc3_gadget_move_started_request(req);
1344                 req->trb = trb;
1345                 req->trb_dma = dwc3_trb_dma_offset(dep, trb);
1346         }
1347
1348         req->num_trbs++;
1349
1350         __dwc3_prepare_one_trb(dep, trb, dma, trb_length, chain, node,
1351                         stream_id, short_not_ok, no_interrupt, is_last,
1352                         must_interrupt);
1353 }
1354
1355 static bool dwc3_needs_extra_trb(struct dwc3_ep *dep, struct dwc3_request *req)
1356 {
1357         unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc);
1358         unsigned int rem = req->request.length % maxp;
1359
1360         if ((req->request.length && req->request.zero && !rem &&
1361                         !usb_endpoint_xfer_isoc(dep->endpoint.desc)) ||
1362                         (!req->direction && rem))
1363                 return true;
1364
1365         return false;
1366 }
1367
1368 /**
1369  * dwc3_prepare_last_sg - prepare TRBs for the last SG entry
1370  * @dep: The endpoint that the request belongs to
1371  * @req: The request to prepare
1372  * @entry_length: The last SG entry size
1373  * @node: Indicates whether this is not the first entry (for isoc only)
1374  *
1375  * Return the number of TRBs prepared.
1376  */
1377 static int dwc3_prepare_last_sg(struct dwc3_ep *dep,
1378                 struct dwc3_request *req, unsigned int entry_length,
1379                 unsigned int node)
1380 {
1381         unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc);
1382         unsigned int rem = req->request.length % maxp;
1383         unsigned int num_trbs = 1;
1384
1385         if (dwc3_needs_extra_trb(dep, req))
1386                 num_trbs++;
1387
1388         if (dwc3_calc_trbs_left(dep) < num_trbs)
1389                 return 0;
1390
1391         req->needs_extra_trb = num_trbs > 1;
1392
1393         /* Prepare a normal TRB */
1394         if (req->direction || req->request.length)
1395                 dwc3_prepare_one_trb(dep, req, entry_length,
1396                                 req->needs_extra_trb, node, false, false);
1397
1398         /* Prepare extra TRBs for ZLP and MPS OUT transfer alignment */
1399         if ((!req->direction && !req->request.length) || req->needs_extra_trb)
1400                 dwc3_prepare_one_trb(dep, req,
1401                                 req->direction ? 0 : maxp - rem,
1402                                 false, 1, true, false);
1403
1404         return num_trbs;
1405 }
1406
1407 static int dwc3_prepare_trbs_sg(struct dwc3_ep *dep,
1408                 struct dwc3_request *req)
1409 {
1410         struct scatterlist *sg = req->start_sg;
1411         struct scatterlist *s;
1412         int             i;
1413         unsigned int length = req->request.length;
1414         unsigned int remaining = req->request.num_mapped_sgs
1415                 - req->num_queued_sgs;
1416         unsigned int num_trbs = req->num_trbs;
1417         bool needs_extra_trb = dwc3_needs_extra_trb(dep, req);
1418
1419         /*
1420          * If we resume preparing the request, then get the remaining length of
1421          * the request and resume where we left off.
1422          */
1423         for_each_sg(req->request.sg, s, req->num_queued_sgs, i)
1424                 length -= sg_dma_len(s);
1425
1426         for_each_sg(sg, s, remaining, i) {
1427                 unsigned int num_trbs_left = dwc3_calc_trbs_left(dep);
1428                 unsigned int trb_length;
1429                 bool must_interrupt = false;
1430                 bool last_sg = false;
1431
1432                 trb_length = min_t(unsigned int, length, sg_dma_len(s));
1433
1434                 length -= trb_length;
1435
1436                 /*
1437                  * IOMMU driver is coalescing the list of sgs which shares a
1438                  * page boundary into one and giving it to USB driver. With
1439                  * this the number of sgs mapped is not equal to the number of
1440                  * sgs passed. So mark the chain bit to false if it isthe last
1441                  * mapped sg.
1442                  */
1443                 if ((i == remaining - 1) || !length)
1444                         last_sg = true;
1445
1446                 if (!num_trbs_left)
1447                         break;
1448
1449                 if (last_sg) {
1450                         if (!dwc3_prepare_last_sg(dep, req, trb_length, i))
1451                                 break;
1452                 } else {
1453                         /*
1454                          * Look ahead to check if we have enough TRBs for the
1455                          * next SG entry. If not, set interrupt on this TRB to
1456                          * resume preparing the next SG entry when more TRBs are
1457                          * free.
1458                          */
1459                         if (num_trbs_left == 1 || (needs_extra_trb &&
1460                                         num_trbs_left <= 2 &&
1461                                         sg_dma_len(sg_next(s)) >= length))
1462                                 must_interrupt = true;
1463
1464                         dwc3_prepare_one_trb(dep, req, trb_length, 1, i, false,
1465                                         must_interrupt);
1466                 }
1467
1468                 /*
1469                  * There can be a situation where all sgs in sglist are not
1470                  * queued because of insufficient trb number. To handle this
1471                  * case, update start_sg to next sg to be queued, so that
1472                  * we have free trbs we can continue queuing from where we
1473                  * previously stopped
1474                  */
1475                 if (!last_sg)
1476                         req->start_sg = sg_next(s);
1477
1478                 req->num_queued_sgs++;
1479                 req->num_pending_sgs--;
1480
1481                 /*
1482                  * The number of pending SG entries may not correspond to the
1483                  * number of mapped SG entries. If all the data are queued, then
1484                  * don't include unused SG entries.
1485                  */
1486                 if (length == 0) {
1487                         req->num_pending_sgs = 0;
1488                         break;
1489                 }
1490
1491                 if (must_interrupt)
1492                         break;
1493         }
1494
1495         return req->num_trbs - num_trbs;
1496 }
1497
1498 static int dwc3_prepare_trbs_linear(struct dwc3_ep *dep,
1499                 struct dwc3_request *req)
1500 {
1501         return dwc3_prepare_last_sg(dep, req, req->request.length, 0);
1502 }
1503
1504 /*
1505  * dwc3_prepare_trbs - setup TRBs from requests
1506  * @dep: endpoint for which requests are being prepared
1507  *
1508  * The function goes through the requests list and sets up TRBs for the
1509  * transfers. The function returns once there are no more TRBs available or
1510  * it runs out of requests.
1511  *
1512  * Returns the number of TRBs prepared or negative errno.
1513  */
1514 static int dwc3_prepare_trbs(struct dwc3_ep *dep)
1515 {
1516         struct dwc3_request     *req, *n;
1517         int                     ret = 0;
1518
1519         BUILD_BUG_ON_NOT_POWER_OF_2(DWC3_TRB_NUM);
1520
1521         /*
1522          * We can get in a situation where there's a request in the started list
1523          * but there weren't enough TRBs to fully kick it in the first time
1524          * around, so it has been waiting for more TRBs to be freed up.
1525          *
1526          * In that case, we should check if we have a request with pending_sgs
1527          * in the started list and prepare TRBs for that request first,
1528          * otherwise we will prepare TRBs completely out of order and that will
1529          * break things.
1530          */
1531         list_for_each_entry(req, &dep->started_list, list) {
1532                 if (req->num_pending_sgs > 0) {
1533                         ret = dwc3_prepare_trbs_sg(dep, req);
1534                         if (!ret || req->num_pending_sgs)
1535                                 return ret;
1536                 }
1537
1538                 if (!dwc3_calc_trbs_left(dep))
1539                         return ret;
1540
1541                 /*
1542                  * Don't prepare beyond a transfer. In DWC_usb32, its transfer
1543                  * burst capability may try to read and use TRBs beyond the
1544                  * active transfer instead of stopping.
1545                  */
1546                 if (dep->stream_capable && req->request.is_last &&
1547                     !DWC3_MST_CAPABLE(&dep->dwc->hwparams))
1548                         return ret;
1549         }
1550
1551         list_for_each_entry_safe(req, n, &dep->pending_list, list) {
1552                 struct dwc3     *dwc = dep->dwc;
1553
1554                 ret = usb_gadget_map_request_by_dev(dwc->sysdev, &req->request,
1555                                                     dep->direction);
1556                 if (ret)
1557                         return ret;
1558
1559                 req->sg                 = req->request.sg;
1560                 req->start_sg           = req->sg;
1561                 req->num_queued_sgs     = 0;
1562                 req->num_pending_sgs    = req->request.num_mapped_sgs;
1563
1564                 if (req->num_pending_sgs > 0) {
1565                         ret = dwc3_prepare_trbs_sg(dep, req);
1566                         if (req->num_pending_sgs)
1567                                 return ret;
1568                 } else {
1569                         ret = dwc3_prepare_trbs_linear(dep, req);
1570                 }
1571
1572                 if (!ret || !dwc3_calc_trbs_left(dep))
1573                         return ret;
1574
1575                 /*
1576                  * Don't prepare beyond a transfer. In DWC_usb32, its transfer
1577                  * burst capability may try to read and use TRBs beyond the
1578                  * active transfer instead of stopping.
1579                  */
1580                 if (dep->stream_capable && req->request.is_last &&
1581                     !DWC3_MST_CAPABLE(&dwc->hwparams))
1582                         return ret;
1583         }
1584
1585         return ret;
1586 }
1587
1588 static void dwc3_gadget_ep_cleanup_cancelled_requests(struct dwc3_ep *dep);
1589
1590 static int __dwc3_gadget_kick_transfer(struct dwc3_ep *dep)
1591 {
1592         struct dwc3_gadget_ep_cmd_params params;
1593         struct dwc3_request             *req;
1594         int                             starting;
1595         int                             ret;
1596         u32                             cmd;
1597
1598         /*
1599          * Note that it's normal to have no new TRBs prepared (i.e. ret == 0).
1600          * This happens when we need to stop and restart a transfer such as in
1601          * the case of reinitiating a stream or retrying an isoc transfer.
1602          */
1603         ret = dwc3_prepare_trbs(dep);
1604         if (ret < 0)
1605                 return ret;
1606
1607         starting = !(dep->flags & DWC3_EP_TRANSFER_STARTED);
1608
1609         /*
1610          * If there's no new TRB prepared and we don't need to restart a
1611          * transfer, there's no need to update the transfer.
1612          */
1613         if (!ret && !starting)
1614                 return ret;
1615
1616         req = next_request(&dep->started_list);
1617         if (!req) {
1618                 dep->flags |= DWC3_EP_PENDING_REQUEST;
1619                 return 0;
1620         }
1621
1622         memset(&params, 0, sizeof(params));
1623
1624         if (starting) {
1625                 params.param0 = upper_32_bits(req->trb_dma);
1626                 params.param1 = lower_32_bits(req->trb_dma);
1627                 cmd = DWC3_DEPCMD_STARTTRANSFER;
1628
1629                 if (dep->stream_capable)
1630                         cmd |= DWC3_DEPCMD_PARAM(req->request.stream_id);
1631
1632                 if (usb_endpoint_xfer_isoc(dep->endpoint.desc))
1633                         cmd |= DWC3_DEPCMD_PARAM(dep->frame_number);
1634         } else {
1635                 cmd = DWC3_DEPCMD_UPDATETRANSFER |
1636                         DWC3_DEPCMD_PARAM(dep->resource_index);
1637         }
1638
1639         ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
1640         if (ret < 0) {
1641                 struct dwc3_request *tmp;
1642
1643                 if (ret == -EAGAIN)
1644                         return ret;
1645
1646                 dwc3_stop_active_transfer(dep, true, true);
1647
1648                 list_for_each_entry_safe(req, tmp, &dep->started_list, list)
1649                         dwc3_gadget_move_cancelled_request(req, DWC3_REQUEST_STATUS_DEQUEUED);
1650
1651                 /* If ep isn't started, then there's no end transfer pending */
1652                 if (!(dep->flags & DWC3_EP_END_TRANSFER_PENDING))
1653                         dwc3_gadget_ep_cleanup_cancelled_requests(dep);
1654
1655                 return ret;
1656         }
1657
1658         if (dep->stream_capable && req->request.is_last &&
1659             !DWC3_MST_CAPABLE(&dep->dwc->hwparams))
1660                 dep->flags |= DWC3_EP_WAIT_TRANSFER_COMPLETE;
1661
1662         return 0;
1663 }
1664
1665 static int __dwc3_gadget_get_frame(struct dwc3 *dwc)
1666 {
1667         u32                     reg;
1668
1669         reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1670         return DWC3_DSTS_SOFFN(reg);
1671 }
1672
1673 /**
1674  * __dwc3_stop_active_transfer - stop the current active transfer
1675  * @dep: isoc endpoint
1676  * @force: set forcerm bit in the command
1677  * @interrupt: command complete interrupt after End Transfer command
1678  *
1679  * When setting force, the ForceRM bit will be set. In that case
1680  * the controller won't update the TRB progress on command
1681  * completion. It also won't clear the HWO bit in the TRB.
1682  * The command will also not complete immediately in that case.
1683  */
1684 static int __dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force, bool interrupt)
1685 {
1686         struct dwc3_gadget_ep_cmd_params params;
1687         u32 cmd;
1688         int ret;
1689
1690         cmd = DWC3_DEPCMD_ENDTRANSFER;
1691         cmd |= force ? DWC3_DEPCMD_HIPRI_FORCERM : 0;
1692         cmd |= interrupt ? DWC3_DEPCMD_CMDIOC : 0;
1693         cmd |= DWC3_DEPCMD_PARAM(dep->resource_index);
1694         memset(&params, 0, sizeof(params));
1695         ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
1696         WARN_ON_ONCE(ret);
1697         dep->resource_index = 0;
1698
1699         if (!interrupt)
1700                 dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
1701         else if (!ret)
1702                 dep->flags |= DWC3_EP_END_TRANSFER_PENDING;
1703
1704         return ret;
1705 }
1706
1707 /**
1708  * dwc3_gadget_start_isoc_quirk - workaround invalid frame number
1709  * @dep: isoc endpoint
1710  *
1711  * This function tests for the correct combination of BIT[15:14] from the 16-bit
1712  * microframe number reported by the XferNotReady event for the future frame
1713  * number to start the isoc transfer.
1714  *
1715  * In DWC_usb31 version 1.70a-ea06 and prior, for highspeed and fullspeed
1716  * isochronous IN, BIT[15:14] of the 16-bit microframe number reported by the
1717  * XferNotReady event are invalid. The driver uses this number to schedule the
1718  * isochronous transfer and passes it to the START TRANSFER command. Because
1719  * this number is invalid, the command may fail. If BIT[15:14] matches the
1720  * internal 16-bit microframe, the START TRANSFER command will pass and the
1721  * transfer will start at the scheduled time, if it is off by 1, the command
1722  * will still pass, but the transfer will start 2 seconds in the future. For all
1723  * other conditions, the START TRANSFER command will fail with bus-expiry.
1724  *
1725  * In order to workaround this issue, we can test for the correct combination of
1726  * BIT[15:14] by sending START TRANSFER commands with different values of
1727  * BIT[15:14]: 'b00, 'b01, 'b10, and 'b11. Each combination is 2^14 uframe apart
1728  * (or 2 seconds). 4 seconds into the future will result in a bus-expiry status.
1729  * As the result, within the 4 possible combinations for BIT[15:14], there will
1730  * be 2 successful and 2 failure START COMMAND status. One of the 2 successful
1731  * command status will result in a 2-second delay start. The smaller BIT[15:14]
1732  * value is the correct combination.
1733  *
1734  * Since there are only 4 outcomes and the results are ordered, we can simply
1735  * test 2 START TRANSFER commands with BIT[15:14] combinations 'b00 and 'b01 to
1736  * deduce the smaller successful combination.
1737  *
1738  * Let test0 = test status for combination 'b00 and test1 = test status for 'b01
1739  * of BIT[15:14]. The correct combination is as follow:
1740  *
1741  * if test0 fails and test1 passes, BIT[15:14] is 'b01
1742  * if test0 fails and test1 fails, BIT[15:14] is 'b10
1743  * if test0 passes and test1 fails, BIT[15:14] is 'b11
1744  * if test0 passes and test1 passes, BIT[15:14] is 'b00
1745  *
1746  * Synopsys STAR 9001202023: Wrong microframe number for isochronous IN
1747  * endpoints.
1748  */
1749 static int dwc3_gadget_start_isoc_quirk(struct dwc3_ep *dep)
1750 {
1751         int cmd_status = 0;
1752         bool test0;
1753         bool test1;
1754
1755         while (dep->combo_num < 2) {
1756                 struct dwc3_gadget_ep_cmd_params params;
1757                 u32 test_frame_number;
1758                 u32 cmd;
1759
1760                 /*
1761                  * Check if we can start isoc transfer on the next interval or
1762                  * 4 uframes in the future with BIT[15:14] as dep->combo_num
1763                  */
1764                 test_frame_number = dep->frame_number & DWC3_FRNUMBER_MASK;
1765                 test_frame_number |= dep->combo_num << 14;
1766                 test_frame_number += max_t(u32, 4, dep->interval);
1767
1768                 params.param0 = upper_32_bits(dep->dwc->bounce_addr);
1769                 params.param1 = lower_32_bits(dep->dwc->bounce_addr);
1770
1771                 cmd = DWC3_DEPCMD_STARTTRANSFER;
1772                 cmd |= DWC3_DEPCMD_PARAM(test_frame_number);
1773                 cmd_status = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
1774
1775                 /* Redo if some other failure beside bus-expiry is received */
1776                 if (cmd_status && cmd_status != -EAGAIN) {
1777                         dep->start_cmd_status = 0;
1778                         dep->combo_num = 0;
1779                         return 0;
1780                 }
1781
1782                 /* Store the first test status */
1783                 if (dep->combo_num == 0)
1784                         dep->start_cmd_status = cmd_status;
1785
1786                 dep->combo_num++;
1787
1788                 /*
1789                  * End the transfer if the START_TRANSFER command is successful
1790                  * to wait for the next XferNotReady to test the command again
1791                  */
1792                 if (cmd_status == 0) {
1793                         dwc3_stop_active_transfer(dep, true, true);
1794                         return 0;
1795                 }
1796         }
1797
1798         /* test0 and test1 are both completed at this point */
1799         test0 = (dep->start_cmd_status == 0);
1800         test1 = (cmd_status == 0);
1801
1802         if (!test0 && test1)
1803                 dep->combo_num = 1;
1804         else if (!test0 && !test1)
1805                 dep->combo_num = 2;
1806         else if (test0 && !test1)
1807                 dep->combo_num = 3;
1808         else if (test0 && test1)
1809                 dep->combo_num = 0;
1810
1811         dep->frame_number &= DWC3_FRNUMBER_MASK;
1812         dep->frame_number |= dep->combo_num << 14;
1813         dep->frame_number += max_t(u32, 4, dep->interval);
1814
1815         /* Reinitialize test variables */
1816         dep->start_cmd_status = 0;
1817         dep->combo_num = 0;
1818
1819         return __dwc3_gadget_kick_transfer(dep);
1820 }
1821
1822 static int __dwc3_gadget_start_isoc(struct dwc3_ep *dep)
1823 {
1824         const struct usb_endpoint_descriptor *desc = dep->endpoint.desc;
1825         struct dwc3 *dwc = dep->dwc;
1826         int ret;
1827         int i;
1828
1829         if (list_empty(&dep->pending_list) &&
1830             list_empty(&dep->started_list)) {
1831                 dep->flags |= DWC3_EP_PENDING_REQUEST;
1832                 return -EAGAIN;
1833         }
1834
1835         if (!dwc->dis_start_transfer_quirk &&
1836             (DWC3_VER_IS_PRIOR(DWC31, 170A) ||
1837              DWC3_VER_TYPE_IS_WITHIN(DWC31, 170A, EA01, EA06))) {
1838                 if (dwc->gadget->speed <= USB_SPEED_HIGH && dep->direction)
1839                         return dwc3_gadget_start_isoc_quirk(dep);
1840         }
1841
1842         if (desc->bInterval <= 14 &&
1843             dwc->gadget->speed >= USB_SPEED_HIGH) {
1844                 u32 frame = __dwc3_gadget_get_frame(dwc);
1845                 bool rollover = frame <
1846                                 (dep->frame_number & DWC3_FRNUMBER_MASK);
1847
1848                 /*
1849                  * frame_number is set from XferNotReady and may be already
1850                  * out of date. DSTS only provides the lower 14 bit of the
1851                  * current frame number. So add the upper two bits of
1852                  * frame_number and handle a possible rollover.
1853                  * This will provide the correct frame_number unless more than
1854                  * rollover has happened since XferNotReady.
1855                  */
1856
1857                 dep->frame_number = (dep->frame_number & ~DWC3_FRNUMBER_MASK) |
1858                                      frame;
1859                 if (rollover)
1860                         dep->frame_number += BIT(14);
1861         }
1862
1863         for (i = 0; i < DWC3_ISOC_MAX_RETRIES; i++) {
1864                 int future_interval = i + 1;
1865
1866                 /* Give the controller at least 500us to schedule transfers */
1867                 if (desc->bInterval < 3)
1868                         future_interval += 3 - desc->bInterval;
1869
1870                 dep->frame_number = DWC3_ALIGN_FRAME(dep, future_interval);
1871
1872                 ret = __dwc3_gadget_kick_transfer(dep);
1873                 if (ret != -EAGAIN)
1874                         break;
1875         }
1876
1877         /*
1878          * After a number of unsuccessful start attempts due to bus-expiry
1879          * status, issue END_TRANSFER command and retry on the next XferNotReady
1880          * event.
1881          */
1882         if (ret == -EAGAIN)
1883                 ret = __dwc3_stop_active_transfer(dep, false, true);
1884
1885         return ret;
1886 }
1887
1888 static int __dwc3_gadget_ep_queue(struct dwc3_ep *dep, struct dwc3_request *req)
1889 {
1890         struct dwc3             *dwc = dep->dwc;
1891
1892         if (!dep->endpoint.desc || !dwc->pullups_connected || !dwc->connected) {
1893                 dev_dbg(dwc->dev, "%s: can't queue to disabled endpoint\n",
1894                                 dep->name);
1895                 return -ESHUTDOWN;
1896         }
1897
1898         if (WARN(req->dep != dep, "request %pK belongs to '%s'\n",
1899                                 &req->request, req->dep->name))
1900                 return -EINVAL;
1901
1902         if (WARN(req->status < DWC3_REQUEST_STATUS_COMPLETED,
1903                                 "%s: request %pK already in flight\n",
1904                                 dep->name, &req->request))
1905                 return -EINVAL;
1906
1907         pm_runtime_get(dwc->dev);
1908
1909         req->request.actual     = 0;
1910         req->request.status     = -EINPROGRESS;
1911
1912         trace_dwc3_ep_queue(req);
1913
1914         list_add_tail(&req->list, &dep->pending_list);
1915         req->status = DWC3_REQUEST_STATUS_QUEUED;
1916
1917         if (dep->flags & DWC3_EP_WAIT_TRANSFER_COMPLETE)
1918                 return 0;
1919
1920         /*
1921          * Start the transfer only after the END_TRANSFER is completed
1922          * and endpoint STALL is cleared.
1923          */
1924         if ((dep->flags & DWC3_EP_END_TRANSFER_PENDING) ||
1925             (dep->flags & DWC3_EP_WEDGE) ||
1926             (dep->flags & DWC3_EP_DELAY_STOP) ||
1927             (dep->flags & DWC3_EP_STALL)) {
1928                 dep->flags |= DWC3_EP_DELAY_START;
1929                 return 0;
1930         }
1931
1932         /*
1933          * NOTICE: Isochronous endpoints should NEVER be prestarted. We must
1934          * wait for a XferNotReady event so we will know what's the current
1935          * (micro-)frame number.
1936          *
1937          * Without this trick, we are very, very likely gonna get Bus Expiry
1938          * errors which will force us issue EndTransfer command.
1939          */
1940         if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
1941                 if (!(dep->flags & DWC3_EP_TRANSFER_STARTED)) {
1942                         if ((dep->flags & DWC3_EP_PENDING_REQUEST))
1943                                 return __dwc3_gadget_start_isoc(dep);
1944
1945                         return 0;
1946                 }
1947         }
1948
1949         __dwc3_gadget_kick_transfer(dep);
1950
1951         return 0;
1952 }
1953
1954 static int dwc3_gadget_ep_queue(struct usb_ep *ep, struct usb_request *request,
1955         gfp_t gfp_flags)
1956 {
1957         struct dwc3_request             *req = to_dwc3_request(request);
1958         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
1959         struct dwc3                     *dwc = dep->dwc;
1960
1961         unsigned long                   flags;
1962
1963         int                             ret;
1964
1965         spin_lock_irqsave(&dwc->lock, flags);
1966         ret = __dwc3_gadget_ep_queue(dep, req);
1967         spin_unlock_irqrestore(&dwc->lock, flags);
1968
1969         return ret;
1970 }
1971
1972 static void dwc3_gadget_ep_skip_trbs(struct dwc3_ep *dep, struct dwc3_request *req)
1973 {
1974         int i;
1975
1976         /* If req->trb is not set, then the request has not started */
1977         if (!req->trb)
1978                 return;
1979
1980         /*
1981          * If request was already started, this means we had to
1982          * stop the transfer. With that we also need to ignore
1983          * all TRBs used by the request, however TRBs can only
1984          * be modified after completion of END_TRANSFER
1985          * command. So what we do here is that we wait for
1986          * END_TRANSFER completion and only after that, we jump
1987          * over TRBs by clearing HWO and incrementing dequeue
1988          * pointer.
1989          */
1990         for (i = 0; i < req->num_trbs; i++) {
1991                 struct dwc3_trb *trb;
1992
1993                 trb = &dep->trb_pool[dep->trb_dequeue];
1994                 trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
1995                 dwc3_ep_inc_deq(dep);
1996         }
1997
1998         req->num_trbs = 0;
1999 }
2000
2001 static void dwc3_gadget_ep_cleanup_cancelled_requests(struct dwc3_ep *dep)
2002 {
2003         struct dwc3_request             *req;
2004         struct dwc3                     *dwc = dep->dwc;
2005
2006         while (!list_empty(&dep->cancelled_list)) {
2007                 req = next_request(&dep->cancelled_list);
2008                 dwc3_gadget_ep_skip_trbs(dep, req);
2009                 switch (req->status) {
2010                 case DWC3_REQUEST_STATUS_DISCONNECTED:
2011                         dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
2012                         break;
2013                 case DWC3_REQUEST_STATUS_DEQUEUED:
2014                         dwc3_gadget_giveback(dep, req, -ECONNRESET);
2015                         break;
2016                 case DWC3_REQUEST_STATUS_STALLED:
2017                         dwc3_gadget_giveback(dep, req, -EPIPE);
2018                         break;
2019                 default:
2020                         dev_err(dwc->dev, "request cancelled with wrong reason:%d\n", req->status);
2021                         dwc3_gadget_giveback(dep, req, -ECONNRESET);
2022                         break;
2023                 }
2024                 /*
2025                  * The endpoint is disabled, let the dwc3_remove_requests()
2026                  * handle the cleanup.
2027                  */
2028                 if (!dep->endpoint.desc)
2029                         break;
2030         }
2031 }
2032
2033 static int dwc3_gadget_ep_dequeue(struct usb_ep *ep,
2034                 struct usb_request *request)
2035 {
2036         struct dwc3_request             *req = to_dwc3_request(request);
2037         struct dwc3_request             *r = NULL;
2038
2039         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
2040         struct dwc3                     *dwc = dep->dwc;
2041
2042         unsigned long                   flags;
2043         int                             ret = 0;
2044
2045         trace_dwc3_ep_dequeue(req);
2046
2047         spin_lock_irqsave(&dwc->lock, flags);
2048
2049         list_for_each_entry(r, &dep->cancelled_list, list) {
2050                 if (r == req)
2051                         goto out;
2052         }
2053
2054         list_for_each_entry(r, &dep->pending_list, list) {
2055                 if (r == req) {
2056                         dwc3_gadget_giveback(dep, req, -ECONNRESET);
2057                         goto out;
2058                 }
2059         }
2060
2061         list_for_each_entry(r, &dep->started_list, list) {
2062                 if (r == req) {
2063                         struct dwc3_request *t;
2064
2065                         /*
2066                          * If a Setup packet is received but yet to DMA out, the controller will
2067                          * not process the End Transfer command of any endpoint. Polling of its
2068                          * DEPCMD.CmdAct may block setting up TRB for Setup packet, causing a
2069                          * timeout. Delay issuing the End Transfer command until the Setup TRB is
2070                          * prepared.
2071                          */
2072                         if (dwc->ep0state != EP0_SETUP_PHASE && !dwc->delayed_status)
2073                                 dep->flags |= DWC3_EP_DELAY_STOP;
2074
2075                         /* wait until it is processed */
2076                         dwc3_stop_active_transfer(dep, true, true);
2077
2078                         /*
2079                          * Remove any started request if the transfer is
2080                          * cancelled.
2081                          */
2082                         list_for_each_entry_safe(r, t, &dep->started_list, list)
2083                                 dwc3_gadget_move_cancelled_request(r,
2084                                                 DWC3_REQUEST_STATUS_DEQUEUED);
2085
2086                         dep->flags &= ~DWC3_EP_WAIT_TRANSFER_COMPLETE;
2087
2088                         goto out;
2089                 }
2090         }
2091
2092         dev_err(dwc->dev, "request %pK was not queued to %s\n",
2093                 request, ep->name);
2094         ret = -EINVAL;
2095 out:
2096         spin_unlock_irqrestore(&dwc->lock, flags);
2097
2098         return ret;
2099 }
2100
2101 int __dwc3_gadget_ep_set_halt(struct dwc3_ep *dep, int value, int protocol)
2102 {
2103         struct dwc3_gadget_ep_cmd_params        params;
2104         struct dwc3                             *dwc = dep->dwc;
2105         struct dwc3_request                     *req;
2106         struct dwc3_request                     *tmp;
2107         int                                     ret;
2108
2109         if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
2110                 dev_err(dwc->dev, "%s is of Isochronous type\n", dep->name);
2111                 return -EINVAL;
2112         }
2113
2114         memset(&params, 0x00, sizeof(params));
2115
2116         if (value) {
2117                 struct dwc3_trb *trb;
2118
2119                 unsigned int transfer_in_flight;
2120                 unsigned int started;
2121
2122                 if (dep->number > 1)
2123                         trb = dwc3_ep_prev_trb(dep, dep->trb_enqueue);
2124                 else
2125                         trb = &dwc->ep0_trb[dep->trb_enqueue];
2126
2127                 transfer_in_flight = trb->ctrl & DWC3_TRB_CTRL_HWO;
2128                 started = !list_empty(&dep->started_list);
2129
2130                 if (!protocol && ((dep->direction && transfer_in_flight) ||
2131                                 (!dep->direction && started))) {
2132                         return -EAGAIN;
2133                 }
2134
2135                 ret = dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETSTALL,
2136                                 &params);
2137                 if (ret)
2138                         dev_err(dwc->dev, "failed to set STALL on %s\n",
2139                                         dep->name);
2140                 else
2141                         dep->flags |= DWC3_EP_STALL;
2142         } else {
2143                 /*
2144                  * Don't issue CLEAR_STALL command to control endpoints. The
2145                  * controller automatically clears the STALL when it receives
2146                  * the SETUP token.
2147                  */
2148                 if (dep->number <= 1) {
2149                         dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE);
2150                         return 0;
2151                 }
2152
2153                 dwc3_stop_active_transfer(dep, true, true);
2154
2155                 list_for_each_entry_safe(req, tmp, &dep->started_list, list)
2156                         dwc3_gadget_move_cancelled_request(req, DWC3_REQUEST_STATUS_STALLED);
2157
2158                 if (dep->flags & DWC3_EP_END_TRANSFER_PENDING ||
2159                     (dep->flags & DWC3_EP_DELAY_STOP)) {
2160                         dep->flags |= DWC3_EP_PENDING_CLEAR_STALL;
2161                         if (protocol)
2162                                 dwc->clear_stall_protocol = dep->number;
2163
2164                         return 0;
2165                 }
2166
2167                 dwc3_gadget_ep_cleanup_cancelled_requests(dep);
2168
2169                 ret = dwc3_send_clear_stall_ep_cmd(dep);
2170                 if (ret) {
2171                         dev_err(dwc->dev, "failed to clear STALL on %s\n",
2172                                         dep->name);
2173                         return ret;
2174                 }
2175
2176                 dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE);
2177
2178                 if ((dep->flags & DWC3_EP_DELAY_START) &&
2179                     !usb_endpoint_xfer_isoc(dep->endpoint.desc))
2180                         __dwc3_gadget_kick_transfer(dep);
2181
2182                 dep->flags &= ~DWC3_EP_DELAY_START;
2183         }
2184
2185         return ret;
2186 }
2187
2188 static int dwc3_gadget_ep_set_halt(struct usb_ep *ep, int value)
2189 {
2190         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
2191         struct dwc3                     *dwc = dep->dwc;
2192
2193         unsigned long                   flags;
2194
2195         int                             ret;
2196
2197         spin_lock_irqsave(&dwc->lock, flags);
2198         ret = __dwc3_gadget_ep_set_halt(dep, value, false);
2199         spin_unlock_irqrestore(&dwc->lock, flags);
2200
2201         return ret;
2202 }
2203
2204 static int dwc3_gadget_ep_set_wedge(struct usb_ep *ep)
2205 {
2206         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
2207         struct dwc3                     *dwc = dep->dwc;
2208         unsigned long                   flags;
2209         int                             ret;
2210
2211         spin_lock_irqsave(&dwc->lock, flags);
2212         dep->flags |= DWC3_EP_WEDGE;
2213
2214         if (dep->number == 0 || dep->number == 1)
2215                 ret = __dwc3_gadget_ep0_set_halt(ep, 1);
2216         else
2217                 ret = __dwc3_gadget_ep_set_halt(dep, 1, false);
2218         spin_unlock_irqrestore(&dwc->lock, flags);
2219
2220         return ret;
2221 }
2222
2223 /* -------------------------------------------------------------------------- */
2224
2225 static struct usb_endpoint_descriptor dwc3_gadget_ep0_desc = {
2226         .bLength        = USB_DT_ENDPOINT_SIZE,
2227         .bDescriptorType = USB_DT_ENDPOINT,
2228         .bmAttributes   = USB_ENDPOINT_XFER_CONTROL,
2229 };
2230
2231 static const struct usb_ep_ops dwc3_gadget_ep0_ops = {
2232         .enable         = dwc3_gadget_ep0_enable,
2233         .disable        = dwc3_gadget_ep0_disable,
2234         .alloc_request  = dwc3_gadget_ep_alloc_request,
2235         .free_request   = dwc3_gadget_ep_free_request,
2236         .queue          = dwc3_gadget_ep0_queue,
2237         .dequeue        = dwc3_gadget_ep_dequeue,
2238         .set_halt       = dwc3_gadget_ep0_set_halt,
2239         .set_wedge      = dwc3_gadget_ep_set_wedge,
2240 };
2241
2242 static const struct usb_ep_ops dwc3_gadget_ep_ops = {
2243         .enable         = dwc3_gadget_ep_enable,
2244         .disable        = dwc3_gadget_ep_disable,
2245         .alloc_request  = dwc3_gadget_ep_alloc_request,
2246         .free_request   = dwc3_gadget_ep_free_request,
2247         .queue          = dwc3_gadget_ep_queue,
2248         .dequeue        = dwc3_gadget_ep_dequeue,
2249         .set_halt       = dwc3_gadget_ep_set_halt,
2250         .set_wedge      = dwc3_gadget_ep_set_wedge,
2251 };
2252
2253 /* -------------------------------------------------------------------------- */
2254
2255 static int dwc3_gadget_get_frame(struct usb_gadget *g)
2256 {
2257         struct dwc3             *dwc = gadget_to_dwc(g);
2258
2259         return __dwc3_gadget_get_frame(dwc);
2260 }
2261
2262 static int __dwc3_gadget_wakeup(struct dwc3 *dwc)
2263 {
2264         int                     retries;
2265
2266         int                     ret;
2267         u32                     reg;
2268
2269         u8                      link_state;
2270
2271         /*
2272          * According to the Databook Remote wakeup request should
2273          * be issued only when the device is in early suspend state.
2274          *
2275          * We can check that via USB Link State bits in DSTS register.
2276          */
2277         reg = dwc3_readl(dwc->regs, DWC3_DSTS);
2278
2279         link_state = DWC3_DSTS_USBLNKST(reg);
2280
2281         switch (link_state) {
2282         case DWC3_LINK_STATE_RESET:
2283         case DWC3_LINK_STATE_RX_DET:    /* in HS, means Early Suspend */
2284         case DWC3_LINK_STATE_U3:        /* in HS, means SUSPEND */
2285         case DWC3_LINK_STATE_U2:        /* in HS, means Sleep (L1) */
2286         case DWC3_LINK_STATE_U1:
2287         case DWC3_LINK_STATE_RESUME:
2288                 break;
2289         default:
2290                 return -EINVAL;
2291         }
2292
2293         ret = dwc3_gadget_set_link_state(dwc, DWC3_LINK_STATE_RECOV);
2294         if (ret < 0) {
2295                 dev_err(dwc->dev, "failed to put link in Recovery\n");
2296                 return ret;
2297         }
2298
2299         /* Recent versions do this automatically */
2300         if (DWC3_VER_IS_PRIOR(DWC3, 194A)) {
2301                 /* write zeroes to Link Change Request */
2302                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2303                 reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;
2304                 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2305         }
2306
2307         /* poll until Link State changes to ON */
2308         retries = 20000;
2309
2310         while (retries--) {
2311                 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
2312
2313                 /* in HS, means ON */
2314                 if (DWC3_DSTS_USBLNKST(reg) == DWC3_LINK_STATE_U0)
2315                         break;
2316         }
2317
2318         if (DWC3_DSTS_USBLNKST(reg) != DWC3_LINK_STATE_U0) {
2319                 dev_err(dwc->dev, "failed to send remote wakeup\n");
2320                 return -EINVAL;
2321         }
2322
2323         return 0;
2324 }
2325
2326 static int dwc3_gadget_wakeup(struct usb_gadget *g)
2327 {
2328         struct dwc3             *dwc = gadget_to_dwc(g);
2329         unsigned long           flags;
2330         int                     ret;
2331
2332         spin_lock_irqsave(&dwc->lock, flags);
2333         ret = __dwc3_gadget_wakeup(dwc);
2334         spin_unlock_irqrestore(&dwc->lock, flags);
2335
2336         return ret;
2337 }
2338
2339 static int dwc3_gadget_set_selfpowered(struct usb_gadget *g,
2340                 int is_selfpowered)
2341 {
2342         struct dwc3             *dwc = gadget_to_dwc(g);
2343         unsigned long           flags;
2344
2345         spin_lock_irqsave(&dwc->lock, flags);
2346         g->is_selfpowered = !!is_selfpowered;
2347         spin_unlock_irqrestore(&dwc->lock, flags);
2348
2349         return 0;
2350 }
2351
2352 static void dwc3_stop_active_transfers(struct dwc3 *dwc)
2353 {
2354         u32 epnum;
2355
2356         for (epnum = 2; epnum < dwc->num_eps; epnum++) {
2357                 struct dwc3_ep *dep;
2358
2359                 dep = dwc->eps[epnum];
2360                 if (!dep)
2361                         continue;
2362
2363                 dwc3_remove_requests(dwc, dep);
2364         }
2365 }
2366
2367 static void __dwc3_gadget_set_ssp_rate(struct dwc3 *dwc)
2368 {
2369         enum usb_ssp_rate       ssp_rate = dwc->gadget_ssp_rate;
2370         u32                     reg;
2371
2372         if (ssp_rate == USB_SSP_GEN_UNKNOWN)
2373                 ssp_rate = dwc->max_ssp_rate;
2374
2375         reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2376         reg &= ~DWC3_DCFG_SPEED_MASK;
2377         reg &= ~DWC3_DCFG_NUMLANES(~0);
2378
2379         if (ssp_rate == USB_SSP_GEN_1x2)
2380                 reg |= DWC3_DCFG_SUPERSPEED;
2381         else if (dwc->max_ssp_rate != USB_SSP_GEN_1x2)
2382                 reg |= DWC3_DCFG_SUPERSPEED_PLUS;
2383
2384         if (ssp_rate != USB_SSP_GEN_2x1 &&
2385             dwc->max_ssp_rate != USB_SSP_GEN_2x1)
2386                 reg |= DWC3_DCFG_NUMLANES(1);
2387
2388         dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2389 }
2390
2391 static void __dwc3_gadget_set_speed(struct dwc3 *dwc)
2392 {
2393         enum usb_device_speed   speed;
2394         u32                     reg;
2395
2396         speed = dwc->gadget_max_speed;
2397         if (speed == USB_SPEED_UNKNOWN || speed > dwc->maximum_speed)
2398                 speed = dwc->maximum_speed;
2399
2400         if (speed == USB_SPEED_SUPER_PLUS &&
2401             DWC3_IP_IS(DWC32)) {
2402                 __dwc3_gadget_set_ssp_rate(dwc);
2403                 return;
2404         }
2405
2406         reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2407         reg &= ~(DWC3_DCFG_SPEED_MASK);
2408
2409         /*
2410          * WORKAROUND: DWC3 revision < 2.20a have an issue
2411          * which would cause metastability state on Run/Stop
2412          * bit if we try to force the IP to USB2-only mode.
2413          *
2414          * Because of that, we cannot configure the IP to any
2415          * speed other than the SuperSpeed
2416          *
2417          * Refers to:
2418          *
2419          * STAR#9000525659: Clock Domain Crossing on DCTL in
2420          * USB 2.0 Mode
2421          */
2422         if (DWC3_VER_IS_PRIOR(DWC3, 220A) &&
2423             !dwc->dis_metastability_quirk) {
2424                 reg |= DWC3_DCFG_SUPERSPEED;
2425         } else {
2426                 switch (speed) {
2427                 case USB_SPEED_FULL:
2428                         reg |= DWC3_DCFG_FULLSPEED;
2429                         break;
2430                 case USB_SPEED_HIGH:
2431                         reg |= DWC3_DCFG_HIGHSPEED;
2432                         break;
2433                 case USB_SPEED_SUPER:
2434                         reg |= DWC3_DCFG_SUPERSPEED;
2435                         break;
2436                 case USB_SPEED_SUPER_PLUS:
2437                         if (DWC3_IP_IS(DWC3))
2438                                 reg |= DWC3_DCFG_SUPERSPEED;
2439                         else
2440                                 reg |= DWC3_DCFG_SUPERSPEED_PLUS;
2441                         break;
2442                 default:
2443                         dev_err(dwc->dev, "invalid speed (%d)\n", speed);
2444
2445                         if (DWC3_IP_IS(DWC3))
2446                                 reg |= DWC3_DCFG_SUPERSPEED;
2447                         else
2448                                 reg |= DWC3_DCFG_SUPERSPEED_PLUS;
2449                 }
2450         }
2451
2452         if (DWC3_IP_IS(DWC32) &&
2453             speed > USB_SPEED_UNKNOWN &&
2454             speed < USB_SPEED_SUPER_PLUS)
2455                 reg &= ~DWC3_DCFG_NUMLANES(~0);
2456
2457         dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2458 }
2459
2460 static int dwc3_gadget_run_stop(struct dwc3 *dwc, int is_on, int suspend)
2461 {
2462         u32                     reg;
2463         u32                     timeout = 500;
2464
2465         if (pm_runtime_suspended(dwc->dev))
2466                 return 0;
2467
2468         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2469         if (is_on) {
2470                 if (DWC3_VER_IS_WITHIN(DWC3, ANY, 187A)) {
2471                         reg &= ~DWC3_DCTL_TRGTULST_MASK;
2472                         reg |= DWC3_DCTL_TRGTULST_RX_DET;
2473                 }
2474
2475                 if (!DWC3_VER_IS_PRIOR(DWC3, 194A))
2476                         reg &= ~DWC3_DCTL_KEEP_CONNECT;
2477                 reg |= DWC3_DCTL_RUN_STOP;
2478
2479                 if (dwc->has_hibernation)
2480                         reg |= DWC3_DCTL_KEEP_CONNECT;
2481
2482                 __dwc3_gadget_set_speed(dwc);
2483                 dwc->pullups_connected = true;
2484         } else {
2485                 reg &= ~DWC3_DCTL_RUN_STOP;
2486
2487                 if (dwc->has_hibernation && !suspend)
2488                         reg &= ~DWC3_DCTL_KEEP_CONNECT;
2489
2490                 dwc->pullups_connected = false;
2491         }
2492
2493         dwc3_gadget_dctl_write_safe(dwc, reg);
2494
2495         do {
2496                 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
2497                 reg &= DWC3_DSTS_DEVCTRLHLT;
2498         } while (--timeout && !(!is_on ^ !reg));
2499
2500         if (!timeout)
2501                 return -ETIMEDOUT;
2502
2503         return 0;
2504 }
2505
2506 static void dwc3_gadget_disable_irq(struct dwc3 *dwc);
2507 static void __dwc3_gadget_stop(struct dwc3 *dwc);
2508 static int __dwc3_gadget_start(struct dwc3 *dwc);
2509
2510 static int dwc3_gadget_pullup(struct usb_gadget *g, int is_on)
2511 {
2512         struct dwc3             *dwc = gadget_to_dwc(g);
2513         unsigned long           flags;
2514         int                     ret;
2515
2516         is_on = !!is_on;
2517         dwc->softconnect = is_on;
2518         /*
2519          * Per databook, when we want to stop the gadget, if a control transfer
2520          * is still in process, complete it and get the core into setup phase.
2521          */
2522         if (!is_on && dwc->ep0state != EP0_SETUP_PHASE) {
2523                 reinit_completion(&dwc->ep0_in_setup);
2524
2525                 ret = wait_for_completion_timeout(&dwc->ep0_in_setup,
2526                                 msecs_to_jiffies(DWC3_PULL_UP_TIMEOUT));
2527                 if (ret == 0)
2528                         dev_warn(dwc->dev, "timed out waiting for SETUP phase\n");
2529         }
2530
2531         /*
2532          * Avoid issuing a runtime resume if the device is already in the
2533          * suspended state during gadget disconnect.  DWC3 gadget was already
2534          * halted/stopped during runtime suspend.
2535          */
2536         if (!is_on) {
2537                 pm_runtime_barrier(dwc->dev);
2538                 if (pm_runtime_suspended(dwc->dev))
2539                         return 0;
2540         }
2541
2542         /*
2543          * Check the return value for successful resume, or error.  For a
2544          * successful resume, the DWC3 runtime PM resume routine will handle
2545          * the run stop sequence, so avoid duplicate operations here.
2546          */
2547         ret = pm_runtime_get_sync(dwc->dev);
2548         if (!ret || ret < 0) {
2549                 pm_runtime_put(dwc->dev);
2550                 return 0;
2551         }
2552
2553         /*
2554          * Synchronize and disable any further event handling while controller
2555          * is being enabled/disabled.
2556          */
2557         disable_irq(dwc->irq_gadget);
2558
2559         spin_lock_irqsave(&dwc->lock, flags);
2560
2561         if (!is_on) {
2562                 u32 count;
2563
2564                 dwc->connected = false;
2565                 /*
2566                  * In the Synopsis DesignWare Cores USB3 Databook Rev. 3.30a
2567                  * Section 4.1.8 Table 4-7, it states that for a device-initiated
2568                  * disconnect, the SW needs to ensure that it sends "a DEPENDXFER
2569                  * command for any active transfers" before clearing the RunStop
2570                  * bit.
2571                  */
2572                 dwc3_stop_active_transfers(dwc);
2573                 __dwc3_gadget_stop(dwc);
2574
2575                 /*
2576                  * In the Synopsis DesignWare Cores USB3 Databook Rev. 3.30a
2577                  * Section 1.3.4, it mentions that for the DEVCTRLHLT bit, the
2578                  * "software needs to acknowledge the events that are generated
2579                  * (by writing to GEVNTCOUNTn) while it is waiting for this bit
2580                  * to be set to '1'."
2581                  */
2582                 count = dwc3_readl(dwc->regs, DWC3_GEVNTCOUNT(0));
2583                 count &= DWC3_GEVNTCOUNT_MASK;
2584                 if (count > 0) {
2585                         dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), count);
2586                         dwc->ev_buf->lpos = (dwc->ev_buf->lpos + count) %
2587                                                 dwc->ev_buf->length;
2588                 }
2589         } else {
2590                 /*
2591                  * In the Synopsys DWC_usb31 1.90a programming guide section
2592                  * 4.1.9, it specifies that for a reconnect after a
2593                  * device-initiated disconnect requires a core soft reset
2594                  * (DCTL.CSftRst) before enabling the run/stop bit.
2595                  */
2596                 spin_unlock_irqrestore(&dwc->lock, flags);
2597                 dwc3_core_soft_reset(dwc);
2598                 spin_lock_irqsave(&dwc->lock, flags);
2599
2600                 dwc3_event_buffers_setup(dwc);
2601                 __dwc3_gadget_start(dwc);
2602         }
2603
2604         ret = dwc3_gadget_run_stop(dwc, is_on, false);
2605         spin_unlock_irqrestore(&dwc->lock, flags);
2606         enable_irq(dwc->irq_gadget);
2607
2608         pm_runtime_put(dwc->dev);
2609
2610         return ret;
2611 }
2612
2613 static void dwc3_gadget_enable_irq(struct dwc3 *dwc)
2614 {
2615         u32                     reg;
2616
2617         /* Enable all but Start and End of Frame IRQs */
2618         reg = (DWC3_DEVTEN_EVNTOVERFLOWEN |
2619                         DWC3_DEVTEN_CMDCMPLTEN |
2620                         DWC3_DEVTEN_ERRTICERREN |
2621                         DWC3_DEVTEN_WKUPEVTEN |
2622                         DWC3_DEVTEN_CONNECTDONEEN |
2623                         DWC3_DEVTEN_USBRSTEN |
2624                         DWC3_DEVTEN_DISCONNEVTEN);
2625
2626         if (DWC3_VER_IS_PRIOR(DWC3, 250A))
2627                 reg |= DWC3_DEVTEN_ULSTCNGEN;
2628
2629         /* On 2.30a and above this bit enables U3/L2-L1 Suspend Events */
2630         if (!DWC3_VER_IS_PRIOR(DWC3, 230A))
2631                 reg |= DWC3_DEVTEN_U3L2L1SUSPEN;
2632
2633         dwc3_writel(dwc->regs, DWC3_DEVTEN, reg);
2634 }
2635
2636 static void dwc3_gadget_disable_irq(struct dwc3 *dwc)
2637 {
2638         /* mask all interrupts */
2639         dwc3_writel(dwc->regs, DWC3_DEVTEN, 0x00);
2640 }
2641
2642 static irqreturn_t dwc3_interrupt(int irq, void *_dwc);
2643 static irqreturn_t dwc3_thread_interrupt(int irq, void *_dwc);
2644
2645 /**
2646  * dwc3_gadget_setup_nump - calculate and initialize NUMP field of %DWC3_DCFG
2647  * @dwc: pointer to our context structure
2648  *
2649  * The following looks like complex but it's actually very simple. In order to
2650  * calculate the number of packets we can burst at once on OUT transfers, we're
2651  * gonna use RxFIFO size.
2652  *
2653  * To calculate RxFIFO size we need two numbers:
2654  * MDWIDTH = size, in bits, of the internal memory bus
2655  * RAM2_DEPTH = depth, in MDWIDTH, of internal RAM2 (where RxFIFO sits)
2656  *
2657  * Given these two numbers, the formula is simple:
2658  *
2659  * RxFIFO Size = (RAM2_DEPTH * MDWIDTH / 8) - 24 - 16;
2660  *
2661  * 24 bytes is for 3x SETUP packets
2662  * 16 bytes is a clock domain crossing tolerance
2663  *
2664  * Given RxFIFO Size, NUMP = RxFIFOSize / 1024;
2665  */
2666 static void dwc3_gadget_setup_nump(struct dwc3 *dwc)
2667 {
2668         u32 ram2_depth;
2669         u32 mdwidth;
2670         u32 nump;
2671         u32 reg;
2672
2673         ram2_depth = DWC3_GHWPARAMS7_RAM2_DEPTH(dwc->hwparams.hwparams7);
2674         mdwidth = dwc3_mdwidth(dwc);
2675
2676         nump = ((ram2_depth * mdwidth / 8) - 24 - 16) / 1024;
2677         nump = min_t(u32, nump, 16);
2678
2679         /* update NumP */
2680         reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2681         reg &= ~DWC3_DCFG_NUMP_MASK;
2682         reg |= nump << DWC3_DCFG_NUMP_SHIFT;
2683         dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2684 }
2685
2686 static int __dwc3_gadget_start(struct dwc3 *dwc)
2687 {
2688         struct dwc3_ep          *dep;
2689         int                     ret = 0;
2690         u32                     reg;
2691
2692         /*
2693          * Use IMOD if enabled via dwc->imod_interval. Otherwise, if
2694          * the core supports IMOD, disable it.
2695          */
2696         if (dwc->imod_interval) {
2697                 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), dwc->imod_interval);
2698                 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), DWC3_GEVNTCOUNT_EHB);
2699         } else if (dwc3_has_imod(dwc)) {
2700                 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), 0);
2701         }
2702
2703         /*
2704          * We are telling dwc3 that we want to use DCFG.NUMP as ACK TP's NUMP
2705          * field instead of letting dwc3 itself calculate that automatically.
2706          *
2707          * This way, we maximize the chances that we'll be able to get several
2708          * bursts of data without going through any sort of endpoint throttling.
2709          */
2710         reg = dwc3_readl(dwc->regs, DWC3_GRXTHRCFG);
2711         if (DWC3_IP_IS(DWC3))
2712                 reg &= ~DWC3_GRXTHRCFG_PKTCNTSEL;
2713         else
2714                 reg &= ~DWC31_GRXTHRCFG_PKTCNTSEL;
2715
2716         dwc3_writel(dwc->regs, DWC3_GRXTHRCFG, reg);
2717
2718         dwc3_gadget_setup_nump(dwc);
2719
2720         /*
2721          * Currently the controller handles single stream only. So, Ignore
2722          * Packet Pending bit for stream selection and don't search for another
2723          * stream if the host sends Data Packet with PP=0 (for OUT direction) or
2724          * ACK with NumP=0 and PP=0 (for IN direction). This slightly improves
2725          * the stream performance.
2726          */
2727         reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2728         reg |= DWC3_DCFG_IGNSTRMPP;
2729         dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2730
2731         /* Enable MST by default if the device is capable of MST */
2732         if (DWC3_MST_CAPABLE(&dwc->hwparams)) {
2733                 reg = dwc3_readl(dwc->regs, DWC3_DCFG1);
2734                 reg &= ~DWC3_DCFG1_DIS_MST_ENH;
2735                 dwc3_writel(dwc->regs, DWC3_DCFG1, reg);
2736         }
2737
2738         /* Start with SuperSpeed Default */
2739         dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
2740
2741         dep = dwc->eps[0];
2742         ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT);
2743         if (ret) {
2744                 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
2745                 goto err0;
2746         }
2747
2748         dep = dwc->eps[1];
2749         ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT);
2750         if (ret) {
2751                 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
2752                 goto err1;
2753         }
2754
2755         /* begin to receive SETUP packets */
2756         dwc->ep0state = EP0_SETUP_PHASE;
2757         dwc->link_state = DWC3_LINK_STATE_SS_DIS;
2758         dwc->delayed_status = false;
2759         dwc3_ep0_out_start(dwc);
2760
2761         dwc3_gadget_enable_irq(dwc);
2762
2763         return 0;
2764
2765 err1:
2766         __dwc3_gadget_ep_disable(dwc->eps[0]);
2767
2768 err0:
2769         return ret;
2770 }
2771
2772 static int dwc3_gadget_start(struct usb_gadget *g,
2773                 struct usb_gadget_driver *driver)
2774 {
2775         struct dwc3             *dwc = gadget_to_dwc(g);
2776         unsigned long           flags;
2777         int                     ret;
2778         int                     irq;
2779
2780         irq = dwc->irq_gadget;
2781         ret = request_threaded_irq(irq, dwc3_interrupt, dwc3_thread_interrupt,
2782                         IRQF_SHARED, "dwc3", dwc->ev_buf);
2783         if (ret) {
2784                 dev_err(dwc->dev, "failed to request irq #%d --> %d\n",
2785                                 irq, ret);
2786                 return ret;
2787         }
2788
2789         spin_lock_irqsave(&dwc->lock, flags);
2790         dwc->gadget_driver      = driver;
2791         spin_unlock_irqrestore(&dwc->lock, flags);
2792
2793         return 0;
2794 }
2795
2796 static void __dwc3_gadget_stop(struct dwc3 *dwc)
2797 {
2798         dwc3_gadget_disable_irq(dwc);
2799         __dwc3_gadget_ep_disable(dwc->eps[0]);
2800         __dwc3_gadget_ep_disable(dwc->eps[1]);
2801 }
2802
2803 static int dwc3_gadget_stop(struct usb_gadget *g)
2804 {
2805         struct dwc3             *dwc = gadget_to_dwc(g);
2806         unsigned long           flags;
2807
2808         spin_lock_irqsave(&dwc->lock, flags);
2809         dwc->gadget_driver      = NULL;
2810         dwc->max_cfg_eps = 0;
2811         spin_unlock_irqrestore(&dwc->lock, flags);
2812
2813         free_irq(dwc->irq_gadget, dwc->ev_buf);
2814
2815         return 0;
2816 }
2817
2818 static void dwc3_gadget_config_params(struct usb_gadget *g,
2819                                       struct usb_dcd_config_params *params)
2820 {
2821         struct dwc3             *dwc = gadget_to_dwc(g);
2822
2823         params->besl_baseline = USB_DEFAULT_BESL_UNSPECIFIED;
2824         params->besl_deep = USB_DEFAULT_BESL_UNSPECIFIED;
2825
2826         /* Recommended BESL */
2827         if (!dwc->dis_enblslpm_quirk) {
2828                 /*
2829                  * If the recommended BESL baseline is 0 or if the BESL deep is
2830                  * less than 2, Microsoft's Windows 10 host usb stack will issue
2831                  * a usb reset immediately after it receives the extended BOS
2832                  * descriptor and the enumeration will fail. To maintain
2833                  * compatibility with the Windows' usb stack, let's set the
2834                  * recommended BESL baseline to 1 and clamp the BESL deep to be
2835                  * within 2 to 15.
2836                  */
2837                 params->besl_baseline = 1;
2838                 if (dwc->is_utmi_l1_suspend)
2839                         params->besl_deep =
2840                                 clamp_t(u8, dwc->hird_threshold, 2, 15);
2841         }
2842
2843         /* U1 Device exit Latency */
2844         if (dwc->dis_u1_entry_quirk)
2845                 params->bU1devExitLat = 0;
2846         else
2847                 params->bU1devExitLat = DWC3_DEFAULT_U1_DEV_EXIT_LAT;
2848
2849         /* U2 Device exit Latency */
2850         if (dwc->dis_u2_entry_quirk)
2851                 params->bU2DevExitLat = 0;
2852         else
2853                 params->bU2DevExitLat =
2854                                 cpu_to_le16(DWC3_DEFAULT_U2_DEV_EXIT_LAT);
2855 }
2856
2857 static void dwc3_gadget_set_speed(struct usb_gadget *g,
2858                                   enum usb_device_speed speed)
2859 {
2860         struct dwc3             *dwc = gadget_to_dwc(g);
2861         unsigned long           flags;
2862
2863         spin_lock_irqsave(&dwc->lock, flags);
2864         dwc->gadget_max_speed = speed;
2865         spin_unlock_irqrestore(&dwc->lock, flags);
2866 }
2867
2868 static void dwc3_gadget_set_ssp_rate(struct usb_gadget *g,
2869                                      enum usb_ssp_rate rate)
2870 {
2871         struct dwc3             *dwc = gadget_to_dwc(g);
2872         unsigned long           flags;
2873
2874         spin_lock_irqsave(&dwc->lock, flags);
2875         dwc->gadget_max_speed = USB_SPEED_SUPER_PLUS;
2876         dwc->gadget_ssp_rate = rate;
2877         spin_unlock_irqrestore(&dwc->lock, flags);
2878 }
2879
2880 static int dwc3_gadget_vbus_draw(struct usb_gadget *g, unsigned int mA)
2881 {
2882         struct dwc3             *dwc = gadget_to_dwc(g);
2883         union power_supply_propval      val = {0};
2884         int                             ret;
2885
2886         if (dwc->usb2_phy)
2887                 return usb_phy_set_power(dwc->usb2_phy, mA);
2888
2889         if (!dwc->usb_psy)
2890                 return -EOPNOTSUPP;
2891
2892         val.intval = 1000 * mA;
2893         ret = power_supply_set_property(dwc->usb_psy, POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT, &val);
2894
2895         return ret;
2896 }
2897
2898 /**
2899  * dwc3_gadget_check_config - ensure dwc3 can support the USB configuration
2900  * @g: pointer to the USB gadget
2901  *
2902  * Used to record the maximum number of endpoints being used in a USB composite
2903  * device. (across all configurations)  This is to be used in the calculation
2904  * of the TXFIFO sizes when resizing internal memory for individual endpoints.
2905  * It will help ensured that the resizing logic reserves enough space for at
2906  * least one max packet.
2907  */
2908 static int dwc3_gadget_check_config(struct usb_gadget *g)
2909 {
2910         struct dwc3 *dwc = gadget_to_dwc(g);
2911         struct usb_ep *ep;
2912         int fifo_size = 0;
2913         int ram1_depth;
2914         int ep_num = 0;
2915
2916         if (!dwc->do_fifo_resize)
2917                 return 0;
2918
2919         list_for_each_entry(ep, &g->ep_list, ep_list) {
2920                 /* Only interested in the IN endpoints */
2921                 if (ep->claimed && (ep->address & USB_DIR_IN))
2922                         ep_num++;
2923         }
2924
2925         if (ep_num <= dwc->max_cfg_eps)
2926                 return 0;
2927
2928         /* Update the max number of eps in the composition */
2929         dwc->max_cfg_eps = ep_num;
2930
2931         fifo_size = dwc3_gadget_calc_tx_fifo_size(dwc, dwc->max_cfg_eps);
2932         /* Based on the equation, increment by one for every ep */
2933         fifo_size += dwc->max_cfg_eps;
2934
2935         /* Check if we can fit a single fifo per endpoint */
2936         ram1_depth = DWC3_RAM1_DEPTH(dwc->hwparams.hwparams7);
2937         if (fifo_size > ram1_depth)
2938                 return -ENOMEM;
2939
2940         return 0;
2941 }
2942
2943 static void dwc3_gadget_async_callbacks(struct usb_gadget *g, bool enable)
2944 {
2945         struct dwc3             *dwc = gadget_to_dwc(g);
2946         unsigned long           flags;
2947
2948         spin_lock_irqsave(&dwc->lock, flags);
2949         dwc->async_callbacks = enable;
2950         spin_unlock_irqrestore(&dwc->lock, flags);
2951 }
2952
2953 static const struct usb_gadget_ops dwc3_gadget_ops = {
2954         .get_frame              = dwc3_gadget_get_frame,
2955         .wakeup                 = dwc3_gadget_wakeup,
2956         .set_selfpowered        = dwc3_gadget_set_selfpowered,
2957         .pullup                 = dwc3_gadget_pullup,
2958         .udc_start              = dwc3_gadget_start,
2959         .udc_stop               = dwc3_gadget_stop,
2960         .udc_set_speed          = dwc3_gadget_set_speed,
2961         .udc_set_ssp_rate       = dwc3_gadget_set_ssp_rate,
2962         .get_config_params      = dwc3_gadget_config_params,
2963         .vbus_draw              = dwc3_gadget_vbus_draw,
2964         .check_config           = dwc3_gadget_check_config,
2965         .udc_async_callbacks    = dwc3_gadget_async_callbacks,
2966 };
2967
2968 /* -------------------------------------------------------------------------- */
2969
2970 static int dwc3_gadget_init_control_endpoint(struct dwc3_ep *dep)
2971 {
2972         struct dwc3 *dwc = dep->dwc;
2973
2974         usb_ep_set_maxpacket_limit(&dep->endpoint, 512);
2975         dep->endpoint.maxburst = 1;
2976         dep->endpoint.ops = &dwc3_gadget_ep0_ops;
2977         if (!dep->direction)
2978                 dwc->gadget->ep0 = &dep->endpoint;
2979
2980         dep->endpoint.caps.type_control = true;
2981
2982         return 0;
2983 }
2984
2985 static int dwc3_gadget_init_in_endpoint(struct dwc3_ep *dep)
2986 {
2987         struct dwc3 *dwc = dep->dwc;
2988         u32 mdwidth;
2989         int size;
2990
2991         mdwidth = dwc3_mdwidth(dwc);
2992
2993         /* MDWIDTH is represented in bits, we need it in bytes */
2994         mdwidth /= 8;
2995
2996         size = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(dep->number >> 1));
2997         if (DWC3_IP_IS(DWC3))
2998                 size = DWC3_GTXFIFOSIZ_TXFDEP(size);
2999         else
3000                 size = DWC31_GTXFIFOSIZ_TXFDEP(size);
3001
3002         /* FIFO Depth is in MDWDITH bytes. Multiply */
3003         size *= mdwidth;
3004
3005         /*
3006          * To meet performance requirement, a minimum TxFIFO size of 3x
3007          * MaxPacketSize is recommended for endpoints that support burst and a
3008          * minimum TxFIFO size of 2x MaxPacketSize for endpoints that don't
3009          * support burst. Use those numbers and we can calculate the max packet
3010          * limit as below.
3011          */
3012         if (dwc->maximum_speed >= USB_SPEED_SUPER)
3013                 size /= 3;
3014         else
3015                 size /= 2;
3016
3017         usb_ep_set_maxpacket_limit(&dep->endpoint, size);
3018
3019         dep->endpoint.max_streams = 16;
3020         dep->endpoint.ops = &dwc3_gadget_ep_ops;
3021         list_add_tail(&dep->endpoint.ep_list,
3022                         &dwc->gadget->ep_list);
3023         dep->endpoint.caps.type_iso = true;
3024         dep->endpoint.caps.type_bulk = true;
3025         dep->endpoint.caps.type_int = true;
3026
3027         return dwc3_alloc_trb_pool(dep);
3028 }
3029
3030 static int dwc3_gadget_init_out_endpoint(struct dwc3_ep *dep)
3031 {
3032         struct dwc3 *dwc = dep->dwc;
3033         u32 mdwidth;
3034         int size;
3035
3036         mdwidth = dwc3_mdwidth(dwc);
3037
3038         /* MDWIDTH is represented in bits, convert to bytes */
3039         mdwidth /= 8;
3040
3041         /* All OUT endpoints share a single RxFIFO space */
3042         size = dwc3_readl(dwc->regs, DWC3_GRXFIFOSIZ(0));
3043         if (DWC3_IP_IS(DWC3))
3044                 size = DWC3_GRXFIFOSIZ_RXFDEP(size);
3045         else
3046                 size = DWC31_GRXFIFOSIZ_RXFDEP(size);
3047
3048         /* FIFO depth is in MDWDITH bytes */
3049         size *= mdwidth;
3050
3051         /*
3052          * To meet performance requirement, a minimum recommended RxFIFO size
3053          * is defined as follow:
3054          * RxFIFO size >= (3 x MaxPacketSize) +
3055          * (3 x 8 bytes setup packets size) + (16 bytes clock crossing margin)
3056          *
3057          * Then calculate the max packet limit as below.
3058          */
3059         size -= (3 * 8) + 16;
3060         if (size < 0)
3061                 size = 0;
3062         else
3063                 size /= 3;
3064
3065         usb_ep_set_maxpacket_limit(&dep->endpoint, size);
3066         dep->endpoint.max_streams = 16;
3067         dep->endpoint.ops = &dwc3_gadget_ep_ops;
3068         list_add_tail(&dep->endpoint.ep_list,
3069                         &dwc->gadget->ep_list);
3070         dep->endpoint.caps.type_iso = true;
3071         dep->endpoint.caps.type_bulk = true;
3072         dep->endpoint.caps.type_int = true;
3073
3074         return dwc3_alloc_trb_pool(dep);
3075 }
3076
3077 static int dwc3_gadget_init_endpoint(struct dwc3 *dwc, u8 epnum)
3078 {
3079         struct dwc3_ep                  *dep;
3080         bool                            direction = epnum & 1;
3081         int                             ret;
3082         u8                              num = epnum >> 1;
3083
3084         dep = kzalloc(sizeof(*dep), GFP_KERNEL);
3085         if (!dep)
3086                 return -ENOMEM;
3087
3088         dep->dwc = dwc;
3089         dep->number = epnum;
3090         dep->direction = direction;
3091         dep->regs = dwc->regs + DWC3_DEP_BASE(epnum);
3092         dwc->eps[epnum] = dep;
3093         dep->combo_num = 0;
3094         dep->start_cmd_status = 0;
3095
3096         snprintf(dep->name, sizeof(dep->name), "ep%u%s", num,
3097                         direction ? "in" : "out");
3098
3099         dep->endpoint.name = dep->name;
3100
3101         if (!(dep->number > 1)) {
3102                 dep->endpoint.desc = &dwc3_gadget_ep0_desc;
3103                 dep->endpoint.comp_desc = NULL;
3104         }
3105
3106         if (num == 0)
3107                 ret = dwc3_gadget_init_control_endpoint(dep);
3108         else if (direction)
3109                 ret = dwc3_gadget_init_in_endpoint(dep);
3110         else
3111                 ret = dwc3_gadget_init_out_endpoint(dep);
3112
3113         if (ret)
3114                 return ret;
3115
3116         dep->endpoint.caps.dir_in = direction;
3117         dep->endpoint.caps.dir_out = !direction;
3118
3119         INIT_LIST_HEAD(&dep->pending_list);
3120         INIT_LIST_HEAD(&dep->started_list);
3121         INIT_LIST_HEAD(&dep->cancelled_list);
3122
3123         dwc3_debugfs_create_endpoint_dir(dep);
3124
3125         return 0;
3126 }
3127
3128 static int dwc3_gadget_init_endpoints(struct dwc3 *dwc, u8 total)
3129 {
3130         u8                              epnum;
3131
3132         INIT_LIST_HEAD(&dwc->gadget->ep_list);
3133
3134         for (epnum = 0; epnum < total; epnum++) {
3135                 int                     ret;
3136
3137                 ret = dwc3_gadget_init_endpoint(dwc, epnum);
3138                 if (ret)
3139                         return ret;
3140         }
3141
3142         return 0;
3143 }
3144
3145 static void dwc3_gadget_free_endpoints(struct dwc3 *dwc)
3146 {
3147         struct dwc3_ep                  *dep;
3148         u8                              epnum;
3149
3150         for (epnum = 0; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
3151                 dep = dwc->eps[epnum];
3152                 if (!dep)
3153                         continue;
3154                 /*
3155                  * Physical endpoints 0 and 1 are special; they form the
3156                  * bi-directional USB endpoint 0.
3157                  *
3158                  * For those two physical endpoints, we don't allocate a TRB
3159                  * pool nor do we add them the endpoints list. Due to that, we
3160                  * shouldn't do these two operations otherwise we would end up
3161                  * with all sorts of bugs when removing dwc3.ko.
3162                  */
3163                 if (epnum != 0 && epnum != 1) {
3164                         dwc3_free_trb_pool(dep);
3165                         list_del(&dep->endpoint.ep_list);
3166                 }
3167
3168                 debugfs_remove_recursive(debugfs_lookup(dep->name,
3169                                 debugfs_lookup(dev_name(dep->dwc->dev),
3170                                                usb_debug_root)));
3171                 kfree(dep);
3172         }
3173 }
3174
3175 /* -------------------------------------------------------------------------- */
3176
3177 static int dwc3_gadget_ep_reclaim_completed_trb(struct dwc3_ep *dep,
3178                 struct dwc3_request *req, struct dwc3_trb *trb,
3179                 const struct dwc3_event_depevt *event, int status, int chain)
3180 {
3181         unsigned int            count;
3182
3183         dwc3_ep_inc_deq(dep);
3184
3185         trace_dwc3_complete_trb(dep, trb);
3186         req->num_trbs--;
3187
3188         /*
3189          * If we're in the middle of series of chained TRBs and we
3190          * receive a short transfer along the way, DWC3 will skip
3191          * through all TRBs including the last TRB in the chain (the
3192          * where CHN bit is zero. DWC3 will also avoid clearing HWO
3193          * bit and SW has to do it manually.
3194          *
3195          * We're going to do that here to avoid problems of HW trying
3196          * to use bogus TRBs for transfers.
3197          */
3198         if (chain && (trb->ctrl & DWC3_TRB_CTRL_HWO))
3199                 trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
3200
3201         /*
3202          * For isochronous transfers, the first TRB in a service interval must
3203          * have the Isoc-First type. Track and report its interval frame number.
3204          */
3205         if (usb_endpoint_xfer_isoc(dep->endpoint.desc) &&
3206             (trb->ctrl & DWC3_TRBCTL_ISOCHRONOUS_FIRST)) {
3207                 unsigned int frame_number;
3208
3209                 frame_number = DWC3_TRB_CTRL_GET_SID_SOFN(trb->ctrl);
3210                 frame_number &= ~(dep->interval - 1);
3211                 req->request.frame_number = frame_number;
3212         }
3213
3214         /*
3215          * We use bounce buffer for requests that needs extra TRB or OUT ZLP. If
3216          * this TRB points to the bounce buffer address, it's a MPS alignment
3217          * TRB. Don't add it to req->remaining calculation.
3218          */
3219         if (trb->bpl == lower_32_bits(dep->dwc->bounce_addr) &&
3220             trb->bph == upper_32_bits(dep->dwc->bounce_addr)) {
3221                 trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
3222                 return 1;
3223         }
3224
3225         count = trb->size & DWC3_TRB_SIZE_MASK;
3226         req->remaining += count;
3227
3228         if ((trb->ctrl & DWC3_TRB_CTRL_HWO) && status != -ESHUTDOWN)
3229                 return 1;
3230
3231         if (event->status & DEPEVT_STATUS_SHORT && !chain)
3232                 return 1;
3233
3234         if ((trb->ctrl & DWC3_TRB_CTRL_IOC) ||
3235             (trb->ctrl & DWC3_TRB_CTRL_LST))
3236                 return 1;
3237
3238         return 0;
3239 }
3240
3241 static int dwc3_gadget_ep_reclaim_trb_sg(struct dwc3_ep *dep,
3242                 struct dwc3_request *req, const struct dwc3_event_depevt *event,
3243                 int status)
3244 {
3245         struct dwc3_trb *trb = &dep->trb_pool[dep->trb_dequeue];
3246         struct scatterlist *sg = req->sg;
3247         struct scatterlist *s;
3248         unsigned int num_queued = req->num_queued_sgs;
3249         unsigned int i;
3250         int ret = 0;
3251
3252         for_each_sg(sg, s, num_queued, i) {
3253                 trb = &dep->trb_pool[dep->trb_dequeue];
3254
3255                 req->sg = sg_next(s);
3256                 req->num_queued_sgs--;
3257
3258                 ret = dwc3_gadget_ep_reclaim_completed_trb(dep, req,
3259                                 trb, event, status, true);
3260                 if (ret)
3261                         break;
3262         }
3263
3264         return ret;
3265 }
3266
3267 static int dwc3_gadget_ep_reclaim_trb_linear(struct dwc3_ep *dep,
3268                 struct dwc3_request *req, const struct dwc3_event_depevt *event,
3269                 int status)
3270 {
3271         struct dwc3_trb *trb = &dep->trb_pool[dep->trb_dequeue];
3272
3273         return dwc3_gadget_ep_reclaim_completed_trb(dep, req, trb,
3274                         event, status, false);
3275 }
3276
3277 static bool dwc3_gadget_ep_request_completed(struct dwc3_request *req)
3278 {
3279         return req->num_pending_sgs == 0 && req->num_queued_sgs == 0;
3280 }
3281
3282 static int dwc3_gadget_ep_cleanup_completed_request(struct dwc3_ep *dep,
3283                 const struct dwc3_event_depevt *event,
3284                 struct dwc3_request *req, int status)
3285 {
3286         int request_status;
3287         int ret;
3288
3289         if (req->request.num_mapped_sgs)
3290                 ret = dwc3_gadget_ep_reclaim_trb_sg(dep, req, event,
3291                                 status);
3292         else
3293                 ret = dwc3_gadget_ep_reclaim_trb_linear(dep, req, event,
3294                                 status);
3295
3296         req->request.actual = req->request.length - req->remaining;
3297
3298         if (!dwc3_gadget_ep_request_completed(req))
3299                 goto out;
3300
3301         if (req->needs_extra_trb) {
3302                 ret = dwc3_gadget_ep_reclaim_trb_linear(dep, req, event,
3303                                 status);
3304                 req->needs_extra_trb = false;
3305         }
3306
3307         /*
3308          * The event status only reflects the status of the TRB with IOC set.
3309          * For the requests that don't set interrupt on completion, the driver
3310          * needs to check and return the status of the completed TRBs associated
3311          * with the request. Use the status of the last TRB of the request.
3312          */
3313         if (req->request.no_interrupt) {
3314                 struct dwc3_trb *trb;
3315
3316                 trb = dwc3_ep_prev_trb(dep, dep->trb_dequeue);
3317                 switch (DWC3_TRB_SIZE_TRBSTS(trb->size)) {
3318                 case DWC3_TRBSTS_MISSED_ISOC:
3319                         /* Isoc endpoint only */
3320                         request_status = -EXDEV;
3321                         break;
3322                 case DWC3_TRB_STS_XFER_IN_PROG:
3323                         /* Applicable when End Transfer with ForceRM=0 */
3324                 case DWC3_TRBSTS_SETUP_PENDING:
3325                         /* Control endpoint only */
3326                 case DWC3_TRBSTS_OK:
3327                 default:
3328                         request_status = 0;
3329                         break;
3330                 }
3331         } else {
3332                 request_status = status;
3333         }
3334
3335         dwc3_gadget_giveback(dep, req, request_status);
3336
3337 out:
3338         return ret;
3339 }
3340
3341 static void dwc3_gadget_ep_cleanup_completed_requests(struct dwc3_ep *dep,
3342                 const struct dwc3_event_depevt *event, int status)
3343 {
3344         struct dwc3_request     *req;
3345
3346         while (!list_empty(&dep->started_list)) {
3347                 int ret;
3348
3349                 req = next_request(&dep->started_list);
3350                 ret = dwc3_gadget_ep_cleanup_completed_request(dep, event,
3351                                 req, status);
3352                 if (ret)
3353                         break;
3354                 /*
3355                  * The endpoint is disabled, let the dwc3_remove_requests()
3356                  * handle the cleanup.
3357                  */
3358                 if (!dep->endpoint.desc)
3359                         break;
3360         }
3361 }
3362
3363 static bool dwc3_gadget_ep_should_continue(struct dwc3_ep *dep)
3364 {
3365         struct dwc3_request     *req;
3366         struct dwc3             *dwc = dep->dwc;
3367
3368         if (!dep->endpoint.desc || !dwc->pullups_connected ||
3369             !dwc->connected)
3370                 return false;
3371
3372         if (!list_empty(&dep->pending_list))
3373                 return true;
3374
3375         /*
3376          * We only need to check the first entry of the started list. We can
3377          * assume the completed requests are removed from the started list.
3378          */
3379         req = next_request(&dep->started_list);
3380         if (!req)
3381                 return false;
3382
3383         return !dwc3_gadget_ep_request_completed(req);
3384 }
3385
3386 static void dwc3_gadget_endpoint_frame_from_event(struct dwc3_ep *dep,
3387                 const struct dwc3_event_depevt *event)
3388 {
3389         dep->frame_number = event->parameters;
3390 }
3391
3392 static bool dwc3_gadget_endpoint_trbs_complete(struct dwc3_ep *dep,
3393                 const struct dwc3_event_depevt *event, int status)
3394 {
3395         struct dwc3             *dwc = dep->dwc;
3396         bool                    no_started_trb = true;
3397
3398         if (!dep->endpoint.desc)
3399                 return no_started_trb;
3400
3401         dwc3_gadget_ep_cleanup_completed_requests(dep, event, status);
3402
3403         if (dep->flags & DWC3_EP_END_TRANSFER_PENDING)
3404                 goto out;
3405
3406         if (usb_endpoint_xfer_isoc(dep->endpoint.desc) &&
3407                 list_empty(&dep->started_list) &&
3408                 (list_empty(&dep->pending_list) || status == -EXDEV))
3409                 dwc3_stop_active_transfer(dep, true, true);
3410         else if (dwc3_gadget_ep_should_continue(dep))
3411                 if (__dwc3_gadget_kick_transfer(dep) == 0)
3412                         no_started_trb = false;
3413
3414 out:
3415         /*
3416          * WORKAROUND: This is the 2nd half of U1/U2 -> U0 workaround.
3417          * See dwc3_gadget_linksts_change_interrupt() for 1st half.
3418          */
3419         if (DWC3_VER_IS_PRIOR(DWC3, 183A)) {
3420                 u32             reg;
3421                 int             i;
3422
3423                 for (i = 0; i < DWC3_ENDPOINTS_NUM; i++) {
3424                         dep = dwc->eps[i];
3425
3426                         if (!(dep->flags & DWC3_EP_ENABLED))
3427                                 continue;
3428
3429                         if (!list_empty(&dep->started_list))
3430                                 return no_started_trb;
3431                 }
3432
3433                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
3434                 reg |= dwc->u1u2;
3435                 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
3436
3437                 dwc->u1u2 = 0;
3438         }
3439
3440         return no_started_trb;
3441 }
3442
3443 static void dwc3_gadget_endpoint_transfer_in_progress(struct dwc3_ep *dep,
3444                 const struct dwc3_event_depevt *event)
3445 {
3446         int status = 0;
3447
3448         if (!dep->endpoint.desc)
3449                 return;
3450
3451         if (usb_endpoint_xfer_isoc(dep->endpoint.desc))
3452                 dwc3_gadget_endpoint_frame_from_event(dep, event);
3453
3454         if (event->status & DEPEVT_STATUS_BUSERR)
3455                 status = -ECONNRESET;
3456
3457         if (event->status & DEPEVT_STATUS_MISSED_ISOC)
3458                 status = -EXDEV;
3459
3460         dwc3_gadget_endpoint_trbs_complete(dep, event, status);
3461 }
3462
3463 static void dwc3_gadget_endpoint_transfer_complete(struct dwc3_ep *dep,
3464                 const struct dwc3_event_depevt *event)
3465 {
3466         int status = 0;
3467
3468         dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
3469
3470         if (event->status & DEPEVT_STATUS_BUSERR)
3471                 status = -ECONNRESET;
3472
3473         if (dwc3_gadget_endpoint_trbs_complete(dep, event, status))
3474                 dep->flags &= ~DWC3_EP_WAIT_TRANSFER_COMPLETE;
3475 }
3476
3477 static void dwc3_gadget_endpoint_transfer_not_ready(struct dwc3_ep *dep,
3478                 const struct dwc3_event_depevt *event)
3479 {
3480         dwc3_gadget_endpoint_frame_from_event(dep, event);
3481
3482         /*
3483          * The XferNotReady event is generated only once before the endpoint
3484          * starts. It will be generated again when END_TRANSFER command is
3485          * issued. For some controller versions, the XferNotReady event may be
3486          * generated while the END_TRANSFER command is still in process. Ignore
3487          * it and wait for the next XferNotReady event after the command is
3488          * completed.
3489          */
3490         if (dep->flags & DWC3_EP_END_TRANSFER_PENDING)
3491                 return;
3492
3493         (void) __dwc3_gadget_start_isoc(dep);
3494 }
3495
3496 static void dwc3_gadget_endpoint_command_complete(struct dwc3_ep *dep,
3497                 const struct dwc3_event_depevt *event)
3498 {
3499         u8 cmd = DEPEVT_PARAMETER_CMD(event->parameters);
3500
3501         if (cmd != DWC3_DEPCMD_ENDTRANSFER)
3502                 return;
3503
3504         /*
3505          * The END_TRANSFER command will cause the controller to generate a
3506          * NoStream Event, and it's not due to the host DP NoStream rejection.
3507          * Ignore the next NoStream event.
3508          */
3509         if (dep->stream_capable)
3510                 dep->flags |= DWC3_EP_IGNORE_NEXT_NOSTREAM;
3511
3512         dep->flags &= ~DWC3_EP_END_TRANSFER_PENDING;
3513         dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
3514         dwc3_gadget_ep_cleanup_cancelled_requests(dep);
3515
3516         if (dep->flags & DWC3_EP_PENDING_CLEAR_STALL) {
3517                 struct dwc3 *dwc = dep->dwc;
3518
3519                 dep->flags &= ~DWC3_EP_PENDING_CLEAR_STALL;
3520                 if (dwc3_send_clear_stall_ep_cmd(dep)) {
3521                         struct usb_ep *ep0 = &dwc->eps[0]->endpoint;
3522
3523                         dev_err(dwc->dev, "failed to clear STALL on %s\n", dep->name);
3524                         if (dwc->delayed_status)
3525                                 __dwc3_gadget_ep0_set_halt(ep0, 1);
3526                         return;
3527                 }
3528
3529                 dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE);
3530                 if (dwc->clear_stall_protocol == dep->number)
3531                         dwc3_ep0_send_delayed_status(dwc);
3532         }
3533
3534         if ((dep->flags & DWC3_EP_DELAY_START) &&
3535             !usb_endpoint_xfer_isoc(dep->endpoint.desc))
3536                 __dwc3_gadget_kick_transfer(dep);
3537
3538         dep->flags &= ~DWC3_EP_DELAY_START;
3539 }
3540
3541 static void dwc3_gadget_endpoint_stream_event(struct dwc3_ep *dep,
3542                 const struct dwc3_event_depevt *event)
3543 {
3544         struct dwc3 *dwc = dep->dwc;
3545
3546         if (event->status == DEPEVT_STREAMEVT_FOUND) {
3547                 dep->flags |= DWC3_EP_FIRST_STREAM_PRIMED;
3548                 goto out;
3549         }
3550
3551         /* Note: NoStream rejection event param value is 0 and not 0xFFFF */
3552         switch (event->parameters) {
3553         case DEPEVT_STREAM_PRIME:
3554                 /*
3555                  * If the host can properly transition the endpoint state from
3556                  * idle to prime after a NoStream rejection, there's no need to
3557                  * force restarting the endpoint to reinitiate the stream. To
3558                  * simplify the check, assume the host follows the USB spec if
3559                  * it primed the endpoint more than once.
3560                  */
3561                 if (dep->flags & DWC3_EP_FORCE_RESTART_STREAM) {
3562                         if (dep->flags & DWC3_EP_FIRST_STREAM_PRIMED)
3563                                 dep->flags &= ~DWC3_EP_FORCE_RESTART_STREAM;
3564                         else
3565                                 dep->flags |= DWC3_EP_FIRST_STREAM_PRIMED;
3566                 }
3567
3568                 break;
3569         case DEPEVT_STREAM_NOSTREAM:
3570                 if ((dep->flags & DWC3_EP_IGNORE_NEXT_NOSTREAM) ||
3571                     !(dep->flags & DWC3_EP_FORCE_RESTART_STREAM) ||
3572                     (!DWC3_MST_CAPABLE(&dwc->hwparams) &&
3573                      !(dep->flags & DWC3_EP_WAIT_TRANSFER_COMPLETE)))
3574                         break;
3575
3576                 /*
3577                  * If the host rejects a stream due to no active stream, by the
3578                  * USB and xHCI spec, the endpoint will be put back to idle
3579                  * state. When the host is ready (buffer added/updated), it will
3580                  * prime the endpoint to inform the usb device controller. This
3581                  * triggers the device controller to issue ERDY to restart the
3582                  * stream. However, some hosts don't follow this and keep the
3583                  * endpoint in the idle state. No prime will come despite host
3584                  * streams are updated, and the device controller will not be
3585                  * triggered to generate ERDY to move the next stream data. To
3586                  * workaround this and maintain compatibility with various
3587                  * hosts, force to reinitate the stream until the host is ready
3588                  * instead of waiting for the host to prime the endpoint.
3589                  */
3590                 if (DWC3_VER_IS_WITHIN(DWC32, 100A, ANY)) {
3591                         unsigned int cmd = DWC3_DGCMD_SET_ENDPOINT_PRIME;
3592
3593                         dwc3_send_gadget_generic_command(dwc, cmd, dep->number);
3594                 } else {
3595                         dep->flags |= DWC3_EP_DELAY_START;
3596                         dwc3_stop_active_transfer(dep, true, true);
3597                         return;
3598                 }
3599                 break;
3600         }
3601
3602 out:
3603         dep->flags &= ~DWC3_EP_IGNORE_NEXT_NOSTREAM;
3604 }
3605
3606 static void dwc3_endpoint_interrupt(struct dwc3 *dwc,
3607                 const struct dwc3_event_depevt *event)
3608 {
3609         struct dwc3_ep          *dep;
3610         u8                      epnum = event->endpoint_number;
3611
3612         dep = dwc->eps[epnum];
3613
3614         if (!(dep->flags & DWC3_EP_ENABLED)) {
3615                 if (!(dep->flags & DWC3_EP_TRANSFER_STARTED))
3616                         return;
3617
3618                 /* Handle only EPCMDCMPLT when EP disabled */
3619                 if (event->endpoint_event != DWC3_DEPEVT_EPCMDCMPLT)
3620                         return;
3621         }
3622
3623         if (epnum == 0 || epnum == 1) {
3624                 dwc3_ep0_interrupt(dwc, event);
3625                 return;
3626         }
3627
3628         switch (event->endpoint_event) {
3629         case DWC3_DEPEVT_XFERINPROGRESS:
3630                 dwc3_gadget_endpoint_transfer_in_progress(dep, event);
3631                 break;
3632         case DWC3_DEPEVT_XFERNOTREADY:
3633                 dwc3_gadget_endpoint_transfer_not_ready(dep, event);
3634                 break;
3635         case DWC3_DEPEVT_EPCMDCMPLT:
3636                 dwc3_gadget_endpoint_command_complete(dep, event);
3637                 break;
3638         case DWC3_DEPEVT_XFERCOMPLETE:
3639                 dwc3_gadget_endpoint_transfer_complete(dep, event);
3640                 break;
3641         case DWC3_DEPEVT_STREAMEVT:
3642                 dwc3_gadget_endpoint_stream_event(dep, event);
3643                 break;
3644         case DWC3_DEPEVT_RXTXFIFOEVT:
3645                 break;
3646         }
3647 }
3648
3649 static void dwc3_disconnect_gadget(struct dwc3 *dwc)
3650 {
3651         if (dwc->async_callbacks && dwc->gadget_driver->disconnect) {
3652                 spin_unlock(&dwc->lock);
3653                 dwc->gadget_driver->disconnect(dwc->gadget);
3654                 spin_lock(&dwc->lock);
3655         }
3656 }
3657
3658 static void dwc3_suspend_gadget(struct dwc3 *dwc)
3659 {
3660         if (dwc->async_callbacks && dwc->gadget_driver->suspend) {
3661                 spin_unlock(&dwc->lock);
3662                 dwc->gadget_driver->suspend(dwc->gadget);
3663                 spin_lock(&dwc->lock);
3664         }
3665 }
3666
3667 static void dwc3_resume_gadget(struct dwc3 *dwc)
3668 {
3669         if (dwc->async_callbacks && dwc->gadget_driver->resume) {
3670                 spin_unlock(&dwc->lock);
3671                 dwc->gadget_driver->resume(dwc->gadget);
3672                 spin_lock(&dwc->lock);
3673         }
3674 }
3675
3676 static void dwc3_reset_gadget(struct dwc3 *dwc)
3677 {
3678         if (!dwc->gadget_driver)
3679                 return;
3680
3681         if (dwc->async_callbacks && dwc->gadget->speed != USB_SPEED_UNKNOWN) {
3682                 spin_unlock(&dwc->lock);
3683                 usb_gadget_udc_reset(dwc->gadget, dwc->gadget_driver);
3684                 spin_lock(&dwc->lock);
3685         }
3686 }
3687
3688 void dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force,
3689         bool interrupt)
3690 {
3691         if (!(dep->flags & DWC3_EP_TRANSFER_STARTED) ||
3692             (dep->flags & DWC3_EP_DELAY_STOP) ||
3693             (dep->flags & DWC3_EP_END_TRANSFER_PENDING))
3694                 return;
3695
3696         /*
3697          * NOTICE: We are violating what the Databook says about the
3698          * EndTransfer command. Ideally we would _always_ wait for the
3699          * EndTransfer Command Completion IRQ, but that's causing too
3700          * much trouble synchronizing between us and gadget driver.
3701          *
3702          * We have discussed this with the IP Provider and it was
3703          * suggested to giveback all requests here.
3704          *
3705          * Note also that a similar handling was tested by Synopsys
3706          * (thanks a lot Paul) and nothing bad has come out of it.
3707          * In short, what we're doing is issuing EndTransfer with
3708          * CMDIOC bit set and delay kicking transfer until the
3709          * EndTransfer command had completed.
3710          *
3711          * As of IP version 3.10a of the DWC_usb3 IP, the controller
3712          * supports a mode to work around the above limitation. The
3713          * software can poll the CMDACT bit in the DEPCMD register
3714          * after issuing a EndTransfer command. This mode is enabled
3715          * by writing GUCTL2[14]. This polling is already done in the
3716          * dwc3_send_gadget_ep_cmd() function so if the mode is
3717          * enabled, the EndTransfer command will have completed upon
3718          * returning from this function.
3719          *
3720          * This mode is NOT available on the DWC_usb31 IP.
3721          */
3722
3723         __dwc3_stop_active_transfer(dep, force, interrupt);
3724 }
3725
3726 static void dwc3_clear_stall_all_ep(struct dwc3 *dwc)
3727 {
3728         u32 epnum;
3729
3730         for (epnum = 1; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
3731                 struct dwc3_ep *dep;
3732                 int ret;
3733
3734                 dep = dwc->eps[epnum];
3735                 if (!dep)
3736                         continue;
3737
3738                 if (!(dep->flags & DWC3_EP_STALL))
3739                         continue;
3740
3741                 dep->flags &= ~DWC3_EP_STALL;
3742
3743                 ret = dwc3_send_clear_stall_ep_cmd(dep);
3744                 WARN_ON_ONCE(ret);
3745         }
3746 }
3747
3748 static void dwc3_gadget_disconnect_interrupt(struct dwc3 *dwc)
3749 {
3750         int                     reg;
3751
3752         dwc3_gadget_set_link_state(dwc, DWC3_LINK_STATE_RX_DET);
3753
3754         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
3755         reg &= ~DWC3_DCTL_INITU1ENA;
3756         reg &= ~DWC3_DCTL_INITU2ENA;
3757         dwc3_gadget_dctl_write_safe(dwc, reg);
3758
3759         dwc3_disconnect_gadget(dwc);
3760
3761         dwc->gadget->speed = USB_SPEED_UNKNOWN;
3762         dwc->setup_packet_pending = false;
3763         usb_gadget_set_state(dwc->gadget, USB_STATE_NOTATTACHED);
3764
3765         dwc->connected = false;
3766 }
3767
3768 static void dwc3_gadget_reset_interrupt(struct dwc3 *dwc)
3769 {
3770         u32                     reg;
3771
3772         /*
3773          * Ideally, dwc3_reset_gadget() would trigger the function
3774          * drivers to stop any active transfers through ep disable.
3775          * However, for functions which defer ep disable, such as mass
3776          * storage, we will need to rely on the call to stop active
3777          * transfers here, and avoid allowing of request queuing.
3778          */
3779         dwc->connected = false;
3780
3781         /*
3782          * WORKAROUND: DWC3 revisions <1.88a have an issue which
3783          * would cause a missing Disconnect Event if there's a
3784          * pending Setup Packet in the FIFO.
3785          *
3786          * There's no suggested workaround on the official Bug
3787          * report, which states that "unless the driver/application
3788          * is doing any special handling of a disconnect event,
3789          * there is no functional issue".
3790          *
3791          * Unfortunately, it turns out that we _do_ some special
3792          * handling of a disconnect event, namely complete all
3793          * pending transfers, notify gadget driver of the
3794          * disconnection, and so on.
3795          *
3796          * Our suggested workaround is to follow the Disconnect
3797          * Event steps here, instead, based on a setup_packet_pending
3798          * flag. Such flag gets set whenever we have a SETUP_PENDING
3799          * status for EP0 TRBs and gets cleared on XferComplete for the
3800          * same endpoint.
3801          *
3802          * Refers to:
3803          *
3804          * STAR#9000466709: RTL: Device : Disconnect event not
3805          * generated if setup packet pending in FIFO
3806          */
3807         if (DWC3_VER_IS_PRIOR(DWC3, 188A)) {
3808                 if (dwc->setup_packet_pending)
3809                         dwc3_gadget_disconnect_interrupt(dwc);
3810         }
3811
3812         dwc3_reset_gadget(dwc);
3813         /*
3814          * In the Synopsis DesignWare Cores USB3 Databook Rev. 3.30a
3815          * Section 4.1.2 Table 4-2, it states that during a USB reset, the SW
3816          * needs to ensure that it sends "a DEPENDXFER command for any active
3817          * transfers."
3818          */
3819         dwc3_stop_active_transfers(dwc);
3820         dwc->connected = true;
3821
3822         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
3823         reg &= ~DWC3_DCTL_TSTCTRL_MASK;
3824         dwc3_gadget_dctl_write_safe(dwc, reg);
3825         dwc->test_mode = false;
3826         dwc3_clear_stall_all_ep(dwc);
3827
3828         /* Reset device address to zero */
3829         reg = dwc3_readl(dwc->regs, DWC3_DCFG);
3830         reg &= ~(DWC3_DCFG_DEVADDR_MASK);
3831         dwc3_writel(dwc->regs, DWC3_DCFG, reg);
3832 }
3833
3834 static void dwc3_gadget_conndone_interrupt(struct dwc3 *dwc)
3835 {
3836         struct dwc3_ep          *dep;
3837         int                     ret;
3838         u32                     reg;
3839         u8                      lanes = 1;
3840         u8                      speed;
3841
3842         reg = dwc3_readl(dwc->regs, DWC3_DSTS);
3843         speed = reg & DWC3_DSTS_CONNECTSPD;
3844         dwc->speed = speed;
3845
3846         if (DWC3_IP_IS(DWC32))
3847                 lanes = DWC3_DSTS_CONNLANES(reg) + 1;
3848
3849         dwc->gadget->ssp_rate = USB_SSP_GEN_UNKNOWN;
3850
3851         /*
3852          * RAMClkSel is reset to 0 after USB reset, so it must be reprogrammed
3853          * each time on Connect Done.
3854          *
3855          * Currently we always use the reset value. If any platform
3856          * wants to set this to a different value, we need to add a
3857          * setting and update GCTL.RAMCLKSEL here.
3858          */
3859
3860         switch (speed) {
3861         case DWC3_DSTS_SUPERSPEED_PLUS:
3862                 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
3863                 dwc->gadget->ep0->maxpacket = 512;
3864                 dwc->gadget->speed = USB_SPEED_SUPER_PLUS;
3865
3866                 if (lanes > 1)
3867                         dwc->gadget->ssp_rate = USB_SSP_GEN_2x2;
3868                 else
3869                         dwc->gadget->ssp_rate = USB_SSP_GEN_2x1;
3870                 break;
3871         case DWC3_DSTS_SUPERSPEED:
3872                 /*
3873                  * WORKAROUND: DWC3 revisions <1.90a have an issue which
3874                  * would cause a missing USB3 Reset event.
3875                  *
3876                  * In such situations, we should force a USB3 Reset
3877                  * event by calling our dwc3_gadget_reset_interrupt()
3878                  * routine.
3879                  *
3880                  * Refers to:
3881                  *
3882                  * STAR#9000483510: RTL: SS : USB3 reset event may
3883                  * not be generated always when the link enters poll
3884                  */
3885                 if (DWC3_VER_IS_PRIOR(DWC3, 190A))
3886                         dwc3_gadget_reset_interrupt(dwc);
3887
3888                 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
3889                 dwc->gadget->ep0->maxpacket = 512;
3890                 dwc->gadget->speed = USB_SPEED_SUPER;
3891
3892                 if (lanes > 1) {
3893                         dwc->gadget->speed = USB_SPEED_SUPER_PLUS;
3894                         dwc->gadget->ssp_rate = USB_SSP_GEN_1x2;
3895                 }
3896                 break;
3897         case DWC3_DSTS_HIGHSPEED:
3898                 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
3899                 dwc->gadget->ep0->maxpacket = 64;
3900                 dwc->gadget->speed = USB_SPEED_HIGH;
3901                 break;
3902         case DWC3_DSTS_FULLSPEED:
3903                 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
3904                 dwc->gadget->ep0->maxpacket = 64;
3905                 dwc->gadget->speed = USB_SPEED_FULL;
3906                 break;
3907         }
3908
3909         dwc->eps[1]->endpoint.maxpacket = dwc->gadget->ep0->maxpacket;
3910
3911         /* Enable USB2 LPM Capability */
3912
3913         if (!DWC3_VER_IS_WITHIN(DWC3, ANY, 194A) &&
3914             !dwc->usb2_gadget_lpm_disable &&
3915             (speed != DWC3_DSTS_SUPERSPEED) &&
3916             (speed != DWC3_DSTS_SUPERSPEED_PLUS)) {
3917                 reg = dwc3_readl(dwc->regs, DWC3_DCFG);
3918                 reg |= DWC3_DCFG_LPM_CAP;
3919                 dwc3_writel(dwc->regs, DWC3_DCFG, reg);
3920
3921                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
3922                 reg &= ~(DWC3_DCTL_HIRD_THRES_MASK | DWC3_DCTL_L1_HIBER_EN);
3923
3924                 reg |= DWC3_DCTL_HIRD_THRES(dwc->hird_threshold |
3925                                             (dwc->is_utmi_l1_suspend << 4));
3926
3927                 /*
3928                  * When dwc3 revisions >= 2.40a, LPM Erratum is enabled and
3929                  * DCFG.LPMCap is set, core responses with an ACK and the
3930                  * BESL value in the LPM token is less than or equal to LPM
3931                  * NYET threshold.
3932                  */
3933                 WARN_ONCE(DWC3_VER_IS_PRIOR(DWC3, 240A) && dwc->has_lpm_erratum,
3934                                 "LPM Erratum not available on dwc3 revisions < 2.40a\n");
3935
3936                 if (dwc->has_lpm_erratum && !DWC3_VER_IS_PRIOR(DWC3, 240A))
3937                         reg |= DWC3_DCTL_NYET_THRES(dwc->lpm_nyet_threshold);
3938
3939                 dwc3_gadget_dctl_write_safe(dwc, reg);
3940         } else {
3941                 if (dwc->usb2_gadget_lpm_disable) {
3942                         reg = dwc3_readl(dwc->regs, DWC3_DCFG);
3943                         reg &= ~DWC3_DCFG_LPM_CAP;
3944                         dwc3_writel(dwc->regs, DWC3_DCFG, reg);
3945                 }
3946
3947                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
3948                 reg &= ~DWC3_DCTL_HIRD_THRES_MASK;
3949                 dwc3_gadget_dctl_write_safe(dwc, reg);
3950         }
3951
3952         dep = dwc->eps[0];
3953         ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_MODIFY);
3954         if (ret) {
3955                 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
3956                 return;
3957         }
3958
3959         dep = dwc->eps[1];
3960         ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_MODIFY);
3961         if (ret) {
3962                 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
3963                 return;
3964         }
3965
3966         /*
3967          * Configure PHY via GUSB3PIPECTLn if required.
3968          *
3969          * Update GTXFIFOSIZn
3970          *
3971          * In both cases reset values should be sufficient.
3972          */
3973 }
3974
3975 static void dwc3_gadget_wakeup_interrupt(struct dwc3 *dwc)
3976 {
3977         /*
3978          * TODO take core out of low power mode when that's
3979          * implemented.
3980          */
3981
3982         if (dwc->async_callbacks && dwc->gadget_driver->resume) {
3983                 spin_unlock(&dwc->lock);
3984                 dwc->gadget_driver->resume(dwc->gadget);
3985                 spin_lock(&dwc->lock);
3986         }
3987 }
3988
3989 static void dwc3_gadget_linksts_change_interrupt(struct dwc3 *dwc,
3990                 unsigned int evtinfo)
3991 {
3992         enum dwc3_link_state    next = evtinfo & DWC3_LINK_STATE_MASK;
3993         unsigned int            pwropt;
3994
3995         /*
3996          * WORKAROUND: DWC3 < 2.50a have an issue when configured without
3997          * Hibernation mode enabled which would show up when device detects
3998          * host-initiated U3 exit.
3999          *
4000          * In that case, device will generate a Link State Change Interrupt
4001          * from U3 to RESUME which is only necessary if Hibernation is
4002          * configured in.
4003          *
4004          * There are no functional changes due to such spurious event and we
4005          * just need to ignore it.
4006          *
4007          * Refers to:
4008          *
4009          * STAR#9000570034 RTL: SS Resume event generated in non-Hibernation
4010          * operational mode
4011          */
4012         pwropt = DWC3_GHWPARAMS1_EN_PWROPT(dwc->hwparams.hwparams1);
4013         if (DWC3_VER_IS_PRIOR(DWC3, 250A) &&
4014                         (pwropt != DWC3_GHWPARAMS1_EN_PWROPT_HIB)) {
4015                 if ((dwc->link_state == DWC3_LINK_STATE_U3) &&
4016                                 (next == DWC3_LINK_STATE_RESUME)) {
4017                         return;
4018                 }
4019         }
4020
4021         /*
4022          * WORKAROUND: DWC3 Revisions <1.83a have an issue which, depending
4023          * on the link partner, the USB session might do multiple entry/exit
4024          * of low power states before a transfer takes place.
4025          *
4026          * Due to this problem, we might experience lower throughput. The
4027          * suggested workaround is to disable DCTL[12:9] bits if we're
4028          * transitioning from U1/U2 to U0 and enable those bits again
4029          * after a transfer completes and there are no pending transfers
4030          * on any of the enabled endpoints.
4031          *
4032          * This is the first half of that workaround.
4033          *
4034          * Refers to:
4035          *
4036          * STAR#9000446952: RTL: Device SS : if U1/U2 ->U0 takes >128us
4037          * core send LGO_Ux entering U0
4038          */
4039         if (DWC3_VER_IS_PRIOR(DWC3, 183A)) {
4040                 if (next == DWC3_LINK_STATE_U0) {
4041                         u32     u1u2;
4042                         u32     reg;
4043
4044                         switch (dwc->link_state) {
4045                         case DWC3_LINK_STATE_U1:
4046                         case DWC3_LINK_STATE_U2:
4047                                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
4048                                 u1u2 = reg & (DWC3_DCTL_INITU2ENA
4049                                                 | DWC3_DCTL_ACCEPTU2ENA
4050                                                 | DWC3_DCTL_INITU1ENA
4051                                                 | DWC3_DCTL_ACCEPTU1ENA);
4052
4053                                 if (!dwc->u1u2)
4054                                         dwc->u1u2 = reg & u1u2;
4055
4056                                 reg &= ~u1u2;
4057
4058                                 dwc3_gadget_dctl_write_safe(dwc, reg);
4059                                 break;
4060                         default:
4061                                 /* do nothing */
4062                                 break;
4063                         }
4064                 }
4065         }
4066
4067         switch (next) {
4068         case DWC3_LINK_STATE_U1:
4069                 if (dwc->speed == USB_SPEED_SUPER)
4070                         dwc3_suspend_gadget(dwc);
4071                 break;
4072         case DWC3_LINK_STATE_U2:
4073         case DWC3_LINK_STATE_U3:
4074                 dwc3_suspend_gadget(dwc);
4075                 break;
4076         case DWC3_LINK_STATE_RESUME:
4077                 dwc3_resume_gadget(dwc);
4078                 break;
4079         default:
4080                 /* do nothing */
4081                 break;
4082         }
4083
4084         dwc->link_state = next;
4085 }
4086
4087 static void dwc3_gadget_suspend_interrupt(struct dwc3 *dwc,
4088                                           unsigned int evtinfo)
4089 {
4090         enum dwc3_link_state next = evtinfo & DWC3_LINK_STATE_MASK;
4091
4092         if (dwc->link_state != next && next == DWC3_LINK_STATE_U3)
4093                 dwc3_suspend_gadget(dwc);
4094
4095         dwc->link_state = next;
4096 }
4097
4098 static void dwc3_gadget_hibernation_interrupt(struct dwc3 *dwc,
4099                 unsigned int evtinfo)
4100 {
4101         unsigned int is_ss = evtinfo & BIT(4);
4102
4103         /*
4104          * WORKAROUND: DWC3 revison 2.20a with hibernation support
4105          * have a known issue which can cause USB CV TD.9.23 to fail
4106          * randomly.
4107          *
4108          * Because of this issue, core could generate bogus hibernation
4109          * events which SW needs to ignore.
4110          *
4111          * Refers to:
4112          *
4113          * STAR#9000546576: Device Mode Hibernation: Issue in USB 2.0
4114          * Device Fallback from SuperSpeed
4115          */
4116         if (is_ss ^ (dwc->speed == USB_SPEED_SUPER))
4117                 return;
4118
4119         /* enter hibernation here */
4120 }
4121
4122 static void dwc3_gadget_interrupt(struct dwc3 *dwc,
4123                 const struct dwc3_event_devt *event)
4124 {
4125         switch (event->type) {
4126         case DWC3_DEVICE_EVENT_DISCONNECT:
4127                 dwc3_gadget_disconnect_interrupt(dwc);
4128                 break;
4129         case DWC3_DEVICE_EVENT_RESET:
4130                 dwc3_gadget_reset_interrupt(dwc);
4131                 break;
4132         case DWC3_DEVICE_EVENT_CONNECT_DONE:
4133                 dwc3_gadget_conndone_interrupt(dwc);
4134                 break;
4135         case DWC3_DEVICE_EVENT_WAKEUP:
4136                 dwc3_gadget_wakeup_interrupt(dwc);
4137                 break;
4138         case DWC3_DEVICE_EVENT_HIBER_REQ:
4139                 if (dev_WARN_ONCE(dwc->dev, !dwc->has_hibernation,
4140                                         "unexpected hibernation event\n"))
4141                         break;
4142
4143                 dwc3_gadget_hibernation_interrupt(dwc, event->event_info);
4144                 break;
4145         case DWC3_DEVICE_EVENT_LINK_STATUS_CHANGE:
4146                 dwc3_gadget_linksts_change_interrupt(dwc, event->event_info);
4147                 break;
4148         case DWC3_DEVICE_EVENT_SUSPEND:
4149                 /* It changed to be suspend event for version 2.30a and above */
4150                 if (!DWC3_VER_IS_PRIOR(DWC3, 230A)) {
4151                         /*
4152                          * Ignore suspend event until the gadget enters into
4153                          * USB_STATE_CONFIGURED state.
4154                          */
4155                         if (dwc->gadget->state >= USB_STATE_CONFIGURED)
4156                                 dwc3_gadget_suspend_interrupt(dwc,
4157                                                 event->event_info);
4158                 }
4159                 break;
4160         case DWC3_DEVICE_EVENT_SOF:
4161         case DWC3_DEVICE_EVENT_ERRATIC_ERROR:
4162         case DWC3_DEVICE_EVENT_CMD_CMPL:
4163         case DWC3_DEVICE_EVENT_OVERFLOW:
4164                 break;
4165         default:
4166                 dev_WARN(dwc->dev, "UNKNOWN IRQ %d\n", event->type);
4167         }
4168 }
4169
4170 static void dwc3_process_event_entry(struct dwc3 *dwc,
4171                 const union dwc3_event *event)
4172 {
4173         trace_dwc3_event(event->raw, dwc);
4174
4175         if (!event->type.is_devspec)
4176                 dwc3_endpoint_interrupt(dwc, &event->depevt);
4177         else if (event->type.type == DWC3_EVENT_TYPE_DEV)
4178                 dwc3_gadget_interrupt(dwc, &event->devt);
4179         else
4180                 dev_err(dwc->dev, "UNKNOWN IRQ type %d\n", event->raw);
4181 }
4182
4183 static irqreturn_t dwc3_process_event_buf(struct dwc3_event_buffer *evt)
4184 {
4185         struct dwc3 *dwc = evt->dwc;
4186         irqreturn_t ret = IRQ_NONE;
4187         int left;
4188
4189         left = evt->count;
4190
4191         if (!(evt->flags & DWC3_EVENT_PENDING))
4192                 return IRQ_NONE;
4193
4194         while (left > 0) {
4195                 union dwc3_event event;
4196
4197                 event.raw = *(u32 *) (evt->cache + evt->lpos);
4198
4199                 dwc3_process_event_entry(dwc, &event);
4200
4201                 /*
4202                  * FIXME we wrap around correctly to the next entry as
4203                  * almost all entries are 4 bytes in size. There is one
4204                  * entry which has 12 bytes which is a regular entry
4205                  * followed by 8 bytes data. ATM I don't know how
4206                  * things are organized if we get next to the a
4207                  * boundary so I worry about that once we try to handle
4208                  * that.
4209                  */
4210                 evt->lpos = (evt->lpos + 4) % evt->length;
4211                 left -= 4;
4212         }
4213
4214         evt->count = 0;
4215         evt->flags &= ~DWC3_EVENT_PENDING;
4216         ret = IRQ_HANDLED;
4217
4218         /* Unmask interrupt */
4219         dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0),
4220                     DWC3_GEVNTSIZ_SIZE(evt->length));
4221
4222         if (dwc->imod_interval) {
4223                 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), DWC3_GEVNTCOUNT_EHB);
4224                 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), dwc->imod_interval);
4225         }
4226
4227         return ret;
4228 }
4229
4230 static irqreturn_t dwc3_thread_interrupt(int irq, void *_evt)
4231 {
4232         struct dwc3_event_buffer *evt = _evt;
4233         struct dwc3 *dwc = evt->dwc;
4234         unsigned long flags;
4235         irqreturn_t ret = IRQ_NONE;
4236
4237         local_bh_disable();
4238         spin_lock_irqsave(&dwc->lock, flags);
4239         ret = dwc3_process_event_buf(evt);
4240         spin_unlock_irqrestore(&dwc->lock, flags);
4241         local_bh_enable();
4242
4243         return ret;
4244 }
4245
4246 static irqreturn_t dwc3_check_event_buf(struct dwc3_event_buffer *evt)
4247 {
4248         struct dwc3 *dwc = evt->dwc;
4249         u32 amount;
4250         u32 count;
4251
4252         if (pm_runtime_suspended(dwc->dev)) {
4253                 pm_runtime_get(dwc->dev);
4254                 disable_irq_nosync(dwc->irq_gadget);
4255                 dwc->pending_events = true;
4256                 return IRQ_HANDLED;
4257         }
4258
4259         /*
4260          * With PCIe legacy interrupt, test shows that top-half irq handler can
4261          * be called again after HW interrupt deassertion. Check if bottom-half
4262          * irq event handler completes before caching new event to prevent
4263          * losing events.
4264          */
4265         if (evt->flags & DWC3_EVENT_PENDING)
4266                 return IRQ_HANDLED;
4267
4268         count = dwc3_readl(dwc->regs, DWC3_GEVNTCOUNT(0));
4269         count &= DWC3_GEVNTCOUNT_MASK;
4270         if (!count)
4271                 return IRQ_NONE;
4272
4273         evt->count = count;
4274         evt->flags |= DWC3_EVENT_PENDING;
4275
4276         /* Mask interrupt */
4277         dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0),
4278                     DWC3_GEVNTSIZ_INTMASK | DWC3_GEVNTSIZ_SIZE(evt->length));
4279
4280         amount = min(count, evt->length - evt->lpos);
4281         memcpy(evt->cache + evt->lpos, evt->buf + evt->lpos, amount);
4282
4283         if (amount < count)
4284                 memcpy(evt->cache, evt->buf, count - amount);
4285
4286         dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), count);
4287
4288         return IRQ_WAKE_THREAD;
4289 }
4290
4291 static irqreturn_t dwc3_interrupt(int irq, void *_evt)
4292 {
4293         struct dwc3_event_buffer        *evt = _evt;
4294
4295         return dwc3_check_event_buf(evt);
4296 }
4297
4298 static int dwc3_gadget_get_irq(struct dwc3 *dwc)
4299 {
4300         struct platform_device *dwc3_pdev = to_platform_device(dwc->dev);
4301         int irq;
4302
4303         irq = platform_get_irq_byname_optional(dwc3_pdev, "peripheral");
4304         if (irq > 0)
4305                 goto out;
4306
4307         if (irq == -EPROBE_DEFER)
4308                 goto out;
4309
4310         irq = platform_get_irq_byname_optional(dwc3_pdev, "dwc_usb3");
4311         if (irq > 0)
4312                 goto out;
4313
4314         if (irq == -EPROBE_DEFER)
4315                 goto out;
4316
4317         irq = platform_get_irq(dwc3_pdev, 0);
4318         if (irq > 0)
4319                 goto out;
4320
4321         if (!irq)
4322                 irq = -EINVAL;
4323
4324 out:
4325         return irq;
4326 }
4327
4328 static void dwc_gadget_release(struct device *dev)
4329 {
4330         struct usb_gadget *gadget = container_of(dev, struct usb_gadget, dev);
4331
4332         kfree(gadget);
4333 }
4334
4335 /**
4336  * dwc3_gadget_init - initializes gadget related registers
4337  * @dwc: pointer to our controller context structure
4338  *
4339  * Returns 0 on success otherwise negative errno.
4340  */
4341 int dwc3_gadget_init(struct dwc3 *dwc)
4342 {
4343         int ret;
4344         int irq;
4345         struct device *dev;
4346
4347         irq = dwc3_gadget_get_irq(dwc);
4348         if (irq < 0) {
4349                 ret = irq;
4350                 goto err0;
4351         }
4352
4353         dwc->irq_gadget = irq;
4354
4355         dwc->ep0_trb = dma_alloc_coherent(dwc->sysdev,
4356                                           sizeof(*dwc->ep0_trb) * 2,
4357                                           &dwc->ep0_trb_addr, GFP_KERNEL);
4358         if (!dwc->ep0_trb) {
4359                 dev_err(dwc->dev, "failed to allocate ep0 trb\n");
4360                 ret = -ENOMEM;
4361                 goto err0;
4362         }
4363
4364         dwc->setup_buf = kzalloc(DWC3_EP0_SETUP_SIZE, GFP_KERNEL);
4365         if (!dwc->setup_buf) {
4366                 ret = -ENOMEM;
4367                 goto err1;
4368         }
4369
4370         dwc->bounce = dma_alloc_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE,
4371                         &dwc->bounce_addr, GFP_KERNEL);
4372         if (!dwc->bounce) {
4373                 ret = -ENOMEM;
4374                 goto err2;
4375         }
4376
4377         init_completion(&dwc->ep0_in_setup);
4378         dwc->gadget = kzalloc(sizeof(struct usb_gadget), GFP_KERNEL);
4379         if (!dwc->gadget) {
4380                 ret = -ENOMEM;
4381                 goto err3;
4382         }
4383
4384
4385         usb_initialize_gadget(dwc->dev, dwc->gadget, dwc_gadget_release);
4386         dev                             = &dwc->gadget->dev;
4387         dev->platform_data              = dwc;
4388         dwc->gadget->ops                = &dwc3_gadget_ops;
4389         dwc->gadget->speed              = USB_SPEED_UNKNOWN;
4390         dwc->gadget->ssp_rate           = USB_SSP_GEN_UNKNOWN;
4391         dwc->gadget->sg_supported       = true;
4392         dwc->gadget->name               = "dwc3-gadget";
4393         dwc->gadget->lpm_capable        = !dwc->usb2_gadget_lpm_disable;
4394
4395         /*
4396          * FIXME We might be setting max_speed to <SUPER, however versions
4397          * <2.20a of dwc3 have an issue with metastability (documented
4398          * elsewhere in this driver) which tells us we can't set max speed to
4399          * anything lower than SUPER.
4400          *
4401          * Because gadget.max_speed is only used by composite.c and function
4402          * drivers (i.e. it won't go into dwc3's registers) we are allowing this
4403          * to happen so we avoid sending SuperSpeed Capability descriptor
4404          * together with our BOS descriptor as that could confuse host into
4405          * thinking we can handle super speed.
4406          *
4407          * Note that, in fact, we won't even support GetBOS requests when speed
4408          * is less than super speed because we don't have means, yet, to tell
4409          * composite.c that we are USB 2.0 + LPM ECN.
4410          */
4411         if (DWC3_VER_IS_PRIOR(DWC3, 220A) &&
4412             !dwc->dis_metastability_quirk)
4413                 dev_info(dwc->dev, "changing max_speed on rev %08x\n",
4414                                 dwc->revision);
4415
4416         dwc->gadget->max_speed          = dwc->maximum_speed;
4417         dwc->gadget->max_ssp_rate       = dwc->max_ssp_rate;
4418
4419         /*
4420          * REVISIT: Here we should clear all pending IRQs to be
4421          * sure we're starting from a well known location.
4422          */
4423
4424         ret = dwc3_gadget_init_endpoints(dwc, dwc->num_eps);
4425         if (ret)
4426                 goto err4;
4427
4428         ret = usb_add_gadget(dwc->gadget);
4429         if (ret) {
4430                 dev_err(dwc->dev, "failed to add gadget\n");
4431                 goto err5;
4432         }
4433
4434         if (DWC3_IP_IS(DWC32) && dwc->maximum_speed == USB_SPEED_SUPER_PLUS)
4435                 dwc3_gadget_set_ssp_rate(dwc->gadget, dwc->max_ssp_rate);
4436         else
4437                 dwc3_gadget_set_speed(dwc->gadget, dwc->maximum_speed);
4438
4439         return 0;
4440
4441 err5:
4442         dwc3_gadget_free_endpoints(dwc);
4443 err4:
4444         usb_put_gadget(dwc->gadget);
4445         dwc->gadget = NULL;
4446 err3:
4447         dma_free_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE, dwc->bounce,
4448                         dwc->bounce_addr);
4449
4450 err2:
4451         kfree(dwc->setup_buf);
4452
4453 err1:
4454         dma_free_coherent(dwc->sysdev, sizeof(*dwc->ep0_trb) * 2,
4455                         dwc->ep0_trb, dwc->ep0_trb_addr);
4456
4457 err0:
4458         return ret;
4459 }
4460
4461 /* -------------------------------------------------------------------------- */
4462
4463 void dwc3_gadget_exit(struct dwc3 *dwc)
4464 {
4465         if (!dwc->gadget)
4466                 return;
4467
4468         usb_del_gadget(dwc->gadget);
4469         dwc3_gadget_free_endpoints(dwc);
4470         usb_put_gadget(dwc->gadget);
4471         dma_free_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE, dwc->bounce,
4472                           dwc->bounce_addr);
4473         kfree(dwc->setup_buf);
4474         dma_free_coherent(dwc->sysdev, sizeof(*dwc->ep0_trb) * 2,
4475                           dwc->ep0_trb, dwc->ep0_trb_addr);
4476 }
4477
4478 int dwc3_gadget_suspend(struct dwc3 *dwc)
4479 {
4480         if (!dwc->gadget_driver)
4481                 return 0;
4482
4483         dwc3_gadget_run_stop(dwc, false, false);
4484         dwc3_disconnect_gadget(dwc);
4485         __dwc3_gadget_stop(dwc);
4486
4487         return 0;
4488 }
4489
4490 int dwc3_gadget_resume(struct dwc3 *dwc)
4491 {
4492         int                     ret;
4493
4494         if (!dwc->gadget_driver || !dwc->softconnect)
4495                 return 0;
4496
4497         ret = __dwc3_gadget_start(dwc);
4498         if (ret < 0)
4499                 goto err0;
4500
4501         ret = dwc3_gadget_run_stop(dwc, true, false);
4502         if (ret < 0)
4503                 goto err1;
4504
4505         return 0;
4506
4507 err1:
4508         __dwc3_gadget_stop(dwc);
4509
4510 err0:
4511         return ret;
4512 }
4513
4514 void dwc3_gadget_process_pending_events(struct dwc3 *dwc)
4515 {
4516         if (dwc->pending_events) {
4517                 dwc3_interrupt(dwc->irq_gadget, dwc->ev_buf);
4518                 dwc->pending_events = false;
4519                 enable_irq(dwc->irq_gadget);
4520         }
4521 }