drivers/staging/octeon-usb/octeon-hcd.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * This file is subject to the terms and conditions of the GNU General Public
   4  * License.  See the file "COPYING" in the main directory of this archive
   5  * for more details.
   6  *
   7  * Copyright (C) 2008 Cavium Networks
   8  *
   9  * Some parts of the code were originally released under BSD license:
  10  *
  11  * Copyright (c) 2003-2010 Cavium Networks (support@cavium.com). All rights
  12  * reserved.
  13  *
  14  * Redistribution and use in source and binary forms, with or without
  15  * modification, are permitted provided that the following conditions are
  16  * met:
  17  *
  18  *   * Redistributions of source code must retain the above copyright
  19  *     notice, this list of conditions and the following disclaimer.
  20  *
  21  *   * Redistributions in binary form must reproduce the above
  22  *     copyright notice, this list of conditions and the following
  23  *     disclaimer in the documentation and/or other materials provided
  24  *     with the distribution.
  25  *
  26  *   * Neither the name of Cavium Networks nor the names of
  27  *     its contributors may be used to endorse or promote products
  28  *     derived from this software without specific prior written
  29  *     permission.
  30  *
  31  * This Software, including technical data, may be subject to U.S. export
  32  * control laws, including the U.S. Export Administration Act and its associated
  33  * regulations, and may be subject to export or import regulations in other
  34  * countries.
  35  *
  36  * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS"
  37  * AND WITH ALL FAULTS AND CAVIUM NETWORKS MAKES NO PROMISES, REPRESENTATIONS OR
  38  * WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT TO
  39  * THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY REPRESENTATION
  40  * OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT DEFECTS, AND CAVIUM
  41  * SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES OF TITLE,
  42  * MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR PURPOSE, LACK OF
  43  * VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT, QUIET POSSESSION OR
  44  * CORRESPONDENCE TO DESCRIPTION. THE ENTIRE RISK ARISING OUT OF USE OR
  45  * PERFORMANCE OF THE SOFTWARE LIES WITH YOU.
  46  */
  47
  48 #include <linux/usb.h>
  49 #include <linux/slab.h>
  50 #include <linux/module.h>
  51 #include <linux/usb/hcd.h>
  52 #include <linux/prefetch.h>
  53 #include <linux/dma-mapping.h>
  54 #include <linux/platform_device.h>
  55
  56 #include <asm/octeon/octeon.h>
  57
  58 #include "octeon-hcd.h"
  59
  60 /**
  61  * enum cvmx_usb_speed - the possible USB device speeds
  62  *
  63  * @CVMX_USB_SPEED_HIGH: Device is operation at 480Mbps
  64  * @CVMX_USB_SPEED_FULL: Device is operation at 12Mbps
  65  * @CVMX_USB_SPEED_LOW:  Device is operation at 1.5Mbps
  66  */
  67 enum cvmx_usb_speed {
  68         CVMX_USB_SPEED_HIGH = 0,
  69         CVMX_USB_SPEED_FULL = 1,
  70         CVMX_USB_SPEED_LOW = 2,
  71 };
  72
  73 /**
  74  * enum cvmx_usb_transfer - the possible USB transfer types
  75  *
  76  * @CVMX_USB_TRANSFER_CONTROL:     USB transfer type control for hub and status
  77  *                                 transfers
  78  * @CVMX_USB_TRANSFER_ISOCHRONOUS: USB transfer type isochronous for low
  79  *                                 priority periodic transfers
  80  * @CVMX_USB_TRANSFER_BULK:        USB transfer type bulk for large low priority
  81  *                                 transfers
  82  * @CVMX_USB_TRANSFER_INTERRUPT:   USB transfer type interrupt for high priority
  83  *                                 periodic transfers
  84  */
  85 enum cvmx_usb_transfer {
  86         CVMX_USB_TRANSFER_CONTROL = 0,
  87         CVMX_USB_TRANSFER_ISOCHRONOUS = 1,
  88         CVMX_USB_TRANSFER_BULK = 2,
  89         CVMX_USB_TRANSFER_INTERRUPT = 3,
  90 };
  91
  92 /**
  93  * enum cvmx_usb_direction - the transfer directions
  94  *
  95  * @CVMX_USB_DIRECTION_OUT: Data is transferring from Octeon to the device/host
  96  * @CVMX_USB_DIRECTION_IN:  Data is transferring from the device/host to Octeon
  97  */
  98 enum cvmx_usb_direction {
  99         CVMX_USB_DIRECTION_OUT,
 100         CVMX_USB_DIRECTION_IN,
 101 };
 102
 103 /**
 104  * enum cvmx_usb_status - possible callback function status codes
 105  *
 106  * @CVMX_USB_STATUS_OK:           The transaction / operation finished without
 107  *                                any errors
 108  * @CVMX_USB_STATUS_SHORT:        FIXME: This is currently not implemented
 109  * @CVMX_USB_STATUS_CANCEL:       The transaction was canceled while in flight
 110  *                                by a user call to cvmx_usb_cancel
 111  * @CVMX_USB_STATUS_ERROR:        The transaction aborted with an unexpected
 112  *                                error status
 113  * @CVMX_USB_STATUS_STALL:        The transaction received a USB STALL response
 114  *                                from the device
 115  * @CVMX_USB_STATUS_XACTERR:      The transaction failed with an error from the
 116  *                                device even after a number of retries
 117  * @CVMX_USB_STATUS_DATATGLERR:   The transaction failed with a data toggle
 118  *                                error even after a number of retries
 119  * @CVMX_USB_STATUS_BABBLEERR:    The transaction failed with a babble error
 120  * @CVMX_USB_STATUS_FRAMEERR:     The transaction failed with a frame error
 121  *                                even after a number of retries
 122  */
 123 enum cvmx_usb_status {
 124         CVMX_USB_STATUS_OK,
 125         CVMX_USB_STATUS_SHORT,
 126         CVMX_USB_STATUS_CANCEL,
 127         CVMX_USB_STATUS_ERROR,
 128         CVMX_USB_STATUS_STALL,
 129         CVMX_USB_STATUS_XACTERR,
 130         CVMX_USB_STATUS_DATATGLERR,
 131         CVMX_USB_STATUS_BABBLEERR,
 132         CVMX_USB_STATUS_FRAMEERR,
 133 };
 134
 135 /**
 136  * struct cvmx_usb_port_status - the USB port status information
 137  *
 138  * @port_enabled:       1 = Usb port is enabled, 0 = disabled
 139  * @port_over_current:  1 = Over current detected, 0 = Over current not
 140  *                      detected. Octeon doesn't support over current detection.
 141  * @port_powered:       1 = Port power is being supplied to the device, 0 =
 142  *                      power is off. Octeon doesn't support turning port power
 143  *                      off.
 144  * @port_speed:         Current port speed.
 145  * @connected:          1 = A device is connected to the port, 0 = No device is
 146  *                      connected.
 147  * @connect_change:     1 = Device connected state changed since the last set
 148  *                      status call.
 149  */
 150 struct cvmx_usb_port_status {
 151         u32 reserved                    : 25;
 152         u32 port_enabled                : 1;
 153         u32 port_over_current           : 1;
 154         u32 port_powered                : 1;
 155         enum cvmx_usb_speed port_speed  : 2;
 156         u32 connected                   : 1;
 157         u32 connect_change              : 1;
 158 };
 159
 160 /**
 161  * struct cvmx_usb_iso_packet - descriptor for Isochronous packets
 162  *
 163  * @offset:     This is the offset in bytes into the main buffer where this data
 164  *              is stored.
 165  * @length:     This is the length in bytes of the data.
 166  * @status:     This is the status of this individual packet transfer.
 167  */
 168 struct cvmx_usb_iso_packet {
 169         int offset;
 170         int length;
 171         enum cvmx_usb_status status;
 172 };
 173
 174 /**
 175  * enum cvmx_usb_initialize_flags - flags used by the initialization function
 176  *
 177  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_XI:    The USB port uses a 12MHz crystal
 178  *                                            as clock source at USB_XO and
 179  *                                            USB_XI.
 180  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_GND:   The USB port uses 12/24/48MHz 2.5V
 181  *                                            board clock source at USB_XO.
 182  *                                            USB_XI should be tied to GND.
 183  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_MHZ_MASK: Mask for clock speed field
 184  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_12MHZ:    Speed of reference clock or
 185  *                                            crystal
 186  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_24MHZ:    Speed of reference clock
 187  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_48MHZ:    Speed of reference clock
 188  * @CVMX_USB_INITIALIZE_FLAGS_NO_DMA:         Disable DMA and used polled IO for
 189  *                                            data transfer use for the USB
 190  */
 191 enum cvmx_usb_initialize_flags {
 192         CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_XI           = 1 << 0,
 193         CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_GND          = 1 << 1,
 194         CVMX_USB_INITIALIZE_FLAGS_CLOCK_MHZ_MASK        = 3 << 3,
 195         CVMX_USB_INITIALIZE_FLAGS_CLOCK_12MHZ           = 1 << 3,
 196         CVMX_USB_INITIALIZE_FLAGS_CLOCK_24MHZ           = 2 << 3,
 197         CVMX_USB_INITIALIZE_FLAGS_CLOCK_48MHZ           = 3 << 3,
 198         /* Bits 3-4 used to encode the clock frequency */
 199         CVMX_USB_INITIALIZE_FLAGS_NO_DMA                = 1 << 5,
 200 };
 201
 202 /**
 203  * enum cvmx_usb_pipe_flags - internal flags for a pipe.
 204  *
 205  * @CVMX_USB_PIPE_FLAGS_SCHEDULED: Used internally to determine if a pipe is
 206  *                                 actively using hardware.
 207  * @CVMX_USB_PIPE_FLAGS_NEED_PING: Used internally to determine if a high speed
 208  *                                 pipe is in the ping state.
 209  */
 210 enum cvmx_usb_pipe_flags {
 211         CVMX_USB_PIPE_FLAGS_SCHEDULED   = 1 << 17,
 212         CVMX_USB_PIPE_FLAGS_NEED_PING   = 1 << 18,
 213 };
 214
 215 /* Maximum number of times to retry failed transactions */
 216 #define MAX_RETRIES             3
 217
 218 /* Maximum number of hardware channels supported by the USB block */
 219 #define MAX_CHANNELS            8
 220
 221 /*
 222  * The low level hardware can transfer a maximum of this number of bytes in each
 223  * transfer. The field is 19 bits wide
 224  */
 225 #define MAX_TRANSFER_BYTES      ((1 << 19) - 1)
 226
 227 /*
 228  * The low level hardware can transfer a maximum of this number of packets in
 229  * each transfer. The field is 10 bits wide
 230  */
 231 #define MAX_TRANSFER_PACKETS    ((1 << 10) - 1)
 232
 233 /**
 234  * Logical transactions may take numerous low level
 235  * transactions, especially when splits are concerned. This
 236  * enum represents all of the possible stages a transaction can
 237  * be in. Note that split completes are always even. This is so
 238  * the NAK handler can backup to the previous low level
 239  * transaction with a simple clearing of bit 0.
 240  */
 241 enum cvmx_usb_stage {
 242         CVMX_USB_STAGE_NON_CONTROL,
 243         CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE,
 244         CVMX_USB_STAGE_SETUP,
 245         CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE,
 246         CVMX_USB_STAGE_DATA,
 247         CVMX_USB_STAGE_DATA_SPLIT_COMPLETE,
 248         CVMX_USB_STAGE_STATUS,
 249         CVMX_USB_STAGE_STATUS_SPLIT_COMPLETE,
 250 };
 251
 252 /**
 253  * struct cvmx_usb_transaction - describes each pending USB transaction
 254  *                               regardless of type. These are linked together
 255  *                               to form a list of pending requests for a pipe.
 256  *
 257  * @node:               List node for transactions in the pipe.
 258  * @type:               Type of transaction, duplicated of the pipe.
 259  * @flags:              State flags for this transaction.
 260  * @buffer:             User's physical buffer address to read/write.
 261  * @buffer_length:      Size of the user's buffer in bytes.
 262  * @control_header:     For control transactions, physical address of the 8
 263  *                      byte standard header.
 264  * @iso_start_frame:    For ISO transactions, the starting frame number.
 265  * @iso_number_packets: For ISO transactions, the number of packets in the
 266  *                      request.
 267  * @iso_packets:        For ISO transactions, the sub packets in the request.
 268  * @actual_bytes:       Actual bytes transfer for this transaction.
 269  * @stage:              For control transactions, the current stage.
 270  * @urb:                URB.
 271  */
 272 struct cvmx_usb_transaction {
 273         struct list_head node;
 274         enum cvmx_usb_transfer type;
 275         u64 buffer;
 276         int buffer_length;
 277         u64 control_header;
 278         int iso_start_frame;
 279         int iso_number_packets;
 280         struct cvmx_usb_iso_packet *iso_packets;
 281         int xfersize;
 282         int pktcnt;
 283         int retries;
 284         int actual_bytes;
 285         enum cvmx_usb_stage stage;
 286         struct urb *urb;
 287 };
 288
 289 /**
 290  * struct cvmx_usb_pipe - a pipe represents a virtual connection between Octeon
 291  *                        and some USB device. It contains a list of pending
 292  *                        request to the device.
 293  *
 294  * @node:               List node for pipe list
 295  * @next:               Pipe after this one in the list
 296  * @transactions:       List of pending transactions
 297  * @interval:           For periodic pipes, the interval between packets in
 298  *                      frames
 299  * @next_tx_frame:      The next frame this pipe is allowed to transmit on
 300  * @flags:              State flags for this pipe
 301  * @device_speed:       Speed of device connected to this pipe
 302  * @transfer_type:      Type of transaction supported by this pipe
 303  * @transfer_dir:       IN or OUT. Ignored for Control
 304  * @multi_count:        Max packet in a row for the device
 305  * @max_packet:         The device's maximum packet size in bytes
 306  * @device_addr:        USB device address at other end of pipe
 307  * @endpoint_num:       USB endpoint number at other end of pipe
 308  * @hub_device_addr:    Hub address this device is connected to
 309  * @hub_port:           Hub port this device is connected to
 310  * @pid_toggle:         This toggles between 0/1 on every packet send to track
 311  *                      the data pid needed
 312  * @channel:            Hardware DMA channel for this pipe
 313  * @split_sc_frame:     The low order bits of the frame number the split
 314  *                      complete should be sent on
 315  */
 316 struct cvmx_usb_pipe {
 317         struct list_head node;
 318         struct list_head transactions;
 319         u64 interval;
 320         u64 next_tx_frame;
 321         enum cvmx_usb_pipe_flags flags;
 322         enum cvmx_usb_speed device_speed;
 323         enum cvmx_usb_transfer transfer_type;
 324         enum cvmx_usb_direction transfer_dir;
 325         int multi_count;
 326         u16 max_packet;
 327         u8 device_addr;
 328         u8 endpoint_num;
 329         u8 hub_device_addr;
 330         u8 hub_port;
 331         u8 pid_toggle;
 332         u8 channel;
 333         s8 split_sc_frame;
 334 };
 335
 336 struct cvmx_usb_tx_fifo {
 337         struct {
 338                 int channel;
 339                 int size;
 340                 u64 address;
 341         } entry[MAX_CHANNELS + 1];
 342         int head;
 343         int tail;
 344 };
 345
 346 /**
 347  * struct octeon_hcd - the state of the USB block
 348  *
 349  * lock:                   Serialization lock.
 350  * init_flags:             Flags passed to initialize.
 351  * index:                  Which USB block this is for.
 352  * idle_hardware_channels: Bit set for every idle hardware channel.
 353  * usbcx_hprt:             Stored port status so we don't need to read a CSR to
 354  *                         determine splits.
 355  * pipe_for_channel:       Map channels to pipes.
 356  * pipe:                   Storage for pipes.
 357  * indent:                 Used by debug output to indent functions.
 358  * port_status:            Last port status used for change notification.
 359  * idle_pipes:             List of open pipes that have no transactions.
 360  * active_pipes:           Active pipes indexed by transfer type.
 361  * frame_number:           Increments every SOF interrupt for time keeping.
 362  * active_split:           Points to the current active split, or NULL.
 363  */
 364 struct octeon_hcd {
 365         spinlock_t lock; /* serialization lock */
 366         int init_flags;
 367         int index;
 368         int idle_hardware_channels;
 369         union cvmx_usbcx_hprt usbcx_hprt;
 370         struct cvmx_usb_pipe *pipe_for_channel[MAX_CHANNELS];
 371         int indent;
 372         struct cvmx_usb_port_status port_status;
 373         struct list_head idle_pipes;
 374         struct list_head active_pipes[4];
 375         u64 frame_number;
 376         struct cvmx_usb_transaction *active_split;
 377         struct cvmx_usb_tx_fifo periodic;
 378         struct cvmx_usb_tx_fifo nonperiodic;
 379 };
 380
 381 /*
 382  * This macro logically sets a single field in a CSR. It does the sequence
 383  * read, modify, and write
 384  */
 385 #define USB_SET_FIELD32(address, _union, field, value)          \
 386         do {                                                    \
 387                 union _union c;                                 \
 388                                                                 \
 389                 c.u32 = cvmx_usb_read_csr32(usb, address);      \
 390                 c.s.field = value;                              \
 391                 cvmx_usb_write_csr32(usb, address, c.u32);      \
 392         } while (0)
 393
 394 /* Returns the IO address to push/pop stuff data from the FIFOs */
 395 #define USB_FIFO_ADDRESS(channel, usb_index) \
 396         (CVMX_USBCX_GOTGCTL(usb_index) + ((channel) + 1) * 0x1000)
 397
 398 /**
 399  * struct octeon_temp_buffer - a bounce buffer for USB transfers
 400  * @orig_buffer: the original buffer passed by the USB stack
 401  * @data:        the newly allocated temporary buffer (excluding meta-data)
 402  *
 403  * Both the DMA engine and FIFO mode will always transfer full 32-bit words. If
 404  * the buffer is too short, we need to allocate a temporary one, and this struct
 405  * represents it.
 406  */
 407 struct octeon_temp_buffer {
 408         void *orig_buffer;
 409         u8 data[0];
 410 };
 411
 412 static inline struct usb_hcd *octeon_to_hcd(struct octeon_hcd *p)
 413 {
 414         return container_of((void *)p, struct usb_hcd, hcd_priv);
 415 }
 416
 417 /**
 418  * octeon_alloc_temp_buffer - allocate a temporary buffer for USB transfer
 419  *                            (if needed)
 420  * @urb:        URB.
 421  * @mem_flags:  Memory allocation flags.
 422  *
 423  * This function allocates a temporary bounce buffer whenever it's needed
 424  * due to HW limitations.
 425  */
 426 static int octeon_alloc_temp_buffer(struct urb *urb, gfp_t mem_flags)
 427 {
 428         struct octeon_temp_buffer *temp;
 429
 430         if (urb->num_sgs || urb->sg ||
 431             (urb->transfer_flags & URB_NO_TRANSFER_DMA_MAP) ||
 432             !(urb->transfer_buffer_length % sizeof(u32)))
 433                 return 0;
 434
 435         temp = kmalloc(ALIGN(urb->transfer_buffer_length, sizeof(u32)) +
 436                        sizeof(*temp), mem_flags);
 437         if (!temp)
 438                 return -ENOMEM;
 439
 440         temp->orig_buffer = urb->transfer_buffer;
 441         if (usb_urb_dir_out(urb))
 442                 memcpy(temp->data, urb->transfer_buffer,
 443                        urb->transfer_buffer_length);
 444         urb->transfer_buffer = temp->data;
 445         urb->transfer_flags |= URB_ALIGNED_TEMP_BUFFER;
 446
 447         return 0;
 448 }
 449
 450 /**
 451  * octeon_free_temp_buffer - free a temporary buffer used by USB transfers.
 452  * @urb: URB.
 453  *
 454  * Frees a buffer allocated by octeon_alloc_temp_buffer().
 455  */
 456 static void octeon_free_temp_buffer(struct urb *urb)
 457 {
 458         struct octeon_temp_buffer *temp;
 459         size_t length;
 460
 461         if (!(urb->transfer_flags & URB_ALIGNED_TEMP_BUFFER))
 462                 return;
 463
 464         temp = container_of(urb->transfer_buffer, struct octeon_temp_buffer,
 465                             data);
 466         if (usb_urb_dir_in(urb)) {
 467                 if (usb_pipeisoc(urb->pipe))
 468                         length = urb->transfer_buffer_length;
 469                 else
 470                         length = urb->actual_length;
 471
 472                 memcpy(temp->orig_buffer, urb->transfer_buffer, length);
 473         }
 474         urb->transfer_buffer = temp->orig_buffer;
 475         urb->transfer_flags &= ~URB_ALIGNED_TEMP_BUFFER;
 476         kfree(temp);
 477 }
 478
 479 /**
 480  * octeon_map_urb_for_dma - Octeon-specific map_urb_for_dma().
 481  * @hcd:        USB HCD structure.
 482  * @urb:        URB.
 483  * @mem_flags:  Memory allocation flags.
 484  */
 485 static int octeon_map_urb_for_dma(struct usb_hcd *hcd, struct urb *urb,
 486                                   gfp_t mem_flags)
 487 {
 488         int ret;
 489
 490         ret = octeon_alloc_temp_buffer(urb, mem_flags);
 491         if (ret)
 492                 return ret;
 493
 494         ret = usb_hcd_map_urb_for_dma(hcd, urb, mem_flags);
 495         if (ret)
 496                 octeon_free_temp_buffer(urb);
 497
 498         return ret;
 499 }
 500
 501 /**
 502  * octeon_unmap_urb_for_dma - Octeon-specific unmap_urb_for_dma()
 503  * @hcd:        USB HCD structure.
 504  * @urb:        URB.
 505  */
 506 static void octeon_unmap_urb_for_dma(struct usb_hcd *hcd, struct urb *urb)
 507 {
 508         usb_hcd_unmap_urb_for_dma(hcd, urb);
 509         octeon_free_temp_buffer(urb);
 510 }
 511
 512 /**
 513  * Read a USB 32bit CSR. It performs the necessary address swizzle
 514  * for 32bit CSRs and logs the value in a readable format if
 515  * debugging is on.
 516  *
 517  * @usb:     USB block this access is for
 518  * @address: 64bit address to read
 519  *
 520  * Returns: Result of the read
 521  */
 522 static inline u32 cvmx_usb_read_csr32(struct octeon_hcd *usb, u64 address)
 523 {
 524         return cvmx_read64_uint32(address ^ 4);
 525 }
 526
 527 /**
 528  * Write a USB 32bit CSR. It performs the necessary address
 529  * swizzle for 32bit CSRs and logs the value in a readable format
 530  * if debugging is on.
 531  *
 532  * @usb:     USB block this access is for
 533  * @address: 64bit address to write
 534  * @value:   Value to write
 535  */
 536 static inline void cvmx_usb_write_csr32(struct octeon_hcd *usb,
 537                                         u64 address, u32 value)
 538 {
 539         cvmx_write64_uint32(address ^ 4, value);
 540         cvmx_read64_uint64(CVMX_USBNX_DMA0_INB_CHN0(usb->index));
 541 }
 542
 543 /**
 544  * Return non zero if this pipe connects to a non HIGH speed
 545  * device through a high speed hub.
 546  *
 547  * @usb:    USB block this access is for
 548  * @pipe:   Pipe to check
 549  *
 550  * Returns: Non zero if we need to do split transactions
 551  */
 552 static inline int cvmx_usb_pipe_needs_split(struct octeon_hcd *usb,
 553                                             struct cvmx_usb_pipe *pipe)
 554 {
 555         return pipe->device_speed != CVMX_USB_SPEED_HIGH &&
 556                usb->usbcx_hprt.s.prtspd == CVMX_USB_SPEED_HIGH;
 557 }
 558
 559 /**
 560  * Trivial utility function to return the correct PID for a pipe
 561  *
 562  * @pipe:   pipe to check
 563  *
 564  * Returns: PID for pipe
 565  */
 566 static inline int cvmx_usb_get_data_pid(struct cvmx_usb_pipe *pipe)
 567 {
 568         if (pipe->pid_toggle)
 569                 return 2; /* Data1 */
 570         return 0; /* Data0 */
 571 }
 572
 573 /* Loops through register until txfflsh or rxfflsh become zero.*/
 574 static int cvmx_wait_tx_rx(struct octeon_hcd *usb, int fflsh_type)
 575 {
 576         int result;
 577         u64 address = CVMX_USBCX_GRSTCTL(usb->index);
 578         u64 done = cvmx_get_cycle() + 100 *
 579                    (u64)octeon_get_clock_rate / 1000000;
 580         union cvmx_usbcx_grstctl c;
 581
 582         while (1) {
 583                 c.u32 = cvmx_usb_read_csr32(usb, address);
 584                 if (fflsh_type == 0 && c.s.txfflsh == 0) {
 585                         result = 0;
 586                         break;
 587                 } else if (fflsh_type == 1 && c.s.rxfflsh == 0) {
 588                         result = 0;
 589                         break;
 590                 } else if (cvmx_get_cycle() > done) {
 591                         result = -1;
 592                         break;
 593                 }
 594
 595                 __delay(100);
 596         }
 597         return result;
 598 }
 599
 600 static void cvmx_fifo_setup(struct octeon_hcd *usb)
 601 {
 602         union cvmx_usbcx_ghwcfg3 usbcx_ghwcfg3;
 603         union cvmx_usbcx_gnptxfsiz npsiz;
 604         union cvmx_usbcx_hptxfsiz psiz;
 605
 606         usbcx_ghwcfg3.u32 = cvmx_usb_read_csr32(usb,
 607                                                 CVMX_USBCX_GHWCFG3(usb->index));
 608
 609         /*
 610          * Program the USBC_GRXFSIZ register to select the size of the receive
 611          * FIFO (25%).
 612          */
 613         USB_SET_FIELD32(CVMX_USBCX_GRXFSIZ(usb->index), cvmx_usbcx_grxfsiz,
 614                         rxfdep, usbcx_ghwcfg3.s.dfifodepth / 4);
 615
 616         /*
 617          * Program the USBC_GNPTXFSIZ register to select the size and the start
 618          * address of the non-periodic transmit FIFO for nonperiodic
 619          * transactions (50%).
 620          */
 621         npsiz.u32 = cvmx_usb_read_csr32(usb, CVMX_USBCX_GNPTXFSIZ(usb->index));
 622         npsiz.s.nptxfdep = usbcx_ghwcfg3.s.dfifodepth / 2;
 623         npsiz.s.nptxfstaddr = usbcx_ghwcfg3.s.dfifodepth / 4;
 624         cvmx_usb_write_csr32(usb, CVMX_USBCX_GNPTXFSIZ(usb->index), npsiz.u32);
 625
 626         /*
 627          * Program the USBC_HPTXFSIZ register to select the size and start
 628          * address of the periodic transmit FIFO for periodic transactions
 629          * (25%).
 630          */
 631         psiz.u32 = cvmx_usb_read_csr32(usb, CVMX_USBCX_HPTXFSIZ(usb->index));
 632         psiz.s.ptxfsize = usbcx_ghwcfg3.s.dfifodepth / 4;
 633         psiz.s.ptxfstaddr = 3 * usbcx_ghwcfg3.s.dfifodepth / 4;
 634         cvmx_usb_write_csr32(usb, CVMX_USBCX_HPTXFSIZ(usb->index), psiz.u32);
 635
 636         /* Flush all FIFOs */
 637         USB_SET_FIELD32(CVMX_USBCX_GRSTCTL(usb->index),
 638                         cvmx_usbcx_grstctl, txfnum, 0x10);
 639         USB_SET_FIELD32(CVMX_USBCX_GRSTCTL(usb->index),
 640                         cvmx_usbcx_grstctl, txfflsh, 1);
 641         cvmx_wait_tx_rx(usb, 0);
 642         USB_SET_FIELD32(CVMX_USBCX_GRSTCTL(usb->index),
 643                         cvmx_usbcx_grstctl, rxfflsh, 1);
 644         cvmx_wait_tx_rx(usb, 1);
 645 }
 646
 647 /**
 648  * Shutdown a USB port after a call to cvmx_usb_initialize().
 649  * The port should be disabled with all pipes closed when this
 650  * function is called.
 651  *
 652  * @usb: USB device state populated by cvmx_usb_initialize().
 653  *
 654  * Returns: 0 or a negative error code.
 655  */
 656 static int cvmx_usb_shutdown(struct octeon_hcd *usb)
 657 {
 658         union cvmx_usbnx_clk_ctl usbn_clk_ctl;
 659
 660         /* Make sure all pipes are closed */
 661         if (!list_empty(&usb->idle_pipes) ||
 662             !list_empty(&usb->active_pipes[CVMX_USB_TRANSFER_ISOCHRONOUS]) ||
 663             !list_empty(&usb->active_pipes[CVMX_USB_TRANSFER_INTERRUPT]) ||
 664             !list_empty(&usb->active_pipes[CVMX_USB_TRANSFER_CONTROL]) ||
 665             !list_empty(&usb->active_pipes[CVMX_USB_TRANSFER_BULK]))
 666                 return -EBUSY;
 667
 668         /* Disable the clocks and put them in power on reset */
 669         usbn_clk_ctl.u64 = cvmx_read64_uint64(CVMX_USBNX_CLK_CTL(usb->index));
 670         usbn_clk_ctl.s.enable = 1;
 671         usbn_clk_ctl.s.por = 1;
 672         usbn_clk_ctl.s.hclk_rst = 1;
 673         usbn_clk_ctl.s.prst = 0;
 674         usbn_clk_ctl.s.hrst = 0;
 675         cvmx_write64_uint64(CVMX_USBNX_CLK_CTL(usb->index), usbn_clk_ctl.u64);
 676         return 0;
 677 }
 678
 679 /**
 680  * Initialize a USB port for use. This must be called before any
 681  * other access to the Octeon USB port is made. The port starts
 682  * off in the disabled state.
 683  *
 684  * @dev:         Pointer to struct device for logging purposes.
 685  * @usb:         Pointer to struct octeon_hcd.
 686  *
 687  * Returns: 0 or a negative error code.
 688  */
 689 static int cvmx_usb_initialize(struct device *dev,
 690                                struct octeon_hcd *usb)
 691 {
 692         int channel;
 693         int divisor;
 694         int retries = 0;
 695         union cvmx_usbcx_hcfg usbcx_hcfg;
 696         union cvmx_usbnx_clk_ctl usbn_clk_ctl;
 697         union cvmx_usbcx_gintsts usbc_gintsts;
 698         union cvmx_usbcx_gahbcfg usbcx_gahbcfg;
 699         union cvmx_usbcx_gintmsk usbcx_gintmsk;
 700         union cvmx_usbcx_gusbcfg usbcx_gusbcfg;
 701         union cvmx_usbnx_usbp_ctl_status usbn_usbp_ctl_status;
 702
 703 retry:
 704         /*
 705          * Power On Reset and PHY Initialization
 706          *
 707          * 1. Wait for DCOK to assert (nothing to do)
 708          *
 709          * 2a. Write USBN0/1_CLK_CTL[POR] = 1 and
 710          *     USBN0/1_CLK_CTL[HRST,PRST,HCLK_RST] = 0
 711          */
 712         usbn_clk_ctl.u64 = cvmx_read64_uint64(CVMX_USBNX_CLK_CTL(usb->index));
 713         usbn_clk_ctl.s.por = 1;
 714         usbn_clk_ctl.s.hrst = 0;
 715         usbn_clk_ctl.s.prst = 0;
 716         usbn_clk_ctl.s.hclk_rst = 0;
 717         usbn_clk_ctl.s.enable = 0;
 718         /*
 719          * 2b. Select the USB reference clock/crystal parameters by writing
 720          *     appropriate values to USBN0/1_CLK_CTL[P_C_SEL, P_RTYPE, P_COM_ON]
 721          */
 722         if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_GND) {
 723                 /*
 724                  * The USB port uses 12/24/48MHz 2.5V board clock
 725                  * source at USB_XO. USB_XI should be tied to GND.
 726                  * Most Octeon evaluation boards require this setting
 727                  */
 728                 if (OCTEON_IS_MODEL(OCTEON_CN3XXX) ||
 729                     OCTEON_IS_MODEL(OCTEON_CN56XX) ||
 730                     OCTEON_IS_MODEL(OCTEON_CN50XX))
 731                         /* From CN56XX,CN50XX,CN31XX,CN30XX manuals */
 732                         usbn_clk_ctl.s.p_rtype = 2; /* p_rclk=1 & p_xenbn=0 */
 733                 else
 734                         /* From CN52XX manual */
 735                         usbn_clk_ctl.s.p_rtype = 1;
 736
 737                 switch (usb->init_flags &
 738                         CVMX_USB_INITIALIZE_FLAGS_CLOCK_MHZ_MASK) {
 739                 case CVMX_USB_INITIALIZE_FLAGS_CLOCK_12MHZ:
 740                         usbn_clk_ctl.s.p_c_sel = 0;
 741                         break;
 742                 case CVMX_USB_INITIALIZE_FLAGS_CLOCK_24MHZ:
 743                         usbn_clk_ctl.s.p_c_sel = 1;
 744                         break;
 745                 case CVMX_USB_INITIALIZE_FLAGS_CLOCK_48MHZ:
 746                         usbn_clk_ctl.s.p_c_sel = 2;
 747                         break;
 748                 }
 749         } else {
 750                 /*
 751                  * The USB port uses a 12MHz crystal as clock source
 752                  * at USB_XO and USB_XI
 753                  */
 754                 if (OCTEON_IS_MODEL(OCTEON_CN3XXX))
 755                         /* From CN31XX,CN30XX manual */
 756                         usbn_clk_ctl.s.p_rtype = 3; /* p_rclk=1 & p_xenbn=1 */
 757                 else
 758                         /* From CN56XX,CN52XX,CN50XX manuals. */
 759                         usbn_clk_ctl.s.p_rtype = 0;
 760
 761                 usbn_clk_ctl.s.p_c_sel = 0;
 762         }
 763         /*
 764          * 2c. Select the HCLK via writing USBN0/1_CLK_CTL[DIVIDE, DIVIDE2] and
 765          *     setting USBN0/1_CLK_CTL[ENABLE] = 1. Divide the core clock down
 766          *     such that USB is as close as possible to 125Mhz
 767          */
 768         divisor = DIV_ROUND_UP(octeon_get_clock_rate(), 125000000);
 769         /* Lower than 4 doesn't seem to work properly */
 770         if (divisor < 4)
 771                 divisor = 4;
 772         usbn_clk_ctl.s.divide = divisor;
 773         usbn_clk_ctl.s.divide2 = 0;
 774         cvmx_write64_uint64(CVMX_USBNX_CLK_CTL(usb->index), usbn_clk_ctl.u64);
 775
 776         /* 2d. Write USBN0/1_CLK_CTL[HCLK_RST] = 1 */
 777         usbn_clk_ctl.s.hclk_rst = 1;
 778         cvmx_write64_uint64(CVMX_USBNX_CLK_CTL(usb->index), usbn_clk_ctl.u64);
 779         /* 2e.  Wait 64 core-clock cycles for HCLK to stabilize */
 780         __delay(64);
 781         /*
 782          * 3. Program the power-on reset field in the USBN clock-control
 783          *    register:
 784          *    USBN_CLK_CTL[POR] = 0
 785          */
 786         usbn_clk_ctl.s.por = 0;
 787         cvmx_write64_uint64(CVMX_USBNX_CLK_CTL(usb->index), usbn_clk_ctl.u64);
 788         /* 4. Wait 1 ms for PHY clock to start */
 789         mdelay(1);
 790         /*
 791          * 5. Program the Reset input from automatic test equipment field in the
 792          *    USBP control and status register:
 793          *    USBN_USBP_CTL_STATUS[ATE_RESET] = 1
 794          */
 795         usbn_usbp_ctl_status.u64 =
 796                 cvmx_read64_uint64(CVMX_USBNX_USBP_CTL_STATUS(usb->index));
 797         usbn_usbp_ctl_status.s.ate_reset = 1;
 798         cvmx_write64_uint64(CVMX_USBNX_USBP_CTL_STATUS(usb->index),
 799                             usbn_usbp_ctl_status.u64);
 800         /* 6. Wait 10 cycles */
 801         __delay(10);
 802         /*
 803          * 7. Clear ATE_RESET field in the USBN clock-control register:
 804          *    USBN_USBP_CTL_STATUS[ATE_RESET] = 0
 805          */
 806         usbn_usbp_ctl_status.s.ate_reset = 0;
 807         cvmx_write64_uint64(CVMX_USBNX_USBP_CTL_STATUS(usb->index),
 808                             usbn_usbp_ctl_status.u64);
 809         /*
 810          * 8. Program the PHY reset field in the USBN clock-control register:
 811          *    USBN_CLK_CTL[PRST] = 1
 812          */
 813         usbn_clk_ctl.s.prst = 1;
 814         cvmx_write64_uint64(CVMX_USBNX_CLK_CTL(usb->index), usbn_clk_ctl.u64);
 815         /*
 816          * 9. Program the USBP control and status register to select host or
 817          *    device mode. USBN_USBP_CTL_STATUS[HST_MODE] = 0 for host, = 1 for
 818          *    device
 819          */
 820         usbn_usbp_ctl_status.s.hst_mode = 0;
 821         cvmx_write64_uint64(CVMX_USBNX_USBP_CTL_STATUS(usb->index),
 822                             usbn_usbp_ctl_status.u64);
 823         /* 10. Wait 1 us */
 824         udelay(1);
 825         /*
 826          * 11. Program the hreset_n field in the USBN clock-control register:
 827          *     USBN_CLK_CTL[HRST] = 1
 828          */
 829         usbn_clk_ctl.s.hrst = 1;
 830         cvmx_write64_uint64(CVMX_USBNX_CLK_CTL(usb->index), usbn_clk_ctl.u64);
 831         /* 12. Proceed to USB core initialization */
 832         usbn_clk_ctl.s.enable = 1;
 833         cvmx_write64_uint64(CVMX_USBNX_CLK_CTL(usb->index), usbn_clk_ctl.u64);
 834         udelay(1);
 835
 836         /*
 837          * USB Core Initialization
 838          *
 839          * 1. Read USBC_GHWCFG1, USBC_GHWCFG2, USBC_GHWCFG3, USBC_GHWCFG4 to
 840          *    determine USB core configuration parameters.
 841          *
 842          *    Nothing needed
 843          *
 844          * 2. Program the following fields in the global AHB configuration
 845          *    register (USBC_GAHBCFG)
 846          *    DMA mode, USBC_GAHBCFG[DMAEn]: 1 = DMA mode, 0 = slave mode
 847          *    Burst length, USBC_GAHBCFG[HBSTLEN] = 0
 848          *    Nonperiodic TxFIFO empty level (slave mode only),
 849          *    USBC_GAHBCFG[NPTXFEMPLVL]
 850          *    Periodic TxFIFO empty level (slave mode only),
 851          *    USBC_GAHBCFG[PTXFEMPLVL]
 852          *    Global interrupt mask, USBC_GAHBCFG[GLBLINTRMSK] = 1
 853          */
 854         usbcx_gahbcfg.u32 = 0;
 855         usbcx_gahbcfg.s.dmaen = !(usb->init_flags &
 856                                   CVMX_USB_INITIALIZE_FLAGS_NO_DMA);
 857         usbcx_gahbcfg.s.hbstlen = 0;
 858         usbcx_gahbcfg.s.nptxfemplvl = 1;
 859         usbcx_gahbcfg.s.ptxfemplvl = 1;
 860         usbcx_gahbcfg.s.glblintrmsk = 1;
 861         cvmx_usb_write_csr32(usb, CVMX_USBCX_GAHBCFG(usb->index),
 862                              usbcx_gahbcfg.u32);
 863
 864         /*
 865          * 3. Program the following fields in USBC_GUSBCFG register.
 866          *    HS/FS timeout calibration, USBC_GUSBCFG[TOUTCAL] = 0
 867          *    ULPI DDR select, USBC_GUSBCFG[DDRSEL] = 0
 868          *    USB turnaround time, USBC_GUSBCFG[USBTRDTIM] = 0x5
 869          *    PHY low-power clock select, USBC_GUSBCFG[PHYLPWRCLKSEL] = 0
 870          */
 871         usbcx_gusbcfg.u32 = cvmx_usb_read_csr32(usb,
 872                                                 CVMX_USBCX_GUSBCFG(usb->index));
 873         usbcx_gusbcfg.s.toutcal = 0;
 874         usbcx_gusbcfg.s.ddrsel = 0;
 875         usbcx_gusbcfg.s.usbtrdtim = 0x5;
 876         usbcx_gusbcfg.s.phylpwrclksel = 0;
 877         cvmx_usb_write_csr32(usb, CVMX_USBCX_GUSBCFG(usb->index),
 878                              usbcx_gusbcfg.u32);
 879
 880         /*
 881          * 4. The software must unmask the following bits in the USBC_GINTMSK
 882          *    register.
 883          *    OTG interrupt mask, USBC_GINTMSK[OTGINTMSK] = 1
 884          *    Mode mismatch interrupt mask, USBC_GINTMSK[MODEMISMSK] = 1
 885          */
 886         usbcx_gintmsk.u32 = cvmx_usb_read_csr32(usb,
 887                                                 CVMX_USBCX_GINTMSK(usb->index));
 888         usbcx_gintmsk.s.otgintmsk = 1;
 889         usbcx_gintmsk.s.modemismsk = 1;
 890         usbcx_gintmsk.s.hchintmsk = 1;
 891         usbcx_gintmsk.s.sofmsk = 0;
 892         /* We need RX FIFO interrupts if we don't have DMA */
 893         if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
 894                 usbcx_gintmsk.s.rxflvlmsk = 1;
 895         cvmx_usb_write_csr32(usb, CVMX_USBCX_GINTMSK(usb->index),
 896                              usbcx_gintmsk.u32);
 897
 898         /*
 899          * Disable all channel interrupts. We'll enable them per channel later.
 900          */
 901         for (channel = 0; channel < 8; channel++)
 902                 cvmx_usb_write_csr32(usb,
 903                                      CVMX_USBCX_HCINTMSKX(channel, usb->index),
 904                                      0);
 905
 906         /*
 907          * Host Port Initialization
 908          *
 909          * 1. Program the host-port interrupt-mask field to unmask,
 910          *    USBC_GINTMSK[PRTINT] = 1
 911          */
 912         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
 913                         cvmx_usbcx_gintmsk, prtintmsk, 1);
 914         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
 915                         cvmx_usbcx_gintmsk, disconnintmsk, 1);
 916
 917         /*
 918          * 2. Program the USBC_HCFG register to select full-speed host
 919          *    or high-speed host.
 920          */
 921         usbcx_hcfg.u32 = cvmx_usb_read_csr32(usb, CVMX_USBCX_HCFG(usb->index));
 922         usbcx_hcfg.s.fslssupp = 0;
 923         usbcx_hcfg.s.fslspclksel = 0;
 924         cvmx_usb_write_csr32(usb, CVMX_USBCX_HCFG(usb->index), usbcx_hcfg.u32);
 925
 926         cvmx_fifo_setup(usb);
 927
 928         /*
 929          * If the controller is getting port events right after the reset, it
 930          * means the initialization failed. Try resetting the controller again
 931          * in such case. This is seen to happen after cold boot on DSR-1000N.
 932          */
 933         usbc_gintsts.u32 = cvmx_usb_read_csr32(usb,
 934                                                CVMX_USBCX_GINTSTS(usb->index));
 935         cvmx_usb_write_csr32(usb, CVMX_USBCX_GINTSTS(usb->index),
 936                              usbc_gintsts.u32);
 937         dev_dbg(dev, "gintsts after reset: 0x%x\n", (int)usbc_gintsts.u32);
 938         if (!usbc_gintsts.s.disconnint && !usbc_gintsts.s.prtint)
 939                 return 0;
 940         if (retries++ >= 5)
 941                 return -EAGAIN;
 942         dev_info(dev, "controller reset failed (gintsts=0x%x) - retrying\n",
 943                  (int)usbc_gintsts.u32);
 944         msleep(50);
 945         cvmx_usb_shutdown(usb);
 946         msleep(50);
 947         goto retry;
 948 }
 949
 950 /**
 951  * Reset a USB port. After this call succeeds, the USB port is
 952  * online and servicing requests.
 953  *
 954  * @usb: USB device state populated by cvmx_usb_initialize().
 955  */
 956 static void cvmx_usb_reset_port(struct octeon_hcd *usb)
 957 {
 958         usb->usbcx_hprt.u32 = cvmx_usb_read_csr32(usb,
 959                                                   CVMX_USBCX_HPRT(usb->index));
 960
 961         /* Program the port reset bit to start the reset process */
 962         USB_SET_FIELD32(CVMX_USBCX_HPRT(usb->index), cvmx_usbcx_hprt,
 963                         prtrst, 1);
 964
 965         /*
 966          * Wait at least 50ms (high speed), or 10ms (full speed) for the reset
 967          * process to complete.
 968          */
 969         mdelay(50);
 970
 971         /* Program the port reset bit to 0, USBC_HPRT[PRTRST] = 0 */
 972         USB_SET_FIELD32(CVMX_USBCX_HPRT(usb->index), cvmx_usbcx_hprt,
 973                         prtrst, 0);
 974
 975         /*
 976          * Read the port speed field to get the enumerated speed,
 977          * USBC_HPRT[PRTSPD].
 978          */
 979         usb->usbcx_hprt.u32 = cvmx_usb_read_csr32(usb,
 980                                                   CVMX_USBCX_HPRT(usb->index));
 981 }
 982
 983 /**
 984  * Disable a USB port. After this call the USB port will not
 985  * generate data transfers and will not generate events.
 986  * Transactions in process will fail and call their
 987  * associated callbacks.
 988  *
 989  * @usb: USB device state populated by cvmx_usb_initialize().
 990  *
 991  * Returns: 0 or a negative error code.
 992  */
 993 static int cvmx_usb_disable(struct octeon_hcd *usb)
 994 {
 995         /* Disable the port */
 996         USB_SET_FIELD32(CVMX_USBCX_HPRT(usb->index), cvmx_usbcx_hprt,
 997                         prtena, 1);
 998         return 0;
 999 }
1000
1001 /**
1002  * Get the current state of the USB port. Use this call to
1003  * determine if the usb port has anything connected, is enabled,
1004  * or has some sort of error condition. The return value of this
1005  * call has "changed" bits to signal of the value of some fields
1006  * have changed between calls.
1007  *
1008  * @usb: USB device state populated by cvmx_usb_initialize().
1009  *
1010  * Returns: Port status information
1011  */
1012 static struct cvmx_usb_port_status cvmx_usb_get_status(struct octeon_hcd *usb)
1013 {
1014         union cvmx_usbcx_hprt usbc_hprt;
1015         struct cvmx_usb_port_status result;
1016
1017         memset(&result, 0, sizeof(result));
1018
1019         usbc_hprt.u32 = cvmx_usb_read_csr32(usb, CVMX_USBCX_HPRT(usb->index));
1020         result.port_enabled = usbc_hprt.s.prtena;
1021         result.port_over_current = usbc_hprt.s.prtovrcurract;
1022         result.port_powered = usbc_hprt.s.prtpwr;
1023         result.port_speed = usbc_hprt.s.prtspd;
1024         result.connected = usbc_hprt.s.prtconnsts;
1025         result.connect_change =
1026                 result.connected != usb->port_status.connected;
1027
1028         return result;
1029 }
1030
1031 /**
1032  * Open a virtual pipe between the host and a USB device. A pipe
1033  * must be opened before data can be transferred between a device
1034  * and Octeon.
1035  *
1036  * @usb:             USB device state populated by cvmx_usb_initialize().
1037  * @device_addr:
1038  *                   USB device address to open the pipe to
1039  *                   (0-127).
1040  * @endpoint_num:
1041  *                   USB endpoint number to open the pipe to
1042  *                   (0-15).
1043  * @device_speed:
1044  *                   The speed of the device the pipe is going
1045  *                   to. This must match the device's speed,
1046  *                   which may be different than the port speed.
1047  * @max_packet:      The maximum packet length the device can
1048  *                   transmit/receive (low speed=0-8, full
1049  *                   speed=0-1023, high speed=0-1024). This value
1050  *                   comes from the standard endpoint descriptor
1051  *                   field wMaxPacketSize bits <10:0>.
1052  * @transfer_type:
1053  *                   The type of transfer this pipe is for.
1054  * @transfer_dir:
1055  *                   The direction the pipe is in. This is not
1056  *                   used for control pipes.
1057  * @interval:        For ISOCHRONOUS and INTERRUPT transfers,
1058  *                   this is how often the transfer is scheduled
1059  *                   for. All other transfers should specify
1060  *                   zero. The units are in frames (8000/sec at
1061  *                   high speed, 1000/sec for full speed).
1062  * @multi_count:
1063  *                   For high speed devices, this is the maximum
1064  *                   allowed number of packet per microframe.
1065  *                   Specify zero for non high speed devices. This
1066  *                   value comes from the standard endpoint descriptor
1067  *                   field wMaxPacketSize bits <12:11>.
1068  * @hub_device_addr:
1069  *                   Hub device address this device is connected
1070  *                   to. Devices connected directly to Octeon
1071  *                   use zero. This is only used when the device
1072  *                   is full/low speed behind a high speed hub.
1073  *                   The address will be of the high speed hub,
1074  *                   not and full speed hubs after it.
1075  * @hub_port:        Which port on the hub the device is
1076  *                   connected. Use zero for devices connected
1077  *                   directly to Octeon. Like hub_device_addr,
1078  *                   this is only used for full/low speed
1079  *                   devices behind a high speed hub.
1080  *
1081  * Returns: A non-NULL value is a pipe. NULL means an error.
1082  */
1083 static struct cvmx_usb_pipe *cvmx_usb_open_pipe(struct octeon_hcd *usb,
1084                                                 int device_addr,
1085                                                 int endpoint_num,
1086                                                 enum cvmx_usb_speed
1087                                                         device_speed,
1088                                                 int max_packet,
1089                                                 enum cvmx_usb_transfer
1090                                                         transfer_type,
1091                                                 enum cvmx_usb_direction
1092                                                         transfer_dir,
1093                                                 int interval, int multi_count,
1094                                                 int hub_device_addr,
1095                                                 int hub_port)
1096 {
1097         struct cvmx_usb_pipe *pipe;
1098
1099         pipe = kzalloc(sizeof(*pipe), GFP_ATOMIC);
1100         if (!pipe)
1101                 return NULL;
1102         if ((device_speed == CVMX_USB_SPEED_HIGH) &&
1103             (transfer_dir == CVMX_USB_DIRECTION_OUT) &&
1104             (transfer_type == CVMX_USB_TRANSFER_BULK))
1105                 pipe->flags |= CVMX_USB_PIPE_FLAGS_NEED_PING;
1106         pipe->device_addr = device_addr;
1107         pipe->endpoint_num = endpoint_num;
1108         pipe->device_speed = device_speed;
1109         pipe->max_packet = max_packet;
1110         pipe->transfer_type = transfer_type;
1111         pipe->transfer_dir = transfer_dir;
1112         INIT_LIST_HEAD(&pipe->transactions);
1113
1114         /*
1115          * All pipes use interval to rate limit NAK processing. Force an
1116          * interval if one wasn't supplied
1117          */
1118         if (!interval)
1119                 interval = 1;
1120         if (cvmx_usb_pipe_needs_split(usb, pipe)) {
1121                 pipe->interval = interval * 8;
1122                 /* Force start splits to be schedule on uFrame 0 */
1123                 pipe->next_tx_frame = ((usb->frame_number + 7) & ~7) +
1124                                         pipe->interval;
1125         } else {
1126                 pipe->interval = interval;
1127                 pipe->next_tx_frame = usb->frame_number + pipe->interval;
1128         }
1129         pipe->multi_count = multi_count;
1130         pipe->hub_device_addr = hub_device_addr;
1131         pipe->hub_port = hub_port;
1132         pipe->pid_toggle = 0;
1133         pipe->split_sc_frame = -1;
1134         list_add_tail(&pipe->node, &usb->idle_pipes);
1135
1136         /*
1137          * We don't need to tell the hardware about this pipe yet since
1138          * it doesn't have any submitted requests
1139          */
1140
1141         return pipe;
1142 }
1143
1144 /**
1145  * Poll the RX FIFOs and remove data as needed. This function is only used
1146  * in non DMA mode. It is very important that this function be called quickly
1147  * enough to prevent FIFO overflow.
1148  *
1149  * @usb:        USB device state populated by cvmx_usb_initialize().
1150  */
1151 static void cvmx_usb_poll_rx_fifo(struct octeon_hcd *usb)
1152 {
1153         union cvmx_usbcx_grxstsph rx_status;
1154         int channel;
1155         int bytes;
1156         u64 address;
1157         u32 *ptr;
1158
1159         rx_status.u32 = cvmx_usb_read_csr32(usb,
1160                                             CVMX_USBCX_GRXSTSPH(usb->index));
1161         /* Only read data if IN data is there */
1162         if (rx_status.s.pktsts != 2)
1163                 return;
1164         /* Check if no data is available */
1165         if (!rx_status.s.bcnt)
1166                 return;
1167
1168         channel = rx_status.s.chnum;
1169         bytes = rx_status.s.bcnt;
1170         if (!bytes)
1171                 return;
1172
1173         /* Get where the DMA engine would have written this data */
1174         address = cvmx_read64_uint64(CVMX_USBNX_DMA0_INB_CHN0(usb->index) +
1175                                      channel * 8);
1176
1177         ptr = cvmx_phys_to_ptr(address);
1178         cvmx_write64_uint64(CVMX_USBNX_DMA0_INB_CHN0(usb->index) + channel * 8,
1179                             address + bytes);
1180
1181         /* Loop writing the FIFO data for this packet into memory */
1182         while (bytes > 0) {
1183                 *ptr++ = cvmx_usb_read_csr32(usb,
1184                                         USB_FIFO_ADDRESS(channel, usb->index));
1185                 bytes -= 4;
1186         }
1187         CVMX_SYNCW;
1188 }
1189
1190 /**
1191  * Fill the TX hardware fifo with data out of the software
1192  * fifos
1193  *
1194  * @usb:            USB device state populated by cvmx_usb_initialize().
1195  * @fifo:           Software fifo to use
1196  * @available:      Amount of space in the hardware fifo
1197  *
1198  * Returns: Non zero if the hardware fifo was too small and needs
1199  *          to be serviced again.
1200  */
1201 static int cvmx_usb_fill_tx_hw(struct octeon_hcd *usb,
1202                                struct cvmx_usb_tx_fifo *fifo, int available)
1203 {
1204         /*
1205          * We're done either when there isn't anymore space or the software FIFO
1206          * is empty
1207          */
1208         while (available && (fifo->head != fifo->tail)) {
1209                 int i = fifo->tail;
1210                 const u32 *ptr = cvmx_phys_to_ptr(fifo->entry[i].address);
1211                 u64 csr_address = USB_FIFO_ADDRESS(fifo->entry[i].channel,
1212                                                    usb->index) ^ 4;
1213                 int words = available;
1214
1215                 /* Limit the amount of data to what the SW fifo has */
1216                 if (fifo->entry[i].size <= available) {
1217                         words = fifo->entry[i].size;
1218                         fifo->tail++;
1219                         if (fifo->tail > MAX_CHANNELS)
1220                                 fifo->tail = 0;
1221                 }
1222
1223                 /* Update the next locations and counts */
1224                 available -= words;
1225                 fifo->entry[i].address += words * 4;
1226                 fifo->entry[i].size -= words;
1227
1228                 /*
1229                  * Write the HW fifo data. The read every three writes is due
1230                  * to an errata on CN3XXX chips
1231                  */
1232                 while (words > 3) {
1233                         cvmx_write64_uint32(csr_address, *ptr++);
1234                         cvmx_write64_uint32(csr_address, *ptr++);
1235                         cvmx_write64_uint32(csr_address, *ptr++);
1236                         cvmx_read64_uint64(
1237                                         CVMX_USBNX_DMA0_INB_CHN0(usb->index));
1238                         words -= 3;
1239                 }
1240                 cvmx_write64_uint32(csr_address, *ptr++);
1241                 if (--words) {
1242                         cvmx_write64_uint32(csr_address, *ptr++);
1243                         if (--words)
1244                                 cvmx_write64_uint32(csr_address, *ptr++);
1245                 }
1246                 cvmx_read64_uint64(CVMX_USBNX_DMA0_INB_CHN0(usb->index));
1247         }
1248         return fifo->head != fifo->tail;
1249 }
1250
1251 /**
1252  * Check the hardware FIFOs and fill them as needed
1253  *
1254  * @usb:        USB device state populated by cvmx_usb_initialize().
1255  */
1256 static void cvmx_usb_poll_tx_fifo(struct octeon_hcd *usb)
1257 {
1258         if (usb->periodic.head != usb->periodic.tail) {
1259                 union cvmx_usbcx_hptxsts tx_status;
1260
1261                 tx_status.u32 = cvmx_usb_read_csr32(usb,
1262                                         CVMX_USBCX_HPTXSTS(usb->index));
1263                 if (cvmx_usb_fill_tx_hw(usb, &usb->periodic,
1264                                         tx_status.s.ptxfspcavail))
1265                         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
1266                                         cvmx_usbcx_gintmsk, ptxfempmsk, 1);
1267                 else
1268                         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
1269                                         cvmx_usbcx_gintmsk, ptxfempmsk, 0);
1270         }
1271
1272         if (usb->nonperiodic.head != usb->nonperiodic.tail) {
1273                 union cvmx_usbcx_gnptxsts tx_status;
1274
1275                 tx_status.u32 = cvmx_usb_read_csr32(usb,
1276                                         CVMX_USBCX_GNPTXSTS(usb->index));
1277                 if (cvmx_usb_fill_tx_hw(usb, &usb->nonperiodic,
1278                                         tx_status.s.nptxfspcavail))
1279                         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
1280                                         cvmx_usbcx_gintmsk, nptxfempmsk, 1);
1281                 else
1282                         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
1283                                         cvmx_usbcx_gintmsk, nptxfempmsk, 0);
1284         }
1285 }
1286
1287 /**
1288  * Fill the TX FIFO with an outgoing packet
1289  *
1290  * @usb:          USB device state populated by cvmx_usb_initialize().
1291  * @channel:      Channel number to get packet from
1292  */
1293 static void cvmx_usb_fill_tx_fifo(struct octeon_hcd *usb, int channel)
1294 {
1295         union cvmx_usbcx_hccharx hcchar;
1296         union cvmx_usbcx_hcspltx usbc_hcsplt;
1297         union cvmx_usbcx_hctsizx usbc_hctsiz;
1298         struct cvmx_usb_tx_fifo *fifo;
1299
1300         /* We only need to fill data on outbound channels */
1301         hcchar.u32 = cvmx_usb_read_csr32(usb,
1302                         CVMX_USBCX_HCCHARX(channel, usb->index));
1303         if (hcchar.s.epdir != CVMX_USB_DIRECTION_OUT)
1304                 return;
1305
1306         /* OUT Splits only have data on the start and not the complete */
1307         usbc_hcsplt.u32 = cvmx_usb_read_csr32(usb,
1308                                 CVMX_USBCX_HCSPLTX(channel, usb->index));
1309         if (usbc_hcsplt.s.spltena && usbc_hcsplt.s.compsplt)
1310                 return;
1311
1312         /*
1313          * Find out how many bytes we need to fill and convert it into 32bit
1314          * words.
1315          */
1316         usbc_hctsiz.u32 = cvmx_usb_read_csr32(usb,
1317                                 CVMX_USBCX_HCTSIZX(channel, usb->index));
1318         if (!usbc_hctsiz.s.xfersize)
1319                 return;
1320
1321         if ((hcchar.s.eptype == CVMX_USB_TRANSFER_INTERRUPT) ||
1322             (hcchar.s.eptype == CVMX_USB_TRANSFER_ISOCHRONOUS))
1323                 fifo = &usb->periodic;
1324         else
1325                 fifo = &usb->nonperiodic;
1326
1327         fifo->entry[fifo->head].channel = channel;
1328         fifo->entry[fifo->head].address =
1329                 cvmx_read64_uint64(CVMX_USBNX_DMA0_OUTB_CHN0(usb->index) +
1330                                    channel * 8);
1331         fifo->entry[fifo->head].size = (usbc_hctsiz.s.xfersize + 3) >> 2;
1332         fifo->head++;
1333         if (fifo->head > MAX_CHANNELS)
1334                 fifo->head = 0;
1335
1336         cvmx_usb_poll_tx_fifo(usb);
1337 }
1338
1339 /**
1340  * Perform channel specific setup for Control transactions. All
1341  * the generic stuff will already have been done in cvmx_usb_start_channel().
1342  *
1343  * @usb:          USB device state populated by cvmx_usb_initialize().
1344  * @channel:      Channel to setup
1345  * @pipe:         Pipe for control transaction
1346  */
1347 static void cvmx_usb_start_channel_control(struct octeon_hcd *usb,
1348                                            int channel,
1349                                            struct cvmx_usb_pipe *pipe)
1350 {
1351         struct usb_hcd *hcd = octeon_to_hcd(usb);
1352         struct device *dev = hcd->self.controller;
1353         struct cvmx_usb_transaction *transaction =
1354                 list_first_entry(&pipe->transactions, typeof(*transaction),
1355                                  node);
1356         struct usb_ctrlrequest *header =
1357                 cvmx_phys_to_ptr(transaction->control_header);
1358         int bytes_to_transfer = transaction->buffer_length -
1359                 transaction->actual_bytes;
1360         int packets_to_transfer;
1361         union cvmx_usbcx_hctsizx usbc_hctsiz;
1362
1363         usbc_hctsiz.u32 = cvmx_usb_read_csr32(usb,
1364                                 CVMX_USBCX_HCTSIZX(channel, usb->index));
1365
1366         switch (transaction->stage) {
1367         case CVMX_USB_STAGE_NON_CONTROL:
1368         case CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE:
1369                 dev_err(dev, "%s: ERROR - Non control stage\n", __func__);
1370                 break;
1371         case CVMX_USB_STAGE_SETUP:
1372                 usbc_hctsiz.s.pid = 3; /* Setup */
1373                 bytes_to_transfer = sizeof(*header);
1374                 /* All Control operations start with a setup going OUT */
1375                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1376                                 cvmx_usbcx_hccharx, epdir,
1377                                 CVMX_USB_DIRECTION_OUT);
1378                 /*
1379                  * Setup send the control header instead of the buffer data. The
1380                  * buffer data will be used in the next stage
1381                  */
1382                 cvmx_write64_uint64(CVMX_USBNX_DMA0_OUTB_CHN0(usb->index) +
1383                                         channel * 8,
1384                                     transaction->control_header);
1385                 break;
1386         case CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE:
1387                 usbc_hctsiz.s.pid = 3; /* Setup */
1388                 bytes_to_transfer = 0;
1389                 /* All Control operations start with a setup going OUT */
1390                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1391                                 cvmx_usbcx_hccharx, epdir,
1392                                 CVMX_USB_DIRECTION_OUT);
1393
1394                 USB_SET_FIELD32(CVMX_USBCX_HCSPLTX(channel, usb->index),
1395                                 cvmx_usbcx_hcspltx, compsplt, 1);
1396                 break;
1397         case CVMX_USB_STAGE_DATA:
1398                 usbc_hctsiz.s.pid = cvmx_usb_get_data_pid(pipe);
1399                 if (cvmx_usb_pipe_needs_split(usb, pipe)) {
1400                         if (header->bRequestType & USB_DIR_IN)
1401                                 bytes_to_transfer = 0;
1402                         else if (bytes_to_transfer > pipe->max_packet)
1403                                 bytes_to_transfer = pipe->max_packet;
1404                 }
1405                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1406                                 cvmx_usbcx_hccharx, epdir,
1407                                 ((header->bRequestType & USB_DIR_IN) ?
1408                                         CVMX_USB_DIRECTION_IN :
1409                                         CVMX_USB_DIRECTION_OUT));
1410                 break;
1411         case CVMX_USB_STAGE_DATA_SPLIT_COMPLETE:
1412                 usbc_hctsiz.s.pid = cvmx_usb_get_data_pid(pipe);
1413                 if (!(header->bRequestType & USB_DIR_IN))
1414                         bytes_to_transfer = 0;
1415                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1416                                 cvmx_usbcx_hccharx, epdir,
1417                                 ((header->bRequestType & USB_DIR_IN) ?
1418                                         CVMX_USB_DIRECTION_IN :
1419                                         CVMX_USB_DIRECTION_OUT));
1420                 USB_SET_FIELD32(CVMX_USBCX_HCSPLTX(channel, usb->index),
1421                                 cvmx_usbcx_hcspltx, compsplt, 1);
1422                 break;
1423         case CVMX_USB_STAGE_STATUS:
1424                 usbc_hctsiz.s.pid = cvmx_usb_get_data_pid(pipe);
1425                 bytes_to_transfer = 0;
1426                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1427                                 cvmx_usbcx_hccharx, epdir,
1428                                 ((header->bRequestType & USB_DIR_IN) ?
1429                                         CVMX_USB_DIRECTION_OUT :
1430                                         CVMX_USB_DIRECTION_IN));
1431                 break;
1432         case CVMX_USB_STAGE_STATUS_SPLIT_COMPLETE:
1433                 usbc_hctsiz.s.pid = cvmx_usb_get_data_pid(pipe);
1434                 bytes_to_transfer = 0;
1435                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1436                                 cvmx_usbcx_hccharx, epdir,
1437                                 ((header->bRequestType & USB_DIR_IN) ?
1438                                         CVMX_USB_DIRECTION_OUT :
1439                                         CVMX_USB_DIRECTION_IN));
1440                 USB_SET_FIELD32(CVMX_USBCX_HCSPLTX(channel, usb->index),
1441                                 cvmx_usbcx_hcspltx, compsplt, 1);
1442                 break;
1443         }
1444
1445         /*
1446          * Make sure the transfer never exceeds the byte limit of the hardware.
1447          * Further bytes will be sent as continued transactions
1448          */
1449         if (bytes_to_transfer > MAX_TRANSFER_BYTES) {
1450                 /* Round MAX_TRANSFER_BYTES to a multiple of out packet size */
1451                 bytes_to_transfer = MAX_TRANSFER_BYTES / pipe->max_packet;
1452                 bytes_to_transfer *= pipe->max_packet;
1453         }
1454
1455         /*
1456          * Calculate the number of packets to transfer. If the length is zero
1457          * we still need to transfer one packet
1458          */
1459         packets_to_transfer = DIV_ROUND_UP(bytes_to_transfer,
1460                                            pipe->max_packet);
1461         if (packets_to_transfer == 0) {
1462                 packets_to_transfer = 1;
1463         } else if ((packets_to_transfer > 1) &&
1464                         (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)) {
1465                 /*
1466                  * Limit to one packet when not using DMA. Channels must be
1467                  * restarted between every packet for IN transactions, so there
1468                  * is no reason to do multiple packets in a row
1469                  */
1470                 packets_to_transfer = 1;
1471                 bytes_to_transfer = packets_to_transfer * pipe->max_packet;
1472         } else if (packets_to_transfer > MAX_TRANSFER_PACKETS) {
1473                 /*
1474                  * Limit the number of packet and data transferred to what the
1475                  * hardware can handle
1476                  */
1477                 packets_to_transfer = MAX_TRANSFER_PACKETS;
1478                 bytes_to_transfer = packets_to_transfer * pipe->max_packet;
1479         }
1480
1481         usbc_hctsiz.s.xfersize = bytes_to_transfer;
1482         usbc_hctsiz.s.pktcnt = packets_to_transfer;
1483
1484         cvmx_usb_write_csr32(usb, CVMX_USBCX_HCTSIZX(channel, usb->index),
1485                              usbc_hctsiz.u32);
1486 }
1487
1488 /**
1489  * Start a channel to perform the pipe's head transaction
1490  *
1491  * @usb:          USB device state populated by cvmx_usb_initialize().
1492  * @channel:      Channel to setup
1493  * @pipe:         Pipe to start
1494  */
1495 static void cvmx_usb_start_channel(struct octeon_hcd *usb, int channel,
1496                                    struct cvmx_usb_pipe *pipe)
1497 {
1498         struct cvmx_usb_transaction *transaction =
1499                 list_first_entry(&pipe->transactions, typeof(*transaction),
1500                                  node);
1501
1502         /* Make sure all writes to the DMA region get flushed */
1503         CVMX_SYNCW;
1504
1505         /* Attach the channel to the pipe */
1506         usb->pipe_for_channel[channel] = pipe;
1507         pipe->channel = channel;
1508         pipe->flags |= CVMX_USB_PIPE_FLAGS_SCHEDULED;
1509
1510         /* Mark this channel as in use */
1511         usb->idle_hardware_channels &= ~(1 << channel);
1512
1513         /* Enable the channel interrupt bits */
1514         {
1515                 union cvmx_usbcx_hcintx usbc_hcint;
1516                 union cvmx_usbcx_hcintmskx usbc_hcintmsk;
1517                 union cvmx_usbcx_haintmsk usbc_haintmsk;
1518
1519                 /* Clear all channel status bits */
1520                 usbc_hcint.u32 = cvmx_usb_read_csr32(usb,
1521                                         CVMX_USBCX_HCINTX(channel, usb->index));
1522
1523                 cvmx_usb_write_csr32(usb,
1524                                      CVMX_USBCX_HCINTX(channel, usb->index),
1525                                      usbc_hcint.u32);
1526
1527                 usbc_hcintmsk.u32 = 0;
1528                 usbc_hcintmsk.s.chhltdmsk = 1;
1529                 if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA) {
1530                         /*
1531                          * Channels need these extra interrupts when we aren't
1532                          * in DMA mode.
1533                          */
1534                         usbc_hcintmsk.s.datatglerrmsk = 1;
1535                         usbc_hcintmsk.s.frmovrunmsk = 1;
1536                         usbc_hcintmsk.s.bblerrmsk = 1;
1537                         usbc_hcintmsk.s.xacterrmsk = 1;
1538                         if (cvmx_usb_pipe_needs_split(usb, pipe)) {
1539                                 /*
1540                                  * Splits don't generate xfercompl, so we need
1541                                  * ACK and NYET.
1542                                  */
1543                                 usbc_hcintmsk.s.nyetmsk = 1;
1544                                 usbc_hcintmsk.s.ackmsk = 1;
1545                         }
1546                         usbc_hcintmsk.s.nakmsk = 1;
1547                         usbc_hcintmsk.s.stallmsk = 1;
1548                         usbc_hcintmsk.s.xfercomplmsk = 1;
1549                 }
1550                 cvmx_usb_write_csr32(usb,
1551                                      CVMX_USBCX_HCINTMSKX(channel, usb->index),
1552                                      usbc_hcintmsk.u32);
1553
1554                 /* Enable the channel interrupt to propagate */
1555                 usbc_haintmsk.u32 = cvmx_usb_read_csr32(usb,
1556                                         CVMX_USBCX_HAINTMSK(usb->index));
1557                 usbc_haintmsk.s.haintmsk |= 1 << channel;
1558                 cvmx_usb_write_csr32(usb, CVMX_USBCX_HAINTMSK(usb->index),
1559                                      usbc_haintmsk.u32);
1560         }
1561
1562         /* Setup the location the DMA engine uses. */
1563         {
1564                 u64 reg;
1565                 u64 dma_address = transaction->buffer +
1566                                   transaction->actual_bytes;
1567
1568                 if (transaction->type == CVMX_USB_TRANSFER_ISOCHRONOUS)
1569                         dma_address = transaction->buffer +
1570                                         transaction->iso_packets[0].offset +
1571                                         transaction->actual_bytes;
1572
1573                 if (pipe->transfer_dir == CVMX_USB_DIRECTION_OUT)
1574                         reg = CVMX_USBNX_DMA0_OUTB_CHN0(usb->index);
1575                 else
1576                         reg = CVMX_USBNX_DMA0_INB_CHN0(usb->index);
1577                 cvmx_write64_uint64(reg + channel * 8, dma_address);
1578         }
1579
1580         /* Setup both the size of the transfer and the SPLIT characteristics */
1581         {
1582                 union cvmx_usbcx_hcspltx usbc_hcsplt = {.u32 = 0};
1583                 union cvmx_usbcx_hctsizx usbc_hctsiz = {.u32 = 0};
1584                 int packets_to_transfer;
1585                 int bytes_to_transfer = transaction->buffer_length -
1586                         transaction->actual_bytes;
1587
1588                 /*
1589                  * ISOCHRONOUS transactions store each individual transfer size
1590                  * in the packet structure, not the global buffer_length
1591                  */
1592                 if (transaction->type == CVMX_USB_TRANSFER_ISOCHRONOUS)
1593                         bytes_to_transfer =
1594                                 transaction->iso_packets[0].length -
1595                                 transaction->actual_bytes;
1596
1597                 /*
1598                  * We need to do split transactions when we are talking to non
1599                  * high speed devices that are behind a high speed hub
1600                  */
1601                 if (cvmx_usb_pipe_needs_split(usb, pipe)) {
1602                         /*
1603                          * On the start split phase (stage is even) record the
1604                          * frame number we will need to send the split complete.
1605                          * We only store the lower two bits since the time ahead
1606                          * can only be two frames
1607                          */
1608                         if ((transaction->stage & 1) == 0) {
1609                                 if (transaction->type == CVMX_USB_TRANSFER_BULK)
1610                                         pipe->split_sc_frame =
1611                                                 (usb->frame_number + 1) & 0x7f;
1612                                 else
1613                                         pipe->split_sc_frame =
1614                                                 (usb->frame_number + 2) & 0x7f;
1615                         } else {
1616                                 pipe->split_sc_frame = -1;
1617                         }
1618
1619                         usbc_hcsplt.s.spltena = 1;
1620                         usbc_hcsplt.s.hubaddr = pipe->hub_device_addr;
1621                         usbc_hcsplt.s.prtaddr = pipe->hub_port;
1622                         usbc_hcsplt.s.compsplt = (transaction->stage ==
1623                                 CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE);
1624
1625                         /*
1626                          * SPLIT transactions can only ever transmit one data
1627                          * packet so limit the transfer size to the max packet
1628                          * size
1629                          */
1630                         if (bytes_to_transfer > pipe->max_packet)
1631                                 bytes_to_transfer = pipe->max_packet;
1632
1633                         /*
1634                          * ISOCHRONOUS OUT splits are unique in that they limit
1635                          * data transfers to 188 byte chunks representing the
1636                          * begin/middle/end of the data or all
1637                          */
1638                         if (!usbc_hcsplt.s.compsplt &&
1639                             (pipe->transfer_dir == CVMX_USB_DIRECTION_OUT) &&
1640                             (pipe->transfer_type ==
1641                              CVMX_USB_TRANSFER_ISOCHRONOUS)) {
1642                                 /*
1643                                  * Clear the split complete frame number as
1644                                  * there isn't going to be a split complete
1645                                  */
1646                                 pipe->split_sc_frame = -1;
1647                                 /*
1648                                  * See if we've started this transfer and sent
1649                                  * data
1650                                  */
1651                                 if (transaction->actual_bytes == 0) {
1652                                         /*
1653                                          * Nothing sent yet, this is either a
1654                                          * begin or the entire payload
1655                                          */
1656                                         if (bytes_to_transfer <= 188)
1657                                                 /* Entire payload in one go */
1658                                                 usbc_hcsplt.s.xactpos = 3;
1659                                         else
1660                                                 /* First part of payload */
1661                                                 usbc_hcsplt.s.xactpos = 2;
1662                                 } else {
1663                                         /*
1664                                          * Continuing the previous data, we must
1665                                          * either be in the middle or at the end
1666                                          */
1667                                         if (bytes_to_transfer <= 188)
1668                                                 /* End of payload */
1669                                                 usbc_hcsplt.s.xactpos = 1;
1670                                         else
1671                                                 /* Middle of payload */
1672                                                 usbc_hcsplt.s.xactpos = 0;
1673                                 }
1674                                 /*
1675                                  * Again, the transfer size is limited to 188
1676                                  * bytes
1677                                  */
1678                                 if (bytes_to_transfer > 188)
1679                                         bytes_to_transfer = 188;
1680                         }
1681                 }
1682
1683                 /*
1684                  * Make sure the transfer never exceeds the byte limit of the
1685                  * hardware. Further bytes will be sent as continued
1686                  * transactions
1687                  */
1688                 if (bytes_to_transfer > MAX_TRANSFER_BYTES) {
1689                         /*
1690                          * Round MAX_TRANSFER_BYTES to a multiple of out packet
1691                          * size
1692                          */
1693                         bytes_to_transfer = MAX_TRANSFER_BYTES /
1694                                 pipe->max_packet;
1695                         bytes_to_transfer *= pipe->max_packet;
1696                 }
1697
1698                 /*
1699                  * Calculate the number of packets to transfer. If the length is
1700                  * zero we still need to transfer one packet
1701                  */
1702                 packets_to_transfer =
1703                         DIV_ROUND_UP(bytes_to_transfer, pipe->max_packet);
1704                 if (packets_to_transfer == 0) {
1705                         packets_to_transfer = 1;
1706                 } else if ((packets_to_transfer > 1) &&
1707                            (usb->init_flags &
1708                             CVMX_USB_INITIALIZE_FLAGS_NO_DMA)) {
1709                         /*
1710                          * Limit to one packet when not using DMA. Channels must
1711                          * be restarted between every packet for IN
1712                          * transactions, so there is no reason to do multiple
1713                          * packets in a row
1714                          */
1715                         packets_to_transfer = 1;
1716                         bytes_to_transfer = packets_to_transfer *
1717                                 pipe->max_packet;
1718                 } else if (packets_to_transfer > MAX_TRANSFER_PACKETS) {
1719                         /*
1720                          * Limit the number of packet and data transferred to
1721                          * what the hardware can handle
1722                          */
1723                         packets_to_transfer = MAX_TRANSFER_PACKETS;
1724                         bytes_to_transfer = packets_to_transfer *
1725                                 pipe->max_packet;
1726                 }
1727
1728                 usbc_hctsiz.s.xfersize = bytes_to_transfer;
1729                 usbc_hctsiz.s.pktcnt = packets_to_transfer;
1730
1731                 /* Update the DATA0/DATA1 toggle */
1732                 usbc_hctsiz.s.pid = cvmx_usb_get_data_pid(pipe);
1733                 /*
1734                  * High speed pipes may need a hardware ping before they start
1735                  */
1736                 if (pipe->flags & CVMX_USB_PIPE_FLAGS_NEED_PING)
1737                         usbc_hctsiz.s.dopng = 1;
1738
1739                 cvmx_usb_write_csr32(usb,
1740                                      CVMX_USBCX_HCSPLTX(channel, usb->index),
1741                                      usbc_hcsplt.u32);
1742                 cvmx_usb_write_csr32(usb,
1743                                      CVMX_USBCX_HCTSIZX(channel, usb->index),
1744                                      usbc_hctsiz.u32);
1745         }
1746
1747         /* Setup the Host Channel Characteristics Register */
1748         {
1749                 union cvmx_usbcx_hccharx usbc_hcchar = {.u32 = 0};
1750
1751                 /*
1752                  * Set the startframe odd/even properly. This is only used for
1753                  * periodic
1754                  */
1755                 usbc_hcchar.s.oddfrm = usb->frame_number & 1;
1756
1757                 /*
1758                  * Set the number of back to back packets allowed by this
1759                  * endpoint. Split transactions interpret "ec" as the number of
1760                  * immediate retries of failure. These retries happen too
1761                  * quickly, so we disable these entirely for splits
1762                  */
1763                 if (cvmx_usb_pipe_needs_split(usb, pipe))
1764                         usbc_hcchar.s.ec = 1;
1765                 else if (pipe->multi_count < 1)
1766                         usbc_hcchar.s.ec = 1;
1767                 else if (pipe->multi_count > 3)
1768                         usbc_hcchar.s.ec = 3;
1769                 else
1770                         usbc_hcchar.s.ec = pipe->multi_count;
1771
1772                 /* Set the rest of the endpoint specific settings */
1773                 usbc_hcchar.s.devaddr = pipe->device_addr;
1774                 usbc_hcchar.s.eptype = transaction->type;
1775                 usbc_hcchar.s.lspddev =
1776                         (pipe->device_speed == CVMX_USB_SPEED_LOW);
1777                 usbc_hcchar.s.epdir = pipe->transfer_dir;
1778                 usbc_hcchar.s.epnum = pipe->endpoint_num;
1779                 usbc_hcchar.s.mps = pipe->max_packet;
1780                 cvmx_usb_write_csr32(usb,
1781                                      CVMX_USBCX_HCCHARX(channel, usb->index),
1782                                      usbc_hcchar.u32);
1783         }
1784
1785         /* Do transaction type specific fixups as needed */
1786         switch (transaction->type) {
1787         case CVMX_USB_TRANSFER_CONTROL:
1788                 cvmx_usb_start_channel_control(usb, channel, pipe);
1789                 break;
1790         case CVMX_USB_TRANSFER_BULK:
1791         case CVMX_USB_TRANSFER_INTERRUPT:
1792                 break;
1793         case CVMX_USB_TRANSFER_ISOCHRONOUS:
1794                 if (!cvmx_usb_pipe_needs_split(usb, pipe)) {
1795                         /*
1796                          * ISO transactions require different PIDs depending on
1797                          * direction and how many packets are needed
1798                          */
1799                         if (pipe->transfer_dir == CVMX_USB_DIRECTION_OUT) {
1800                                 if (pipe->multi_count < 2) /* Need DATA0 */
1801                                         USB_SET_FIELD32(
1802                                                 CVMX_USBCX_HCTSIZX(channel,
1803                                                                    usb->index),
1804                                                 cvmx_usbcx_hctsizx, pid, 0);
1805                                 else /* Need MDATA */
1806                                         USB_SET_FIELD32(
1807                                                 CVMX_USBCX_HCTSIZX(channel,
1808                                                                    usb->index),
1809                                                 cvmx_usbcx_hctsizx, pid, 3);
1810                         }
1811                 }
1812                 break;
1813         }
1814         {
1815                 union cvmx_usbcx_hctsizx usbc_hctsiz = { .u32 =
1816                         cvmx_usb_read_csr32(usb,
1817                                             CVMX_USBCX_HCTSIZX(channel,
1818                                                                usb->index))
1819                 };
1820                 transaction->xfersize = usbc_hctsiz.s.xfersize;
1821                 transaction->pktcnt = usbc_hctsiz.s.pktcnt;
1822         }
1823         /* Remember when we start a split transaction */
1824         if (cvmx_usb_pipe_needs_split(usb, pipe))
1825                 usb->active_split = transaction;
1826         USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1827                         cvmx_usbcx_hccharx, chena, 1);
1828         if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
1829                 cvmx_usb_fill_tx_fifo(usb, channel);
1830 }
1831
1832 /**
1833  * Find a pipe that is ready to be scheduled to hardware.
1834  * @usb:         USB device state populated by cvmx_usb_initialize().
1835  * @xfer_type:   Transfer type
1836  *
1837  * Returns: Pipe or NULL if none are ready
1838  */
1839 static struct cvmx_usb_pipe *cvmx_usb_find_ready_pipe(struct octeon_hcd *usb,
1840                 enum cvmx_usb_transfer xfer_type)
1841 {
1842         struct list_head *list = usb->active_pipes + xfer_type;
1843         u64 current_frame = usb->frame_number;
1844         struct cvmx_usb_pipe *pipe;
1845
1846         list_for_each_entry(pipe, list, node) {
1847                 struct cvmx_usb_transaction *t =
1848                         list_first_entry(&pipe->transactions, typeof(*t),
1849                                          node);
1850                 if (!(pipe->flags & CVMX_USB_PIPE_FLAGS_SCHEDULED) && t &&
1851                     (pipe->next_tx_frame <= current_frame) &&
1852                     ((pipe->split_sc_frame == -1) ||
1853                      ((((int)current_frame - pipe->split_sc_frame) & 0x7f) <
1854                       0x40)) &&
1855                     (!usb->active_split || (usb->active_split == t))) {
1856                         prefetch(t);
1857                         return pipe;
1858                 }
1859         }
1860         return NULL;
1861 }
1862
1863 static struct cvmx_usb_pipe *cvmx_usb_next_pipe(struct octeon_hcd *usb,
1864                                                 int is_sof)
1865 {
1866         struct cvmx_usb_pipe *pipe;
1867
1868         /* Find a pipe needing service. */
1869         if (is_sof) {
1870                 /*
1871                  * Only process periodic pipes on SOF interrupts. This way we
1872                  * are sure that the periodic data is sent in the beginning of
1873                  * the frame.
1874                  */
1875                 pipe = cvmx_usb_find_ready_pipe(usb,
1876                                                 CVMX_USB_TRANSFER_ISOCHRONOUS);
1877                 if (pipe)
1878                         return pipe;
1879                 pipe = cvmx_usb_find_ready_pipe(usb,
1880                                                 CVMX_USB_TRANSFER_INTERRUPT);
1881                 if (pipe)
1882                         return pipe;
1883         }
1884         pipe = cvmx_usb_find_ready_pipe(usb, CVMX_USB_TRANSFER_CONTROL);
1885         if (pipe)
1886                 return pipe;
1887         return cvmx_usb_find_ready_pipe(usb, CVMX_USB_TRANSFER_BULK);
1888 }
1889
1890 /**
1891  * Called whenever a pipe might need to be scheduled to the
1892  * hardware.
1893  *
1894  * @usb:         USB device state populated by cvmx_usb_initialize().
1895  * @is_sof:      True if this schedule was called on a SOF interrupt.
1896  */
1897 static void cvmx_usb_schedule(struct octeon_hcd *usb, int is_sof)
1898 {
1899         int channel;
1900         struct cvmx_usb_pipe *pipe;
1901         int need_sof;
1902         enum cvmx_usb_transfer ttype;
1903
1904         if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA) {
1905                 /*
1906                  * Without DMA we need to be careful to not schedule something
1907                  * at the end of a frame and cause an overrun.
1908                  */
1909                 union cvmx_usbcx_hfnum hfnum = {
1910                         .u32 = cvmx_usb_read_csr32(usb,
1911                                                 CVMX_USBCX_HFNUM(usb->index))
1912                 };
1913
1914                 union cvmx_usbcx_hfir hfir = {
1915                         .u32 = cvmx_usb_read_csr32(usb,
1916                                                 CVMX_USBCX_HFIR(usb->index))
1917                 };
1918
1919                 if (hfnum.s.frrem < hfir.s.frint / 4)
1920                         goto done;
1921         }
1922
1923         while (usb->idle_hardware_channels) {
1924                 /* Find an idle channel */
1925                 channel = __fls(usb->idle_hardware_channels);
1926                 if (unlikely(channel > 7))
1927                         break;
1928
1929                 pipe = cvmx_usb_next_pipe(usb, is_sof);
1930                 if (!pipe)
1931                         break;
1932
1933                 cvmx_usb_start_channel(usb, channel, pipe);
1934         }
1935
1936 done:
1937         /*
1938          * Only enable SOF interrupts when we have transactions pending in the
1939          * future that might need to be scheduled
1940          */
1941         need_sof = 0;
1942         for (ttype = CVMX_USB_TRANSFER_CONTROL;
1943              ttype <= CVMX_USB_TRANSFER_INTERRUPT; ttype++) {
1944                 list_for_each_entry(pipe, &usb->active_pipes[ttype], node) {
1945                         if (pipe->next_tx_frame > usb->frame_number) {
1946                                 need_sof = 1;
1947                                 break;
1948                         }
1949                 }
1950         }
1951         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
1952                         cvmx_usbcx_gintmsk, sofmsk, need_sof);
1953 }
1954
1955 static void octeon_usb_urb_complete_callback(struct octeon_hcd *usb,
1956                                              enum cvmx_usb_status status,
1957                                              struct cvmx_usb_pipe *pipe,
1958                                              struct cvmx_usb_transaction
1959                                                 *transaction,
1960                                              int bytes_transferred,
1961                                              struct urb *urb)
1962 {
1963         struct usb_hcd *hcd = octeon_to_hcd(usb);
1964         struct device *dev = hcd->self.controller;
1965
1966         if (likely(status == CVMX_USB_STATUS_OK))
1967                 urb->actual_length = bytes_transferred;
1968         else
1969                 urb->actual_length = 0;
1970
1971         urb->hcpriv = NULL;
1972
1973         /* For Isochronous transactions we need to update the URB packet status
1974          * list from data in our private copy
1975          */
1976         if (usb_pipetype(urb->pipe) == PIPE_ISOCHRONOUS) {
1977                 int i;
1978                 /*
1979                  * The pointer to the private list is stored in the setup_packet
1980                  * field.
1981                  */
1982                 struct cvmx_usb_iso_packet *iso_packet =
1983                         (struct cvmx_usb_iso_packet *)urb->setup_packet;
1984                 /* Recalculate the transfer size by adding up each packet */
1985                 urb->actual_length = 0;
1986                 for (i = 0; i < urb->number_of_packets; i++) {
1987                         if (iso_packet[i].status == CVMX_USB_STATUS_OK) {
1988                                 urb->iso_frame_desc[i].status = 0;
1989                                 urb->iso_frame_desc[i].actual_length =
1990                                         iso_packet[i].length;
1991                                 urb->actual_length +=
1992                                         urb->iso_frame_desc[i].actual_length;
1993                         } else {
1994                                 dev_dbg(dev, "ISOCHRONOUS packet=%d of %d status=%d pipe=%p transaction=%p size=%d\n",
1995                                         i, urb->number_of_packets,
1996                                         iso_packet[i].status, pipe,
1997                                         transaction, iso_packet[i].length);
1998                                 urb->iso_frame_desc[i].status = -EREMOTEIO;
1999                         }
2000                 }
2001                 /* Free the private list now that we don't need it anymore */
2002                 kfree(iso_packet);
2003                 urb->setup_packet = NULL;
2004         }
2005
2006         switch (status) {
2007         case CVMX_USB_STATUS_OK:
2008                 urb->status = 0;
2009                 break;
2010         case CVMX_USB_STATUS_CANCEL:
2011                 if (urb->status == 0)
2012                         urb->status = -ENOENT;
2013                 break;
2014         case CVMX_USB_STATUS_STALL:
2015                 dev_dbg(dev, "status=stall pipe=%p transaction=%p size=%d\n",
2016                         pipe, transaction, bytes_transferred);
2017                 urb->status = -EPIPE;
2018                 break;
2019         case CVMX_USB_STATUS_BABBLEERR:
2020                 dev_dbg(dev, "status=babble pipe=%p transaction=%p size=%d\n",
2021                         pipe, transaction, bytes_transferred);
2022                 urb->status = -EPIPE;
2023                 break;
2024         case CVMX_USB_STATUS_SHORT:
2025                 dev_dbg(dev, "status=short pipe=%p transaction=%p size=%d\n",
2026                         pipe, transaction, bytes_transferred);
2027                 urb->status = -EREMOTEIO;
2028                 break;
2029         case CVMX_USB_STATUS_ERROR:
2030         case CVMX_USB_STATUS_XACTERR:
2031         case CVMX_USB_STATUS_DATATGLERR:
2032         case CVMX_USB_STATUS_FRAMEERR:
2033                 dev_dbg(dev, "status=%d pipe=%p transaction=%p size=%d\n",
2034                         status, pipe, transaction, bytes_transferred);
2035                 urb->status = -EPROTO;
2036                 break;
2037         }
2038         usb_hcd_unlink_urb_from_ep(octeon_to_hcd(usb), urb);
2039         spin_unlock(&usb->lock);
2040         usb_hcd_giveback_urb(octeon_to_hcd(usb), urb, urb->status);
2041         spin_lock(&usb->lock);
2042 }
2043
2044 /**
2045  * Signal the completion of a transaction and free it. The
2046  * transaction will be removed from the pipe transaction list.
2047  *
2048  * @usb:         USB device state populated by cvmx_usb_initialize().
2049  * @pipe:        Pipe the transaction is on
2050  * @transaction:
2051  *               Transaction that completed
2052  * @complete_code:
2053  *               Completion code
2054  */
2055 static void cvmx_usb_complete(struct octeon_hcd *usb,
2056                               struct cvmx_usb_pipe *pipe,
2057                               struct cvmx_usb_transaction *transaction,
2058                               enum cvmx_usb_status complete_code)
2059 {
2060         /* If this was a split then clear our split in progress marker */
2061         if (usb->active_split == transaction)
2062                 usb->active_split = NULL;
2063
2064         /*
2065          * Isochronous transactions need extra processing as they might not be
2066          * done after a single data transfer
2067          */
2068         if (unlikely(transaction->type == CVMX_USB_TRANSFER_ISOCHRONOUS)) {
2069                 /* Update the number of bytes transferred in this ISO packet */
2070                 transaction->iso_packets[0].length = transaction->actual_bytes;
2071                 transaction->iso_packets[0].status = complete_code;
2072
2073                 /*
2074                  * If there are more ISOs pending and we succeeded, schedule the
2075                  * next one
2076                  */
2077                 if ((transaction->iso_number_packets > 1) &&
2078                     (complete_code == CVMX_USB_STATUS_OK)) {
2079                         /* No bytes transferred for this packet as of yet */
2080                         transaction->actual_bytes = 0;
2081                         /* One less ISO waiting to transfer */
2082                         transaction->iso_number_packets--;
2083                         /* Increment to the next location in our packet array */
2084                         transaction->iso_packets++;
2085                         transaction->stage = CVMX_USB_STAGE_NON_CONTROL;
2086                         return;
2087                 }
2088         }
2089
2090         /* Remove the transaction from the pipe list */
2091         list_del(&transaction->node);
2092         if (list_empty(&pipe->transactions))
2093                 list_move_tail(&pipe->node, &usb->idle_pipes);
2094         octeon_usb_urb_complete_callback(usb, complete_code, pipe,
2095                                          transaction,
2096                                          transaction->actual_bytes,
2097                                          transaction->urb);
2098         kfree(transaction);
2099 }
2100
2101 /**
2102  * Submit a usb transaction to a pipe. Called for all types
2103  * of transactions.
2104  *
2105  * @usb:
2106  * @pipe:           Which pipe to submit to.
2107  * @type:           Transaction type
2108  * @buffer:         User buffer for the transaction
2109  * @buffer_length:
2110  *                  User buffer's length in bytes
2111  * @control_header:
2112  *                  For control transactions, the 8 byte standard header
2113  * @iso_start_frame:
2114  *                  For ISO transactions, the start frame
2115  * @iso_number_packets:
2116  *                  For ISO, the number of packet in the transaction.
2117  * @iso_packets:
2118  *                  A description of each ISO packet
2119  * @urb:            URB for the callback
2120  *
2121  * Returns: Transaction or NULL on failure.
2122  */
2123 static struct cvmx_usb_transaction *cvmx_usb_submit_transaction(
2124                                 struct octeon_hcd *usb,
2125                                 struct cvmx_usb_pipe *pipe,
2126                                 enum cvmx_usb_transfer type,
2127                                 u64 buffer,
2128                                 int buffer_length,
2129                                 u64 control_header,
2130                                 int iso_start_frame,
2131                                 int iso_number_packets,
2132                                 struct cvmx_usb_iso_packet *iso_packets,
2133                                 struct urb *urb)
2134 {
2135         struct cvmx_usb_transaction *transaction;
2136
2137         if (unlikely(pipe->transfer_type != type))
2138                 return NULL;
2139
2140         transaction = kzalloc(sizeof(*transaction), GFP_ATOMIC);
2141         if (unlikely(!transaction))
2142                 return NULL;
2143
2144         transaction->type = type;
2145         transaction->buffer = buffer;
2146         transaction->buffer_length = buffer_length;
2147         transaction->control_header = control_header;
2148         /* FIXME: This is not used, implement it. */
2149         transaction->iso_start_frame = iso_start_frame;
2150         transaction->iso_number_packets = iso_number_packets;
2151         transaction->iso_packets = iso_packets;
2152         transaction->urb = urb;
2153         if (transaction->type == CVMX_USB_TRANSFER_CONTROL)
2154                 transaction->stage = CVMX_USB_STAGE_SETUP;
2155         else
2156                 transaction->stage = CVMX_USB_STAGE_NON_CONTROL;
2157
2158         if (!list_empty(&pipe->transactions)) {
2159                 list_add_tail(&transaction->node, &pipe->transactions);
2160         } else {
2161                 list_add_tail(&transaction->node, &pipe->transactions);
2162                 list_move_tail(&pipe->node,
2163                                &usb->active_pipes[pipe->transfer_type]);
2164
2165                 /*
2166                  * We may need to schedule the pipe if this was the head of the
2167                  * pipe.
2168                  */
2169                 cvmx_usb_schedule(usb, 0);
2170         }
2171
2172         return transaction;
2173 }
2174
2175 /**
2176  * Call to submit a USB Bulk transfer to a pipe.
2177  *
2178  * @usb:            USB device state populated by cvmx_usb_initialize().
2179  * @pipe:           Handle to the pipe for the transfer.
2180  * @urb:            URB.
2181  *
2182  * Returns: A submitted transaction or NULL on failure.
2183  */
2184 static struct cvmx_usb_transaction *cvmx_usb_submit_bulk(
2185                                                 struct octeon_hcd *usb,
2186                                                 struct cvmx_usb_pipe *pipe,
2187                                                 struct urb *urb)
2188 {
2189         return cvmx_usb_submit_transaction(usb, pipe, CVMX_USB_TRANSFER_BULK,
2190                                            urb->transfer_dma,
2191                                            urb->transfer_buffer_length,
2192                                            0, /* control_header */
2193                                            0, /* iso_start_frame */
2194                                            0, /* iso_number_packets */
2195                                            NULL, /* iso_packets */
2196                                            urb);
2197 }
2198
2199 /**
2200  * Call to submit a USB Interrupt transfer to a pipe.
2201  *
2202  * @usb:            USB device state populated by cvmx_usb_initialize().
2203  * @pipe:           Handle to the pipe for the transfer.
2204  * @urb:            URB returned when the callback is called.
2205  *
2206  * Returns: A submitted transaction or NULL on failure.
2207  */
2208 static struct cvmx_usb_transaction *cvmx_usb_submit_interrupt(
2209                                                 struct octeon_hcd *usb,
2210                                                 struct cvmx_usb_pipe *pipe,
2211                                                 struct urb *urb)
2212 {
2213         return cvmx_usb_submit_transaction(usb, pipe,
2214                                            CVMX_USB_TRANSFER_INTERRUPT,
2215                                            urb->transfer_dma,
2216                                            urb->transfer_buffer_length,
2217                                            0, /* control_header */
2218                                            0, /* iso_start_frame */
2219                                            0, /* iso_number_packets */
2220                                            NULL, /* iso_packets */
2221                                            urb);
2222 }
2223
2224 /**
2225  * Call to submit a USB Control transfer to a pipe.
2226  *
2227  * @usb:            USB device state populated by cvmx_usb_initialize().
2228  * @pipe:           Handle to the pipe for the transfer.
2229  * @urb:            URB.
2230  *
2231  * Returns: A submitted transaction or NULL on failure.
2232  */
2233 static struct cvmx_usb_transaction *cvmx_usb_submit_control(
2234                                                 struct octeon_hcd *usb,
2235                                                 struct cvmx_usb_pipe *pipe,
2236                                                 struct urb *urb)
2237 {
2238         int buffer_length = urb->transfer_buffer_length;
2239         u64 control_header = urb->setup_dma;
2240         struct usb_ctrlrequest *header = cvmx_phys_to_ptr(control_header);
2241
2242         if ((header->bRequestType & USB_DIR_IN) == 0)
2243                 buffer_length = le16_to_cpu(header->wLength);
2244
2245         return cvmx_usb_submit_transaction(usb, pipe,
2246                                            CVMX_USB_TRANSFER_CONTROL,
2247                                            urb->transfer_dma, buffer_length,
2248                                            control_header,
2249                                            0, /* iso_start_frame */
2250                                            0, /* iso_number_packets */
2251                                            NULL, /* iso_packets */
2252                                            urb);
2253 }
2254
2255 /**
2256  * Call to submit a USB Isochronous transfer to a pipe.
2257  *
2258  * @usb:            USB device state populated by cvmx_usb_initialize().
2259  * @pipe:           Handle to the pipe for the transfer.
2260  * @urb:            URB returned when the callback is called.
2261  *
2262  * Returns: A submitted transaction or NULL on failure.
2263  */
2264 static struct cvmx_usb_transaction *cvmx_usb_submit_isochronous(
2265                                                 struct octeon_hcd *usb,
2266                                                 struct cvmx_usb_pipe *pipe,
2267                                                 struct urb *urb)
2268 {
2269         struct cvmx_usb_iso_packet *packets;
2270
2271         packets = (struct cvmx_usb_iso_packet *)urb->setup_packet;
2272         return cvmx_usb_submit_transaction(usb, pipe,
2273                                            CVMX_USB_TRANSFER_ISOCHRONOUS,
2274                                            urb->transfer_dma,
2275                                            urb->transfer_buffer_length,
2276                                            0, /* control_header */
2277                                            urb->start_frame,
2278                                            urb->number_of_packets,
2279                                            packets, urb);
2280 }
2281
2282 /**
2283  * Cancel one outstanding request in a pipe. Canceling a request
2284  * can fail if the transaction has already completed before cancel
2285  * is called. Even after a successful cancel call, it may take
2286  * a frame or two for the cvmx_usb_poll() function to call the
2287  * associated callback.
2288  *
2289  * @usb:         USB device state populated by cvmx_usb_initialize().
2290  * @pipe:        Pipe to cancel requests in.
2291  * @transaction: Transaction to cancel, returned by the submit function.
2292  *
2293  * Returns: 0 or a negative error code.
2294  */
2295 static int cvmx_usb_cancel(struct octeon_hcd *usb,
2296                            struct cvmx_usb_pipe *pipe,
2297                            struct cvmx_usb_transaction *transaction)
2298 {
2299         /*
2300          * If the transaction is the HEAD of the queue and scheduled. We need to
2301          * treat it special
2302          */
2303         if (list_first_entry(&pipe->transactions, typeof(*transaction), node) ==
2304             transaction && (pipe->flags & CVMX_USB_PIPE_FLAGS_SCHEDULED)) {
2305                 union cvmx_usbcx_hccharx usbc_hcchar;
2306
2307                 usb->pipe_for_channel[pipe->channel] = NULL;
2308                 pipe->flags &= ~CVMX_USB_PIPE_FLAGS_SCHEDULED;
2309
2310                 CVMX_SYNCW;
2311
2312                 usbc_hcchar.u32 = cvmx_usb_read_csr32(usb,
2313                                 CVMX_USBCX_HCCHARX(pipe->channel, usb->index));
2314                 /*
2315                  * If the channel isn't enabled then the transaction already
2316                  * completed.
2317                  */
2318                 if (usbc_hcchar.s.chena) {
2319                         usbc_hcchar.s.chdis = 1;
2320                         cvmx_usb_write_csr32(usb,
2321                                              CVMX_USBCX_HCCHARX(pipe->channel,
2322                                                                 usb->index),
2323                                              usbc_hcchar.u32);
2324                 }
2325         }
2326         cvmx_usb_complete(usb, pipe, transaction, CVMX_USB_STATUS_CANCEL);
2327         return 0;
2328 }
2329
2330 /**
2331  * Cancel all outstanding requests in a pipe. Logically all this
2332  * does is call cvmx_usb_cancel() in a loop.
2333  *
2334  * @usb:         USB device state populated by cvmx_usb_initialize().
2335  * @pipe:        Pipe to cancel requests in.
2336  *
2337  * Returns: 0 or a negative error code.
2338  */
2339 static int cvmx_usb_cancel_all(struct octeon_hcd *usb,
2340                                struct cvmx_usb_pipe *pipe)
2341 {
2342         struct cvmx_usb_transaction *transaction, *next;
2343
2344         /* Simply loop through and attempt to cancel each transaction */
2345         list_for_each_entry_safe(transaction, next, &pipe->transactions, node) {
2346                 int result = cvmx_usb_cancel(usb, pipe, transaction);
2347
2348                 if (unlikely(result != 0))
2349                         return result;
2350         }
2351         return 0;
2352 }
2353
2354 /**
2355  * Close a pipe created with cvmx_usb_open_pipe().
2356  *
2357  * @usb:         USB device state populated by cvmx_usb_initialize().
2358  * @pipe:        Pipe to close.
2359  *
2360  * Returns: 0 or a negative error code. EBUSY is returned if the pipe has
2361  *          outstanding transfers.
2362  */
2363 static int cvmx_usb_close_pipe(struct octeon_hcd *usb,
2364                                struct cvmx_usb_pipe *pipe)
2365 {
2366         /* Fail if the pipe has pending transactions */
2367         if (!list_empty(&pipe->transactions))
2368                 return -EBUSY;
2369
2370         list_del(&pipe->node);
2371         kfree(pipe);
2372
2373         return 0;
2374 }
2375
2376 /**
2377  * Get the current USB protocol level frame number. The frame
2378  * number is always in the range of 0-0x7ff.
2379  *
2380  * @usb: USB device state populated by cvmx_usb_initialize().
2381  *
2382  * Returns: USB frame number
2383  */
2384 static int cvmx_usb_get_frame_number(struct octeon_hcd *usb)
2385 {
2386         union cvmx_usbcx_hfnum usbc_hfnum;
2387
2388         usbc_hfnum.u32 = cvmx_usb_read_csr32(usb, CVMX_USBCX_HFNUM(usb->index));
2389
2390         return usbc_hfnum.s.frnum;
2391 }
2392
2393 static void cvmx_usb_transfer_control(struct octeon_hcd *usb,
2394                                       struct cvmx_usb_pipe *pipe,
2395                                       struct cvmx_usb_transaction *transaction,
2396                                       union cvmx_usbcx_hccharx usbc_hcchar,
2397                                       int buffer_space_left,
2398                                       int bytes_in_last_packet)
2399 {
2400         switch (transaction->stage) {
2401         case CVMX_USB_STAGE_NON_CONTROL:
2402         case CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE:
2403                 /* This should be impossible */
2404                 cvmx_usb_complete(usb, pipe, transaction,
2405                                   CVMX_USB_STATUS_ERROR);
2406                 break;
2407         case CVMX_USB_STAGE_SETUP:
2408                 pipe->pid_toggle = 1;
2409                 if (cvmx_usb_pipe_needs_split(usb, pipe)) {
2410                         transaction->stage =
2411                                 CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE;
2412                 } else {
2413                         struct usb_ctrlrequest *header =
2414                                 cvmx_phys_to_ptr(transaction->control_header);
2415                         if (header->wLength)
2416                                 transaction->stage = CVMX_USB_STAGE_DATA;
2417                         else
2418                                 transaction->stage = CVMX_USB_STAGE_STATUS;
2419                 }
2420                 break;
2421         case CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE:
2422                 {
2423                         struct usb_ctrlrequest *header =
2424                                 cvmx_phys_to_ptr(transaction->control_header);
2425                         if (header->wLength)
2426                                 transaction->stage = CVMX_USB_STAGE_DATA;
2427                         else
2428                                 transaction->stage = CVMX_USB_STAGE_STATUS;
2429                 }
2430                 break;
2431         case CVMX_USB_STAGE_DATA:
2432                 if (cvmx_usb_pipe_needs_split(usb, pipe)) {
2433                         transaction->stage = CVMX_USB_STAGE_DATA_SPLIT_COMPLETE;
2434                         /*
2435                          * For setup OUT data that are splits,
2436                          * the hardware doesn't appear to count
2437                          * transferred data. Here we manually
2438                          * update the data transferred
2439                          */
2440                         if (!usbc_hcchar.s.epdir) {
2441                                 if (buffer_space_left < pipe->max_packet)
2442                                         transaction->actual_bytes +=
2443                                                 buffer_space_left;
2444                                 else
2445                                         transaction->actual_bytes +=
2446                                                 pipe->max_packet;
2447                         }
2448                 } else if ((buffer_space_left == 0) ||
2449                            (bytes_in_last_packet < pipe->max_packet)) {
2450                         pipe->pid_toggle = 1;
2451                         transaction->stage = CVMX_USB_STAGE_STATUS;
2452                 }
2453                 break;
2454         case CVMX_USB_STAGE_DATA_SPLIT_COMPLETE:
2455                 if ((buffer_space_left == 0) ||
2456                     (bytes_in_last_packet < pipe->max_packet)) {
2457                         pipe->pid_toggle = 1;
2458                         transaction->stage = CVMX_USB_STAGE_STATUS;
2459                 } else {
2460                         transaction->stage = CVMX_USB_STAGE_DATA;
2461                 }
2462                 break;
2463         case CVMX_USB_STAGE_STATUS:
2464                 if (cvmx_usb_pipe_needs_split(usb, pipe))
2465                         transaction->stage =
2466                                 CVMX_USB_STAGE_STATUS_SPLIT_COMPLETE;
2467                 else
2468                         cvmx_usb_complete(usb, pipe, transaction,
2469                                           CVMX_USB_STATUS_OK);
2470                 break;
2471         case CVMX_USB_STAGE_STATUS_SPLIT_COMPLETE:
2472                 cvmx_usb_complete(usb, pipe, transaction, CVMX_USB_STATUS_OK);
2473                 break;
2474         }
2475 }
2476
2477 static void cvmx_usb_transfer_bulk(struct octeon_hcd *usb,
2478                                    struct cvmx_usb_pipe *pipe,
2479                                    struct cvmx_usb_transaction *transaction,
2480                                    union cvmx_usbcx_hcintx usbc_hcint,
2481                                    int buffer_space_left,
2482                                    int bytes_in_last_packet)
2483 {
2484         /*
2485          * The only time a bulk transfer isn't complete when it finishes with
2486          * an ACK is during a split transaction. For splits we need to continue
2487          * the transfer if more data is needed.
2488          */
2489         if (cvmx_usb_pipe_needs_split(usb, pipe)) {
2490                 if (transaction->stage == CVMX_USB_STAGE_NON_CONTROL)
2491                         transaction->stage =
2492                                 CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE;
2493                 else if (buffer_space_left &&
2494                          (bytes_in_last_packet == pipe->max_packet))
2495                         transaction->stage = CVMX_USB_STAGE_NON_CONTROL;
2496                 else
2497                         cvmx_usb_complete(usb, pipe, transaction,
2498                                           CVMX_USB_STATUS_OK);
2499         } else {
2500                 if ((pipe->device_speed == CVMX_USB_SPEED_HIGH) &&
2501                     (pipe->transfer_dir == CVMX_USB_DIRECTION_OUT) &&
2502                     (usbc_hcint.s.nak))
2503                         pipe->flags |= CVMX_USB_PIPE_FLAGS_NEED_PING;
2504                 if (!buffer_space_left ||
2505                     (bytes_in_last_packet < pipe->max_packet))
2506                         cvmx_usb_complete(usb, pipe, transaction,
2507                                           CVMX_USB_STATUS_OK);
2508         }
2509 }
2510
2511 static void cvmx_usb_transfer_intr(struct octeon_hcd *usb,
2512                                    struct cvmx_usb_pipe *pipe,
2513                                    struct cvmx_usb_transaction *transaction,
2514                                    int buffer_space_left,
2515                                    int bytes_in_last_packet)
2516 {
2517         if (cvmx_usb_pipe_needs_split(usb, pipe)) {
2518                 if (transaction->stage == CVMX_USB_STAGE_NON_CONTROL) {
2519                         transaction->stage =
2520                                 CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE;
2521                 } else if (buffer_space_left &&
2522                            (bytes_in_last_packet == pipe->max_packet)) {
2523                         transaction->stage = CVMX_USB_STAGE_NON_CONTROL;
2524                 } else {
2525                         pipe->next_tx_frame += pipe->interval;
2526                         cvmx_usb_complete(usb, pipe, transaction,
2527                                           CVMX_USB_STATUS_OK);
2528                 }
2529         } else if (!buffer_space_left ||
2530                    (bytes_in_last_packet < pipe->max_packet)) {
2531                 pipe->next_tx_frame += pipe->interval;
2532                 cvmx_usb_complete(usb, pipe, transaction, CVMX_USB_STATUS_OK);
2533         }
2534 }
2535
2536 static void cvmx_usb_transfer_isoc(struct octeon_hcd *usb,
2537                                    struct cvmx_usb_pipe *pipe,
2538                                    struct cvmx_usb_transaction *transaction,
2539                                    int buffer_space_left,
2540                                    int bytes_in_last_packet,
2541                                    int bytes_this_transfer)
2542 {
2543         if (cvmx_usb_pipe_needs_split(usb, pipe)) {
2544                 /*
2545                  * ISOCHRONOUS OUT splits don't require a complete split stage.
2546                  * Instead they use a sequence of begin OUT splits to transfer
2547                  * the data 188 bytes at a time. Once the transfer is complete,
2548                  * the pipe sleeps until the next schedule interval.
2549                  */
2550                 if (pipe->transfer_dir == CVMX_USB_DIRECTION_OUT) {
2551                         /*
2552                          * If no space left or this wasn't a max size packet
2553                          * then this transfer is complete. Otherwise start it
2554                          * again to send the next 188 bytes
2555                          */
2556                         if (!buffer_space_left || (bytes_this_transfer < 188)) {
2557                                 pipe->next_tx_frame += pipe->interval;
2558                                 cvmx_usb_complete(usb, pipe, transaction,
2559                                                   CVMX_USB_STATUS_OK);
2560                         }
2561                         return;
2562                 }
2563                 if (transaction->stage ==
2564                     CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE) {
2565                         /*
2566                          * We are in the incoming data phase. Keep getting data
2567                          * until we run out of space or get a small packet
2568                          */
2569                         if ((buffer_space_left == 0) ||
2570                             (bytes_in_last_packet < pipe->max_packet)) {
2571                                 pipe->next_tx_frame += pipe->interval;
2572                                 cvmx_usb_complete(usb, pipe, transaction,
2573                                                   CVMX_USB_STATUS_OK);
2574                         }
2575                 } else {
2576                         transaction->stage =
2577                                 CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE;
2578                 }
2579         } else {
2580                 pipe->next_tx_frame += pipe->interval;
2581                 cvmx_usb_complete(usb, pipe, transaction, CVMX_USB_STATUS_OK);
2582         }
2583 }
2584
2585 /**
2586  * Poll a channel for status
2587  *
2588  * @usb:     USB device
2589  * @channel: Channel to poll
2590  *
2591  * Returns: Zero on success
2592  */
2593 static int cvmx_usb_poll_channel(struct octeon_hcd *usb, int channel)
2594 {
2595         struct usb_hcd *hcd = octeon_to_hcd(usb);
2596         struct device *dev = hcd->self.controller;
2597         union cvmx_usbcx_hcintx usbc_hcint;
2598         union cvmx_usbcx_hctsizx usbc_hctsiz;
2599         union cvmx_usbcx_hccharx usbc_hcchar;
2600         struct cvmx_usb_pipe *pipe;
2601         struct cvmx_usb_transaction *transaction;
2602         int bytes_this_transfer;
2603         int bytes_in_last_packet;
2604         int packets_processed;
2605         int buffer_space_left;
2606
2607         /* Read the interrupt status bits for the channel */
2608         usbc_hcint.u32 = cvmx_usb_read_csr32(usb,
2609                                 CVMX_USBCX_HCINTX(channel, usb->index));
2610
2611         if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA) {
2612                 usbc_hcchar.u32 = cvmx_usb_read_csr32(usb,
2613                                 CVMX_USBCX_HCCHARX(channel, usb->index));
2614
2615                 if (usbc_hcchar.s.chena && usbc_hcchar.s.chdis) {
2616                         /*
2617                          * There seems to be a bug in CN31XX which can cause
2618                          * interrupt IN transfers to get stuck until we do a
2619                          * write of HCCHARX without changing things
2620                          */
2621                         cvmx_usb_write_csr32(usb,
2622                                              CVMX_USBCX_HCCHARX(channel,
2623                                                                 usb->index),
2624                                              usbc_hcchar.u32);
2625                         return 0;
2626                 }
2627
2628                 /*
2629                  * In non DMA mode the channels don't halt themselves. We need
2630                  * to manually disable channels that are left running
2631                  */
2632                 if (!usbc_hcint.s.chhltd) {
2633                         if (usbc_hcchar.s.chena) {
2634                                 union cvmx_usbcx_hcintmskx hcintmsk;
2635                                 /* Disable all interrupts except CHHLTD */
2636                                 hcintmsk.u32 = 0;
2637                                 hcintmsk.s.chhltdmsk = 1;
2638                                 cvmx_usb_write_csr32(usb,
2639                                                      CVMX_USBCX_HCINTMSKX(channel, usb->index),
2640                                                      hcintmsk.u32);
2641                                 usbc_hcchar.s.chdis = 1;
2642                                 cvmx_usb_write_csr32(usb,
2643                                                      CVMX_USBCX_HCCHARX(channel, usb->index),
2644                                                      usbc_hcchar.u32);
2645                                 return 0;
2646                         } else if (usbc_hcint.s.xfercompl) {
2647                                 /*
2648                                  * Successful IN/OUT with transfer complete.
2649                                  * Channel halt isn't needed.
2650                                  */
2651                         } else {
2652                                 dev_err(dev, "USB%d: Channel %d interrupt without halt\n",
2653                                         usb->index, channel);
2654                                 return 0;
2655                         }
2656                 }
2657         } else {
2658                 /*
2659                  * There is are no interrupts that we need to process when the
2660                  * channel is still running
2661                  */
2662                 if (!usbc_hcint.s.chhltd)
2663                         return 0;
2664         }
2665
2666         /* Disable the channel interrupts now that it is done */
2667         cvmx_usb_write_csr32(usb, CVMX_USBCX_HCINTMSKX(channel, usb->index), 0);
2668         usb->idle_hardware_channels |= (1 << channel);
2669
2670         /* Make sure this channel is tied to a valid pipe */
2671         pipe = usb->pipe_for_channel[channel];
2672         prefetch(pipe);
2673         if (!pipe)
2674                 return 0;
2675         transaction = list_first_entry(&pipe->transactions,
2676                                        typeof(*transaction),
2677                                        node);
2678         prefetch(transaction);
2679
2680         /*
2681          * Disconnect this pipe from the HW channel. Later the schedule
2682          * function will figure out which pipe needs to go
2683          */
2684         usb->pipe_for_channel[channel] = NULL;
2685         pipe->flags &= ~CVMX_USB_PIPE_FLAGS_SCHEDULED;
2686
2687         /*
2688          * Read the channel config info so we can figure out how much data
2689          * transferred
2690          */
2691         usbc_hcchar.u32 = cvmx_usb_read_csr32(usb,
2692                         CVMX_USBCX_HCCHARX(channel, usb->index));
2693         usbc_hctsiz.u32 = cvmx_usb_read_csr32(usb,
2694                         CVMX_USBCX_HCTSIZX(channel, usb->index));
2695
2696         /*
2697          * Calculating the number of bytes successfully transferred is dependent
2698          * on the transfer direction
2699          */
2700         packets_processed = transaction->pktcnt - usbc_hctsiz.s.pktcnt;
2701         if (usbc_hcchar.s.epdir) {
2702                 /*
2703                  * IN transactions are easy. For every byte received the
2704                  * hardware decrements xfersize. All we need to do is subtract
2705                  * the current value of xfersize from its starting value and we
2706                  * know how many bytes were written to the buffer
2707                  */
2708                 bytes_this_transfer = transaction->xfersize -
2709                         usbc_hctsiz.s.xfersize;
2710         } else {
2711                 /*
2712                  * OUT transaction don't decrement xfersize. Instead pktcnt is
2713                  * decremented on every successful packet send. The hardware
2714                  * does this when it receives an ACK, or NYET. If it doesn't
2715                  * receive one of these responses pktcnt doesn't change
2716                  */
2717                 bytes_this_transfer = packets_processed * usbc_hcchar.s.mps;
2718                 /*
2719                  * The last packet may not be a full transfer if we didn't have
2720                  * enough data
2721                  */
2722                 if (bytes_this_transfer > transaction->xfersize)
2723                         bytes_this_transfer = transaction->xfersize;
2724         }
2725         /* Figure out how many bytes were in the last packet of the transfer */
2726         if (packets_processed)
2727                 bytes_in_last_packet = bytes_this_transfer -
2728                         (packets_processed - 1) * usbc_hcchar.s.mps;
2729         else
2730                 bytes_in_last_packet = bytes_this_transfer;
2731
2732         /*
2733          * As a special case, setup transactions output the setup header, not
2734          * the user's data. For this reason we don't count setup data as bytes
2735          * transferred
2736          */
2737         if ((transaction->stage == CVMX_USB_STAGE_SETUP) ||
2738             (transaction->stage == CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE))
2739                 bytes_this_transfer = 0;
2740
2741         /*
2742          * Add the bytes transferred to the running total. It is important that
2743          * bytes_this_transfer doesn't count any data that needs to be
2744          * retransmitted
2745          */
2746         transaction->actual_bytes += bytes_this_transfer;
2747         if (transaction->type == CVMX_USB_TRANSFER_ISOCHRONOUS)
2748                 buffer_space_left = transaction->iso_packets[0].length -
2749                         transaction->actual_bytes;
2750         else
2751                 buffer_space_left = transaction->buffer_length -
2752                         transaction->actual_bytes;
2753
2754         /*
2755          * We need to remember the PID toggle state for the next transaction.
2756          * The hardware already updated it for the next transaction
2757          */
2758         pipe->pid_toggle = !(usbc_hctsiz.s.pid == 0);
2759
2760         /*
2761          * For high speed bulk out, assume the next transaction will need to do
2762          * a ping before proceeding. If this isn't true the ACK processing below
2763          * will clear this flag
2764          */
2765         if ((pipe->device_speed == CVMX_USB_SPEED_HIGH) &&
2766             (pipe->transfer_type == CVMX_USB_TRANSFER_BULK) &&
2767             (pipe->transfer_dir == CVMX_USB_DIRECTION_OUT))
2768                 pipe->flags |= CVMX_USB_PIPE_FLAGS_NEED_PING;
2769
2770         if (WARN_ON_ONCE(bytes_this_transfer < 0)) {
2771                 /*
2772                  * In some rare cases the DMA engine seems to get stuck and
2773                  * keeps substracting same byte count over and over again. In
2774                  * such case we just need to fail every transaction.
2775                  */
2776                 cvmx_usb_complete(usb, pipe, transaction,
2777                                   CVMX_USB_STATUS_ERROR);
2778                 return 0;
2779         }
2780
2781         if (usbc_hcint.s.stall) {
2782                 /*
2783                  * STALL as a response means this transaction cannot be
2784                  * completed because the device can't process transactions. Tell
2785                  * the user. Any data that was transferred will be counted on
2786                  * the actual bytes transferred
2787                  */
2788                 pipe->pid_toggle = 0;
2789                 cvmx_usb_complete(usb, pipe, transaction,
2790                                   CVMX_USB_STATUS_STALL);
2791         } else if (usbc_hcint.s.xacterr) {
2792                 /*
2793                  * XactErr as a response means the device signaled
2794                  * something wrong with the transfer. For example, PID
2795                  * toggle errors cause these.
2796                  */
2797                 cvmx_usb_complete(usb, pipe, transaction,
2798                                   CVMX_USB_STATUS_XACTERR);
2799         } else if (usbc_hcint.s.bblerr) {
2800                 /* Babble Error (BblErr) */
2801                 cvmx_usb_complete(usb, pipe, transaction,
2802                                   CVMX_USB_STATUS_BABBLEERR);
2803         } else if (usbc_hcint.s.datatglerr) {
2804                 /* Data toggle error */
2805                 cvmx_usb_complete(usb, pipe, transaction,
2806                                   CVMX_USB_STATUS_DATATGLERR);
2807         } else if (usbc_hcint.s.nyet) {
2808                 /*
2809                  * NYET as a response is only allowed in three cases: as a
2810                  * response to a ping, as a response to a split transaction, and
2811                  * as a response to a bulk out. The ping case is handled by
2812                  * hardware, so we only have splits and bulk out
2813                  */
2814                 if (!cvmx_usb_pipe_needs_split(usb, pipe)) {
2815                         transaction->retries = 0;
2816                         /*
2817                          * If there is more data to go then we need to try
2818                          * again. Otherwise this transaction is complete
2819                          */
2820                         if ((buffer_space_left == 0) ||
2821                             (bytes_in_last_packet < pipe->max_packet))
2822                                 cvmx_usb_complete(usb, pipe,
2823                                                   transaction,
2824                                                   CVMX_USB_STATUS_OK);
2825                 } else {
2826                         /*
2827                          * Split transactions retry the split complete 4 times
2828                          * then rewind to the start split and do the entire
2829                          * transactions again
2830                          */
2831                         transaction->retries++;
2832                         if ((transaction->retries & 0x3) == 0) {
2833                                 /*
2834                                  * Rewind to the beginning of the transaction by
2835                                  * anding off the split complete bit
2836                                  */
2837                                 transaction->stage &= ~1;
2838                                 pipe->split_sc_frame = -1;
2839                         }
2840                 }
2841         } else if (usbc_hcint.s.ack) {
2842                 transaction->retries = 0;
2843                 /*
2844                  * The ACK bit can only be checked after the other error bits.
2845                  * This is because a multi packet transfer may succeed in a
2846                  * number of packets and then get a different response on the
2847                  * last packet. In this case both ACK and the last response bit
2848                  * will be set. If none of the other response bits is set, then
2849                  * the last packet must have been an ACK
2850                  *
2851                  * Since we got an ACK, we know we don't need to do a ping on
2852                  * this pipe
2853                  */
2854                 pipe->flags &= ~CVMX_USB_PIPE_FLAGS_NEED_PING;
2855
2856                 switch (transaction->type) {
2857                 case CVMX_USB_TRANSFER_CONTROL:
2858                         cvmx_usb_transfer_control(usb, pipe, transaction,
2859                                                   usbc_hcchar,
2860                                                   buffer_space_left,
2861                                                   bytes_in_last_packet);
2862                         break;
2863                 case CVMX_USB_TRANSFER_BULK:
2864                         cvmx_usb_transfer_bulk(usb, pipe, transaction,
2865                                                usbc_hcint, buffer_space_left,
2866                                                bytes_in_last_packet);
2867                         break;
2868                 case CVMX_USB_TRANSFER_INTERRUPT:
2869                         cvmx_usb_transfer_intr(usb, pipe, transaction,
2870                                                buffer_space_left,
2871                                                bytes_in_last_packet);
2872                         break;
2873                 case CVMX_USB_TRANSFER_ISOCHRONOUS:
2874                         cvmx_usb_transfer_isoc(usb, pipe, transaction,
2875                                                buffer_space_left,
2876                                                bytes_in_last_packet,
2877                                                bytes_this_transfer);
2878                         break;
2879                 }
2880         } else if (usbc_hcint.s.nak) {
2881                 /*
2882                  * If this was a split then clear our split in progress marker.
2883                  */
2884                 if (usb->active_split == transaction)
2885                         usb->active_split = NULL;
2886                 /*
2887                  * NAK as a response means the device couldn't accept the
2888                  * transaction, but it should be retried in the future. Rewind
2889                  * to the beginning of the transaction by anding off the split
2890                  * complete bit. Retry in the next interval
2891                  */
2892                 transaction->retries = 0;
2893                 transaction->stage &= ~1;
2894                 pipe->next_tx_frame += pipe->interval;
2895                 if (pipe->next_tx_frame < usb->frame_number)
2896                         pipe->next_tx_frame = usb->frame_number +
2897                                 pipe->interval -
2898                                 (usb->frame_number - pipe->next_tx_frame) %
2899                                 pipe->interval;
2900         } else {
2901                 struct cvmx_usb_port_status port;
2902
2903                 port = cvmx_usb_get_status(usb);
2904                 if (port.port_enabled) {
2905                         /* We'll retry the exact same transaction again */
2906                         transaction->retries++;
2907                 } else {
2908                         /*
2909                          * We get channel halted interrupts with no result bits
2910                          * sets when the cable is unplugged
2911                          */
2912                         cvmx_usb_complete(usb, pipe, transaction,
2913                                           CVMX_USB_STATUS_ERROR);
2914                 }
2915         }
2916         return 0;
2917 }
2918
2919 static void octeon_usb_port_callback(struct octeon_hcd *usb)
2920 {
2921         spin_unlock(&usb->lock);
2922         usb_hcd_poll_rh_status(octeon_to_hcd(usb));
2923         spin_lock(&usb->lock);
2924 }
2925
2926 /**
2927  * Poll the USB block for status and call all needed callback
2928  * handlers. This function is meant to be called in the interrupt
2929  * handler for the USB controller. It can also be called
2930  * periodically in a loop for non-interrupt based operation.
2931  *
2932  * @usb: USB device state populated by cvmx_usb_initialize().
2933  *
2934  * Returns: 0 or a negative error code.
2935  */
2936 static int cvmx_usb_poll(struct octeon_hcd *usb)
2937 {
2938         union cvmx_usbcx_hfnum usbc_hfnum;
2939         union cvmx_usbcx_gintsts usbc_gintsts;
2940
2941         prefetch_range(usb, sizeof(*usb));
2942
2943         /* Update the frame counter */
2944         usbc_hfnum.u32 = cvmx_usb_read_csr32(usb, CVMX_USBCX_HFNUM(usb->index));
2945         if ((usb->frame_number & 0x3fff) > usbc_hfnum.s.frnum)
2946                 usb->frame_number += 0x4000;
2947         usb->frame_number &= ~0x3fffull;
2948         usb->frame_number |= usbc_hfnum.s.frnum;
2949
2950         /* Read the pending interrupts */
2951         usbc_gintsts.u32 = cvmx_usb_read_csr32(usb,
2952                                                CVMX_USBCX_GINTSTS(usb->index));
2953
2954         /* Clear the interrupts now that we know about them */
2955         cvmx_usb_write_csr32(usb, CVMX_USBCX_GINTSTS(usb->index),
2956                              usbc_gintsts.u32);
2957
2958         if (usbc_gintsts.s.rxflvl) {
2959                 /*
2960                  * RxFIFO Non-Empty (RxFLvl)
2961                  * Indicates that there is at least one packet pending to be
2962                  * read from the RxFIFO.
2963                  *
2964                  * In DMA mode this is handled by hardware
2965                  */
2966                 if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
2967                         cvmx_usb_poll_rx_fifo(usb);
2968         }
2969         if (usbc_gintsts.s.ptxfemp || usbc_gintsts.s.nptxfemp) {
2970                 /* Fill the Tx FIFOs when not in DMA mode */
2971                 if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
2972                         cvmx_usb_poll_tx_fifo(usb);
2973         }
2974         if (usbc_gintsts.s.disconnint || usbc_gintsts.s.prtint) {
2975                 union cvmx_usbcx_hprt usbc_hprt;
2976                 /*
2977                  * Disconnect Detected Interrupt (DisconnInt)
2978                  * Asserted when a device disconnect is detected.
2979                  *
2980                  * Host Port Interrupt (PrtInt)
2981                  * The core sets this bit to indicate a change in port status of
2982                  * one of the O2P USB core ports in Host mode. The application
2983                  * must read the Host Port Control and Status (HPRT) register to
2984                  * determine the exact event that caused this interrupt. The
2985                  * application must clear the appropriate status bit in the Host
2986                  * Port Control and Status register to clear this bit.
2987                  *
2988                  * Call the user's port callback
2989                  */
2990                 octeon_usb_port_callback(usb);
2991                 /* Clear the port change bits */
2992                 usbc_hprt.u32 =
2993                         cvmx_usb_read_csr32(usb, CVMX_USBCX_HPRT(usb->index));
2994                 usbc_hprt.s.prtena = 0;
2995                 cvmx_usb_write_csr32(usb, CVMX_USBCX_HPRT(usb->index),
2996                                      usbc_hprt.u32);
2997         }
2998         if (usbc_gintsts.s.hchint) {
2999                 /*
3000                  * Host Channels Interrupt (HChInt)
3001                  * The core sets this bit to indicate that an interrupt is
3002                  * pending on one of the channels of the core (in Host mode).
3003                  * The application must read the Host All Channels Interrupt
3004                  * (HAINT) register to determine the exact number of the channel
3005                  * on which the interrupt occurred, and then read the
3006                  * corresponding Host Channel-n Interrupt (HCINTn) register to
3007                  * determine the exact cause of the interrupt. The application
3008                  * must clear the appropriate status bit in the HCINTn register
3009                  * to clear this bit.
3010                  */
3011                 union cvmx_usbcx_haint usbc_haint;
3012
3013                 usbc_haint.u32 = cvmx_usb_read_csr32(usb,
3014                                         CVMX_USBCX_HAINT(usb->index));
3015                 while (usbc_haint.u32) {
3016                         int channel;
3017
3018                         channel = __fls(usbc_haint.u32);
3019                         cvmx_usb_poll_channel(usb, channel);
3020                         usbc_haint.u32 ^= 1 << channel;
3021                 }
3022         }
3023
3024         cvmx_usb_schedule(usb, usbc_gintsts.s.sof);
3025
3026         return 0;
3027 }
3028
3029 /* convert between an HCD pointer and the corresponding struct octeon_hcd */
3030 static inline struct octeon_hcd *hcd_to_octeon(struct usb_hcd *hcd)
3031 {
3032         return (struct octeon_hcd *)(hcd->hcd_priv);
3033 }
3034
3035 static irqreturn_t octeon_usb_irq(struct usb_hcd *hcd)
3036 {
3037         struct octeon_hcd *usb = hcd_to_octeon(hcd);
3038         unsigned long flags;
3039
3040         spin_lock_irqsave(&usb->lock, flags);
3041         cvmx_usb_poll(usb);
3042         spin_unlock_irqrestore(&usb->lock, flags);
3043         return IRQ_HANDLED;
3044 }
3045
3046 static int octeon_usb_start(struct usb_hcd *hcd)
3047 {
3048         hcd->state = HC_STATE_RUNNING;
3049         return 0;
3050 }
3051
3052 static void octeon_usb_stop(struct usb_hcd *hcd)
3053 {
3054         hcd->state = HC_STATE_HALT;
3055 }
3056
3057 static int octeon_usb_get_frame_number(struct usb_hcd *hcd)
3058 {
3059         struct octeon_hcd *usb = hcd_to_octeon(hcd);
3060
3061         return cvmx_usb_get_frame_number(usb);
3062 }
3063
3064 static int octeon_usb_urb_enqueue(struct usb_hcd *hcd,
3065                                   struct urb *urb,
3066                                   gfp_t mem_flags)
3067 {
3068         struct octeon_hcd *usb = hcd_to_octeon(hcd);
3069         struct device *dev = hcd->self.controller;
3070         struct cvmx_usb_transaction *transaction = NULL;
3071         struct cvmx_usb_pipe *pipe;
3072         unsigned long flags;
3073         struct cvmx_usb_iso_packet *iso_packet;
3074         struct usb_host_endpoint *ep = urb->ep;
3075         int rc;
3076
3077         urb->status = 0;
3078         spin_lock_irqsave(&usb->lock, flags);
3079
3080         rc = usb_hcd_link_urb_to_ep(hcd, urb);
3081         if (rc) {
3082                 spin_unlock_irqrestore(&usb->lock, flags);
3083                 return rc;
3084         }
3085
3086         if (!ep->hcpriv) {
3087                 enum cvmx_usb_transfer transfer_type;
3088                 enum cvmx_usb_speed speed;
3089                 int split_device = 0;
3090                 int split_port = 0;
3091
3092                 switch (usb_pipetype(urb->pipe)) {
3093                 case PIPE_ISOCHRONOUS:
3094                         transfer_type = CVMX_USB_TRANSFER_ISOCHRONOUS;
3095                         break;
3096                 case PIPE_INTERRUPT:
3097                         transfer_type = CVMX_USB_TRANSFER_INTERRUPT;
3098                         break;
3099                 case PIPE_CONTROL:
3100                         transfer_type = CVMX_USB_TRANSFER_CONTROL;
3101                         break;
3102                 default:
3103                         transfer_type = CVMX_USB_TRANSFER_BULK;
3104                         break;
3105                 }
3106                 switch (urb->dev->speed) {
3107                 case USB_SPEED_LOW:
3108                         speed = CVMX_USB_SPEED_LOW;
3109                         break;
3110                 case USB_SPEED_FULL:
3111                         speed = CVMX_USB_SPEED_FULL;
3112                         break;
3113                 default:
3114                         speed = CVMX_USB_SPEED_HIGH;
3115                         break;
3116                 }
3117                 /*
3118                  * For slow devices on high speed ports we need to find the hub
3119                  * that does the speed translation so we know where to send the
3120                  * split transactions.
3121                  */
3122                 if (speed != CVMX_USB_SPEED_HIGH) {
3123                         /*
3124                          * Start at this device and work our way up the usb
3125                          * tree.
3126                          */
3127                         struct usb_device *dev = urb->dev;
3128
3129                         while (dev->parent) {
3130                                 /*
3131                                  * If our parent is high speed then he'll
3132                                  * receive the splits.
3133                                  */
3134                                 if (dev->parent->speed == USB_SPEED_HIGH) {
3135                                         split_device = dev->parent->devnum;
3136                                         split_port = dev->portnum;
3137                                         break;
3138                                 }
3139                                 /*
3140                                  * Move up the tree one level. If we make it all
3141                                  * the way up the tree, then the port must not
3142                                  * be in high speed mode and we don't need a
3143                                  * split.
3144                                  */
3145                                 dev = dev->parent;
3146                         }
3147                 }
3148                 pipe = cvmx_usb_open_pipe(usb, usb_pipedevice(urb->pipe),
3149                                           usb_pipeendpoint(urb->pipe), speed,
3150                                           le16_to_cpu(ep->desc.wMaxPacketSize)
3151                                           & 0x7ff,
3152                                           transfer_type,
3153                                           usb_pipein(urb->pipe) ?
3154                                                 CVMX_USB_DIRECTION_IN :
3155                                                 CVMX_USB_DIRECTION_OUT,
3156                                           urb->interval,
3157                                           (le16_to_cpu(ep->desc.wMaxPacketSize)
3158                                            >> 11) & 0x3,
3159                                           split_device, split_port);
3160                 if (!pipe) {
3161                         usb_hcd_unlink_urb_from_ep(hcd, urb);
3162                         spin_unlock_irqrestore(&usb->lock, flags);
3163                         dev_dbg(dev, "Failed to create pipe\n");
3164                         return -ENOMEM;
3165                 }
3166                 ep->hcpriv = pipe;
3167         } else {
3168                 pipe = ep->hcpriv;
3169         }
3170
3171         switch (usb_pipetype(urb->pipe)) {
3172         case PIPE_ISOCHRONOUS:
3173                 dev_dbg(dev, "Submit isochronous to %d.%d\n",
3174                         usb_pipedevice(urb->pipe),
3175                         usb_pipeendpoint(urb->pipe));
3176                 /*
3177                  * Allocate a structure to use for our private list of
3178                  * isochronous packets.
3179                  */
3180                 iso_packet = kmalloc_array(urb->number_of_packets,
3181                                            sizeof(struct cvmx_usb_iso_packet),
3182                                            GFP_ATOMIC);
3183                 if (iso_packet) {
3184                         int i;
3185                         /* Fill the list with the data from the URB */
3186                         for (i = 0; i < urb->number_of_packets; i++) {
3187                                 iso_packet[i].offset =
3188                                         urb->iso_frame_desc[i].offset;
3189                                 iso_packet[i].length =
3190                                         urb->iso_frame_desc[i].length;
3191                                 iso_packet[i].status = CVMX_USB_STATUS_ERROR;
3192                         }
3193                         /*
3194                          * Store a pointer to the list in the URB setup_packet
3195                          * field. We know this currently isn't being used and
3196                          * this saves us a bunch of logic.
3197                          */
3198                         urb->setup_packet = (char *)iso_packet;
3199                         transaction = cvmx_usb_submit_isochronous(usb,
3200                                                                   pipe, urb);
3201                         /*
3202                          * If submit failed we need to free our private packet
3203                          * list.
3204                          */
3205                         if (!transaction) {
3206                                 urb->setup_packet = NULL;
3207                                 kfree(iso_packet);
3208                         }
3209                 }
3210                 break;
3211         case PIPE_INTERRUPT:
3212                 dev_dbg(dev, "Submit interrupt to %d.%d\n",
3213                         usb_pipedevice(urb->pipe),
3214                         usb_pipeendpoint(urb->pipe));
3215                 transaction = cvmx_usb_submit_interrupt(usb, pipe, urb);
3216                 break;
3217         case PIPE_CONTROL:
3218                 dev_dbg(dev, "Submit control to %d.%d\n",
3219                         usb_pipedevice(urb->pipe),
3220                         usb_pipeendpoint(urb->pipe));
3221                 transaction = cvmx_usb_submit_control(usb, pipe, urb);
3222                 break;
3223         case PIPE_BULK:
3224                 dev_dbg(dev, "Submit bulk to %d.%d\n",
3225                         usb_pipedevice(urb->pipe),
3226                         usb_pipeendpoint(urb->pipe));
3227                 transaction = cvmx_usb_submit_bulk(usb, pipe, urb);
3228                 break;
3229         }
3230         if (!transaction) {
3231                 usb_hcd_unlink_urb_from_ep(hcd, urb);
3232                 spin_unlock_irqrestore(&usb->lock, flags);
3233                 dev_dbg(dev, "Failed to submit\n");
3234                 return -ENOMEM;
3235         }
3236         urb->hcpriv = transaction;
3237         spin_unlock_irqrestore(&usb->lock, flags);
3238         return 0;
3239 }
3240
3241 static int octeon_usb_urb_dequeue(struct usb_hcd *hcd,
3242                                   struct urb *urb,
3243                                   int status)
3244 {
3245         struct octeon_hcd *usb = hcd_to_octeon(hcd);
3246         unsigned long flags;
3247         int rc;
3248
3249         if (!urb->dev)
3250                 return -EINVAL;
3251
3252         spin_lock_irqsave(&usb->lock, flags);
3253
3254         rc = usb_hcd_check_unlink_urb(hcd, urb, status);
3255         if (rc)
3256                 goto out;
3257
3258         urb->status = status;
3259         cvmx_usb_cancel(usb, urb->ep->hcpriv, urb->hcpriv);
3260
3261 out:
3262         spin_unlock_irqrestore(&usb->lock, flags);
3263
3264         return rc;
3265 }
3266
3267 static void octeon_usb_endpoint_disable(struct usb_hcd *hcd,
3268                                         struct usb_host_endpoint *ep)
3269 {
3270         struct device *dev = hcd->self.controller;
3271
3272         if (ep->hcpriv) {
3273                 struct octeon_hcd *usb = hcd_to_octeon(hcd);
3274                 struct cvmx_usb_pipe *pipe = ep->hcpriv;
3275                 unsigned long flags;
3276
3277                 spin_lock_irqsave(&usb->lock, flags);
3278                 cvmx_usb_cancel_all(usb, pipe);
3279                 if (cvmx_usb_close_pipe(usb, pipe))
3280                         dev_dbg(dev, "Closing pipe %p failed\n", pipe);
3281                 spin_unlock_irqrestore(&usb->lock, flags);
3282                 ep->hcpriv = NULL;
3283         }
3284 }
3285
3286 static int octeon_usb_hub_status_data(struct usb_hcd *hcd, char *buf)
3287 {
3288         struct octeon_hcd *usb = hcd_to_octeon(hcd);
3289         struct cvmx_usb_port_status port_status;
3290         unsigned long flags;
3291
3292         spin_lock_irqsave(&usb->lock, flags);
3293         port_status = cvmx_usb_get_status(usb);
3294         spin_unlock_irqrestore(&usb->lock, flags);
3295         buf[0] = port_status.connect_change << 1;
3296
3297         return buf[0] != 0;
3298 }
3299
3300 static int octeon_usb_hub_control(struct usb_hcd *hcd, u16 typeReq, u16 wValue,
3301                                   u16 wIndex, char *buf, u16 wLength)
3302 {
3303         struct octeon_hcd *usb = hcd_to_octeon(hcd);
3304         struct device *dev = hcd->self.controller;
3305         struct cvmx_usb_port_status usb_port_status;
3306         int port_status;
3307         struct usb_hub_descriptor *desc;
3308         unsigned long flags;
3309
3310         switch (typeReq) {
3311         case ClearHubFeature:
3312                 dev_dbg(dev, "ClearHubFeature\n");
3313                 switch (wValue) {
3314                 case C_HUB_LOCAL_POWER:
3315                 case C_HUB_OVER_CURRENT:
3316                         /* Nothing required here */
3317                         break;
3318                 default:
3319                         return -EINVAL;
3320                 }
3321                 break;
3322         case ClearPortFeature:
3323                 dev_dbg(dev, "ClearPortFeature\n");
3324                 if (wIndex != 1) {
3325                         dev_dbg(dev, " INVALID\n");
3326                         return -EINVAL;
3327                 }
3328
3329                 switch (wValue) {
3330                 case USB_PORT_FEAT_ENABLE:
3331                         dev_dbg(dev, " ENABLE\n");
3332                         spin_lock_irqsave(&usb->lock, flags);
3333                         cvmx_usb_disable(usb);
3334                         spin_unlock_irqrestore(&usb->lock, flags);
3335                         break;
3336                 case USB_PORT_FEAT_SUSPEND:
3337                         dev_dbg(dev, " SUSPEND\n");
3338                         /* Not supported on Octeon */
3339                         break;
3340                 case USB_PORT_FEAT_POWER:
3341                         dev_dbg(dev, " POWER\n");
3342                         /* Not supported on Octeon */
3343                         break;
3344                 case USB_PORT_FEAT_INDICATOR:
3345                         dev_dbg(dev, " INDICATOR\n");
3346                         /* Port inidicator not supported */
3347                         break;
3348                 case USB_PORT_FEAT_C_CONNECTION:
3349                         dev_dbg(dev, " C_CONNECTION\n");
3350                         /* Clears drivers internal connect status change flag */
3351                         spin_lock_irqsave(&usb->lock, flags);
3352                         usb->port_status = cvmx_usb_get_status(usb);
3353                         spin_unlock_irqrestore(&usb->lock, flags);
3354                         break;
3355                 case USB_PORT_FEAT_C_RESET:
3356                         dev_dbg(dev, " C_RESET\n");
3357                         /*
3358                          * Clears the driver's internal Port Reset Change flag.
3359                          */
3360                         spin_lock_irqsave(&usb->lock, flags);
3361                         usb->port_status = cvmx_usb_get_status(usb);
3362                         spin_unlock_irqrestore(&usb->lock, flags);
3363                         break;
3364                 case USB_PORT_FEAT_C_ENABLE:
3365                         dev_dbg(dev, " C_ENABLE\n");
3366                         /*
3367                          * Clears the driver's internal Port Enable/Disable
3368                          * Change flag.
3369                          */
3370                         spin_lock_irqsave(&usb->lock, flags);
3371                         usb->port_status = cvmx_usb_get_status(usb);
3372                         spin_unlock_irqrestore(&usb->lock, flags);
3373                         break;
3374                 case USB_PORT_FEAT_C_SUSPEND:
3375                         dev_dbg(dev, " C_SUSPEND\n");
3376                         /*
3377                          * Clears the driver's internal Port Suspend Change
3378                          * flag, which is set when resume signaling on the host
3379                          * port is complete.
3380                          */
3381                         break;
3382                 case USB_PORT_FEAT_C_OVER_CURRENT:
3383                         dev_dbg(dev, " C_OVER_CURRENT\n");
3384                         /* Clears the driver's overcurrent Change flag */
3385                         spin_lock_irqsave(&usb->lock, flags);
3386                         usb->port_status = cvmx_usb_get_status(usb);
3387                         spin_unlock_irqrestore(&usb->lock, flags);
3388                         break;
3389                 default:
3390                         dev_dbg(dev, " UNKNOWN\n");
3391                         return -EINVAL;
3392                 }
3393                 break;
3394         case GetHubDescriptor:
3395                 dev_dbg(dev, "GetHubDescriptor\n");
3396                 desc = (struct usb_hub_descriptor *)buf;
3397                 desc->bDescLength = 9;
3398                 desc->bDescriptorType = 0x29;
3399                 desc->bNbrPorts = 1;
3400                 desc->wHubCharacteristics = cpu_to_le16(0x08);
3401                 desc->bPwrOn2PwrGood = 1;
3402                 desc->bHubContrCurrent = 0;
3403                 desc->u.hs.DeviceRemovable[0] = 0;
3404                 desc->u.hs.DeviceRemovable[1] = 0xff;
3405                 break;
3406         case GetHubStatus:
3407                 dev_dbg(dev, "GetHubStatus\n");
3408                 *(__le32 *)buf = 0;
3409                 break;
3410         case GetPortStatus:
3411                 dev_dbg(dev, "GetPortStatus\n");
3412                 if (wIndex != 1) {
3413                         dev_dbg(dev, " INVALID\n");
3414                         return -EINVAL;
3415                 }
3416
3417                 spin_lock_irqsave(&usb->lock, flags);
3418                 usb_port_status = cvmx_usb_get_status(usb);
3419                 spin_unlock_irqrestore(&usb->lock, flags);
3420                 port_status = 0;
3421
3422                 if (usb_port_status.connect_change) {
3423                         port_status |= (1 << USB_PORT_FEAT_C_CONNECTION);
3424                         dev_dbg(dev, " C_CONNECTION\n");
3425                 }
3426
3427                 if (usb_port_status.port_enabled) {
3428                         port_status |= (1 << USB_PORT_FEAT_C_ENABLE);
3429                         dev_dbg(dev, " C_ENABLE\n");
3430                 }
3431
3432                 if (usb_port_status.connected) {
3433                         port_status |= (1 << USB_PORT_FEAT_CONNECTION);
3434                         dev_dbg(dev, " CONNECTION\n");
3435                 }
3436
3437                 if (usb_port_status.port_enabled) {
3438                         port_status |= (1 << USB_PORT_FEAT_ENABLE);
3439                         dev_dbg(dev, " ENABLE\n");
3440                 }
3441
3442                 if (usb_port_status.port_over_current) {
3443                         port_status |= (1 << USB_PORT_FEAT_OVER_CURRENT);
3444                         dev_dbg(dev, " OVER_CURRENT\n");
3445                 }
3446
3447                 if (usb_port_status.port_powered) {
3448                         port_status |= (1 << USB_PORT_FEAT_POWER);
3449                         dev_dbg(dev, " POWER\n");
3450                 }
3451
3452                 if (usb_port_status.port_speed == CVMX_USB_SPEED_HIGH) {
3453                         port_status |= USB_PORT_STAT_HIGH_SPEED;
3454                         dev_dbg(dev, " HIGHSPEED\n");
3455                 } else if (usb_port_status.port_speed == CVMX_USB_SPEED_LOW) {
3456                         port_status |= (1 << USB_PORT_FEAT_LOWSPEED);
3457                         dev_dbg(dev, " LOWSPEED\n");
3458                 }
3459
3460                 *((__le32 *)buf) = cpu_to_le32(port_status);
3461                 break;
3462         case SetHubFeature:
3463                 dev_dbg(dev, "SetHubFeature\n");
3464                 /* No HUB features supported */
3465                 break;
3466         case SetPortFeature:
3467                 dev_dbg(dev, "SetPortFeature\n");
3468                 if (wIndex != 1) {
3469                         dev_dbg(dev, " INVALID\n");
3470                         return -EINVAL;
3471                 }
3472
3473                 switch (wValue) {
3474                 case USB_PORT_FEAT_SUSPEND:
3475                         dev_dbg(dev, " SUSPEND\n");
3476                         return -EINVAL;
3477                 case USB_PORT_FEAT_POWER:
3478                         dev_dbg(dev, " POWER\n");
3479                         /*
3480                          * Program the port power bit to drive VBUS on the USB.
3481                          */
3482                         spin_lock_irqsave(&usb->lock, flags);
3483                         USB_SET_FIELD32(CVMX_USBCX_HPRT(usb->index),
3484                                         cvmx_usbcx_hprt, prtpwr, 1);
3485                         spin_unlock_irqrestore(&usb->lock, flags);
3486                         return 0;
3487                 case USB_PORT_FEAT_RESET:
3488                         dev_dbg(dev, " RESET\n");
3489                         spin_lock_irqsave(&usb->lock, flags);
3490                         cvmx_usb_reset_port(usb);
3491                         spin_unlock_irqrestore(&usb->lock, flags);
3492                         return 0;
3493                 case USB_PORT_FEAT_INDICATOR:
3494                         dev_dbg(dev, " INDICATOR\n");
3495                         /* Not supported */
3496                         break;
3497                 default:
3498                         dev_dbg(dev, " UNKNOWN\n");
3499                         return -EINVAL;
3500                 }
3501                 break;
3502         default:
3503                 dev_dbg(dev, "Unknown root hub request\n");
3504                 return -EINVAL;
3505         }
3506         return 0;
3507 }
3508
3509 static const struct hc_driver octeon_hc_driver = {
3510         .description            = "Octeon USB",
3511         .product_desc           = "Octeon Host Controller",
3512         .hcd_priv_size          = sizeof(struct octeon_hcd),
3513         .irq                    = octeon_usb_irq,
3514         .flags                  = HCD_MEMORY | HCD_DMA | HCD_USB2,
3515         .start                  = octeon_usb_start,
3516         .stop                   = octeon_usb_stop,
3517         .urb_enqueue            = octeon_usb_urb_enqueue,
3518         .urb_dequeue            = octeon_usb_urb_dequeue,
3519         .endpoint_disable       = octeon_usb_endpoint_disable,
3520         .get_frame_number       = octeon_usb_get_frame_number,
3521         .hub_status_data        = octeon_usb_hub_status_data,
3522         .hub_control            = octeon_usb_hub_control,
3523         .map_urb_for_dma        = octeon_map_urb_for_dma,
3524         .unmap_urb_for_dma      = octeon_unmap_urb_for_dma,
3525 };
3526
3527 static int octeon_usb_probe(struct platform_device *pdev)
3528 {
3529         int status;
3530         int initialize_flags;
3531         int usb_num;
3532         struct resource *res_mem;
3533         struct device_node *usbn_node;
3534         int irq = platform_get_irq(pdev, 0);
3535         struct device *dev = &pdev->dev;
3536         struct octeon_hcd *usb;
3537         struct usb_hcd *hcd;
3538         u32 clock_rate = 48000000;
3539         bool is_crystal_clock = false;
3540         const char *clock_type;
3541         int i;
3542
3543         if (!dev->of_node) {
3544                 dev_err(dev, "Error: empty of_node\n");
3545                 return -ENXIO;
3546         }
3547         usbn_node = dev->of_node->parent;
3548
3549         i = of_property_read_u32(usbn_node,
3550                                  "clock-frequency", &clock_rate);
3551         if (i)
3552                 i = of_property_read_u32(usbn_node,
3553                                          "refclk-frequency", &clock_rate);
3554         if (i) {
3555                 dev_err(dev, "No USBN \"clock-frequency\"\n");
3556                 return -ENXIO;
3557         }
3558         switch (clock_rate) {
3559         case 12000000:
3560                 initialize_flags = CVMX_USB_INITIALIZE_FLAGS_CLOCK_12MHZ;
3561                 break;
3562         case 24000000:
3563                 initialize_flags = CVMX_USB_INITIALIZE_FLAGS_CLOCK_24MHZ;
3564                 break;
3565         case 48000000:
3566                 initialize_flags = CVMX_USB_INITIALIZE_FLAGS_CLOCK_48MHZ;
3567                 break;
3568         default:
3569                 dev_err(dev, "Illegal USBN \"clock-frequency\" %u\n",
3570                         clock_rate);
3571                 return -ENXIO;
3572         }
3573
3574         i = of_property_read_string(usbn_node,
3575                                     "cavium,refclk-type", &clock_type);
3576         if (i)
3577                 i = of_property_read_string(usbn_node,
3578                                             "refclk-type", &clock_type);
3579
3580         if (!i && strcmp("crystal", clock_type) == 0)
3581                 is_crystal_clock = true;
3582
3583         if (is_crystal_clock)
3584                 initialize_flags |= CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_XI;
3585         else
3586                 initialize_flags |= CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_GND;
3587
3588         res_mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
3589         if (!res_mem) {
3590                 dev_err(dev, "found no memory resource\n");
3591                 return -ENXIO;
3592         }
3593         usb_num = (res_mem->start >> 44) & 1;
3594
3595         if (irq < 0) {
3596                 /* Defective device tree, but we know how to fix it. */
3597                 irq_hw_number_t hwirq = usb_num ? (1 << 6) + 17 : 56;
3598
3599                 irq = irq_create_mapping(NULL, hwirq);
3600         }
3601
3602         /*
3603          * Set the DMA mask to 64bits so we get buffers already translated for
3604          * DMA.
3605          */
3606         i = dma_coerce_mask_and_coherent(dev, DMA_BIT_MASK(64));
3607         if (i)
3608                 return i;
3609
3610         /*
3611          * Only cn52XX and cn56XX have DWC_OTG USB hardware and the
3612          * IOB priority registers.  Under heavy network load USB
3613          * hardware can be starved by the IOB causing a crash.  Give
3614          * it a priority boost if it has been waiting more than 400
3615          * cycles to avoid this situation.
3616          *
3617          * Testing indicates that a cnt_val of 8192 is not sufficient,
3618          * but no failures are seen with 4096.  We choose a value of
3619          * 400 to give a safety factor of 10.
3620          */
3621         if (OCTEON_IS_MODEL(OCTEON_CN52XX) || OCTEON_IS_MODEL(OCTEON_CN56XX)) {
3622                 union cvmx_iob_n2c_l2c_pri_cnt pri_cnt;
3623
3624                 pri_cnt.u64 = 0;
3625                 pri_cnt.s.cnt_enb = 1;
3626                 pri_cnt.s.cnt_val = 400;
3627                 cvmx_write_csr(CVMX_IOB_N2C_L2C_PRI_CNT, pri_cnt.u64);
3628         }
3629
3630         hcd = usb_create_hcd(&octeon_hc_driver, dev, dev_name(dev));
3631         if (!hcd) {
3632                 dev_dbg(dev, "Failed to allocate memory for HCD\n");
3633                 return -1;
3634         }
3635         hcd->uses_new_polling = 1;
3636         usb = (struct octeon_hcd *)hcd->hcd_priv;
3637
3638         spin_lock_init(&usb->lock);
3639
3640         usb->init_flags = initialize_flags;
3641
3642         /* Initialize the USB state structure */
3643         usb->index = usb_num;
3644         INIT_LIST_HEAD(&usb->idle_pipes);
3645         for (i = 0; i < ARRAY_SIZE(usb->active_pipes); i++)
3646                 INIT_LIST_HEAD(&usb->active_pipes[i]);
3647
3648         /* Due to an errata, CN31XX doesn't support DMA */
3649         if (OCTEON_IS_MODEL(OCTEON_CN31XX)) {
3650                 usb->init_flags |= CVMX_USB_INITIALIZE_FLAGS_NO_DMA;
3651                 /* Only use one channel with non DMA */
3652                 usb->idle_hardware_channels = 0x1;
3653         } else if (OCTEON_IS_MODEL(OCTEON_CN5XXX)) {
3654                 /* CN5XXX have an errata with channel 3 */
3655                 usb->idle_hardware_channels = 0xf7;
3656         } else {
3657                 usb->idle_hardware_channels = 0xff;
3658         }
3659
3660         status = cvmx_usb_initialize(dev, usb);
3661         if (status) {
3662                 dev_dbg(dev, "USB initialization failed with %d\n", status);
3663                 usb_put_hcd(hcd);
3664                 return -1;
3665         }
3666
3667         status = usb_add_hcd(hcd, irq, 0);
3668         if (status) {
3669                 dev_dbg(dev, "USB add HCD failed with %d\n", status);
3670                 usb_put_hcd(hcd);
3671                 return -1;
3672         }
3673         device_wakeup_enable(hcd->self.controller);
3674
3675         dev_info(dev, "Registered HCD for port %d on irq %d\n", usb_num, irq);
3676
3677         return 0;
3678 }
3679
3680 static int octeon_usb_remove(struct platform_device *pdev)
3681 {
3682         int status;
3683         struct device *dev = &pdev->dev;
3684         struct usb_hcd *hcd = dev_get_drvdata(dev);
3685         struct octeon_hcd *usb = hcd_to_octeon(hcd);
3686         unsigned long flags;
3687
3688         usb_remove_hcd(hcd);
3689         spin_lock_irqsave(&usb->lock, flags);
3690         status = cvmx_usb_shutdown(usb);
3691         spin_unlock_irqrestore(&usb->lock, flags);
3692         if (status)
3693                 dev_dbg(dev, "USB shutdown failed with %d\n", status);
3694
3695         usb_put_hcd(hcd);
3696
3697         return 0;
3698 }
3699
3700 static const struct of_device_id octeon_usb_match[] = {
3701         {
3702                 .compatible = "cavium,octeon-5750-usbc",
3703         },
3704         {},
3705 };
3706 MODULE_DEVICE_TABLE(of, octeon_usb_match);
3707
3708 static struct platform_driver octeon_usb_driver = {
3709         .driver = {
3710                 .name           = "octeon-hcd",
3711                 .of_match_table = octeon_usb_match,
3712         },
3713         .probe      = octeon_usb_probe,
3714         .remove     = octeon_usb_remove,
3715 };
3716
3717 static int __init octeon_usb_driver_init(void)
3718 {
3719         if (usb_disabled())
3720                 return 0;
3721
3722         return platform_driver_register(&octeon_usb_driver);
3723 }
3724 module_init(octeon_usb_driver_init);
3725
3726 static void __exit octeon_usb_driver_exit(void)
3727 {
3728         if (usb_disabled())
3729                 return;
3730
3731         platform_driver_unregister(&octeon_usb_driver);
3732 }
3733 module_exit(octeon_usb_driver_exit);
3734
3735 MODULE_LICENSE("GPL");
3736 MODULE_AUTHOR("Cavium, Inc. <support@cavium.com>");
3737 MODULE_DESCRIPTION("Cavium Inc. OCTEON USB Host driver.");