drivers/tty/serial/8250/8250_bcm7271.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /* Copyright (c) 2020, Broadcom */
   3 /*
   4  * 8250-core based driver for Broadcom ns16550a UARTs
   5  *
   6  * This driver uses the standard 8250 driver core but adds additional
   7  * optional features including the ability to use a baud rate clock
   8  * mux for more accurate high speed baud rate selection and also
   9  * an optional DMA engine.
  10  *
  11  */
  12
  13 #include <linux/module.h>
  14 #include <linux/types.h>
  15 #include <linux/tty.h>
  16 #include <linux/errno.h>
  17 #include <linux/device.h>
  18 #include <linux/io.h>
  19 #include <linux/of.h>
  20 #include <linux/dma-mapping.h>
  21 #include <linux/tty_flip.h>
  22 #include <linux/delay.h>
  23 #include <linux/clk.h>
  24 #include <linux/debugfs.h>
  25
  26 #include "8250.h"
  27
  28 /* Register definitions for UART DMA block. Version 1.1 or later. */
  29 #define UDMA_ARB_RX             0x00
  30 #define UDMA_ARB_TX             0x04
  31 #define         UDMA_ARB_REQ                            0x00000001
  32 #define         UDMA_ARB_GRANT                          0x00000002
  33
  34 #define UDMA_RX_REVISION        0x00
  35 #define UDMA_RX_REVISION_REQUIRED                       0x00000101
  36 #define UDMA_RX_CTRL            0x04
  37 #define         UDMA_RX_CTRL_BUF_CLOSE_MODE             0x00010000
  38 #define         UDMA_RX_CTRL_MASK_WR_DONE               0x00008000
  39 #define         UDMA_RX_CTRL_ENDIAN_OVERRIDE            0x00004000
  40 #define         UDMA_RX_CTRL_ENDIAN                     0x00002000
  41 #define         UDMA_RX_CTRL_OE_IS_ERR                  0x00001000
  42 #define         UDMA_RX_CTRL_PE_IS_ERR                  0x00000800
  43 #define         UDMA_RX_CTRL_FE_IS_ERR                  0x00000400
  44 #define         UDMA_RX_CTRL_NUM_BUF_USED_MASK          0x000003c0
  45 #define         UDMA_RX_CTRL_NUM_BUF_USED_SHIFT 6
  46 #define         UDMA_RX_CTRL_BUF_CLOSE_CLK_SEL_SYS      0x00000020
  47 #define         UDMA_RX_CTRL_BUF_CLOSE_ENA              0x00000010
  48 #define         UDMA_RX_CTRL_TIMEOUT_CLK_SEL_SYS        0x00000008
  49 #define         UDMA_RX_CTRL_TIMEOUT_ENA                0x00000004
  50 #define         UDMA_RX_CTRL_ABORT                      0x00000002
  51 #define         UDMA_RX_CTRL_ENA                        0x00000001
  52 #define UDMA_RX_STATUS          0x08
  53 #define         UDMA_RX_STATUS_ACTIVE_BUF_MASK          0x0000000f
  54 #define UDMA_RX_TRANSFER_LEN    0x0c
  55 #define UDMA_RX_TRANSFER_TOTAL  0x10
  56 #define UDMA_RX_BUFFER_SIZE     0x14
  57 #define UDMA_RX_SRC_ADDR        0x18
  58 #define UDMA_RX_TIMEOUT         0x1c
  59 #define UDMA_RX_BUFFER_CLOSE    0x20
  60 #define UDMA_RX_BLOCKOUT_COUNTER 0x24
  61 #define UDMA_RX_BUF0_PTR_LO     0x28
  62 #define UDMA_RX_BUF0_PTR_HI     0x2c
  63 #define UDMA_RX_BUF0_STATUS     0x30
  64 #define         UDMA_RX_BUFX_STATUS_OVERRUN_ERR         0x00000010
  65 #define         UDMA_RX_BUFX_STATUS_FRAME_ERR           0x00000008
  66 #define         UDMA_RX_BUFX_STATUS_PARITY_ERR          0x00000004
  67 #define         UDMA_RX_BUFX_STATUS_CLOSE_EXPIRED       0x00000002
  68 #define         UDMA_RX_BUFX_STATUS_DATA_RDY            0x00000001
  69 #define UDMA_RX_BUF0_DATA_LEN   0x34
  70 #define UDMA_RX_BUF1_PTR_LO     0x38
  71 #define UDMA_RX_BUF1_PTR_HI     0x3c
  72 #define UDMA_RX_BUF1_STATUS     0x40
  73 #define UDMA_RX_BUF1_DATA_LEN   0x44
  74
  75 #define UDMA_TX_REVISION        0x00
  76 #define UDMA_TX_REVISION_REQUIRED                       0x00000101
  77 #define UDMA_TX_CTRL            0x04
  78 #define         UDMA_TX_CTRL_ENDIAN_OVERRIDE            0x00000080
  79 #define         UDMA_TX_CTRL_ENDIAN                     0x00000040
  80 #define         UDMA_TX_CTRL_NUM_BUF_USED_MASK          0x00000030
  81 #define         UDMA_TX_CTRL_NUM_BUF_USED_1             0x00000010
  82 #define         UDMA_TX_CTRL_ABORT                      0x00000002
  83 #define         UDMA_TX_CTRL_ENA                        0x00000001
  84 #define UDMA_TX_DST_ADDR        0x08
  85 #define UDMA_TX_BLOCKOUT_COUNTER 0x10
  86 #define UDMA_TX_TRANSFER_LEN    0x14
  87 #define UDMA_TX_TRANSFER_TOTAL  0x18
  88 #define UDMA_TX_STATUS          0x20
  89 #define UDMA_TX_BUF0_PTR_LO     0x24
  90 #define UDMA_TX_BUF0_PTR_HI     0x28
  91 #define UDMA_TX_BUF0_STATUS     0x2c
  92 #define         UDMA_TX_BUFX_LAST                       0x00000002
  93 #define         UDMA_TX_BUFX_EMPTY                      0x00000001
  94 #define UDMA_TX_BUF0_DATA_LEN   0x30
  95 #define UDMA_TX_BUF0_DATA_SENT  0x34
  96 #define UDMA_TX_BUF1_PTR_LO     0x38
  97
  98 #define UDMA_INTR_STATUS        0x00
  99 #define         UDMA_INTR_ARB_TX_GRANT                  0x00040000
 100 #define         UDMA_INTR_ARB_RX_GRANT                  0x00020000
 101 #define         UDMA_INTR_TX_ALL_EMPTY                  0x00010000
 102 #define         UDMA_INTR_TX_EMPTY_BUF1                 0x00008000
 103 #define         UDMA_INTR_TX_EMPTY_BUF0                 0x00004000
 104 #define         UDMA_INTR_TX_ABORT                      0x00002000
 105 #define         UDMA_INTR_TX_DONE                       0x00001000
 106 #define         UDMA_INTR_RX_ERROR                      0x00000800
 107 #define         UDMA_INTR_RX_TIMEOUT                    0x00000400
 108 #define         UDMA_INTR_RX_READY_BUF7                 0x00000200
 109 #define         UDMA_INTR_RX_READY_BUF6                 0x00000100
 110 #define         UDMA_INTR_RX_READY_BUF5                 0x00000080
 111 #define         UDMA_INTR_RX_READY_BUF4                 0x00000040
 112 #define         UDMA_INTR_RX_READY_BUF3                 0x00000020
 113 #define         UDMA_INTR_RX_READY_BUF2                 0x00000010
 114 #define         UDMA_INTR_RX_READY_BUF1                 0x00000008
 115 #define         UDMA_INTR_RX_READY_BUF0                 0x00000004
 116 #define         UDMA_INTR_RX_READY_MASK                 0x000003fc
 117 #define         UDMA_INTR_RX_READY_SHIFT                2
 118 #define         UDMA_INTR_RX_ABORT                      0x00000002
 119 #define         UDMA_INTR_RX_DONE                       0x00000001
 120 #define UDMA_INTR_SET           0x04
 121 #define UDMA_INTR_CLEAR         0x08
 122 #define UDMA_INTR_MASK_STATUS   0x0c
 123 #define UDMA_INTR_MASK_SET      0x10
 124 #define UDMA_INTR_MASK_CLEAR    0x14
 125
 126
 127 #define UDMA_RX_INTERRUPTS ( \
 128         UDMA_INTR_RX_ERROR | \
 129         UDMA_INTR_RX_TIMEOUT | \
 130         UDMA_INTR_RX_READY_BUF0 | \
 131         UDMA_INTR_RX_READY_BUF1 | \
 132         UDMA_INTR_RX_READY_BUF2 | \
 133         UDMA_INTR_RX_READY_BUF3 | \
 134         UDMA_INTR_RX_READY_BUF4 | \
 135         UDMA_INTR_RX_READY_BUF5 | \
 136         UDMA_INTR_RX_READY_BUF6 | \
 137         UDMA_INTR_RX_READY_BUF7 | \
 138         UDMA_INTR_RX_ABORT | \
 139         UDMA_INTR_RX_DONE)
 140
 141 #define UDMA_RX_ERR_INTERRUPTS ( \
 142         UDMA_INTR_RX_ERROR | \
 143         UDMA_INTR_RX_TIMEOUT | \
 144         UDMA_INTR_RX_ABORT | \
 145         UDMA_INTR_RX_DONE)
 146
 147 #define UDMA_TX_INTERRUPTS ( \
 148         UDMA_INTR_TX_ABORT | \
 149         UDMA_INTR_TX_DONE)
 150
 151 #define UDMA_IS_RX_INTERRUPT(status) ((status) & UDMA_RX_INTERRUPTS)
 152 #define UDMA_IS_TX_INTERRUPT(status) ((status) & UDMA_TX_INTERRUPTS)
 153
 154
 155 /* Current devices have 8 sets of RX buffer registers */
 156 #define UDMA_RX_BUFS_COUNT      8
 157 #define UDMA_RX_BUFS_REG_OFFSET (UDMA_RX_BUF1_PTR_LO - UDMA_RX_BUF0_PTR_LO)
 158 #define UDMA_RX_BUFx_PTR_LO(x)  (UDMA_RX_BUF0_PTR_LO + \
 159                                  ((x) * UDMA_RX_BUFS_REG_OFFSET))
 160 #define UDMA_RX_BUFx_PTR_HI(x)  (UDMA_RX_BUF0_PTR_HI + \
 161                                  ((x) * UDMA_RX_BUFS_REG_OFFSET))
 162 #define UDMA_RX_BUFx_STATUS(x)  (UDMA_RX_BUF0_STATUS + \
 163                                  ((x) * UDMA_RX_BUFS_REG_OFFSET))
 164 #define UDMA_RX_BUFx_DATA_LEN(x) (UDMA_RX_BUF0_DATA_LEN + \
 165                                   ((x) * UDMA_RX_BUFS_REG_OFFSET))
 166
 167 /* Current devices have 2 sets of TX buffer registers */
 168 #define UDMA_TX_BUFS_COUNT      2
 169 #define UDMA_TX_BUFS_REG_OFFSET (UDMA_TX_BUF1_PTR_LO - UDMA_TX_BUF0_PTR_LO)
 170 #define UDMA_TX_BUFx_PTR_LO(x)  (UDMA_TX_BUF0_PTR_LO + \
 171                                  ((x) * UDMA_TX_BUFS_REG_OFFSET))
 172 #define UDMA_TX_BUFx_PTR_HI(x)  (UDMA_TX_BUF0_PTR_HI + \
 173                                  ((x) * UDMA_TX_BUFS_REG_OFFSET))
 174 #define UDMA_TX_BUFx_STATUS(x)  (UDMA_TX_BUF0_STATUS + \
 175                                  ((x) * UDMA_TX_BUFS_REG_OFFSET))
 176 #define UDMA_TX_BUFx_DATA_LEN(x) (UDMA_TX_BUF0_DATA_LEN + \
 177                                   ((x) * UDMA_TX_BUFS_REG_OFFSET))
 178 #define UDMA_TX_BUFx_DATA_SENT(x) (UDMA_TX_BUF0_DATA_SENT + \
 179                                    ((x) * UDMA_TX_BUFS_REG_OFFSET))
 180 #define REGS_8250 0
 181 #define REGS_DMA_RX 1
 182 #define REGS_DMA_TX 2
 183 #define REGS_DMA_ISR 3
 184 #define REGS_DMA_ARB 4
 185 #define REGS_MAX 5
 186
 187 #define TX_BUF_SIZE 4096
 188 #define RX_BUF_SIZE 4096
 189 #define RX_BUFS_COUNT 2
 190 #define KHZ    1000
 191 #define MHZ(x) ((x) * KHZ * KHZ)
 192
 193 static const u32 brcmstb_rate_table[] = {
 194         MHZ(81),
 195         MHZ(108),
 196         MHZ(64),                /* Actually 64285715 for some chips */
 197         MHZ(48),
 198 };
 199
 200 static const u32 brcmstb_rate_table_7278[] = {
 201         MHZ(81),
 202         MHZ(108),
 203         0,
 204         MHZ(48),
 205 };
 206
 207 struct brcmuart_priv {
 208         int             line;
 209         struct clk      *baud_mux_clk;
 210         unsigned long   default_mux_rate;
 211         u32             real_rates[ARRAY_SIZE(brcmstb_rate_table)];
 212         const u32       *rate_table;
 213         ktime_t         char_wait;
 214         struct uart_port *up;
 215         struct hrtimer  hrt;
 216         bool            shutdown;
 217         bool            dma_enabled;
 218         struct uart_8250_dma dma;
 219         void __iomem    *regs[REGS_MAX];
 220         dma_addr_t      rx_addr;
 221         void            *rx_bufs;
 222         size_t          rx_size;
 223         int             rx_next_buf;
 224         dma_addr_t      tx_addr;
 225         void            *tx_buf;
 226         size_t          tx_size;
 227         bool            tx_running;
 228         bool            rx_running;
 229         struct dentry   *debugfs_dir;
 230
 231         /* stats exposed through debugfs */
 232         u64             dma_rx_partial_buf;
 233         u64             dma_rx_full_buf;
 234         u32             rx_bad_timeout_late_char;
 235         u32             rx_bad_timeout_no_char;
 236         u32             rx_missing_close_timeout;
 237         u32             rx_err;
 238         u32             rx_timeout;
 239         u32             rx_abort;
 240         u32             saved_mctrl;
 241 };
 242
 243 static struct dentry *brcmuart_debugfs_root;
 244
 245 /*
 246  * Register access routines
 247  */
 248 static u32 udma_readl(struct brcmuart_priv *priv,
 249                 int reg_type, int offset)
 250 {
 251         return readl(priv->regs[reg_type] + offset);
 252 }
 253
 254 static void udma_writel(struct brcmuart_priv *priv,
 255                         int reg_type, int offset, u32 value)
 256 {
 257         writel(value, priv->regs[reg_type] + offset);
 258 }
 259
 260 static void udma_set(struct brcmuart_priv *priv,
 261                 int reg_type, int offset, u32 bits)
 262 {
 263         void __iomem *reg = priv->regs[reg_type] + offset;
 264         u32 value;
 265
 266         value = readl(reg);
 267         value |= bits;
 268         writel(value, reg);
 269 }
 270
 271 static void udma_unset(struct brcmuart_priv *priv,
 272                 int reg_type, int offset, u32 bits)
 273 {
 274         void __iomem *reg = priv->regs[reg_type] + offset;
 275         u32 value;
 276
 277         value = readl(reg);
 278         value &= ~bits;
 279         writel(value, reg);
 280 }
 281
 282 /*
 283  * The UART DMA engine hardware can be used by multiple UARTS, but
 284  * only one at a time. Sharing is not currently supported so
 285  * the first UART to request the DMA engine will get it and any
 286  * subsequent requests by other UARTS will fail.
 287  */
 288 static int brcmuart_arbitration(struct brcmuart_priv *priv, bool acquire)
 289 {
 290         u32 rx_grant;
 291         u32 tx_grant;
 292         int waits;
 293         int ret = 0;
 294
 295         if (acquire) {
 296                 udma_set(priv, REGS_DMA_ARB, UDMA_ARB_RX, UDMA_ARB_REQ);
 297                 udma_set(priv, REGS_DMA_ARB, UDMA_ARB_TX, UDMA_ARB_REQ);
 298
 299                 waits = 1;
 300                 while (1) {
 301                         rx_grant = udma_readl(priv, REGS_DMA_ARB, UDMA_ARB_RX);
 302                         tx_grant = udma_readl(priv, REGS_DMA_ARB, UDMA_ARB_TX);
 303                         if (rx_grant & tx_grant & UDMA_ARB_GRANT)
 304                                 return 0;
 305                         if (waits-- == 0)
 306                                 break;
 307                         msleep(1);
 308                 }
 309                 ret = 1;
 310         }
 311
 312         udma_unset(priv, REGS_DMA_ARB, UDMA_ARB_RX, UDMA_ARB_REQ);
 313         udma_unset(priv, REGS_DMA_ARB, UDMA_ARB_TX, UDMA_ARB_REQ);
 314         return ret;
 315 }
 316
 317 static void brcmuart_init_dma_hardware(struct brcmuart_priv *priv)
 318 {
 319         u32 daddr;
 320         u32 value;
 321         int x;
 322
 323         /* Start with all interrupts disabled */
 324         udma_writel(priv, REGS_DMA_ISR, UDMA_INTR_MASK_SET, 0xffffffff);
 325
 326         udma_writel(priv, REGS_DMA_RX, UDMA_RX_BUFFER_SIZE, RX_BUF_SIZE);
 327
 328         /*
 329          * Setup buffer close to happen when 32 character times have
 330          * elapsed since the last character was received.
 331          */
 332         udma_writel(priv, REGS_DMA_RX, UDMA_RX_BUFFER_CLOSE, 16*10*32);
 333         value = (RX_BUFS_COUNT << UDMA_RX_CTRL_NUM_BUF_USED_SHIFT)
 334                 | UDMA_RX_CTRL_BUF_CLOSE_MODE
 335                 | UDMA_RX_CTRL_BUF_CLOSE_ENA;
 336         udma_writel(priv, REGS_DMA_RX, UDMA_RX_CTRL, value);
 337
 338         udma_writel(priv, REGS_DMA_RX, UDMA_RX_BLOCKOUT_COUNTER, 0);
 339         daddr = priv->rx_addr;
 340         for (x = 0; x < RX_BUFS_COUNT; x++) {
 341
 342                 /* Set RX transfer length to 0 for unknown */
 343                 udma_writel(priv, REGS_DMA_RX, UDMA_RX_TRANSFER_LEN, 0);
 344
 345                 udma_writel(priv, REGS_DMA_RX, UDMA_RX_BUFx_PTR_LO(x),
 346                             lower_32_bits(daddr));
 347                 udma_writel(priv, REGS_DMA_RX, UDMA_RX_BUFx_PTR_HI(x),
 348                             upper_32_bits(daddr));
 349                 daddr += RX_BUF_SIZE;
 350         }
 351
 352         daddr = priv->tx_addr;
 353         udma_writel(priv, REGS_DMA_TX, UDMA_TX_BUFx_PTR_LO(0),
 354                     lower_32_bits(daddr));
 355         udma_writel(priv, REGS_DMA_TX, UDMA_TX_BUFx_PTR_HI(0),
 356                     upper_32_bits(daddr));
 357         udma_writel(priv, REGS_DMA_TX, UDMA_TX_CTRL,
 358                     UDMA_TX_CTRL_NUM_BUF_USED_1);
 359
 360         /* clear all interrupts then enable them */
 361         udma_writel(priv, REGS_DMA_ISR, UDMA_INTR_CLEAR, 0xffffffff);
 362         udma_writel(priv, REGS_DMA_ISR, UDMA_INTR_MASK_CLEAR,
 363                 UDMA_RX_INTERRUPTS | UDMA_TX_INTERRUPTS);
 364
 365 }
 366
 367 static void start_rx_dma(struct uart_8250_port *p)
 368 {
 369         struct brcmuart_priv *priv = p->port.private_data;
 370         int x;
 371
 372         udma_unset(priv, REGS_DMA_RX, UDMA_RX_CTRL, UDMA_RX_CTRL_ENA);
 373
 374         /* Clear the RX ready bit for all buffers */
 375         for (x = 0; x < RX_BUFS_COUNT; x++)
 376                 udma_unset(priv, REGS_DMA_RX, UDMA_RX_BUFx_STATUS(x),
 377                         UDMA_RX_BUFX_STATUS_DATA_RDY);
 378
 379         /* always start with buffer 0 */
 380         udma_unset(priv, REGS_DMA_RX, UDMA_RX_STATUS,
 381                    UDMA_RX_STATUS_ACTIVE_BUF_MASK);
 382         priv->rx_next_buf = 0;
 383
 384         udma_set(priv, REGS_DMA_RX, UDMA_RX_CTRL, UDMA_RX_CTRL_ENA);
 385         priv->rx_running = true;
 386 }
 387
 388 static void stop_rx_dma(struct uart_8250_port *p)
 389 {
 390         struct brcmuart_priv *priv = p->port.private_data;
 391
 392         /* If RX is running, set the RX ABORT */
 393         if (priv->rx_running)
 394                 udma_set(priv, REGS_DMA_RX, UDMA_RX_CTRL, UDMA_RX_CTRL_ABORT);
 395 }
 396
 397 static int stop_tx_dma(struct uart_8250_port *p)
 398 {
 399         struct brcmuart_priv *priv = p->port.private_data;
 400         u32 value;
 401
 402         /* If TX is running, set the TX ABORT */
 403         value = udma_readl(priv, REGS_DMA_TX, UDMA_TX_CTRL);
 404         if (value & UDMA_TX_CTRL_ENA)
 405                 udma_set(priv, REGS_DMA_TX, UDMA_TX_CTRL, UDMA_TX_CTRL_ABORT);
 406         priv->tx_running = false;
 407         return 0;
 408 }
 409
 410 /*
 411  * NOTE: printk's in this routine will hang the system if this is
 412  * the console tty
 413  */
 414 static int brcmuart_tx_dma(struct uart_8250_port *p)
 415 {
 416         struct brcmuart_priv *priv = p->port.private_data;
 417         struct circ_buf *xmit = &p->port.state->xmit;
 418         u32 tx_size;
 419
 420         if (uart_tx_stopped(&p->port) || priv->tx_running ||
 421                 uart_circ_empty(xmit)) {
 422                 return 0;
 423         }
 424         tx_size = CIRC_CNT_TO_END(xmit->head, xmit->tail, UART_XMIT_SIZE);
 425
 426         priv->dma.tx_err = 0;
 427         memcpy(priv->tx_buf, &xmit->buf[xmit->tail], tx_size);
 428         xmit->tail += tx_size;
 429         xmit->tail &= UART_XMIT_SIZE - 1;
 430         p->port.icount.tx += tx_size;
 431
 432         if (uart_circ_chars_pending(xmit) < WAKEUP_CHARS)
 433                 uart_write_wakeup(&p->port);
 434
 435         udma_writel(priv, REGS_DMA_TX, UDMA_TX_TRANSFER_LEN, tx_size);
 436         udma_writel(priv, REGS_DMA_TX, UDMA_TX_BUF0_DATA_LEN, tx_size);
 437         udma_unset(priv, REGS_DMA_TX, UDMA_TX_BUF0_STATUS, UDMA_TX_BUFX_EMPTY);
 438         udma_set(priv, REGS_DMA_TX, UDMA_TX_CTRL, UDMA_TX_CTRL_ENA);
 439         priv->tx_running = true;
 440
 441         return 0;
 442 }
 443
 444 static void brcmuart_rx_buf_done_isr(struct uart_port *up, int index)
 445 {
 446         struct brcmuart_priv *priv = up->private_data;
 447         struct tty_port *tty_port = &up->state->port;
 448         u32 status;
 449         u32 length;
 450         u32 copied;
 451
 452         /* Make sure we're still in sync with the hardware */
 453         status = udma_readl(priv, REGS_DMA_RX, UDMA_RX_BUFx_STATUS(index));
 454         length = udma_readl(priv, REGS_DMA_RX, UDMA_RX_BUFx_DATA_LEN(index));
 455
 456         if ((status & UDMA_RX_BUFX_STATUS_DATA_RDY) == 0) {
 457                 dev_err(up->dev, "RX done interrupt but DATA_RDY not found\n");
 458                 return;
 459         }
 460         if (status & (UDMA_RX_BUFX_STATUS_OVERRUN_ERR |
 461                       UDMA_RX_BUFX_STATUS_FRAME_ERR |
 462                       UDMA_RX_BUFX_STATUS_PARITY_ERR)) {
 463                 if (status & UDMA_RX_BUFX_STATUS_OVERRUN_ERR) {
 464                         up->icount.overrun++;
 465                         dev_warn(up->dev, "RX OVERRUN Error\n");
 466                 }
 467                 if (status & UDMA_RX_BUFX_STATUS_FRAME_ERR) {
 468                         up->icount.frame++;
 469                         dev_warn(up->dev, "RX FRAMING Error\n");
 470                 }
 471                 if (status & UDMA_RX_BUFX_STATUS_PARITY_ERR) {
 472                         up->icount.parity++;
 473                         dev_warn(up->dev, "RX PARITY Error\n");
 474                 }
 475         }
 476         copied = (u32)tty_insert_flip_string(
 477                 tty_port,
 478                 priv->rx_bufs + (index * RX_BUF_SIZE),
 479                 length);
 480         if (copied != length) {
 481                 dev_warn(up->dev, "Flip buffer overrun of %d bytes\n",
 482                          length - copied);
 483                 up->icount.overrun += length - copied;
 484         }
 485         up->icount.rx += length;
 486         if (status & UDMA_RX_BUFX_STATUS_CLOSE_EXPIRED)
 487                 priv->dma_rx_partial_buf++;
 488         else if (length != RX_BUF_SIZE)
 489                 /*
 490                  * This is a bug in the controller that doesn't cause
 491                  * any problems but will be fixed in the future.
 492                  */
 493                 priv->rx_missing_close_timeout++;
 494         else
 495                 priv->dma_rx_full_buf++;
 496
 497         tty_flip_buffer_push(tty_port);
 498 }
 499
 500 static void brcmuart_rx_isr(struct uart_port *up, u32 rx_isr)
 501 {
 502         struct brcmuart_priv *priv = up->private_data;
 503         struct device *dev = up->dev;
 504         u32 rx_done_isr;
 505         u32 check_isr;
 506
 507         rx_done_isr = (rx_isr & UDMA_INTR_RX_READY_MASK);
 508         while (rx_done_isr) {
 509                 check_isr = UDMA_INTR_RX_READY_BUF0 << priv->rx_next_buf;
 510                 if (check_isr & rx_done_isr) {
 511                         brcmuart_rx_buf_done_isr(up, priv->rx_next_buf);
 512                 } else {
 513                         dev_err(dev,
 514                                 "RX buffer ready out of sequence, restarting RX DMA\n");
 515                         start_rx_dma(up_to_u8250p(up));
 516                         break;
 517                 }
 518                 if (rx_isr & UDMA_RX_ERR_INTERRUPTS) {
 519                         if (rx_isr & UDMA_INTR_RX_ERROR)
 520                                 priv->rx_err++;
 521                         if (rx_isr & UDMA_INTR_RX_TIMEOUT) {
 522                                 priv->rx_timeout++;
 523                                 dev_err(dev, "RX TIMEOUT Error\n");
 524                         }
 525                         if (rx_isr & UDMA_INTR_RX_ABORT)
 526                                 priv->rx_abort++;
 527                         priv->rx_running = false;
 528                 }
 529                 /* If not ABORT, re-enable RX buffer */
 530                 if (!(rx_isr & UDMA_INTR_RX_ABORT))
 531                         udma_unset(priv, REGS_DMA_RX,
 532                                    UDMA_RX_BUFx_STATUS(priv->rx_next_buf),
 533                                    UDMA_RX_BUFX_STATUS_DATA_RDY);
 534                 rx_done_isr &= ~check_isr;
 535                 priv->rx_next_buf++;
 536                 if (priv->rx_next_buf == RX_BUFS_COUNT)
 537                         priv->rx_next_buf = 0;
 538         }
 539 }
 540
 541 static void brcmuart_tx_isr(struct uart_port *up, u32 isr)
 542 {
 543         struct brcmuart_priv *priv = up->private_data;
 544         struct device *dev = up->dev;
 545         struct uart_8250_port *port_8250 = up_to_u8250p(up);
 546         struct circ_buf *xmit = &port_8250->port.state->xmit;
 547
 548         if (isr & UDMA_INTR_TX_ABORT) {
 549                 if (priv->tx_running)
 550                         dev_err(dev, "Unexpected TX_ABORT interrupt\n");
 551                 return;
 552         }
 553         priv->tx_running = false;
 554         if (!uart_circ_empty(xmit) && !uart_tx_stopped(up))
 555                 brcmuart_tx_dma(port_8250);
 556 }
 557
 558 static irqreturn_t brcmuart_isr(int irq, void *dev_id)
 559 {
 560         struct uart_port *up = dev_id;
 561         struct device *dev = up->dev;
 562         struct brcmuart_priv *priv = up->private_data;
 563         unsigned long flags;
 564         u32 interrupts;
 565         u32 rval;
 566         u32 tval;
 567
 568         interrupts = udma_readl(priv, REGS_DMA_ISR, UDMA_INTR_STATUS);
 569         if (interrupts == 0)
 570                 return IRQ_NONE;
 571
 572         spin_lock_irqsave(&up->lock, flags);
 573
 574         /* Clear all interrupts */
 575         udma_writel(priv, REGS_DMA_ISR, UDMA_INTR_CLEAR, interrupts);
 576
 577         rval = UDMA_IS_RX_INTERRUPT(interrupts);
 578         if (rval)
 579                 brcmuart_rx_isr(up, rval);
 580         tval = UDMA_IS_TX_INTERRUPT(interrupts);
 581         if (tval)
 582                 brcmuart_tx_isr(up, tval);
 583         if ((rval | tval) == 0)
 584                 dev_warn(dev, "Spurious interrupt: 0x%x\n", interrupts);
 585
 586         spin_unlock_irqrestore(&up->lock, flags);
 587         return IRQ_HANDLED;
 588 }
 589
 590 static int brcmuart_startup(struct uart_port *port)
 591 {
 592         int res;
 593         struct uart_8250_port *up = up_to_u8250p(port);
 594         struct brcmuart_priv *priv = up->port.private_data;
 595
 596         priv->shutdown = false;
 597
 598         /*
 599          * prevent serial8250_do_startup() from allocating non-existent
 600          * DMA resources
 601          */
 602         up->dma = NULL;
 603
 604         res = serial8250_do_startup(port);
 605         if (!priv->dma_enabled)
 606                 return res;
 607         /*
 608          * Disable the Receive Data Interrupt because the DMA engine
 609          * will handle this.
 610          */
 611         up->ier &= ~UART_IER_RDI;
 612         serial_port_out(port, UART_IER, up->ier);
 613
 614         priv->tx_running = false;
 615         priv->dma.rx_dma = NULL;
 616         priv->dma.tx_dma = brcmuart_tx_dma;
 617         up->dma = &priv->dma;
 618
 619         brcmuart_init_dma_hardware(priv);
 620         start_rx_dma(up);
 621         return res;
 622 }
 623
 624 static void brcmuart_shutdown(struct uart_port *port)
 625 {
 626         struct uart_8250_port *up = up_to_u8250p(port);
 627         struct brcmuart_priv *priv = up->port.private_data;
 628         unsigned long flags;
 629
 630         spin_lock_irqsave(&port->lock, flags);
 631         priv->shutdown = true;
 632         if (priv->dma_enabled) {
 633                 stop_rx_dma(up);
 634                 stop_tx_dma(up);
 635                 /* disable all interrupts */
 636                 udma_writel(priv, REGS_DMA_ISR, UDMA_INTR_MASK_SET,
 637                         UDMA_RX_INTERRUPTS | UDMA_TX_INTERRUPTS);
 638         }
 639
 640         /*
 641          * prevent serial8250_do_shutdown() from trying to free
 642          * DMA resources that we never alloc'd for this driver.
 643          */
 644         up->dma = NULL;
 645
 646         spin_unlock_irqrestore(&port->lock, flags);
 647         serial8250_do_shutdown(port);
 648 }
 649
 650 /*
 651  * Not all clocks run at the exact specified rate, so set each requested
 652  * rate and then get the actual rate.
 653  */
 654 static void init_real_clk_rates(struct device *dev, struct brcmuart_priv *priv)
 655 {
 656         int x;
 657         int rc;
 658
 659         priv->default_mux_rate = clk_get_rate(priv->baud_mux_clk);
 660         for (x = 0; x < ARRAY_SIZE(priv->real_rates); x++) {
 661                 if (priv->rate_table[x] == 0) {
 662                         priv->real_rates[x] = 0;
 663                         continue;
 664                 }
 665                 rc = clk_set_rate(priv->baud_mux_clk, priv->rate_table[x]);
 666                 if (rc) {
 667                         dev_err(dev, "Error selecting BAUD MUX clock for %u\n",
 668                                 priv->rate_table[x]);
 669                         priv->real_rates[x] = priv->rate_table[x];
 670                 } else {
 671                         priv->real_rates[x] = clk_get_rate(priv->baud_mux_clk);
 672                 }
 673         }
 674         clk_set_rate(priv->baud_mux_clk, priv->default_mux_rate);
 675 }
 676
 677 static void set_clock_mux(struct uart_port *up, struct brcmuart_priv *priv,
 678                         u32 baud)
 679 {
 680         u32 percent;
 681         u32 best_percent = UINT_MAX;
 682         u32 quot;
 683         u32 best_quot = 1;
 684         u32 rate;
 685         int best_index = -1;
 686         u64 hires_rate;
 687         u64 hires_baud;
 688         u64 hires_err;
 689         int rc;
 690         int i;
 691         int real_baud;
 692
 693         /* If the Baud Mux Clock was not specified, just return */
 694         if (priv->baud_mux_clk == NULL)
 695                 return;
 696
 697         /* Find the closest match for specified baud */
 698         for (i = 0; i < ARRAY_SIZE(priv->real_rates); i++) {
 699                 if (priv->real_rates[i] == 0)
 700                         continue;
 701                 rate = priv->real_rates[i] / 16;
 702                 quot = DIV_ROUND_CLOSEST(rate, baud);
 703                 if (!quot)
 704                         continue;
 705
 706                 /* increase resolution to get xx.xx percent */
 707                 hires_rate = (u64)rate * 10000;
 708                 hires_baud = (u64)baud * 10000;
 709
 710                 hires_err = div_u64(hires_rate, (u64)quot);
 711
 712                 /* get the delta */
 713                 if (hires_err > hires_baud)
 714                         hires_err = (hires_err - hires_baud);
 715                 else
 716                         hires_err = (hires_baud - hires_err);
 717
 718                 percent = (unsigned long)DIV_ROUND_CLOSEST_ULL(hires_err, baud);
 719                 dev_dbg(up->dev,
 720                         "Baud rate: %u, MUX Clk: %u, Error: %u.%u%%\n",
 721                         baud, priv->real_rates[i], percent / 100,
 722                         percent % 100);
 723                 if (percent < best_percent) {
 724                         best_percent = percent;
 725                         best_index = i;
 726                         best_quot = quot;
 727                 }
 728         }
 729         if (best_index == -1) {
 730                 dev_err(up->dev, "Error, %d BAUD rate is too fast.\n", baud);
 731                 return;
 732         }
 733         rate = priv->real_rates[best_index];
 734         rc = clk_set_rate(priv->baud_mux_clk, rate);
 735         if (rc)
 736                 dev_err(up->dev, "Error selecting BAUD MUX clock\n");
 737
 738         /* Error over 3 percent will cause data errors */
 739         if (best_percent > 300)
 740                 dev_err(up->dev, "Error, baud: %d has %u.%u%% error\n",
 741                         baud, percent / 100, percent % 100);
 742
 743         real_baud = rate / 16 / best_quot;
 744         dev_dbg(up->dev, "Selecting BAUD MUX rate: %u\n", rate);
 745         dev_dbg(up->dev, "Requested baud: %u, Actual baud: %u\n",
 746                 baud, real_baud);
 747
 748         /* calc nanoseconds for 1.5 characters time at the given baud rate */
 749         i = NSEC_PER_SEC / real_baud / 10;
 750         i += (i / 2);
 751         priv->char_wait = ns_to_ktime(i);
 752
 753         up->uartclk = rate;
 754 }
 755
 756 static void brcmstb_set_termios(struct uart_port *up,
 757                                 struct ktermios *termios,
 758                                 struct ktermios *old)
 759 {
 760         struct uart_8250_port *p8250 = up_to_u8250p(up);
 761         struct brcmuart_priv *priv = up->private_data;
 762
 763         if (priv->dma_enabled)
 764                 stop_rx_dma(p8250);
 765         set_clock_mux(up, priv, tty_termios_baud_rate(termios));
 766         serial8250_do_set_termios(up, termios, old);
 767         if (p8250->mcr & UART_MCR_AFE)
 768                 p8250->port.status |= UPSTAT_AUTOCTS;
 769         if (priv->dma_enabled)
 770                 start_rx_dma(p8250);
 771 }
 772
 773 static int brcmuart_handle_irq(struct uart_port *p)
 774 {
 775         unsigned int iir = serial_port_in(p, UART_IIR);
 776         struct brcmuart_priv *priv = p->private_data;
 777         struct uart_8250_port *up = up_to_u8250p(p);
 778         unsigned int status;
 779         unsigned long flags;
 780         unsigned int ier;
 781         unsigned int mcr;
 782         int handled = 0;
 783
 784         /*
 785          * There's a bug in some 8250 cores where we get a timeout
 786          * interrupt but there is no data ready.
 787          */
 788         if (((iir & UART_IIR_ID) == UART_IIR_RX_TIMEOUT) && !(priv->shutdown)) {
 789                 spin_lock_irqsave(&p->lock, flags);
 790                 status = serial_port_in(p, UART_LSR);
 791                 if ((status & UART_LSR_DR) == 0) {
 792
 793                         ier = serial_port_in(p, UART_IER);
 794                         /*
 795                          * if Receive Data Interrupt is enabled and
 796                          * we're uing hardware flow control, deassert
 797                          * RTS and wait for any chars in the pipline to
 798                          * arrive and then check for DR again.
 799                          */
 800                         if ((ier & UART_IER_RDI) && (up->mcr & UART_MCR_AFE)) {
 801                                 ier &= ~(UART_IER_RLSI | UART_IER_RDI);
 802                                 serial_port_out(p, UART_IER, ier);
 803                                 mcr = serial_port_in(p, UART_MCR);
 804                                 mcr &= ~UART_MCR_RTS;
 805                                 serial_port_out(p, UART_MCR, mcr);
 806                                 hrtimer_start(&priv->hrt, priv->char_wait,
 807                                               HRTIMER_MODE_REL);
 808                         } else {
 809                                 serial_port_in(p, UART_RX);
 810                         }
 811
 812                         handled = 1;
 813                 }
 814                 spin_unlock_irqrestore(&p->lock, flags);
 815                 if (handled)
 816                         return 1;
 817         }
 818         return serial8250_handle_irq(p, iir);
 819 }
 820
 821 static enum hrtimer_restart brcmuart_hrtimer_func(struct hrtimer *t)
 822 {
 823         struct brcmuart_priv *priv = container_of(t, struct brcmuart_priv, hrt);
 824         struct uart_port *p = priv->up;
 825         struct uart_8250_port *up = up_to_u8250p(p);
 826         unsigned int status;
 827         unsigned long flags;
 828
 829         if (priv->shutdown)
 830                 return HRTIMER_NORESTART;
 831
 832         spin_lock_irqsave(&p->lock, flags);
 833         status = serial_port_in(p, UART_LSR);
 834
 835         /*
 836          * If a character did not arrive after the timeout, clear the false
 837          * receive timeout.
 838          */
 839         if ((status & UART_LSR_DR) == 0) {
 840                 serial_port_in(p, UART_RX);
 841                 priv->rx_bad_timeout_no_char++;
 842         } else {
 843                 priv->rx_bad_timeout_late_char++;
 844         }
 845
 846         /* re-enable receive unless upper layer has disabled it */
 847         if ((up->ier & (UART_IER_RLSI | UART_IER_RDI)) ==
 848             (UART_IER_RLSI | UART_IER_RDI)) {
 849                 status = serial_port_in(p, UART_IER);
 850                 status |= (UART_IER_RLSI | UART_IER_RDI);
 851                 serial_port_out(p, UART_IER, status);
 852                 status = serial_port_in(p, UART_MCR);
 853                 status |= UART_MCR_RTS;
 854                 serial_port_out(p, UART_MCR, status);
 855         }
 856         spin_unlock_irqrestore(&p->lock, flags);
 857         return HRTIMER_NORESTART;
 858 }
 859
 860 static const struct of_device_id brcmuart_dt_ids[] = {
 861         {
 862                 .compatible = "brcm,bcm7278-uart",
 863                 .data = brcmstb_rate_table_7278,
 864         },
 865         {
 866                 .compatible = "brcm,bcm7271-uart",
 867                 .data = brcmstb_rate_table,
 868         },
 869         {},
 870 };
 871
 872 MODULE_DEVICE_TABLE(of, brcmuart_dt_ids);
 873
 874 static void brcmuart_free_bufs(struct device *dev, struct brcmuart_priv *priv)
 875 {
 876         if (priv->rx_bufs)
 877                 dma_free_coherent(dev, priv->rx_size, priv->rx_bufs,
 878                                   priv->rx_addr);
 879         if (priv->tx_buf)
 880                 dma_free_coherent(dev, priv->tx_size, priv->tx_buf,
 881                                   priv->tx_addr);
 882 }
 883
 884 static void brcmuart_throttle(struct uart_port *port)
 885 {
 886         struct brcmuart_priv *priv = port->private_data;
 887
 888         udma_writel(priv, REGS_DMA_ISR, UDMA_INTR_MASK_SET, UDMA_RX_INTERRUPTS);
 889 }
 890
 891 static void brcmuart_unthrottle(struct uart_port *port)
 892 {
 893         struct brcmuart_priv *priv = port->private_data;
 894
 895         udma_writel(priv, REGS_DMA_ISR, UDMA_INTR_MASK_CLEAR,
 896                     UDMA_RX_INTERRUPTS);
 897 }
 898
 899 static int debugfs_stats_show(struct seq_file *s, void *unused)
 900 {
 901         struct brcmuart_priv *priv = s->private;
 902
 903         seq_printf(s, "rx_err:\t\t\t\t%u\n",
 904                    priv->rx_err);
 905         seq_printf(s, "rx_timeout:\t\t\t%u\n",
 906                    priv->rx_timeout);
 907         seq_printf(s, "rx_abort:\t\t\t%u\n",
 908                    priv->rx_abort);
 909         seq_printf(s, "rx_bad_timeout_late_char:\t%u\n",
 910                    priv->rx_bad_timeout_late_char);
 911         seq_printf(s, "rx_bad_timeout_no_char:\t\t%u\n",
 912                    priv->rx_bad_timeout_no_char);
 913         seq_printf(s, "rx_missing_close_timeout:\t%u\n",
 914                    priv->rx_missing_close_timeout);
 915         if (priv->dma_enabled) {
 916                 seq_printf(s, "dma_rx_partial_buf:\t\t%llu\n",
 917                            priv->dma_rx_partial_buf);
 918                 seq_printf(s, "dma_rx_full_buf:\t\t%llu\n",
 919                            priv->dma_rx_full_buf);
 920         }
 921         return 0;
 922 }
 923 DEFINE_SHOW_ATTRIBUTE(debugfs_stats);
 924
 925 static void brcmuart_init_debugfs(struct brcmuart_priv *priv,
 926                                   const char *device)
 927 {
 928         priv->debugfs_dir = debugfs_create_dir(device, brcmuart_debugfs_root);
 929         debugfs_create_file("stats", 0444, priv->debugfs_dir, priv,
 930                             &debugfs_stats_fops);
 931 }
 932
 933
 934 static int brcmuart_probe(struct platform_device *pdev)
 935 {
 936         struct resource *regs;
 937         struct device_node *np = pdev->dev.of_node;
 938         const struct of_device_id *of_id = NULL;
 939         struct uart_8250_port *new_port;
 940         struct device *dev = &pdev->dev;
 941         struct brcmuart_priv *priv;
 942         struct clk *baud_mux_clk;
 943         struct uart_8250_port up;
 944         int irq;
 945         void __iomem *membase = NULL;
 946         resource_size_t mapbase = 0;
 947         u32 clk_rate = 0;
 948         int ret;
 949         int x;
 950         int dma_irq;
 951         static const char * const reg_names[REGS_MAX] = {
 952                 "uart", "dma_rx", "dma_tx", "dma_intr2", "dma_arb"
 953         };
 954
 955         irq = platform_get_irq(pdev, 0);
 956         if (irq < 0)
 957                 return irq;
 958         priv = devm_kzalloc(dev, sizeof(struct brcmuart_priv),
 959                         GFP_KERNEL);
 960         if (!priv)
 961                 return -ENOMEM;
 962
 963         of_id = of_match_node(brcmuart_dt_ids, np);
 964         if (!of_id || !of_id->data)
 965                 priv->rate_table = brcmstb_rate_table;
 966         else
 967                 priv->rate_table = of_id->data;
 968
 969         for (x = 0; x < REGS_MAX; x++) {
 970                 regs = platform_get_resource_byname(pdev, IORESOURCE_MEM,
 971                                                 reg_names[x]);
 972                 if (!regs)
 973                         break;
 974                 priv->regs[x] = devm_ioremap(dev, regs->start,
 975                                              resource_size(regs));
 976                 if (!priv->regs[x])
 977                         return -ENOMEM;
 978                 if (x == REGS_8250) {
 979                         mapbase = regs->start;
 980                         membase = priv->regs[x];
 981                 }
 982         }
 983
 984         /* We should have just the uart base registers or all the registers */
 985         if (x != 1 && x != REGS_MAX) {
 986                 dev_warn(dev, "%s registers not specified\n", reg_names[x]);
 987                 return -EINVAL;
 988         }
 989
 990         /* if the DMA registers were specified, try to enable DMA */
 991         if (x > REGS_DMA_RX) {
 992                 if (brcmuart_arbitration(priv, 1) == 0) {
 993                         u32 txrev = 0;
 994                         u32 rxrev = 0;
 995
 996                         txrev = udma_readl(priv, REGS_DMA_RX, UDMA_RX_REVISION);
 997                         rxrev = udma_readl(priv, REGS_DMA_TX, UDMA_TX_REVISION);
 998                         if ((txrev >= UDMA_TX_REVISION_REQUIRED) &&
 999                                 (rxrev >= UDMA_RX_REVISION_REQUIRED)) {
1000
1001                                 /* Enable the use of the DMA hardware */
1002                                 priv->dma_enabled = true;
1003                         } else {
1004                                 brcmuart_arbitration(priv, 0);
1005                                 dev_err(dev,
1006                                         "Unsupported DMA Hardware Revision\n");
1007                         }
1008                 } else {
1009                         dev_err(dev,
1010                                 "Timeout arbitrating for UART DMA hardware\n");
1011                 }
1012         }
1013
1014         of_property_read_u32(np, "clock-frequency", &clk_rate);
1015
1016         /* See if a Baud clock has been specified */
1017         baud_mux_clk = of_clk_get_by_name(np, "sw_baud");
1018         if (IS_ERR(baud_mux_clk)) {
1019                 if (PTR_ERR(baud_mux_clk) == -EPROBE_DEFER)
1020                         return -EPROBE_DEFER;
1021                 dev_dbg(dev, "BAUD MUX clock not specified\n");
1022         } else {
1023                 dev_dbg(dev, "BAUD MUX clock found\n");
1024                 ret = clk_prepare_enable(baud_mux_clk);
1025                 if (ret)
1026                         return ret;
1027                 priv->baud_mux_clk = baud_mux_clk;
1028                 init_real_clk_rates(dev, priv);
1029                 clk_rate = priv->default_mux_rate;
1030         }
1031
1032         if (clk_rate == 0) {
1033                 dev_err(dev, "clock-frequency or clk not defined\n");
1034                 return -EINVAL;
1035         }
1036
1037         dev_dbg(dev, "DMA is %senabled\n", priv->dma_enabled ? "" : "not ");
1038
1039         memset(&up, 0, sizeof(up));
1040         up.port.type = PORT_16550A;
1041         up.port.uartclk = clk_rate;
1042         up.port.dev = dev;
1043         up.port.mapbase = mapbase;
1044         up.port.membase = membase;
1045         up.port.irq = irq;
1046         up.port.handle_irq = brcmuart_handle_irq;
1047         up.port.regshift = 2;
1048         up.port.iotype = of_device_is_big_endian(np) ?
1049                 UPIO_MEM32BE : UPIO_MEM32;
1050         up.port.flags = UPF_SHARE_IRQ | UPF_BOOT_AUTOCONF
1051                 | UPF_FIXED_PORT | UPF_FIXED_TYPE;
1052         up.port.dev = dev;
1053         up.port.private_data = priv;
1054         up.capabilities = UART_CAP_FIFO | UART_CAP_AFE;
1055         up.port.fifosize = 32;
1056
1057         /* Check for a fixed line number */
1058         ret = of_alias_get_id(np, "serial");
1059         if (ret >= 0)
1060                 up.port.line = ret;
1061
1062         /* setup HR timer */
1063         hrtimer_init(&priv->hrt, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
1064         priv->hrt.function = brcmuart_hrtimer_func;
1065
1066         up.port.shutdown = brcmuart_shutdown;
1067         up.port.startup = brcmuart_startup;
1068         up.port.throttle = brcmuart_throttle;
1069         up.port.unthrottle = brcmuart_unthrottle;
1070         up.port.set_termios = brcmstb_set_termios;
1071
1072         if (priv->dma_enabled) {
1073                 priv->rx_size = RX_BUF_SIZE * RX_BUFS_COUNT;
1074                 priv->rx_bufs = dma_alloc_coherent(dev,
1075                                                    priv->rx_size,
1076                                                    &priv->rx_addr, GFP_KERNEL);
1077                 if (!priv->rx_bufs) {
1078                         ret = -ENOMEM;
1079                         goto err;
1080                 }
1081                 priv->tx_size = UART_XMIT_SIZE;
1082                 priv->tx_buf = dma_alloc_coherent(dev,
1083                                                   priv->tx_size,
1084                                                   &priv->tx_addr, GFP_KERNEL);
1085                 if (!priv->tx_buf) {
1086                         ret = -ENOMEM;
1087                         goto err;
1088                 }
1089         }
1090
1091         ret = serial8250_register_8250_port(&up);
1092         if (ret < 0) {
1093                 dev_err(dev, "unable to register 8250 port\n");
1094                 goto err;
1095         }
1096         priv->line = ret;
1097         new_port = serial8250_get_port(ret);
1098         priv->up = &new_port->port;
1099         if (priv->dma_enabled) {
1100                 dma_irq = platform_get_irq_byname(pdev,  "dma");
1101                 if (dma_irq < 0) {
1102                         ret = dma_irq;
1103                         dev_err(dev, "no IRQ resource info\n");
1104                         goto err1;
1105                 }
1106                 ret = devm_request_irq(dev, dma_irq, brcmuart_isr,
1107                                 IRQF_SHARED, "uart DMA irq", &new_port->port);
1108                 if (ret) {
1109                         dev_err(dev, "unable to register IRQ handler\n");
1110                         goto err1;
1111                 }
1112         }
1113         platform_set_drvdata(pdev, priv);
1114         brcmuart_init_debugfs(priv, dev_name(&pdev->dev));
1115         return 0;
1116
1117 err1:
1118         serial8250_unregister_port(priv->line);
1119 err:
1120         brcmuart_free_bufs(dev, priv);
1121         brcmuart_arbitration(priv, 0);
1122         return ret;
1123 }
1124
1125 static int brcmuart_remove(struct platform_device *pdev)
1126 {
1127         struct brcmuart_priv *priv = platform_get_drvdata(pdev);
1128
1129         debugfs_remove_recursive(priv->debugfs_dir);
1130         hrtimer_cancel(&priv->hrt);
1131         serial8250_unregister_port(priv->line);
1132         brcmuart_free_bufs(&pdev->dev, priv);
1133         brcmuart_arbitration(priv, 0);
1134         return 0;
1135 }
1136
1137 static int __maybe_unused brcmuart_suspend(struct device *dev)
1138 {
1139         struct brcmuart_priv *priv = dev_get_drvdata(dev);
1140         struct uart_8250_port *up = serial8250_get_port(priv->line);
1141         struct uart_port *port = &up->port;
1142
1143         serial8250_suspend_port(priv->line);
1144         clk_disable_unprepare(priv->baud_mux_clk);
1145
1146         /*
1147          * This will prevent resume from enabling RTS before the
1148          *  baud rate has been resored.
1149          */
1150         priv->saved_mctrl = port->mctrl;
1151         port->mctrl = 0;
1152
1153         return 0;
1154 }
1155
1156 static int __maybe_unused brcmuart_resume(struct device *dev)
1157 {
1158         struct brcmuart_priv *priv = dev_get_drvdata(dev);
1159         struct uart_8250_port *up = serial8250_get_port(priv->line);
1160         struct uart_port *port = &up->port;
1161         int ret;
1162
1163         ret = clk_prepare_enable(priv->baud_mux_clk);
1164         if (ret)
1165                 dev_err(dev, "Error enabling BAUD MUX clock\n");
1166
1167         /*
1168          * The hardware goes back to it's default after suspend
1169          * so get the "clk" back in sync.
1170          */
1171         ret = clk_set_rate(priv->baud_mux_clk, priv->default_mux_rate);
1172         if (ret)
1173                 dev_err(dev, "Error restoring default BAUD MUX clock\n");
1174         if (priv->dma_enabled) {
1175                 if (brcmuart_arbitration(priv, 1)) {
1176                         dev_err(dev, "Timeout arbitrating for DMA hardware on resume\n");
1177                         return(-EBUSY);
1178                 }
1179                 brcmuart_init_dma_hardware(priv);
1180                 start_rx_dma(serial8250_get_port(priv->line));
1181         }
1182         serial8250_resume_port(priv->line);
1183         port->mctrl = priv->saved_mctrl;
1184         return 0;
1185 }
1186
1187 static const struct dev_pm_ops brcmuart_dev_pm_ops = {
1188         SET_SYSTEM_SLEEP_PM_OPS(brcmuart_suspend, brcmuart_resume)
1189 };
1190
1191 static struct platform_driver brcmuart_platform_driver = {
1192         .driver = {
1193                 .name   = "bcm7271-uart",
1194                 .pm             = &brcmuart_dev_pm_ops,
1195                 .of_match_table = brcmuart_dt_ids,
1196         },
1197         .probe          = brcmuart_probe,
1198         .remove         = brcmuart_remove,
1199 };
1200
1201 static int __init brcmuart_init(void)
1202 {
1203         brcmuart_debugfs_root = debugfs_create_dir(
1204                 brcmuart_platform_driver.driver.name, NULL);
1205         return platform_driver_register(&brcmuart_platform_driver);
1206 }
1207 module_init(brcmuart_init);
1208
1209 static void __exit brcmuart_deinit(void)
1210 {
1211         platform_driver_unregister(&brcmuart_platform_driver);
1212         debugfs_remove_recursive(brcmuart_debugfs_root);
1213 }
1214 module_exit(brcmuart_deinit);
1215
1216 MODULE_AUTHOR("Al Cooper");
1217 MODULE_DESCRIPTION("Broadcom NS16550A compatible serial port driver");
1218 MODULE_LICENSE("GPL v2");