drivers/bluetooth/hci_bcm4377.c

   1 // SPDX-License-Identifier: GPL-2.0-only OR MIT
   2 /*
   3  * Bluetooth HCI driver for Broadcom 4377/4378/4387 devices attached via PCIe
   4  *
   5  * Copyright (C) The Asahi Linux Contributors
   6  */
   7
   8 #include <linux/async.h>
   9 #include <linux/bitfield.h>
  10 #include <linux/completion.h>
  11 #include <linux/dma-mapping.h>
  12 #include <linux/dmi.h>
  13 #include <linux/firmware.h>
  14 #include <linux/module.h>
  15 #include <linux/msi.h>
  16 #include <linux/of.h>
  17 #include <linux/pci.h>
  18 #include <linux/printk.h>
  19
  20 #include <asm/unaligned.h>
  21
  22 #include <net/bluetooth/bluetooth.h>
  23 #include <net/bluetooth/hci_core.h>
  24
  25 enum bcm4377_chip {
  26         BCM4377 = 0,
  27         BCM4378,
  28         BCM4387,
  29 };
  30
  31 #define BCM4377_DEVICE_ID 0x5fa0
  32 #define BCM4378_DEVICE_ID 0x5f69
  33 #define BCM4387_DEVICE_ID 0x5f71
  34
  35 #define BCM4377_TIMEOUT 1000
  36
  37 /*
  38  * These devices only support DMA transactions inside a 32bit window
  39  * (possibly to avoid 64 bit arithmetic). The window size cannot exceed
  40  * 0xffffffff but is always aligned down to the previous 0x200 byte boundary
  41  * which effectively limits the window to [start, start+0xfffffe00].
  42  * We just limit the DMA window to [0, 0xfffffe00] to make sure we don't
  43  * run into this limitation.
  44  */
  45 #define BCM4377_DMA_MASK 0xfffffe00
  46
  47 #define BCM4377_PCIECFG_BAR0_WINDOW1       0x80
  48 #define BCM4377_PCIECFG_BAR0_WINDOW2       0x70
  49 #define BCM4377_PCIECFG_BAR0_CORE2_WINDOW1 0x74
  50 #define BCM4377_PCIECFG_BAR0_CORE2_WINDOW2 0x78
  51 #define BCM4377_PCIECFG_BAR2_WINDOW        0x84
  52
  53 #define BCM4377_PCIECFG_BAR0_CORE2_WINDOW1_DEFAULT 0x18011000
  54 #define BCM4377_PCIECFG_BAR2_WINDOW_DEFAULT        0x19000000
  55
  56 #define BCM4377_PCIECFG_SUBSYSTEM_CTRL 0x88
  57
  58 #define BCM4377_BAR0_FW_DOORBELL 0x140
  59 #define BCM4377_BAR0_RTI_CONTROL 0x144
  60
  61 #define BCM4377_BAR0_SLEEP_CONTROL            0x150
  62 #define BCM4377_BAR0_SLEEP_CONTROL_UNQUIESCE  0
  63 #define BCM4377_BAR0_SLEEP_CONTROL_AWAKE      2
  64 #define BCM4377_BAR0_SLEEP_CONTROL_QUIESCE    3
  65
  66 #define BCM4377_BAR0_DOORBELL       0x174
  67 #define BCM4377_BAR0_DOORBELL_VALUE GENMASK(31, 16)
  68 #define BCM4377_BAR0_DOORBELL_IDX   GENMASK(15, 8)
  69 #define BCM4377_BAR0_DOORBELL_RING  BIT(5)
  70
  71 #define BCM4377_BAR0_HOST_WINDOW_LO   0x590
  72 #define BCM4377_BAR0_HOST_WINDOW_HI   0x594
  73 #define BCM4377_BAR0_HOST_WINDOW_SIZE 0x598
  74
  75 #define BCM4377_BAR2_BOOTSTAGE 0x200454
  76
  77 #define BCM4377_BAR2_FW_LO   0x200478
  78 #define BCM4377_BAR2_FW_HI   0x20047c
  79 #define BCM4377_BAR2_FW_SIZE 0x200480
  80
  81 #define BCM4377_BAR2_CONTEXT_ADDR_LO 0x20048c
  82 #define BCM4377_BAR2_CONTEXT_ADDR_HI 0x200450
  83
  84 #define BCM4377_BAR2_RTI_STATUS      0x20045c
  85 #define BCM4377_BAR2_RTI_WINDOW_LO   0x200494
  86 #define BCM4377_BAR2_RTI_WINDOW_HI   0x200498
  87 #define BCM4377_BAR2_RTI_WINDOW_SIZE 0x20049c
  88
  89 #define BCM4377_OTP_SIZE          0xe0
  90 #define BCM4377_OTP_SYS_VENDOR    0x15
  91 #define BCM4377_OTP_CIS           0x80
  92 #define BCM4377_OTP_VENDOR_HDR    0x00000008
  93 #define BCM4377_OTP_MAX_PARAM_LEN 16
  94
  95 #define BCM4377_N_TRANSFER_RINGS   9
  96 #define BCM4377_N_COMPLETION_RINGS 6
  97
  98 #define BCM4377_MAX_RING_SIZE 256
  99
 100 #define BCM4377_MSGID_GENERATION GENMASK(15, 8)
 101 #define BCM4377_MSGID_ID         GENMASK(7, 0)
 102
 103 #define BCM4377_RING_N_ENTRIES 128
 104
 105 #define BCM4377_CONTROL_MSG_SIZE                   0x34
 106 #define BCM4377_XFER_RING_MAX_INPLACE_PAYLOAD_SIZE (4 * 0xff)
 107
 108 #define MAX_ACL_PAYLOAD_SIZE   (HCI_MAX_FRAME_SIZE + HCI_ACL_HDR_SIZE)
 109 #define MAX_SCO_PAYLOAD_SIZE   (HCI_MAX_SCO_SIZE + HCI_SCO_HDR_SIZE)
 110 #define MAX_EVENT_PAYLOAD_SIZE (HCI_MAX_EVENT_SIZE + HCI_EVENT_HDR_SIZE)
 111
 112 enum bcm4377_otp_params_type {
 113         BCM4377_OTP_BOARD_PARAMS,
 114         BCM4377_OTP_CHIP_PARAMS
 115 };
 116
 117 enum bcm4377_transfer_ring_id {
 118         BCM4377_XFER_RING_CONTROL = 0,
 119         BCM4377_XFER_RING_HCI_H2D = 1,
 120         BCM4377_XFER_RING_HCI_D2H = 2,
 121         BCM4377_XFER_RING_SCO_H2D = 3,
 122         BCM4377_XFER_RING_SCO_D2H = 4,
 123         BCM4377_XFER_RING_ACL_H2D = 5,
 124         BCM4377_XFER_RING_ACL_D2H = 6,
 125 };
 126
 127 enum bcm4377_completion_ring_id {
 128         BCM4377_ACK_RING_CONTROL = 0,
 129         BCM4377_ACK_RING_HCI_ACL = 1,
 130         BCM4377_EVENT_RING_HCI_ACL = 2,
 131         BCM4377_ACK_RING_SCO = 3,
 132         BCM4377_EVENT_RING_SCO = 4,
 133 };
 134
 135 enum bcm4377_doorbell {
 136         BCM4377_DOORBELL_CONTROL = 0,
 137         BCM4377_DOORBELL_HCI_H2D = 1,
 138         BCM4377_DOORBELL_HCI_D2H = 2,
 139         BCM4377_DOORBELL_ACL_H2D = 3,
 140         BCM4377_DOORBELL_ACL_D2H = 4,
 141         BCM4377_DOORBELL_SCO = 6,
 142 };
 143
 144 /*
 145  * Transfer ring entry
 146  *
 147  * flags: Flags to indicate if the payload is appended or mapped
 148  * len: Payload length
 149  * payload: Optional payload DMA address
 150  * id: Message id to recognize the answer in the completion ring entry
 151  */
 152 struct bcm4377_xfer_ring_entry {
 153 #define BCM4377_XFER_RING_FLAG_PAYLOAD_MAPPED    BIT(0)
 154 #define BCM4377_XFER_RING_FLAG_PAYLOAD_IN_FOOTER BIT(1)
 155         u8 flags;
 156         __le16 len;
 157         u8 _unk0;
 158         __le64 payload;
 159         __le16 id;
 160         u8 _unk1[2];
 161 } __packed;
 162 static_assert(sizeof(struct bcm4377_xfer_ring_entry) == 0x10);
 163
 164 /*
 165  * Completion ring entry
 166  *
 167  * flags: Flags to indicate if the payload is appended or mapped. If the payload
 168  *        is mapped it can be found in the buffer of the corresponding transfer
 169  *        ring message.
 170  * ring_id: Transfer ring ID which required this message
 171  * msg_id: Message ID specified in transfer ring entry
 172  * len: Payload length
 173  */
 174 struct bcm4377_completion_ring_entry {
 175         u8 flags;
 176         u8 _unk0;
 177         __le16 ring_id;
 178         __le16 msg_id;
 179         __le32 len;
 180         u8 _unk1[6];
 181 } __packed;
 182 static_assert(sizeof(struct bcm4377_completion_ring_entry) == 0x10);
 183
 184 enum bcm4377_control_message_type {
 185         BCM4377_CONTROL_MSG_CREATE_XFER_RING = 1,
 186         BCM4377_CONTROL_MSG_CREATE_COMPLETION_RING = 2,
 187         BCM4377_CONTROL_MSG_DESTROY_XFER_RING = 3,
 188         BCM4377_CONTROL_MSG_DESTROY_COMPLETION_RING = 4,
 189 };
 190
 191 /*
 192  * Control message used to create a completion ring
 193  *
 194  * msg_type: Must be BCM4377_CONTROL_MSG_CREATE_COMPLETION_RING
 195  * header_size: Unknown, but probably reserved space in front of the entry
 196  * footer_size: Number of 32 bit words reserved for payloads after the entry
 197  * id/id_again: Completion ring index
 198  * ring_iova: DMA address of the ring buffer
 199  * n_elements: Number of elements inside the ring buffer
 200  * msi: MSI index, doesn't work for all rings though and should be zero
 201  * intmod_delay: Unknown delay
 202  * intmod_bytes: Unknown
 203  */
 204 struct bcm4377_create_completion_ring_msg {
 205         u8 msg_type;
 206         u8 header_size;
 207         u8 footer_size;
 208         u8 _unk0;
 209         __le16 id;
 210         __le16 id_again;
 211         __le64 ring_iova;
 212         __le16 n_elements;
 213         __le32 unk;
 214         u8 _unk1[6];
 215         __le16 msi;
 216         __le16 intmod_delay;
 217         __le32 intmod_bytes;
 218         __le16 _unk2;
 219         __le32 _unk3;
 220         u8 _unk4[10];
 221 } __packed;
 222 static_assert(sizeof(struct bcm4377_create_completion_ring_msg) ==
 223               BCM4377_CONTROL_MSG_SIZE);
 224
 225 /*
 226  * Control ring message used to destroy a completion ring
 227  *
 228  * msg_type: Must be BCM4377_CONTROL_MSG_DESTROY_COMPLETION_RING
 229  * ring_id: Completion ring to be destroyed
 230  */
 231 struct bcm4377_destroy_completion_ring_msg {
 232         u8 msg_type;
 233         u8 _pad0;
 234         __le16 ring_id;
 235         u8 _pad1[48];
 236 } __packed;
 237 static_assert(sizeof(struct bcm4377_destroy_completion_ring_msg) ==
 238               BCM4377_CONTROL_MSG_SIZE);
 239
 240 /*
 241  * Control message used to create a transfer ring
 242  *
 243  * msg_type: Must be BCM4377_CONTROL_MSG_CREATE_XFER_RING
 244  * header_size: Number of 32 bit words reserved for unknown content before the
 245  *              entry
 246  * footer_size: Number of 32 bit words reserved for payloads after the entry
 247  * ring_id/ring_id_again: Transfer ring index
 248  * ring_iova: DMA address of the ring buffer
 249  * n_elements: Number of elements inside the ring buffer
 250  * completion_ring_id: Completion ring index for acknowledgements and events
 251  * doorbell: Doorbell index used to notify device of new entries
 252  * flags: Transfer ring flags
 253  *          - virtual: set if there is no associated shared memory and only the
 254  *                     corresponding completion ring is used
 255  *          - sync: only set for the SCO rings
 256  */
 257 struct bcm4377_create_transfer_ring_msg {
 258         u8 msg_type;
 259         u8 header_size;
 260         u8 footer_size;
 261         u8 _unk0;
 262         __le16 ring_id;
 263         __le16 ring_id_again;
 264         __le64 ring_iova;
 265         u8 _unk1[8];
 266         __le16 n_elements;
 267         __le16 completion_ring_id;
 268         __le16 doorbell;
 269 #define BCM4377_XFER_RING_FLAG_VIRTUAL BIT(7)
 270 #define BCM4377_XFER_RING_FLAG_SYNC    BIT(8)
 271         __le16 flags;
 272         u8 _unk2[20];
 273 } __packed;
 274 static_assert(sizeof(struct bcm4377_create_transfer_ring_msg) ==
 275               BCM4377_CONTROL_MSG_SIZE);
 276
 277 /*
 278  * Control ring message used to destroy a transfer ring
 279  *
 280  * msg_type: Must be BCM4377_CONTROL_MSG_DESTROY_XFER_RING
 281  * ring_id: Transfer ring to be destroyed
 282  */
 283 struct bcm4377_destroy_transfer_ring_msg {
 284         u8 msg_type;
 285         u8 _pad0;
 286         __le16 ring_id;
 287         u8 _pad1[48];
 288 } __packed;
 289 static_assert(sizeof(struct bcm4377_destroy_transfer_ring_msg) ==
 290               BCM4377_CONTROL_MSG_SIZE);
 291
 292 /*
 293  * "Converged IPC" context struct used to make the device aware of all other
 294  * shared memory structures. A pointer to this structure is configured inside a
 295  * MMIO register.
 296  *
 297  * version: Protocol version, must be 2.
 298  * size: Size of this structure, must be 0x68.
 299  * enabled_caps: Enabled capabilities. Unknown bitfield but should be 2.
 300  * peripheral_info_addr: DMA address for a 0x20 buffer to which the device will
 301  *                       write unknown contents
 302  * {completion,xfer}_ring_{tails,heads}_addr: DMA pointers to ring heads/tails
 303  * n_completion_rings: Number of completion rings, the firmware only works if
 304  *                     this is set to BCM4377_N_COMPLETION_RINGS.
 305  * n_xfer_rings: Number of transfer rings, the firmware only works if
 306  *               this is set to BCM4377_N_TRANSFER_RINGS.
 307  * control_completion_ring_addr: Control completion ring buffer DMA address
 308  * control_xfer_ring_addr: Control transfer ring buffer DMA address
 309  * control_xfer_ring_n_entries: Number of control transfer ring entries
 310  * control_completion_ring_n_entries: Number of control completion ring entries
 311  * control_xfer_ring_doorbell: Control transfer ring doorbell
 312  * control_completion_ring_doorbell: Control completion ring doorbell,
 313  *                                   must be set to 0xffff
 314  * control_xfer_ring_msi: Control completion ring MSI index, must be 0
 315  * control_completion_ring_msi: Control completion ring MSI index, must be 0.
 316  * control_xfer_ring_header_size: Number of 32 bit words reserved in front of
 317  *                                every control transfer ring entry
 318  * control_xfer_ring_footer_size: Number of 32 bit words reserved after every
 319  *                                control transfer ring entry
 320  * control_completion_ring_header_size: Number of 32 bit words reserved in front
 321  *                                      of every control completion ring entry
 322  * control_completion_ring_footer_size: Number of 32 bit words reserved after
 323  *                                      every control completion ring entry
 324  * scratch_pad: Optional scratch pad DMA address
 325  * scratch_pad_size: Scratch pad size
 326  */
 327 struct bcm4377_context {
 328         __le16 version;
 329         __le16 size;
 330         __le32 enabled_caps;
 331
 332         __le64 peripheral_info_addr;
 333
 334         /* ring heads and tails */
 335         __le64 completion_ring_heads_addr;
 336         __le64 xfer_ring_tails_addr;
 337         __le64 completion_ring_tails_addr;
 338         __le64 xfer_ring_heads_addr;
 339         __le16 n_completion_rings;
 340         __le16 n_xfer_rings;
 341
 342         /* control ring configuration */
 343         __le64 control_completion_ring_addr;
 344         __le64 control_xfer_ring_addr;
 345         __le16 control_xfer_ring_n_entries;
 346         __le16 control_completion_ring_n_entries;
 347         __le16 control_xfer_ring_doorbell;
 348         __le16 control_completion_ring_doorbell;
 349         __le16 control_xfer_ring_msi;
 350         __le16 control_completion_ring_msi;
 351         u8 control_xfer_ring_header_size;
 352         u8 control_xfer_ring_footer_size;
 353         u8 control_completion_ring_header_size;
 354         u8 control_completion_ring_footer_size;
 355
 356         __le16 _unk0;
 357         __le16 _unk1;
 358
 359         __le64 scratch_pad;
 360         __le32 scratch_pad_size;
 361
 362         __le32 _unk3;
 363 } __packed;
 364 static_assert(sizeof(struct bcm4377_context) == 0x68);
 365
 366 #define BCM4378_CALIBRATION_CHUNK_SIZE 0xe6
 367 struct bcm4378_hci_send_calibration_cmd {
 368         u8 unk;
 369         __le16 blocks_left;
 370         u8 data[BCM4378_CALIBRATION_CHUNK_SIZE];
 371 } __packed;
 372
 373 #define BCM4378_PTB_CHUNK_SIZE 0xcf
 374 struct bcm4378_hci_send_ptb_cmd {
 375         __le16 blocks_left;
 376         u8 data[BCM4378_PTB_CHUNK_SIZE];
 377 } __packed;
 378
 379 /*
 380  * Shared memory structure used to store the ring head and tail pointers.
 381  */
 382 struct bcm4377_ring_state {
 383         __le16 completion_ring_head[BCM4377_N_COMPLETION_RINGS];
 384         __le16 completion_ring_tail[BCM4377_N_COMPLETION_RINGS];
 385         __le16 xfer_ring_head[BCM4377_N_TRANSFER_RINGS];
 386         __le16 xfer_ring_tail[BCM4377_N_TRANSFER_RINGS];
 387 };
 388
 389 /*
 390  * A transfer ring can be used in two configurations:
 391  *  1) Send control or HCI messages to the device which are then acknowledged
 392  *     in the corresponding completion ring
 393  *  2) Receiving HCI frames from the devices. In this case the transfer ring
 394  *     itself contains empty messages that are acknowledged once data is
 395  *     available from the device. If the payloads fit inside the footers
 396  *     of the completion ring the transfer ring can be configured to be
 397  *     virtual such that it has no ring buffer.
 398  *
 399  * ring_id: ring index hardcoded in the firmware
 400  * doorbell: doorbell index to notify device of new entries
 401  * payload_size: optional in-place payload size
 402  * mapped_payload_size: optional out-of-place payload size
 403  * completion_ring: index of corresponding completion ring
 404  * n_entries: number of entries inside this ring
 405  * generation: ring generation; incremented on hci_open to detect stale messages
 406  * sync: set to true for SCO rings
 407  * virtual: set to true if this ring has no entries and is just required to
 408  *          setup a corresponding completion ring for device->host messages
 409  * d2h_buffers_only: set to true if this ring is only used to provide large
 410  *                   buffers used by device->host messages in the completion
 411  *                   ring
 412  * allow_wait: allow to wait for messages to be acknowledged
 413  * enabled: true once the ring has been created and can be used
 414  * ring: ring buffer for entries (struct bcm4377_xfer_ring_entry)
 415  * ring_dma: DMA address for ring entry buffer
 416  * payloads: payload buffer for mapped_payload_size payloads
 417  * payloads_dma:DMA address for payload buffer
 418  * events: pointer to array of completions if waiting is allowed
 419  * msgids: bitmap to keep track of used message ids
 420  * lock: Spinlock to protect access to ring structurs used in the irq handler
 421  */
 422 struct bcm4377_transfer_ring {
 423         enum bcm4377_transfer_ring_id ring_id;
 424         enum bcm4377_doorbell doorbell;
 425         size_t payload_size;
 426         size_t mapped_payload_size;
 427         u8 completion_ring;
 428         u16 n_entries;
 429         u8 generation;
 430
 431         bool sync;
 432         bool virtual;
 433         bool d2h_buffers_only;
 434         bool allow_wait;
 435         bool enabled;
 436
 437         void *ring;
 438         dma_addr_t ring_dma;
 439
 440         void *payloads;
 441         dma_addr_t payloads_dma;
 442
 443         struct completion **events;
 444         DECLARE_BITMAP(msgids, BCM4377_MAX_RING_SIZE);
 445         spinlock_t lock;
 446 };
 447
 448 /*
 449  * A completion ring can be either used to either acknowledge messages sent in
 450  * the corresponding transfer ring or to receive messages associated with the
 451  * transfer ring. When used to receive messages the transfer ring either
 452  * has no ring buffer and is only advanced ("virtual transfer ring") or it
 453  * only contains empty DMA buffers to be used for the payloads.
 454  *
 455  * ring_id: completion ring id, hardcoded in firmware
 456  * payload_size: optional payload size after each entry
 457  * delay: unknown delay
 458  * n_entries: number of entries in this ring
 459  * enabled: true once the ring has been created and can be used
 460  * ring: ring buffer for entries (struct bcm4377_completion_ring_entry)
 461  * ring_dma: DMA address of ring buffer
 462  * transfer_rings: bitmap of corresponding transfer ring ids
 463  */
 464 struct bcm4377_completion_ring {
 465         enum bcm4377_completion_ring_id ring_id;
 466         u16 payload_size;
 467         u16 delay;
 468         u16 n_entries;
 469         bool enabled;
 470
 471         void *ring;
 472         dma_addr_t ring_dma;
 473
 474         unsigned long transfer_rings;
 475 };
 476
 477 struct bcm4377_data;
 478
 479 /*
 480  * Chip-specific configuration struct
 481  *
 482  * id: Chip id (e.g. 0x4377 for BCM4377)
 483  * otp_offset: Offset to the start of the OTP inside BAR0
 484  * bar0_window1: Backplane address mapped to the first window in BAR0
 485  * bar0_window2: Backplane address mapped to the second window in BAR0
 486  * bar0_core2_window2: Optional backplane address mapped to the second core's
 487  *                     second window in BAR0
 488  * has_bar0_core2_window2: Set to true if this chip requires the second core's
 489  *                         second window to be configured
 490  * clear_pciecfg_subsystem_ctrl_bit19: Set to true if bit 19 in the
 491  *                                     vendor-specific subsystem control
 492  *                                     register has to be cleared
 493  * disable_aspm: Set to true if ASPM must be disabled due to hardware errata
 494  * broken_ext_scan: Set to true if the chip erroneously claims to support
 495  *                  extended scanning
 496  * broken_mws_transport_config: Set to true if the chip erroneously claims to
 497  *                              support MWS Transport Configuration
 498  * send_calibration: Optional callback to send calibration data
 499  * send_ptb: Callback to send "PTB" regulatory/calibration data
 500  */
 501 struct bcm4377_hw {
 502         unsigned int id;
 503
 504         u32 otp_offset;
 505
 506         u32 bar0_window1;
 507         u32 bar0_window2;
 508         u32 bar0_core2_window2;
 509
 510         unsigned long has_bar0_core2_window2 : 1;
 511         unsigned long clear_pciecfg_subsystem_ctrl_bit19 : 1;
 512         unsigned long disable_aspm : 1;
 513         unsigned long broken_ext_scan : 1;
 514         unsigned long broken_mws_transport_config : 1;
 515
 516         int (*send_calibration)(struct bcm4377_data *bcm4377);
 517         int (*send_ptb)(struct bcm4377_data *bcm4377,
 518                         const struct firmware *fw);
 519 };
 520
 521 static const struct bcm4377_hw bcm4377_hw_variants[];
 522 static const struct dmi_system_id bcm4377_dmi_board_table[];
 523
 524 /*
 525  * Private struct associated with each device containing global state
 526  *
 527  * pdev: Pointer to associated struct pci_dev
 528  * hdev: Pointer to associated strucy hci_dev
 529  * bar0: iomem pointing to BAR0
 530  * bar1: iomem pointing to BAR2
 531  * bootstage: Current value of the bootstage
 532  * rti_status: Current "RTI" status value
 533  * hw: Pointer to chip-specific struct bcm4377_hw
 534  * taurus_cal_blob: "Taurus" calibration blob used for some chips
 535  * taurus_cal_size: "Taurus" calibration blob size
 536  * taurus_beamforming_cal_blob: "Taurus" beamforming calibration blob used for
 537  *                              some chips
 538  * taurus_beamforming_cal_size: "Taurus" beamforming calibration blob size
 539  * stepping: Chip stepping read from OTP; used for firmware selection
 540  * vendor: Antenna vendor read from OTP; used for firmware selection
 541  * board_type: Board type from FDT or DMI match; used for firmware selection
 542  * event: Event for changed bootstage or rti_status; used for booting firmware
 543  * ctx: "Converged IPC" context
 544  * ctx_dma: "Converged IPC" context DMA address
 545  * ring_state: Shared memory buffer containing ring head and tail indexes
 546  * ring_state_dma: DMA address for ring_state
 547  * {control,hci_acl,sco}_ack_ring: Completion rings used to acknowledge messages
 548  * {hci_acl,sco}_event_ring: Completion rings used for device->host messages
 549  * control_h2d_ring: Transfer ring used for control messages
 550  * {hci,sco,acl}_h2d_ring: Transfer ring used to transfer HCI frames
 551  * {hci,sco,acl}_d2h_ring: Transfer ring used to receive HCI frames in the
 552  *                         corresponding completion ring
 553  */
 554 struct bcm4377_data {
 555         struct pci_dev *pdev;
 556         struct hci_dev *hdev;
 557
 558         void __iomem *bar0;
 559         void __iomem *bar2;
 560
 561         u32 bootstage;
 562         u32 rti_status;
 563
 564         const struct bcm4377_hw *hw;
 565
 566         const void *taurus_cal_blob;
 567         int taurus_cal_size;
 568         const void *taurus_beamforming_cal_blob;
 569         int taurus_beamforming_cal_size;
 570
 571         char stepping[BCM4377_OTP_MAX_PARAM_LEN];
 572         char vendor[BCM4377_OTP_MAX_PARAM_LEN];
 573         const char *board_type;
 574
 575         struct completion event;
 576
 577         struct bcm4377_context *ctx;
 578         dma_addr_t ctx_dma;
 579
 580         struct bcm4377_ring_state *ring_state;
 581         dma_addr_t ring_state_dma;
 582
 583         /*
 584          * The HCI and ACL rings have to be merged because this structure is
 585          * hardcoded in the firmware.
 586          */
 587         struct bcm4377_completion_ring control_ack_ring;
 588         struct bcm4377_completion_ring hci_acl_ack_ring;
 589         struct bcm4377_completion_ring hci_acl_event_ring;
 590         struct bcm4377_completion_ring sco_ack_ring;
 591         struct bcm4377_completion_ring sco_event_ring;
 592
 593         struct bcm4377_transfer_ring control_h2d_ring;
 594         struct bcm4377_transfer_ring hci_h2d_ring;
 595         struct bcm4377_transfer_ring hci_d2h_ring;
 596         struct bcm4377_transfer_ring sco_h2d_ring;
 597         struct bcm4377_transfer_ring sco_d2h_ring;
 598         struct bcm4377_transfer_ring acl_h2d_ring;
 599         struct bcm4377_transfer_ring acl_d2h_ring;
 600 };
 601
 602 static void bcm4377_ring_doorbell(struct bcm4377_data *bcm4377, u8 doorbell,
 603                                   u16 val)
 604 {
 605         u32 db = 0;
 606
 607         db |= FIELD_PREP(BCM4377_BAR0_DOORBELL_VALUE, val);
 608         db |= FIELD_PREP(BCM4377_BAR0_DOORBELL_IDX, doorbell);
 609         db |= BCM4377_BAR0_DOORBELL_RING;
 610
 611         dev_dbg(&bcm4377->pdev->dev, "write %d to doorbell #%d (0x%x)\n", val,
 612                 doorbell, db);
 613         iowrite32(db, bcm4377->bar0 + BCM4377_BAR0_DOORBELL);
 614 }
 615
 616 static int bcm4377_extract_msgid(struct bcm4377_data *bcm4377,
 617                                  struct bcm4377_transfer_ring *ring,
 618                                  u16 raw_msgid, u8 *msgid)
 619 {
 620         u8 generation = FIELD_GET(BCM4377_MSGID_GENERATION, raw_msgid);
 621         *msgid = FIELD_GET(BCM4377_MSGID_ID, raw_msgid);
 622
 623         if (generation != ring->generation) {
 624                 dev_warn(
 625                         &bcm4377->pdev->dev,
 626                         "invalid message generation %d should be %d in entry for ring %d\n",
 627                         generation, ring->generation, ring->ring_id);
 628                 return -EINVAL;
 629         }
 630
 631         if (*msgid >= ring->n_entries) {
 632                 dev_warn(&bcm4377->pdev->dev,
 633                          "invalid message id in entry for ring %d: %d > %d\n",
 634                          ring->ring_id, *msgid, ring->n_entries);
 635                 return -EINVAL;
 636         }
 637
 638         return 0;
 639 }
 640
 641 static void bcm4377_handle_event(struct bcm4377_data *bcm4377,
 642                                  struct bcm4377_transfer_ring *ring,
 643                                  u16 raw_msgid, u8 entry_flags, u8 type,
 644                                  void *payload, size_t len)
 645 {
 646         struct sk_buff *skb;
 647         u16 head;
 648         u8 msgid;
 649         unsigned long flags;
 650
 651         spin_lock_irqsave(&ring->lock, flags);
 652         if (!ring->enabled) {
 653                 dev_warn(&bcm4377->pdev->dev,
 654                          "event for disabled transfer ring %d\n",
 655                          ring->ring_id);
 656                 goto out;
 657         }
 658
 659         if (ring->d2h_buffers_only &&
 660             entry_flags & BCM4377_XFER_RING_FLAG_PAYLOAD_MAPPED) {
 661                 if (bcm4377_extract_msgid(bcm4377, ring, raw_msgid, &msgid))
 662                         goto out;
 663
 664                 if (len > ring->mapped_payload_size) {
 665                         dev_warn(
 666                                 &bcm4377->pdev->dev,
 667                                 "invalid payload len in event for ring %d: %zu > %zu\n",
 668                                 ring->ring_id, len, ring->mapped_payload_size);
 669                         goto out;
 670                 }
 671
 672                 payload = ring->payloads + msgid * ring->mapped_payload_size;
 673         }
 674
 675         skb = bt_skb_alloc(len, GFP_ATOMIC);
 676         if (!skb)
 677                 goto out;
 678
 679         memcpy(skb_put(skb, len), payload, len);
 680         hci_skb_pkt_type(skb) = type;
 681         hci_recv_frame(bcm4377->hdev, skb);
 682
 683 out:
 684         head = le16_to_cpu(bcm4377->ring_state->xfer_ring_head[ring->ring_id]);
 685         head = (head + 1) % ring->n_entries;
 686         bcm4377->ring_state->xfer_ring_head[ring->ring_id] = cpu_to_le16(head);
 687
 688         bcm4377_ring_doorbell(bcm4377, ring->doorbell, head);
 689
 690         spin_unlock_irqrestore(&ring->lock, flags);
 691 }
 692
 693 static void bcm4377_handle_ack(struct bcm4377_data *bcm4377,
 694                                struct bcm4377_transfer_ring *ring,
 695                                u16 raw_msgid)
 696 {
 697         unsigned long flags;
 698         u8 msgid;
 699
 700         spin_lock_irqsave(&ring->lock, flags);
 701
 702         if (bcm4377_extract_msgid(bcm4377, ring, raw_msgid, &msgid))
 703                 goto unlock;
 704
 705         if (!test_bit(msgid, ring->msgids)) {
 706                 dev_warn(
 707                         &bcm4377->pdev->dev,
 708                         "invalid message id in ack for ring %d: %d is not used\n",
 709                         ring->ring_id, msgid);
 710                 goto unlock;
 711         }
 712
 713         if (ring->allow_wait && ring->events[msgid]) {
 714                 complete(ring->events[msgid]);
 715                 ring->events[msgid] = NULL;
 716         }
 717
 718         bitmap_release_region(ring->msgids, msgid, ring->n_entries);
 719
 720 unlock:
 721         spin_unlock_irqrestore(&ring->lock, flags);
 722 }
 723
 724 static void bcm4377_handle_completion(struct bcm4377_data *bcm4377,
 725                                       struct bcm4377_completion_ring *ring,
 726                                       u16 pos)
 727 {
 728         struct bcm4377_completion_ring_entry *entry;
 729         u16 msg_id, transfer_ring;
 730         size_t entry_size, data_len;
 731         void *data;
 732
 733         if (pos >= ring->n_entries) {
 734                 dev_warn(&bcm4377->pdev->dev,
 735                          "invalid offset %d for completion ring %d\n", pos,
 736                          ring->ring_id);
 737                 return;
 738         }
 739
 740         entry_size = sizeof(*entry) + ring->payload_size;
 741         entry = ring->ring + pos * entry_size;
 742         data = ring->ring + pos * entry_size + sizeof(*entry);
 743         data_len = le32_to_cpu(entry->len);
 744         msg_id = le16_to_cpu(entry->msg_id);
 745         transfer_ring = le16_to_cpu(entry->ring_id);
 746
 747         if ((ring->transfer_rings & BIT(transfer_ring)) == 0) {
 748                 dev_warn(
 749                         &bcm4377->pdev->dev,
 750                         "invalid entry at offset %d for transfer ring %d in completion ring %d\n",
 751                         pos, transfer_ring, ring->ring_id);
 752                 return;
 753         }
 754
 755         dev_dbg(&bcm4377->pdev->dev,
 756                 "entry in completion ring %d for transfer ring %d with msg_id %d\n",
 757                 ring->ring_id, transfer_ring, msg_id);
 758
 759         switch (transfer_ring) {
 760         case BCM4377_XFER_RING_CONTROL:
 761                 bcm4377_handle_ack(bcm4377, &bcm4377->control_h2d_ring, msg_id);
 762                 break;
 763         case BCM4377_XFER_RING_HCI_H2D:
 764                 bcm4377_handle_ack(bcm4377, &bcm4377->hci_h2d_ring, msg_id);
 765                 break;
 766         case BCM4377_XFER_RING_SCO_H2D:
 767                 bcm4377_handle_ack(bcm4377, &bcm4377->sco_h2d_ring, msg_id);
 768                 break;
 769         case BCM4377_XFER_RING_ACL_H2D:
 770                 bcm4377_handle_ack(bcm4377, &bcm4377->acl_h2d_ring, msg_id);
 771                 break;
 772
 773         case BCM4377_XFER_RING_HCI_D2H:
 774                 bcm4377_handle_event(bcm4377, &bcm4377->hci_d2h_ring, msg_id,
 775                                      entry->flags, HCI_EVENT_PKT, data,
 776                                      data_len);
 777                 break;
 778         case BCM4377_XFER_RING_SCO_D2H:
 779                 bcm4377_handle_event(bcm4377, &bcm4377->sco_d2h_ring, msg_id,
 780                                      entry->flags, HCI_SCODATA_PKT, data,
 781                                      data_len);
 782                 break;
 783         case BCM4377_XFER_RING_ACL_D2H:
 784                 bcm4377_handle_event(bcm4377, &bcm4377->acl_d2h_ring, msg_id,
 785                                      entry->flags, HCI_ACLDATA_PKT, data,
 786                                      data_len);
 787                 break;
 788
 789         default:
 790                 dev_warn(
 791                         &bcm4377->pdev->dev,
 792                         "entry in completion ring %d for unknown transfer ring %d with msg_id %d\n",
 793                         ring->ring_id, transfer_ring, msg_id);
 794         }
 795 }
 796
 797 static void bcm4377_poll_completion_ring(struct bcm4377_data *bcm4377,
 798                                          struct bcm4377_completion_ring *ring)
 799 {
 800         u16 tail;
 801         __le16 *heads = bcm4377->ring_state->completion_ring_head;
 802         __le16 *tails = bcm4377->ring_state->completion_ring_tail;
 803
 804         if (!ring->enabled)
 805                 return;
 806
 807         tail = le16_to_cpu(tails[ring->ring_id]);
 808         dev_dbg(&bcm4377->pdev->dev,
 809                 "completion ring #%d: head: %d, tail: %d\n", ring->ring_id,
 810                 le16_to_cpu(heads[ring->ring_id]), tail);
 811
 812         while (tail != le16_to_cpu(READ_ONCE(heads[ring->ring_id]))) {
 813                 /*
 814                  * ensure the CPU doesn't speculate through the comparison.
 815                  * otherwise it might already read the (empty) queue entry
 816                  * before the updated head has been loaded and checked.
 817                  */
 818                 dma_rmb();
 819
 820                 bcm4377_handle_completion(bcm4377, ring, tail);
 821
 822                 tail = (tail + 1) % ring->n_entries;
 823                 tails[ring->ring_id] = cpu_to_le16(tail);
 824         }
 825 }
 826
 827 static irqreturn_t bcm4377_irq(int irq, void *data)
 828 {
 829         struct bcm4377_data *bcm4377 = data;
 830         u32 bootstage, rti_status;
 831
 832         bootstage = ioread32(bcm4377->bar2 + BCM4377_BAR2_BOOTSTAGE);
 833         rti_status = ioread32(bcm4377->bar2 + BCM4377_BAR2_RTI_STATUS);
 834
 835         if (bootstage != bcm4377->bootstage ||
 836             rti_status != bcm4377->rti_status) {
 837                 dev_dbg(&bcm4377->pdev->dev,
 838                         "bootstage = %d -> %d, rti state = %d -> %d\n",
 839                         bcm4377->bootstage, bootstage, bcm4377->rti_status,
 840                         rti_status);
 841                 complete(&bcm4377->event);
 842                 bcm4377->bootstage = bootstage;
 843                 bcm4377->rti_status = rti_status;
 844         }
 845
 846         if (rti_status > 2)
 847                 dev_err(&bcm4377->pdev->dev, "RTI status is %d\n", rti_status);
 848
 849         bcm4377_poll_completion_ring(bcm4377, &bcm4377->control_ack_ring);
 850         bcm4377_poll_completion_ring(bcm4377, &bcm4377->hci_acl_event_ring);
 851         bcm4377_poll_completion_ring(bcm4377, &bcm4377->hci_acl_ack_ring);
 852         bcm4377_poll_completion_ring(bcm4377, &bcm4377->sco_ack_ring);
 853         bcm4377_poll_completion_ring(bcm4377, &bcm4377->sco_event_ring);
 854
 855         return IRQ_HANDLED;
 856 }
 857
 858 static int bcm4377_enqueue(struct bcm4377_data *bcm4377,
 859                            struct bcm4377_transfer_ring *ring, void *data,
 860                            size_t len, bool wait)
 861 {
 862         unsigned long flags;
 863         struct bcm4377_xfer_ring_entry *entry;
 864         void *payload;
 865         size_t offset;
 866         u16 head, tail, new_head;
 867         u16 raw_msgid;
 868         int ret, msgid;
 869         DECLARE_COMPLETION_ONSTACK(event);
 870
 871         if (len > ring->payload_size && len > ring->mapped_payload_size) {
 872                 dev_warn(
 873                         &bcm4377->pdev->dev,
 874                         "payload len %zu is too large for ring %d (max is %zu or %zu)\n",
 875                         len, ring->ring_id, ring->payload_size,
 876                         ring->mapped_payload_size);
 877                 return -EINVAL;
 878         }
 879         if (wait && !ring->allow_wait)
 880                 return -EINVAL;
 881         if (ring->virtual)
 882                 return -EINVAL;
 883
 884         spin_lock_irqsave(&ring->lock, flags);
 885
 886         head = le16_to_cpu(bcm4377->ring_state->xfer_ring_head[ring->ring_id]);
 887         tail = le16_to_cpu(bcm4377->ring_state->xfer_ring_tail[ring->ring_id]);
 888
 889         new_head = (head + 1) % ring->n_entries;
 890
 891         if (new_head == tail) {
 892                 dev_warn(&bcm4377->pdev->dev,
 893                          "can't send message because ring %d is full\n",
 894                          ring->ring_id);
 895                 ret = -EINVAL;
 896                 goto out;
 897         }
 898
 899         msgid = bitmap_find_free_region(ring->msgids, ring->n_entries, 0);
 900         if (msgid < 0) {
 901                 dev_warn(&bcm4377->pdev->dev,
 902                          "can't find message id for ring %d\n", ring->ring_id);
 903                 ret = -EINVAL;
 904                 goto out;
 905         }
 906
 907         raw_msgid = FIELD_PREP(BCM4377_MSGID_GENERATION, ring->generation);
 908         raw_msgid |= FIELD_PREP(BCM4377_MSGID_ID, msgid);
 909
 910         offset = head * (sizeof(*entry) + ring->payload_size);
 911         entry = ring->ring + offset;
 912
 913         memset(entry, 0, sizeof(*entry));
 914         entry->id = cpu_to_le16(raw_msgid);
 915         entry->len = cpu_to_le16(len);
 916
 917         if (len <= ring->payload_size) {
 918                 entry->flags = BCM4377_XFER_RING_FLAG_PAYLOAD_IN_FOOTER;
 919                 payload = ring->ring + offset + sizeof(*entry);
 920         } else {
 921                 entry->flags = BCM4377_XFER_RING_FLAG_PAYLOAD_MAPPED;
 922                 entry->payload = cpu_to_le64(ring->payloads_dma +
 923                                              msgid * ring->mapped_payload_size);
 924                 payload = ring->payloads + msgid * ring->mapped_payload_size;
 925         }
 926
 927         memcpy(payload, data, len);
 928
 929         if (wait)
 930                 ring->events[msgid] = &event;
 931
 932         /*
 933          * The 4377 chips stop responding to any commands as soon as they
 934          * have been idle for a while. Poking the sleep control register here
 935          * makes them come alive again.
 936          */
 937         iowrite32(BCM4377_BAR0_SLEEP_CONTROL_AWAKE,
 938                   bcm4377->bar0 + BCM4377_BAR0_SLEEP_CONTROL);
 939
 940         dev_dbg(&bcm4377->pdev->dev,
 941                 "updating head for transfer queue #%d to %d\n", ring->ring_id,
 942                 new_head);
 943         bcm4377->ring_state->xfer_ring_head[ring->ring_id] =
 944                 cpu_to_le16(new_head);
 945
 946         if (!ring->sync)
 947                 bcm4377_ring_doorbell(bcm4377, ring->doorbell, new_head);
 948         ret = 0;
 949
 950 out:
 951         spin_unlock_irqrestore(&ring->lock, flags);
 952
 953         if (ret == 0 && wait) {
 954                 ret = wait_for_completion_interruptible_timeout(
 955                         &event, BCM4377_TIMEOUT);
 956                 if (ret == 0)
 957                         ret = -ETIMEDOUT;
 958                 else if (ret > 0)
 959                         ret = 0;
 960
 961                 spin_lock_irqsave(&ring->lock, flags);
 962                 ring->events[msgid] = NULL;
 963                 spin_unlock_irqrestore(&ring->lock, flags);
 964         }
 965
 966         return ret;
 967 }
 968
 969 static int bcm4377_create_completion_ring(struct bcm4377_data *bcm4377,
 970                                           struct bcm4377_completion_ring *ring)
 971 {
 972         struct bcm4377_create_completion_ring_msg msg;
 973         int ret;
 974
 975         if (ring->enabled) {
 976                 dev_warn(&bcm4377->pdev->dev,
 977                          "completion ring %d already enabled\n", ring->ring_id);
 978                 return 0;
 979         }
 980
 981         memset(ring->ring, 0,
 982                ring->n_entries * (sizeof(struct bcm4377_completion_ring_entry) +
 983                                   ring->payload_size));
 984         memset(&msg, 0, sizeof(msg));
 985         msg.msg_type = BCM4377_CONTROL_MSG_CREATE_COMPLETION_RING;
 986         msg.id = cpu_to_le16(ring->ring_id);
 987         msg.id_again = cpu_to_le16(ring->ring_id);
 988         msg.ring_iova = cpu_to_le64(ring->ring_dma);
 989         msg.n_elements = cpu_to_le16(ring->n_entries);
 990         msg.intmod_bytes = cpu_to_le32(0xffffffff);
 991         msg.unk = cpu_to_le32(0xffffffff);
 992         msg.intmod_delay = cpu_to_le16(ring->delay);
 993         msg.footer_size = ring->payload_size / 4;
 994
 995         ret = bcm4377_enqueue(bcm4377, &bcm4377->control_h2d_ring, &msg,
 996                               sizeof(msg), true);
 997         if (!ret)
 998                 ring->enabled = true;
 999
1000         return ret;
1001 }
1002
1003 static int bcm4377_destroy_completion_ring(struct bcm4377_data *bcm4377,
1004                                            struct bcm4377_completion_ring *ring)
1005 {
1006         struct bcm4377_destroy_completion_ring_msg msg;
1007         int ret;
1008
1009         memset(&msg, 0, sizeof(msg));
1010         msg.msg_type = BCM4377_CONTROL_MSG_DESTROY_COMPLETION_RING;
1011         msg.ring_id = cpu_to_le16(ring->ring_id);
1012
1013         ret = bcm4377_enqueue(bcm4377, &bcm4377->control_h2d_ring, &msg,
1014                               sizeof(msg), true);
1015         if (ret)
1016                 dev_warn(&bcm4377->pdev->dev,
1017                          "failed to destroy completion ring %d\n",
1018                          ring->ring_id);
1019
1020         ring->enabled = false;
1021         return ret;
1022 }
1023
1024 static int bcm4377_create_transfer_ring(struct bcm4377_data *bcm4377,
1025                                         struct bcm4377_transfer_ring *ring)
1026 {
1027         struct bcm4377_create_transfer_ring_msg msg;
1028         u16 flags = 0;
1029         int ret, i;
1030         unsigned long spinlock_flags;
1031
1032         if (ring->virtual)
1033                 flags |= BCM4377_XFER_RING_FLAG_VIRTUAL;
1034         if (ring->sync)
1035                 flags |= BCM4377_XFER_RING_FLAG_SYNC;
1036
1037         spin_lock_irqsave(&ring->lock, spinlock_flags);
1038         memset(&msg, 0, sizeof(msg));
1039         msg.msg_type = BCM4377_CONTROL_MSG_CREATE_XFER_RING;
1040         msg.ring_id = cpu_to_le16(ring->ring_id);
1041         msg.ring_id_again = cpu_to_le16(ring->ring_id);
1042         msg.ring_iova = cpu_to_le64(ring->ring_dma);
1043         msg.n_elements = cpu_to_le16(ring->n_entries);
1044         msg.completion_ring_id = cpu_to_le16(ring->completion_ring);
1045         msg.doorbell = cpu_to_le16(ring->doorbell);
1046         msg.flags = cpu_to_le16(flags);
1047         msg.footer_size = ring->payload_size / 4;
1048
1049         bcm4377->ring_state->xfer_ring_head[ring->ring_id] = 0;
1050         bcm4377->ring_state->xfer_ring_tail[ring->ring_id] = 0;
1051         ring->generation++;
1052         spin_unlock_irqrestore(&ring->lock, spinlock_flags);
1053
1054         ret = bcm4377_enqueue(bcm4377, &bcm4377->control_h2d_ring, &msg,
1055                               sizeof(msg), true);
1056
1057         spin_lock_irqsave(&ring->lock, spinlock_flags);
1058
1059         if (ring->d2h_buffers_only) {
1060                 for (i = 0; i < ring->n_entries; ++i) {
1061                         struct bcm4377_xfer_ring_entry *entry =
1062                                 ring->ring + i * sizeof(*entry);
1063                         u16 raw_msgid = FIELD_PREP(BCM4377_MSGID_GENERATION,
1064                                                    ring->generation);
1065                         raw_msgid |= FIELD_PREP(BCM4377_MSGID_ID, i);
1066
1067                         memset(entry, 0, sizeof(*entry));
1068                         entry->id = cpu_to_le16(raw_msgid);
1069                         entry->len = cpu_to_le16(ring->mapped_payload_size);
1070                         entry->flags = BCM4377_XFER_RING_FLAG_PAYLOAD_MAPPED;
1071                         entry->payload =
1072                                 cpu_to_le64(ring->payloads_dma +
1073                                             i * ring->mapped_payload_size);
1074                 }
1075         }
1076
1077         /*
1078          * send some messages if this is a device->host ring to allow the device
1079          * to reply by acknowledging them in the completion ring
1080          */
1081         if (ring->virtual || ring->d2h_buffers_only) {
1082                 bcm4377->ring_state->xfer_ring_head[ring->ring_id] =
1083                         cpu_to_le16(0xf);
1084                 bcm4377_ring_doorbell(bcm4377, ring->doorbell, 0xf);
1085         }
1086
1087         ring->enabled = true;
1088         spin_unlock_irqrestore(&ring->lock, spinlock_flags);
1089
1090         return ret;
1091 }
1092
1093 static int bcm4377_destroy_transfer_ring(struct bcm4377_data *bcm4377,
1094                                          struct bcm4377_transfer_ring *ring)
1095 {
1096         struct bcm4377_destroy_transfer_ring_msg msg;
1097         int ret;
1098
1099         memset(&msg, 0, sizeof(msg));
1100         msg.msg_type = BCM4377_CONTROL_MSG_DESTROY_XFER_RING;
1101         msg.ring_id = cpu_to_le16(ring->ring_id);
1102
1103         ret = bcm4377_enqueue(bcm4377, &bcm4377->control_h2d_ring, &msg,
1104                               sizeof(msg), true);
1105         if (ret)
1106                 dev_warn(&bcm4377->pdev->dev,
1107                          "failed to destroy transfer ring %d\n", ring->ring_id);
1108
1109         ring->enabled = false;
1110         return ret;
1111 }
1112
1113 static int __bcm4378_send_calibration_chunk(struct bcm4377_data *bcm4377,
1114                                             const void *data, size_t data_len,
1115                                             u16 blocks_left)
1116 {
1117         struct bcm4378_hci_send_calibration_cmd cmd;
1118         struct sk_buff *skb;
1119
1120         if (data_len > sizeof(cmd.data))
1121                 return -EINVAL;
1122
1123         memset(&cmd, 0, sizeof(cmd));
1124         cmd.unk = 0x03;
1125         cmd.blocks_left = cpu_to_le16(blocks_left);
1126         memcpy(cmd.data, data, data_len);
1127
1128         skb = __hci_cmd_sync(bcm4377->hdev, 0xfd97, sizeof(cmd), &cmd,
1129                              HCI_INIT_TIMEOUT);
1130         if (IS_ERR(skb))
1131                 return PTR_ERR(skb);
1132
1133         kfree_skb(skb);
1134         return 0;
1135 }
1136
1137 static int __bcm4378_send_calibration(struct bcm4377_data *bcm4377,
1138                                       const void *data, size_t data_size)
1139 {
1140         int ret;
1141         size_t i, left, transfer_len;
1142         size_t blocks =
1143                 DIV_ROUND_UP(data_size, (size_t)BCM4378_CALIBRATION_CHUNK_SIZE);
1144
1145         if (!data) {
1146                 dev_err(&bcm4377->pdev->dev,
1147                         "no calibration data available.\n");
1148                 return -ENOENT;
1149         }
1150
1151         for (i = 0, left = data_size; i < blocks; ++i, left -= transfer_len) {
1152                 transfer_len =
1153                         min_t(size_t, left, BCM4378_CALIBRATION_CHUNK_SIZE);
1154
1155                 ret = __bcm4378_send_calibration_chunk(
1156                         bcm4377, data + i * BCM4378_CALIBRATION_CHUNK_SIZE,
1157                         transfer_len, blocks - i - 1);
1158                 if (ret) {
1159                         dev_err(&bcm4377->pdev->dev,
1160                                 "send calibration chunk failed with %d\n", ret);
1161                         return ret;
1162                 }
1163         }
1164
1165         return 0;
1166 }
1167
1168 static int bcm4378_send_calibration(struct bcm4377_data *bcm4377)
1169 {
1170         if ((strcmp(bcm4377->stepping, "b1") == 0) ||
1171             strcmp(bcm4377->stepping, "b3") == 0)
1172                 return __bcm4378_send_calibration(
1173                         bcm4377, bcm4377->taurus_beamforming_cal_blob,
1174                         bcm4377->taurus_beamforming_cal_size);
1175         else
1176                 return __bcm4378_send_calibration(bcm4377,
1177                                                   bcm4377->taurus_cal_blob,
1178                                                   bcm4377->taurus_cal_size);
1179 }
1180
1181 static int bcm4387_send_calibration(struct bcm4377_data *bcm4377)
1182 {
1183         if (strcmp(bcm4377->stepping, "c2") == 0)
1184                 return __bcm4378_send_calibration(
1185                         bcm4377, bcm4377->taurus_beamforming_cal_blob,
1186                         bcm4377->taurus_beamforming_cal_size);
1187         else
1188                 return __bcm4378_send_calibration(bcm4377,
1189                                                   bcm4377->taurus_cal_blob,
1190                                                   bcm4377->taurus_cal_size);
1191 }
1192
1193 static const struct firmware *bcm4377_request_blob(struct bcm4377_data *bcm4377,
1194                                                    const char *suffix)
1195 {
1196         const struct firmware *fw;
1197         char name0[64], name1[64];
1198         int ret;
1199
1200         snprintf(name0, sizeof(name0), "brcm/brcmbt%04x%s-%s-%s.%s",
1201                  bcm4377->hw->id, bcm4377->stepping, bcm4377->board_type,
1202                  bcm4377->vendor, suffix);
1203         snprintf(name1, sizeof(name1), "brcm/brcmbt%04x%s-%s.%s",
1204                  bcm4377->hw->id, bcm4377->stepping, bcm4377->board_type,
1205                  suffix);
1206         dev_dbg(&bcm4377->pdev->dev, "Trying to load firmware: '%s' or '%s'\n",
1207                 name0, name1);
1208
1209         ret = firmware_request_nowarn(&fw, name0, &bcm4377->pdev->dev);
1210         if (!ret)
1211                 return fw;
1212         ret = firmware_request_nowarn(&fw, name1, &bcm4377->pdev->dev);
1213         if (!ret)
1214                 return fw;
1215
1216         dev_err(&bcm4377->pdev->dev,
1217                 "Unable to load firmware; tried '%s' and '%s'\n", name0, name1);
1218         return NULL;
1219 }
1220
1221 static int bcm4377_send_ptb(struct bcm4377_data *bcm4377,
1222                             const struct firmware *fw)
1223 {
1224         struct sk_buff *skb;
1225
1226         skb = __hci_cmd_sync(bcm4377->hdev, 0xfd98, fw->size, fw->data,
1227                              HCI_INIT_TIMEOUT);
1228         /*
1229          * This command seems to always fail on more recent firmware versions
1230          * (even in traces taken from the macOS driver). It's unclear why this
1231          * happens but because the PTB file contains calibration and/or
1232          * regulatory data and may be required on older firmware we still try to
1233          * send it here just in case and just ignore if it fails.
1234          */
1235         if (!IS_ERR(skb))
1236                 kfree_skb(skb);
1237         return 0;
1238 }
1239
1240 static int bcm4378_send_ptb_chunk(struct bcm4377_data *bcm4377,
1241                                   const void *data, size_t data_len,
1242                                   u16 blocks_left)
1243 {
1244         struct bcm4378_hci_send_ptb_cmd cmd;
1245         struct sk_buff *skb;
1246
1247         if (data_len > BCM4378_PTB_CHUNK_SIZE)
1248                 return -EINVAL;
1249
1250         memset(&cmd, 0, sizeof(cmd));
1251         cmd.blocks_left = cpu_to_le16(blocks_left);
1252         memcpy(cmd.data, data, data_len);
1253
1254         skb = __hci_cmd_sync(bcm4377->hdev, 0xfe0d, sizeof(cmd), &cmd,
1255                              HCI_INIT_TIMEOUT);
1256         if (IS_ERR(skb))
1257                 return PTR_ERR(skb);
1258
1259         kfree_skb(skb);
1260         return 0;
1261 }
1262
1263 static int bcm4378_send_ptb(struct bcm4377_data *bcm4377,
1264                             const struct firmware *fw)
1265 {
1266         size_t chunks = DIV_ROUND_UP(fw->size, (size_t)BCM4378_PTB_CHUNK_SIZE);
1267         size_t i, left, transfer_len;
1268         int ret;
1269
1270         for (i = 0, left = fw->size; i < chunks; ++i, left -= transfer_len) {
1271                 transfer_len = min_t(size_t, left, BCM4378_PTB_CHUNK_SIZE);
1272
1273                 dev_dbg(&bcm4377->pdev->dev, "sending ptb chunk %zu/%zu\n",
1274                         i + 1, chunks);
1275                 ret = bcm4378_send_ptb_chunk(
1276                         bcm4377, fw->data + i * BCM4378_PTB_CHUNK_SIZE,
1277                         transfer_len, chunks - i - 1);
1278                 if (ret) {
1279                         dev_err(&bcm4377->pdev->dev,
1280                                 "sending ptb chunk %zu failed (%d)", i, ret);
1281                         return ret;
1282                 }
1283         }
1284
1285         return 0;
1286 }
1287
1288 static int bcm4377_hci_open(struct hci_dev *hdev)
1289 {
1290         struct bcm4377_data *bcm4377 = hci_get_drvdata(hdev);
1291         int ret;
1292
1293         dev_dbg(&bcm4377->pdev->dev, "creating rings\n");
1294
1295         ret = bcm4377_create_completion_ring(bcm4377,
1296                                              &bcm4377->hci_acl_ack_ring);
1297         if (ret)
1298                 return ret;
1299         ret = bcm4377_create_completion_ring(bcm4377,
1300                                              &bcm4377->hci_acl_event_ring);
1301         if (ret)
1302                 goto destroy_hci_acl_ack;
1303         ret = bcm4377_create_completion_ring(bcm4377, &bcm4377->sco_ack_ring);
1304         if (ret)
1305                 goto destroy_hci_acl_event;
1306         ret = bcm4377_create_completion_ring(bcm4377, &bcm4377->sco_event_ring);
1307         if (ret)
1308                 goto destroy_sco_ack;
1309         dev_dbg(&bcm4377->pdev->dev,
1310                 "all completion rings successfully created!\n");
1311
1312         ret = bcm4377_create_transfer_ring(bcm4377, &bcm4377->hci_h2d_ring);
1313         if (ret)
1314                 goto destroy_sco_event;
1315         ret = bcm4377_create_transfer_ring(bcm4377, &bcm4377->hci_d2h_ring);
1316         if (ret)
1317                 goto destroy_hci_h2d;
1318         ret = bcm4377_create_transfer_ring(bcm4377, &bcm4377->sco_h2d_ring);
1319         if (ret)
1320                 goto destroy_hci_d2h;
1321         ret = bcm4377_create_transfer_ring(bcm4377, &bcm4377->sco_d2h_ring);
1322         if (ret)
1323                 goto destroy_sco_h2d;
1324         ret = bcm4377_create_transfer_ring(bcm4377, &bcm4377->acl_h2d_ring);
1325         if (ret)
1326                 goto destroy_sco_d2h;
1327         ret = bcm4377_create_transfer_ring(bcm4377, &bcm4377->acl_d2h_ring);
1328         if (ret)
1329                 goto destroy_acl_h2d;
1330         dev_dbg(&bcm4377->pdev->dev,
1331                 "all transfer rings successfully created!\n");
1332
1333         return 0;
1334
1335 destroy_acl_h2d:
1336         bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->acl_h2d_ring);
1337 destroy_sco_d2h:
1338         bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->sco_d2h_ring);
1339 destroy_sco_h2d:
1340         bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->sco_h2d_ring);
1341 destroy_hci_d2h:
1342         bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->hci_h2d_ring);
1343 destroy_hci_h2d:
1344         bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->hci_d2h_ring);
1345 destroy_sco_event:
1346         bcm4377_destroy_completion_ring(bcm4377, &bcm4377->sco_event_ring);
1347 destroy_sco_ack:
1348         bcm4377_destroy_completion_ring(bcm4377, &bcm4377->sco_ack_ring);
1349 destroy_hci_acl_event:
1350         bcm4377_destroy_completion_ring(bcm4377, &bcm4377->hci_acl_event_ring);
1351 destroy_hci_acl_ack:
1352         bcm4377_destroy_completion_ring(bcm4377, &bcm4377->hci_acl_ack_ring);
1353
1354         dev_err(&bcm4377->pdev->dev, "Creating rings failed with %d\n", ret);
1355         return ret;
1356 }
1357
1358 static int bcm4377_hci_close(struct hci_dev *hdev)
1359 {
1360         struct bcm4377_data *bcm4377 = hci_get_drvdata(hdev);
1361
1362         dev_dbg(&bcm4377->pdev->dev, "destroying rings in hci_close\n");
1363
1364         bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->acl_d2h_ring);
1365         bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->acl_h2d_ring);
1366         bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->sco_d2h_ring);
1367         bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->sco_h2d_ring);
1368         bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->hci_d2h_ring);
1369         bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->hci_h2d_ring);
1370
1371         bcm4377_destroy_completion_ring(bcm4377, &bcm4377->sco_event_ring);
1372         bcm4377_destroy_completion_ring(bcm4377, &bcm4377->sco_ack_ring);
1373         bcm4377_destroy_completion_ring(bcm4377, &bcm4377->hci_acl_event_ring);
1374         bcm4377_destroy_completion_ring(bcm4377, &bcm4377->hci_acl_ack_ring);
1375
1376         return 0;
1377 }
1378
1379 static bool bcm4377_is_valid_bdaddr(struct bcm4377_data *bcm4377,
1380                                     bdaddr_t *addr)
1381 {
1382         if (addr->b[0] != 0x93)
1383                 return true;
1384         if (addr->b[1] != 0x76)
1385                 return true;
1386         if (addr->b[2] != 0x00)
1387                 return true;
1388         if (addr->b[4] != (bcm4377->hw->id & 0xff))
1389                 return true;
1390         if (addr->b[5] != (bcm4377->hw->id >> 8))
1391                 return true;
1392         return false;
1393 }
1394
1395 static int bcm4377_check_bdaddr(struct bcm4377_data *bcm4377)
1396 {
1397         struct hci_rp_read_bd_addr *bda;
1398         struct sk_buff *skb;
1399
1400         skb = __hci_cmd_sync(bcm4377->hdev, HCI_OP_READ_BD_ADDR, 0, NULL,
1401                              HCI_INIT_TIMEOUT);
1402         if (IS_ERR(skb)) {
1403                 int err = PTR_ERR(skb);
1404
1405                 dev_err(&bcm4377->pdev->dev, "HCI_OP_READ_BD_ADDR failed (%d)",
1406                         err);
1407                 return err;
1408         }
1409
1410         if (skb->len != sizeof(*bda)) {
1411                 dev_err(&bcm4377->pdev->dev,
1412                         "HCI_OP_READ_BD_ADDR reply length invalid");
1413                 kfree_skb(skb);
1414                 return -EIO;
1415         }
1416
1417         bda = (struct hci_rp_read_bd_addr *)skb->data;
1418         if (!bcm4377_is_valid_bdaddr(bcm4377, &bda->bdaddr))
1419                 set_bit(HCI_QUIRK_INVALID_BDADDR, &bcm4377->hdev->quirks);
1420
1421         kfree_skb(skb);
1422         return 0;
1423 }
1424
1425 static int bcm4377_hci_setup(struct hci_dev *hdev)
1426 {
1427         struct bcm4377_data *bcm4377 = hci_get_drvdata(hdev);
1428         const struct firmware *fw;
1429         int ret;
1430
1431         if (bcm4377->hw->send_calibration) {
1432                 ret = bcm4377->hw->send_calibration(bcm4377);
1433                 if (ret)
1434                         return ret;
1435         }
1436
1437         fw = bcm4377_request_blob(bcm4377, "ptb");
1438         if (!fw) {
1439                 dev_err(&bcm4377->pdev->dev, "failed to load PTB data");
1440                 return -ENOENT;
1441         }
1442
1443         ret = bcm4377->hw->send_ptb(bcm4377, fw);
1444         release_firmware(fw);
1445         if (ret)
1446                 return ret;
1447
1448         return bcm4377_check_bdaddr(bcm4377);
1449 }
1450
1451 static int bcm4377_hci_send_frame(struct hci_dev *hdev, struct sk_buff *skb)
1452 {
1453         struct bcm4377_data *bcm4377 = hci_get_drvdata(hdev);
1454         struct bcm4377_transfer_ring *ring;
1455         int ret;
1456
1457         switch (hci_skb_pkt_type(skb)) {
1458         case HCI_COMMAND_PKT:
1459                 hdev->stat.cmd_tx++;
1460                 ring = &bcm4377->hci_h2d_ring;
1461                 break;
1462
1463         case HCI_ACLDATA_PKT:
1464                 hdev->stat.acl_tx++;
1465                 ring = &bcm4377->acl_h2d_ring;
1466                 break;
1467
1468         case HCI_SCODATA_PKT:
1469                 hdev->stat.sco_tx++;
1470                 ring = &bcm4377->sco_h2d_ring;
1471                 break;
1472
1473         default:
1474                 return -EILSEQ;
1475         }
1476
1477         ret = bcm4377_enqueue(bcm4377, ring, skb->data, skb->len, false);
1478         if (ret < 0) {
1479                 hdev->stat.err_tx++;
1480                 return ret;
1481         }
1482
1483         hdev->stat.byte_tx += skb->len;
1484         kfree_skb(skb);
1485         return ret;
1486 }
1487
1488 static int bcm4377_hci_set_bdaddr(struct hci_dev *hdev, const bdaddr_t *bdaddr)
1489 {
1490         struct bcm4377_data *bcm4377 = hci_get_drvdata(hdev);
1491         struct sk_buff *skb;
1492         int err;
1493
1494         skb = __hci_cmd_sync(hdev, 0xfc01, 6, bdaddr, HCI_INIT_TIMEOUT);
1495         if (IS_ERR(skb)) {
1496                 err = PTR_ERR(skb);
1497                 dev_err(&bcm4377->pdev->dev,
1498                         "Change address command failed (%d)", err);
1499                 return err;
1500         }
1501         kfree_skb(skb);
1502
1503         return 0;
1504 }
1505
1506 static int bcm4377_alloc_transfer_ring(struct bcm4377_data *bcm4377,
1507                                        struct bcm4377_transfer_ring *ring)
1508 {
1509         size_t entry_size;
1510
1511         spin_lock_init(&ring->lock);
1512         ring->payload_size = ALIGN(ring->payload_size, 4);
1513         ring->mapped_payload_size = ALIGN(ring->mapped_payload_size, 4);
1514
1515         if (ring->payload_size > BCM4377_XFER_RING_MAX_INPLACE_PAYLOAD_SIZE)
1516                 return -EINVAL;
1517         if (ring->n_entries > BCM4377_MAX_RING_SIZE)
1518                 return -EINVAL;
1519         if (ring->virtual && ring->allow_wait)
1520                 return -EINVAL;
1521
1522         if (ring->d2h_buffers_only) {
1523                 if (ring->virtual)
1524                         return -EINVAL;
1525                 if (ring->payload_size)
1526                         return -EINVAL;
1527                 if (!ring->mapped_payload_size)
1528                         return -EINVAL;
1529         }
1530         if (ring->virtual)
1531                 return 0;
1532
1533         entry_size =
1534                 ring->payload_size + sizeof(struct bcm4377_xfer_ring_entry);
1535         ring->ring = dmam_alloc_coherent(&bcm4377->pdev->dev,
1536                                          ring->n_entries * entry_size,
1537                                          &ring->ring_dma, GFP_KERNEL);
1538         if (!ring->ring)
1539                 return -ENOMEM;
1540
1541         if (ring->allow_wait) {
1542                 ring->events = devm_kcalloc(&bcm4377->pdev->dev,
1543                                             ring->n_entries,
1544                                             sizeof(*ring->events), GFP_KERNEL);
1545                 if (!ring->events)
1546                         return -ENOMEM;
1547         }
1548
1549         if (ring->mapped_payload_size) {
1550                 ring->payloads = dmam_alloc_coherent(
1551                         &bcm4377->pdev->dev,
1552                         ring->n_entries * ring->mapped_payload_size,
1553                         &ring->payloads_dma, GFP_KERNEL);
1554                 if (!ring->payloads)
1555                         return -ENOMEM;
1556         }
1557
1558         return 0;
1559 }
1560
1561 static int bcm4377_alloc_completion_ring(struct bcm4377_data *bcm4377,
1562                                          struct bcm4377_completion_ring *ring)
1563 {
1564         size_t entry_size;
1565
1566         ring->payload_size = ALIGN(ring->payload_size, 4);
1567         if (ring->payload_size > BCM4377_XFER_RING_MAX_INPLACE_PAYLOAD_SIZE)
1568                 return -EINVAL;
1569         if (ring->n_entries > BCM4377_MAX_RING_SIZE)
1570                 return -EINVAL;
1571
1572         entry_size = ring->payload_size +
1573                      sizeof(struct bcm4377_completion_ring_entry);
1574
1575         ring->ring = dmam_alloc_coherent(&bcm4377->pdev->dev,
1576                                          ring->n_entries * entry_size,
1577                                          &ring->ring_dma, GFP_KERNEL);
1578         if (!ring->ring)
1579                 return -ENOMEM;
1580         return 0;
1581 }
1582
1583 static int bcm4377_init_context(struct bcm4377_data *bcm4377)
1584 {
1585         struct device *dev = &bcm4377->pdev->dev;
1586         dma_addr_t peripheral_info_dma;
1587
1588         bcm4377->ctx = dmam_alloc_coherent(dev, sizeof(*bcm4377->ctx),
1589                                            &bcm4377->ctx_dma, GFP_KERNEL);
1590         if (!bcm4377->ctx)
1591                 return -ENOMEM;
1592         memset(bcm4377->ctx, 0, sizeof(*bcm4377->ctx));
1593
1594         bcm4377->ring_state =
1595                 dmam_alloc_coherent(dev, sizeof(*bcm4377->ring_state),
1596                                     &bcm4377->ring_state_dma, GFP_KERNEL);
1597         if (!bcm4377->ring_state)
1598                 return -ENOMEM;
1599         memset(bcm4377->ring_state, 0, sizeof(*bcm4377->ring_state));
1600
1601         bcm4377->ctx->version = cpu_to_le16(1);
1602         bcm4377->ctx->size = cpu_to_le16(sizeof(*bcm4377->ctx));
1603         bcm4377->ctx->enabled_caps = cpu_to_le32(2);
1604
1605         /*
1606          * The BT device will write 0x20 bytes of data to this buffer but
1607          * the exact contents are unknown. It only needs to exist for BT
1608          * to work such that we can just allocate and then ignore it.
1609          */
1610         if (!dmam_alloc_coherent(&bcm4377->pdev->dev, 0x20,
1611                                  &peripheral_info_dma, GFP_KERNEL))
1612                 return -ENOMEM;
1613         bcm4377->ctx->peripheral_info_addr = cpu_to_le64(peripheral_info_dma);
1614
1615         bcm4377->ctx->xfer_ring_heads_addr = cpu_to_le64(
1616                 bcm4377->ring_state_dma +
1617                 offsetof(struct bcm4377_ring_state, xfer_ring_head));
1618         bcm4377->ctx->xfer_ring_tails_addr = cpu_to_le64(
1619                 bcm4377->ring_state_dma +
1620                 offsetof(struct bcm4377_ring_state, xfer_ring_tail));
1621         bcm4377->ctx->completion_ring_heads_addr = cpu_to_le64(
1622                 bcm4377->ring_state_dma +
1623                 offsetof(struct bcm4377_ring_state, completion_ring_head));
1624         bcm4377->ctx->completion_ring_tails_addr = cpu_to_le64(
1625                 bcm4377->ring_state_dma +
1626                 offsetof(struct bcm4377_ring_state, completion_ring_tail));
1627
1628         bcm4377->ctx->n_completion_rings =
1629                 cpu_to_le16(BCM4377_N_COMPLETION_RINGS);
1630         bcm4377->ctx->n_xfer_rings = cpu_to_le16(BCM4377_N_TRANSFER_RINGS);
1631
1632         bcm4377->ctx->control_completion_ring_addr =
1633                 cpu_to_le64(bcm4377->control_ack_ring.ring_dma);
1634         bcm4377->ctx->control_completion_ring_n_entries =
1635                 cpu_to_le16(bcm4377->control_ack_ring.n_entries);
1636         bcm4377->ctx->control_completion_ring_doorbell = cpu_to_le16(0xffff);
1637         bcm4377->ctx->control_completion_ring_msi = 0;
1638         bcm4377->ctx->control_completion_ring_header_size = 0;
1639         bcm4377->ctx->control_completion_ring_footer_size = 0;
1640
1641         bcm4377->ctx->control_xfer_ring_addr =
1642                 cpu_to_le64(bcm4377->control_h2d_ring.ring_dma);
1643         bcm4377->ctx->control_xfer_ring_n_entries =
1644                 cpu_to_le16(bcm4377->control_h2d_ring.n_entries);
1645         bcm4377->ctx->control_xfer_ring_doorbell =
1646                 cpu_to_le16(bcm4377->control_h2d_ring.doorbell);
1647         bcm4377->ctx->control_xfer_ring_msi = 0;
1648         bcm4377->ctx->control_xfer_ring_header_size = 0;
1649         bcm4377->ctx->control_xfer_ring_footer_size =
1650                 bcm4377->control_h2d_ring.payload_size / 4;
1651
1652         dev_dbg(&bcm4377->pdev->dev, "context initialized at IOVA %pad",
1653                 &bcm4377->ctx_dma);
1654
1655         return 0;
1656 }
1657
1658 static int bcm4377_prepare_rings(struct bcm4377_data *bcm4377)
1659 {
1660         int ret;
1661
1662         /*
1663          * Even though many of these settings appear to be configurable
1664          * when sending the "create ring" messages most of these are
1665          * actually hardcoded in some (and quite possibly all) firmware versions
1666          * and changing them on the host has no effect.
1667          * Specifically, this applies to at least the doorbells, the transfer
1668          * and completion ring ids and their mapping (e.g. both HCI and ACL
1669          * entries will always be queued in completion rings 1 and 2 no matter
1670          * what we configure here).
1671          */
1672         bcm4377->control_ack_ring.ring_id = BCM4377_ACK_RING_CONTROL;
1673         bcm4377->control_ack_ring.n_entries = 32;
1674         bcm4377->control_ack_ring.transfer_rings =
1675                 BIT(BCM4377_XFER_RING_CONTROL);
1676
1677         bcm4377->hci_acl_ack_ring.ring_id = BCM4377_ACK_RING_HCI_ACL;
1678         bcm4377->hci_acl_ack_ring.n_entries = 2 * BCM4377_RING_N_ENTRIES;
1679         bcm4377->hci_acl_ack_ring.transfer_rings =
1680                 BIT(BCM4377_XFER_RING_HCI_H2D) | BIT(BCM4377_XFER_RING_ACL_H2D);
1681         bcm4377->hci_acl_ack_ring.delay = 1000;
1682
1683         /*
1684          * A payload size of MAX_EVENT_PAYLOAD_SIZE is enough here since large
1685          * ACL packets will be transmitted inside buffers mapped via
1686          * acl_d2h_ring anyway.
1687          */
1688         bcm4377->hci_acl_event_ring.ring_id = BCM4377_EVENT_RING_HCI_ACL;
1689         bcm4377->hci_acl_event_ring.payload_size = MAX_EVENT_PAYLOAD_SIZE;
1690         bcm4377->hci_acl_event_ring.n_entries = 2 * BCM4377_RING_N_ENTRIES;
1691         bcm4377->hci_acl_event_ring.transfer_rings =
1692                 BIT(BCM4377_XFER_RING_HCI_D2H) | BIT(BCM4377_XFER_RING_ACL_D2H);
1693         bcm4377->hci_acl_event_ring.delay = 1000;
1694
1695         bcm4377->sco_ack_ring.ring_id = BCM4377_ACK_RING_SCO;
1696         bcm4377->sco_ack_ring.n_entries = BCM4377_RING_N_ENTRIES;
1697         bcm4377->sco_ack_ring.transfer_rings = BIT(BCM4377_XFER_RING_SCO_H2D);
1698
1699         bcm4377->sco_event_ring.ring_id = BCM4377_EVENT_RING_SCO;
1700         bcm4377->sco_event_ring.payload_size = MAX_SCO_PAYLOAD_SIZE;
1701         bcm4377->sco_event_ring.n_entries = BCM4377_RING_N_ENTRIES;
1702         bcm4377->sco_event_ring.transfer_rings = BIT(BCM4377_XFER_RING_SCO_D2H);
1703
1704         bcm4377->control_h2d_ring.ring_id = BCM4377_XFER_RING_CONTROL;
1705         bcm4377->control_h2d_ring.doorbell = BCM4377_DOORBELL_CONTROL;
1706         bcm4377->control_h2d_ring.payload_size = BCM4377_CONTROL_MSG_SIZE;
1707         bcm4377->control_h2d_ring.completion_ring = BCM4377_ACK_RING_CONTROL;
1708         bcm4377->control_h2d_ring.allow_wait = true;
1709         bcm4377->control_h2d_ring.n_entries = BCM4377_RING_N_ENTRIES;
1710
1711         bcm4377->hci_h2d_ring.ring_id = BCM4377_XFER_RING_HCI_H2D;
1712         bcm4377->hci_h2d_ring.doorbell = BCM4377_DOORBELL_HCI_H2D;
1713         bcm4377->hci_h2d_ring.payload_size = MAX_EVENT_PAYLOAD_SIZE;
1714         bcm4377->hci_h2d_ring.completion_ring = BCM4377_ACK_RING_HCI_ACL;
1715         bcm4377->hci_h2d_ring.n_entries = BCM4377_RING_N_ENTRIES;
1716
1717         bcm4377->hci_d2h_ring.ring_id = BCM4377_XFER_RING_HCI_D2H;
1718         bcm4377->hci_d2h_ring.doorbell = BCM4377_DOORBELL_HCI_D2H;
1719         bcm4377->hci_d2h_ring.completion_ring = BCM4377_EVENT_RING_HCI_ACL;
1720         bcm4377->hci_d2h_ring.virtual = true;
1721         bcm4377->hci_d2h_ring.n_entries = BCM4377_RING_N_ENTRIES;
1722
1723         bcm4377->sco_h2d_ring.ring_id = BCM4377_XFER_RING_SCO_H2D;
1724         bcm4377->sco_h2d_ring.doorbell = BCM4377_DOORBELL_SCO;
1725         bcm4377->sco_h2d_ring.payload_size = MAX_SCO_PAYLOAD_SIZE;
1726         bcm4377->sco_h2d_ring.completion_ring = BCM4377_ACK_RING_SCO;
1727         bcm4377->sco_h2d_ring.sync = true;
1728         bcm4377->sco_h2d_ring.n_entries = BCM4377_RING_N_ENTRIES;
1729
1730         bcm4377->sco_d2h_ring.ring_id = BCM4377_XFER_RING_SCO_D2H;
1731         bcm4377->sco_d2h_ring.doorbell = BCM4377_DOORBELL_SCO;
1732         bcm4377->sco_d2h_ring.completion_ring = BCM4377_EVENT_RING_SCO;
1733         bcm4377->sco_d2h_ring.virtual = true;
1734         bcm4377->sco_d2h_ring.sync = true;
1735         bcm4377->sco_d2h_ring.n_entries = BCM4377_RING_N_ENTRIES;
1736
1737         /*
1738          * This ring has to use mapped_payload_size because the largest ACL
1739          * packet doesn't fit inside the largest possible footer
1740          */
1741         bcm4377->acl_h2d_ring.ring_id = BCM4377_XFER_RING_ACL_H2D;
1742         bcm4377->acl_h2d_ring.doorbell = BCM4377_DOORBELL_ACL_H2D;
1743         bcm4377->acl_h2d_ring.mapped_payload_size = MAX_ACL_PAYLOAD_SIZE;
1744         bcm4377->acl_h2d_ring.completion_ring = BCM4377_ACK_RING_HCI_ACL;
1745         bcm4377->acl_h2d_ring.n_entries = BCM4377_RING_N_ENTRIES;
1746
1747         /*
1748          * This ring only contains empty buffers to be used by incoming
1749          * ACL packets that do not fit inside the footer of hci_acl_event_ring
1750          */
1751         bcm4377->acl_d2h_ring.ring_id = BCM4377_XFER_RING_ACL_D2H;
1752         bcm4377->acl_d2h_ring.doorbell = BCM4377_DOORBELL_ACL_D2H;
1753         bcm4377->acl_d2h_ring.completion_ring = BCM4377_EVENT_RING_HCI_ACL;
1754         bcm4377->acl_d2h_ring.d2h_buffers_only = true;
1755         bcm4377->acl_d2h_ring.mapped_payload_size = MAX_ACL_PAYLOAD_SIZE;
1756         bcm4377->acl_d2h_ring.n_entries = BCM4377_RING_N_ENTRIES;
1757
1758         /*
1759          * no need for any cleanup since this is only called from _probe
1760          * and only devres-managed allocations are used
1761          */
1762         ret = bcm4377_alloc_transfer_ring(bcm4377, &bcm4377->control_h2d_ring);
1763         if (ret)
1764                 return ret;
1765         ret = bcm4377_alloc_transfer_ring(bcm4377, &bcm4377->hci_h2d_ring);
1766         if (ret)
1767                 return ret;
1768         ret = bcm4377_alloc_transfer_ring(bcm4377, &bcm4377->hci_d2h_ring);
1769         if (ret)
1770                 return ret;
1771         ret = bcm4377_alloc_transfer_ring(bcm4377, &bcm4377->sco_h2d_ring);
1772         if (ret)
1773                 return ret;
1774         ret = bcm4377_alloc_transfer_ring(bcm4377, &bcm4377->sco_d2h_ring);
1775         if (ret)
1776                 return ret;
1777         ret = bcm4377_alloc_transfer_ring(bcm4377, &bcm4377->acl_h2d_ring);
1778         if (ret)
1779                 return ret;
1780         ret = bcm4377_alloc_transfer_ring(bcm4377, &bcm4377->acl_d2h_ring);
1781         if (ret)
1782                 return ret;
1783
1784         ret = bcm4377_alloc_completion_ring(bcm4377,
1785                                             &bcm4377->control_ack_ring);
1786         if (ret)
1787                 return ret;
1788         ret = bcm4377_alloc_completion_ring(bcm4377,
1789                                             &bcm4377->hci_acl_ack_ring);
1790         if (ret)
1791                 return ret;
1792         ret = bcm4377_alloc_completion_ring(bcm4377,
1793                                             &bcm4377->hci_acl_event_ring);
1794         if (ret)
1795                 return ret;
1796         ret = bcm4377_alloc_completion_ring(bcm4377, &bcm4377->sco_ack_ring);
1797         if (ret)
1798                 return ret;
1799         ret = bcm4377_alloc_completion_ring(bcm4377, &bcm4377->sco_event_ring);
1800         if (ret)
1801                 return ret;
1802
1803         dev_dbg(&bcm4377->pdev->dev, "all rings allocated and prepared\n");
1804
1805         return 0;
1806 }
1807
1808 static int bcm4377_boot(struct bcm4377_data *bcm4377)
1809 {
1810         const struct firmware *fw;
1811         void *bfr;
1812         dma_addr_t fw_dma;
1813         int ret = 0;
1814         u32 bootstage, rti_status;
1815
1816         bootstage = ioread32(bcm4377->bar2 + BCM4377_BAR2_BOOTSTAGE);
1817         rti_status = ioread32(bcm4377->bar2 + BCM4377_BAR2_RTI_STATUS);
1818
1819         if (bootstage != 0) {
1820                 dev_err(&bcm4377->pdev->dev, "bootstage is %d and not 0\n",
1821                         bootstage);
1822                 return -EINVAL;
1823         }
1824
1825         if (rti_status != 0) {
1826                 dev_err(&bcm4377->pdev->dev, "RTI status is %d and not 0\n",
1827                         rti_status);
1828                 return -EINVAL;
1829         }
1830
1831         fw = bcm4377_request_blob(bcm4377, "bin");
1832         if (!fw) {
1833                 dev_err(&bcm4377->pdev->dev, "Failed to load firmware\n");
1834                 return -ENOENT;
1835         }
1836
1837         bfr = dma_alloc_coherent(&bcm4377->pdev->dev, fw->size, &fw_dma,
1838                                  GFP_KERNEL);
1839         if (!bfr) {
1840                 ret = -ENOMEM;
1841                 goto out_release_fw;
1842         }
1843
1844         memcpy(bfr, fw->data, fw->size);
1845
1846         iowrite32(0, bcm4377->bar0 + BCM4377_BAR0_HOST_WINDOW_LO);
1847         iowrite32(0, bcm4377->bar0 + BCM4377_BAR0_HOST_WINDOW_HI);
1848         iowrite32(BCM4377_DMA_MASK,
1849                   bcm4377->bar0 + BCM4377_BAR0_HOST_WINDOW_SIZE);
1850
1851         iowrite32(lower_32_bits(fw_dma), bcm4377->bar2 + BCM4377_BAR2_FW_LO);
1852         iowrite32(upper_32_bits(fw_dma), bcm4377->bar2 + BCM4377_BAR2_FW_HI);
1853         iowrite32(fw->size, bcm4377->bar2 + BCM4377_BAR2_FW_SIZE);
1854         iowrite32(0, bcm4377->bar0 + BCM4377_BAR0_FW_DOORBELL);
1855
1856         dev_dbg(&bcm4377->pdev->dev, "waiting for firmware to boot\n");
1857
1858         ret = wait_for_completion_interruptible_timeout(&bcm4377->event,
1859                                                         BCM4377_TIMEOUT);
1860         if (ret == 0) {
1861                 ret = -ETIMEDOUT;
1862                 goto out_dma_free;
1863         } else if (ret < 0) {
1864                 goto out_dma_free;
1865         }
1866
1867         if (bcm4377->bootstage != 2) {
1868                 dev_err(&bcm4377->pdev->dev, "boostage %d != 2\n",
1869                         bcm4377->bootstage);
1870                 ret = -ENXIO;
1871                 goto out_dma_free;
1872         }
1873
1874         dev_dbg(&bcm4377->pdev->dev, "firmware has booted (stage = %x)\n",
1875                 bcm4377->bootstage);
1876         ret = 0;
1877
1878 out_dma_free:
1879         dma_free_coherent(&bcm4377->pdev->dev, fw->size, bfr, fw_dma);
1880 out_release_fw:
1881         release_firmware(fw);
1882         return ret;
1883 }
1884
1885 static int bcm4377_setup_rti(struct bcm4377_data *bcm4377)
1886 {
1887         int ret;
1888
1889         dev_dbg(&bcm4377->pdev->dev, "starting RTI\n");
1890         iowrite32(1, bcm4377->bar0 + BCM4377_BAR0_RTI_CONTROL);
1891
1892         ret = wait_for_completion_interruptible_timeout(&bcm4377->event,
1893                                                         BCM4377_TIMEOUT);
1894         if (ret == 0) {
1895                 dev_err(&bcm4377->pdev->dev,
1896                         "timed out while waiting for RTI to transition to state 1");
1897                 return -ETIMEDOUT;
1898         } else if (ret < 0) {
1899                 return ret;
1900         }
1901
1902         if (bcm4377->rti_status != 1) {
1903                 dev_err(&bcm4377->pdev->dev, "RTI did not ack state 1 (%d)\n",
1904                         bcm4377->rti_status);
1905                 return -ENODEV;
1906         }
1907         dev_dbg(&bcm4377->pdev->dev, "RTI is in state 1\n");
1908
1909         /* allow access to the entire IOVA space again */
1910         iowrite32(0, bcm4377->bar2 + BCM4377_BAR2_RTI_WINDOW_LO);
1911         iowrite32(0, bcm4377->bar2 + BCM4377_BAR2_RTI_WINDOW_HI);
1912         iowrite32(BCM4377_DMA_MASK,
1913                   bcm4377->bar2 + BCM4377_BAR2_RTI_WINDOW_SIZE);
1914
1915         /* setup "Converged IPC" context */
1916         iowrite32(lower_32_bits(bcm4377->ctx_dma),
1917                   bcm4377->bar2 + BCM4377_BAR2_CONTEXT_ADDR_LO);
1918         iowrite32(upper_32_bits(bcm4377->ctx_dma),
1919                   bcm4377->bar2 + BCM4377_BAR2_CONTEXT_ADDR_HI);
1920         iowrite32(2, bcm4377->bar0 + BCM4377_BAR0_RTI_CONTROL);
1921
1922         ret = wait_for_completion_interruptible_timeout(&bcm4377->event,
1923                                                         BCM4377_TIMEOUT);
1924         if (ret == 0) {
1925                 dev_err(&bcm4377->pdev->dev,
1926                         "timed out while waiting for RTI to transition to state 2");
1927                 return -ETIMEDOUT;
1928         } else if (ret < 0) {
1929                 return ret;
1930         }
1931
1932         if (bcm4377->rti_status != 2) {
1933                 dev_err(&bcm4377->pdev->dev, "RTI did not ack state 2 (%d)\n",
1934                         bcm4377->rti_status);
1935                 return -ENODEV;
1936         }
1937
1938         dev_dbg(&bcm4377->pdev->dev,
1939                 "RTI is in state 2; control ring is ready\n");
1940         bcm4377->control_ack_ring.enabled = true;
1941
1942         return 0;
1943 }
1944
1945 static int bcm4377_parse_otp_board_params(struct bcm4377_data *bcm4377,
1946                                           char tag, const char *val, size_t len)
1947 {
1948         if (tag != 'V')
1949                 return 0;
1950         if (len >= sizeof(bcm4377->vendor))
1951                 return -EINVAL;
1952
1953         strscpy(bcm4377->vendor, val, len + 1);
1954         return 0;
1955 }
1956
1957 static int bcm4377_parse_otp_chip_params(struct bcm4377_data *bcm4377, char tag,
1958                                          const char *val, size_t len)
1959 {
1960         size_t idx = 0;
1961
1962         if (tag != 's')
1963                 return 0;
1964         if (len >= sizeof(bcm4377->stepping))
1965                 return -EINVAL;
1966
1967         while (len != 0) {
1968                 bcm4377->stepping[idx] = tolower(val[idx]);
1969                 if (val[idx] == '\0')
1970                         return 0;
1971
1972                 idx++;
1973                 len--;
1974         }
1975
1976         bcm4377->stepping[idx] = '\0';
1977         return 0;
1978 }
1979
1980 static int bcm4377_parse_otp_str(struct bcm4377_data *bcm4377, const u8 *str,
1981                                  enum bcm4377_otp_params_type type)
1982 {
1983         const char *p;
1984         int ret;
1985
1986         p = skip_spaces(str);
1987         while (*p) {
1988                 char tag = *p++;
1989                 const char *end;
1990                 size_t len;
1991
1992                 if (*p++ != '=') /* implicit NUL check */
1993                         return -EINVAL;
1994
1995                 /* *p might be NUL here, if so end == p and len == 0 */
1996                 end = strchrnul(p, ' ');
1997                 len = end - p;
1998
1999                 /* leave 1 byte for NUL in destination string */
2000                 if (len > (BCM4377_OTP_MAX_PARAM_LEN - 1))
2001                         return -EINVAL;
2002
2003                 switch (type) {
2004                 case BCM4377_OTP_BOARD_PARAMS:
2005                         ret = bcm4377_parse_otp_board_params(bcm4377, tag, p,
2006                                                              len);
2007                         break;
2008                 case BCM4377_OTP_CHIP_PARAMS:
2009                         ret = bcm4377_parse_otp_chip_params(bcm4377, tag, p,
2010                                                             len);
2011                         break;
2012                 default:
2013                         ret = -EINVAL;
2014                         break;
2015                 }
2016
2017                 if (ret)
2018                         return ret;
2019
2020                 /* Skip to next arg, if any */
2021                 p = skip_spaces(end);
2022         }
2023
2024         return 0;
2025 }
2026
2027 static int bcm4377_parse_otp_sys_vendor(struct bcm4377_data *bcm4377, u8 *otp,
2028                                         size_t size)
2029 {
2030         int idx = 4;
2031         const char *chip_params;
2032         const char *board_params;
2033         int ret;
2034
2035         /* 4-byte header and two empty strings */
2036         if (size < 6)
2037                 return -EINVAL;
2038
2039         if (get_unaligned_le32(otp) != BCM4377_OTP_VENDOR_HDR)
2040                 return -EINVAL;
2041
2042         chip_params = &otp[idx];
2043
2044         /* Skip first string, including terminator */
2045         idx += strnlen(chip_params, size - idx) + 1;
2046         if (idx >= size)
2047                 return -EINVAL;
2048
2049         board_params = &otp[idx];
2050
2051         /* Skip to terminator of second string */
2052         idx += strnlen(board_params, size - idx);
2053         if (idx >= size)
2054                 return -EINVAL;
2055
2056         /* At this point both strings are guaranteed NUL-terminated */
2057         dev_dbg(&bcm4377->pdev->dev,
2058                 "OTP: chip_params='%s' board_params='%s'\n", chip_params,
2059                 board_params);
2060
2061         ret = bcm4377_parse_otp_str(bcm4377, chip_params,
2062                                     BCM4377_OTP_CHIP_PARAMS);
2063         if (ret)
2064                 return ret;
2065
2066         ret = bcm4377_parse_otp_str(bcm4377, board_params,
2067                                     BCM4377_OTP_BOARD_PARAMS);
2068         if (ret)
2069                 return ret;
2070
2071         if (!bcm4377->stepping[0] || !bcm4377->vendor[0])
2072                 return -EINVAL;
2073
2074         dev_dbg(&bcm4377->pdev->dev, "OTP: stepping=%s, vendor=%s\n",
2075                 bcm4377->stepping, bcm4377->vendor);
2076         return 0;
2077 }
2078
2079 static int bcm4377_parse_otp(struct bcm4377_data *bcm4377)
2080 {
2081         u8 *otp;
2082         int i;
2083         int ret = -ENOENT;
2084
2085         otp = kzalloc(BCM4377_OTP_SIZE, GFP_KERNEL);
2086         if (!otp)
2087                 return -ENOMEM;
2088
2089         for (i = 0; i < BCM4377_OTP_SIZE; ++i)
2090                 otp[i] = ioread8(bcm4377->bar0 + bcm4377->hw->otp_offset + i);
2091
2092         i = 0;
2093         while (i < (BCM4377_OTP_SIZE - 1)) {
2094                 u8 type = otp[i];
2095                 u8 length = otp[i + 1];
2096
2097                 if (type == 0)
2098                         break;
2099
2100                 if ((i + 2 + length) > BCM4377_OTP_SIZE)
2101                         break;
2102
2103                 switch (type) {
2104                 case BCM4377_OTP_SYS_VENDOR:
2105                         dev_dbg(&bcm4377->pdev->dev,
2106                                 "OTP @ 0x%x (%d): SYS_VENDOR", i, length);
2107                         ret = bcm4377_parse_otp_sys_vendor(bcm4377, &otp[i + 2],
2108                                                            length);
2109                         break;
2110                 case BCM4377_OTP_CIS:
2111                         dev_dbg(&bcm4377->pdev->dev, "OTP @ 0x%x (%d): CIS", i,
2112                                 length);
2113                         break;
2114                 default:
2115                         dev_dbg(&bcm4377->pdev->dev, "OTP @ 0x%x (%d): unknown",
2116                                 i, length);
2117                         break;
2118                 }
2119
2120                 i += 2 + length;
2121         }
2122
2123         kfree(otp);
2124         return ret;
2125 }
2126
2127 static int bcm4377_init_cfg(struct bcm4377_data *bcm4377)
2128 {
2129         int ret;
2130         u32 ctrl;
2131
2132         ret = pci_write_config_dword(bcm4377->pdev,
2133                                      BCM4377_PCIECFG_BAR0_WINDOW1,
2134                                      bcm4377->hw->bar0_window1);
2135         if (ret)
2136                 return ret;
2137
2138         ret = pci_write_config_dword(bcm4377->pdev,
2139                                      BCM4377_PCIECFG_BAR0_WINDOW2,
2140                                      bcm4377->hw->bar0_window2);
2141         if (ret)
2142                 return ret;
2143
2144         ret = pci_write_config_dword(
2145                 bcm4377->pdev, BCM4377_PCIECFG_BAR0_CORE2_WINDOW1,
2146                 BCM4377_PCIECFG_BAR0_CORE2_WINDOW1_DEFAULT);
2147         if (ret)
2148                 return ret;
2149
2150         if (bcm4377->hw->has_bar0_core2_window2) {
2151                 ret = pci_write_config_dword(bcm4377->pdev,
2152                                              BCM4377_PCIECFG_BAR0_CORE2_WINDOW2,
2153                                              bcm4377->hw->bar0_core2_window2);
2154                 if (ret)
2155                         return ret;
2156         }
2157
2158         ret = pci_write_config_dword(bcm4377->pdev, BCM4377_PCIECFG_BAR2_WINDOW,
2159                                      BCM4377_PCIECFG_BAR2_WINDOW_DEFAULT);
2160         if (ret)
2161                 return ret;
2162
2163         ret = pci_read_config_dword(bcm4377->pdev,
2164                                     BCM4377_PCIECFG_SUBSYSTEM_CTRL, &ctrl);
2165         if (ret)
2166                 return ret;
2167
2168         if (bcm4377->hw->clear_pciecfg_subsystem_ctrl_bit19)
2169                 ctrl &= ~BIT(19);
2170         ctrl |= BIT(16);
2171
2172         return pci_write_config_dword(bcm4377->pdev,
2173                                       BCM4377_PCIECFG_SUBSYSTEM_CTRL, ctrl);
2174 }
2175
2176 static int bcm4377_probe_dmi(struct bcm4377_data *bcm4377)
2177 {
2178         const struct dmi_system_id *board_type_dmi_id;
2179
2180         board_type_dmi_id = dmi_first_match(bcm4377_dmi_board_table);
2181         if (board_type_dmi_id && board_type_dmi_id->driver_data) {
2182                 bcm4377->board_type = board_type_dmi_id->driver_data;
2183                 dev_dbg(&bcm4377->pdev->dev,
2184                         "found board type via DMI match: %s\n",
2185                         bcm4377->board_type);
2186         }
2187
2188         return 0;
2189 }
2190
2191 static int bcm4377_probe_of(struct bcm4377_data *bcm4377)
2192 {
2193         struct device_node *np = bcm4377->pdev->dev.of_node;
2194         int ret;
2195
2196         if (!np)
2197                 return 0;
2198
2199         ret = of_property_read_string(np, "brcm,board-type",
2200                                       &bcm4377->board_type);
2201         if (ret) {
2202                 dev_err(&bcm4377->pdev->dev, "no brcm,board-type property\n");
2203                 return ret;
2204         }
2205
2206         bcm4377->taurus_beamforming_cal_blob =
2207                 of_get_property(np, "brcm,taurus-bf-cal-blob",
2208                                 &bcm4377->taurus_beamforming_cal_size);
2209         if (!bcm4377->taurus_beamforming_cal_blob) {
2210                 dev_err(&bcm4377->pdev->dev,
2211                         "no brcm,taurus-bf-cal-blob property\n");
2212                 return -ENOENT;
2213         }
2214         bcm4377->taurus_cal_blob = of_get_property(np, "brcm,taurus-cal-blob",
2215                                                    &bcm4377->taurus_cal_size);
2216         if (!bcm4377->taurus_cal_blob) {
2217                 dev_err(&bcm4377->pdev->dev,
2218                         "no brcm,taurus-cal-blob property\n");
2219                 return -ENOENT;
2220         }
2221
2222         return 0;
2223 }
2224
2225 static void bcm4377_disable_aspm(struct bcm4377_data *bcm4377)
2226 {
2227         pci_disable_link_state(bcm4377->pdev,
2228                                PCIE_LINK_STATE_L0S | PCIE_LINK_STATE_L1);
2229
2230         /*
2231          * pci_disable_link_state can fail if either CONFIG_PCIEASPM is disabled
2232          * or if the BIOS hasn't handed over control to us. We must *always*
2233          * disable ASPM for this device due to hardware errata though.
2234          */
2235         pcie_capability_clear_word(bcm4377->pdev, PCI_EXP_LNKCTL,
2236                                    PCI_EXP_LNKCTL_ASPMC);
2237 }
2238
2239 static void bcm4377_pci_free_irq_vectors(void *data)
2240 {
2241         pci_free_irq_vectors(data);
2242 }
2243
2244 static void bcm4377_hci_free_dev(void *data)
2245 {
2246         hci_free_dev(data);
2247 }
2248
2249 static void bcm4377_hci_unregister_dev(void *data)
2250 {
2251         hci_unregister_dev(data);
2252 }
2253
2254 static int bcm4377_probe(struct pci_dev *pdev, const struct pci_device_id *id)
2255 {
2256         struct bcm4377_data *bcm4377;
2257         struct hci_dev *hdev;
2258         int ret, irq;
2259
2260         ret = dma_set_mask_and_coherent(&pdev->dev, BCM4377_DMA_MASK);
2261         if (ret)
2262                 return ret;
2263
2264         bcm4377 = devm_kzalloc(&pdev->dev, sizeof(*bcm4377), GFP_KERNEL);
2265         if (!bcm4377)
2266                 return -ENOMEM;
2267
2268         bcm4377->pdev = pdev;
2269         bcm4377->hw = &bcm4377_hw_variants[id->driver_data];
2270         init_completion(&bcm4377->event);
2271
2272         ret = bcm4377_prepare_rings(bcm4377);
2273         if (ret)
2274                 return ret;
2275
2276         ret = bcm4377_init_context(bcm4377);
2277         if (ret)
2278                 return ret;
2279
2280         ret = bcm4377_probe_dmi(bcm4377);
2281         if (ret)
2282                 return ret;
2283         ret = bcm4377_probe_of(bcm4377);
2284         if (ret)
2285                 return ret;
2286         if (!bcm4377->board_type) {
2287                 dev_err(&pdev->dev, "unable to determine board type\n");
2288                 return -ENODEV;
2289         }
2290
2291         if (bcm4377->hw->disable_aspm)
2292                 bcm4377_disable_aspm(bcm4377);
2293
2294         ret = pci_reset_function_locked(pdev);
2295         if (ret)
2296                 dev_warn(
2297                         &pdev->dev,
2298                         "function level reset failed with %d; trying to continue anyway\n",
2299                         ret);
2300
2301         /*
2302          * If this number is too low and we try to access any BAR too
2303          * early the device will crash. Experiments have shown that
2304          * approximately 50 msec is the minimum amount we have to wait.
2305          * Let's double that to be safe.
2306          */
2307         msleep(100);
2308
2309         ret = pcim_enable_device(pdev);
2310         if (ret)
2311                 return ret;
2312         pci_set_master(pdev);
2313
2314         ret = bcm4377_init_cfg(bcm4377);
2315         if (ret)
2316                 return ret;
2317
2318         bcm4377->bar0 = pcim_iomap(pdev, 0, 0);
2319         if (!bcm4377->bar0)
2320                 return -EBUSY;
2321         bcm4377->bar2 = pcim_iomap(pdev, 2, 0);
2322         if (!bcm4377->bar2)
2323                 return -EBUSY;
2324
2325         ret = bcm4377_parse_otp(bcm4377);
2326         if (ret) {
2327                 dev_err(&pdev->dev, "Reading OTP failed with %d\n", ret);
2328                 return ret;
2329         }
2330
2331         /*
2332          * Legacy interrupts result in an IRQ storm because we don't know where
2333          * the interrupt mask and status registers for these chips are.
2334          * MSIs are acked automatically instead.
2335          */
2336         ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_MSI);
2337         if (ret < 0)
2338                 return -ENODEV;
2339         ret = devm_add_action_or_reset(&pdev->dev, bcm4377_pci_free_irq_vectors,
2340                                        pdev);
2341         if (ret)
2342                 return ret;
2343
2344         irq = pci_irq_vector(pdev, 0);
2345         if (irq <= 0)
2346                 return -ENODEV;
2347
2348         ret = devm_request_irq(&pdev->dev, irq, bcm4377_irq, 0, "bcm4377",
2349                                bcm4377);
2350         if (ret)
2351                 return ret;
2352
2353         hdev = hci_alloc_dev();
2354         if (!hdev)
2355                 return -ENOMEM;
2356         ret = devm_add_action_or_reset(&pdev->dev, bcm4377_hci_free_dev, hdev);
2357         if (ret)
2358                 return ret;
2359
2360         bcm4377->hdev = hdev;
2361
2362         hdev->bus = HCI_PCI;
2363         hdev->dev_type = HCI_PRIMARY;
2364         hdev->open = bcm4377_hci_open;
2365         hdev->close = bcm4377_hci_close;
2366         hdev->send = bcm4377_hci_send_frame;
2367         hdev->set_bdaddr = bcm4377_hci_set_bdaddr;
2368         hdev->setup = bcm4377_hci_setup;
2369
2370         set_bit(HCI_QUIRK_USE_BDADDR_PROPERTY, &hdev->quirks);
2371         if (bcm4377->hw->broken_mws_transport_config)
2372                 set_bit(HCI_QUIRK_BROKEN_MWS_TRANSPORT_CONFIG, &hdev->quirks);
2373         if (bcm4377->hw->broken_ext_scan)
2374                 set_bit(HCI_QUIRK_BROKEN_EXT_SCAN, &hdev->quirks);
2375
2376         pci_set_drvdata(pdev, bcm4377);
2377         hci_set_drvdata(hdev, bcm4377);
2378         SET_HCIDEV_DEV(hdev, &pdev->dev);
2379
2380         ret = bcm4377_boot(bcm4377);
2381         if (ret)
2382                 return ret;
2383
2384         ret = bcm4377_setup_rti(bcm4377);
2385         if (ret)
2386                 return ret;
2387
2388         ret = hci_register_dev(hdev);
2389         if (ret)
2390                 return ret;
2391         return devm_add_action_or_reset(&pdev->dev, bcm4377_hci_unregister_dev,
2392                                         hdev);
2393 }
2394
2395 static int bcm4377_suspend(struct pci_dev *pdev, pm_message_t state)
2396 {
2397         struct bcm4377_data *bcm4377 = pci_get_drvdata(pdev);
2398         int ret;
2399
2400         ret = hci_suspend_dev(bcm4377->hdev);
2401         if (ret)
2402                 return ret;
2403
2404         iowrite32(BCM4377_BAR0_SLEEP_CONTROL_QUIESCE,
2405                   bcm4377->bar0 + BCM4377_BAR0_SLEEP_CONTROL);
2406
2407         return 0;
2408 }
2409
2410 static int bcm4377_resume(struct pci_dev *pdev)
2411 {
2412         struct bcm4377_data *bcm4377 = pci_get_drvdata(pdev);
2413
2414         iowrite32(BCM4377_BAR0_SLEEP_CONTROL_UNQUIESCE,
2415                   bcm4377->bar0 + BCM4377_BAR0_SLEEP_CONTROL);
2416
2417         return hci_resume_dev(bcm4377->hdev);
2418 }
2419
2420 static const struct dmi_system_id bcm4377_dmi_board_table[] = {
2421         {
2422                 .matches = {
2423                         DMI_MATCH(DMI_BOARD_VENDOR, "Apple Inc."),
2424                         DMI_MATCH(DMI_PRODUCT_NAME, "MacBookAir9,1"),
2425                 },
2426                 .driver_data = "apple,formosa",
2427         },
2428         {
2429                 .matches = {
2430                         DMI_MATCH(DMI_BOARD_VENDOR, "Apple Inc."),
2431                         DMI_MATCH(DMI_PRODUCT_NAME, "MacBookPro15,4"),
2432                 },
2433                 .driver_data = "apple,formosa",
2434         },
2435         {
2436                 .matches = {
2437                         DMI_MATCH(DMI_BOARD_VENDOR, "Apple Inc."),
2438                         DMI_MATCH(DMI_PRODUCT_NAME, "MacBookPro16,3"),
2439                 },
2440                 .driver_data = "apple,formosa",
2441         },
2442         {}
2443 };
2444
2445 static const struct bcm4377_hw bcm4377_hw_variants[] = {
2446         [BCM4377] = {
2447                 .id = 0x4377,
2448                 .otp_offset = 0x4120,
2449                 .bar0_window1 = 0x1800b000,
2450                 .bar0_window2 = 0x1810c000,
2451                 .disable_aspm = true,
2452                 .broken_ext_scan = true,
2453                 .send_ptb = bcm4377_send_ptb,
2454         },
2455
2456         [BCM4378] = {
2457                 .id = 0x4378,
2458                 .otp_offset = 0x4120,
2459                 .bar0_window1 = 0x18002000,
2460                 .bar0_window2 = 0x1810a000,
2461                 .bar0_core2_window2 = 0x18107000,
2462                 .has_bar0_core2_window2 = true,
2463                 .broken_mws_transport_config = true,
2464                 .send_calibration = bcm4378_send_calibration,
2465                 .send_ptb = bcm4378_send_ptb,
2466         },
2467
2468         [BCM4387] = {
2469                 .id = 0x4387,
2470                 .otp_offset = 0x413c,
2471                 .bar0_window1 = 0x18002000,
2472                 .bar0_window2 = 0x18109000,
2473                 .bar0_core2_window2 = 0x18106000,
2474                 .has_bar0_core2_window2 = true,
2475                 .clear_pciecfg_subsystem_ctrl_bit19 = true,
2476                 .broken_mws_transport_config = true,
2477                 .send_calibration = bcm4387_send_calibration,
2478                 .send_ptb = bcm4378_send_ptb,
2479         },
2480 };
2481
2482 #define BCM4377_DEVID_ENTRY(id)                                             \
2483         {                                                                   \
2484                 PCI_VENDOR_ID_BROADCOM, BCM##id##_DEVICE_ID, PCI_ANY_ID,    \
2485                         PCI_ANY_ID, PCI_CLASS_NETWORK_OTHER << 8, 0xffff00, \
2486                         BCM##id                                             \
2487         }
2488
2489 static const struct pci_device_id bcm4377_devid_table[] = {
2490         BCM4377_DEVID_ENTRY(4377),
2491         BCM4377_DEVID_ENTRY(4378),
2492         BCM4377_DEVID_ENTRY(4387),
2493         {},
2494 };
2495 MODULE_DEVICE_TABLE(pci, bcm4377_devid_table);
2496
2497 static struct pci_driver bcm4377_pci_driver = {
2498         .name = "hci_bcm4377",
2499         .id_table = bcm4377_devid_table,
2500         .probe = bcm4377_probe,
2501         .suspend = bcm4377_suspend,
2502         .resume = bcm4377_resume,
2503 };
2504 module_pci_driver(bcm4377_pci_driver);
2505
2506 MODULE_AUTHOR("Sven Peter <sven@svenpeter.dev>");
2507 MODULE_DESCRIPTION("Bluetooth support for Broadcom 4377/4378/4387 devices");
2508 MODULE_LICENSE("Dual MIT/GPL");
2509 MODULE_FIRMWARE("brcm/brcmbt4377*.bin");
2510 MODULE_FIRMWARE("brcm/brcmbt4377*.ptb");
2511 MODULE_FIRMWARE("brcm/brcmbt4378*.bin");
2512 MODULE_FIRMWARE("brcm/brcmbt4378*.ptb");
2513 MODULE_FIRMWARE("brcm/brcmbt4387*.bin");
2514 MODULE_FIRMWARE("brcm/brcmbt4387*.ptb");