drivers/thunderbolt/icm.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Internal Thunderbolt Connection Manager. This is a firmware running on
   4  * the Thunderbolt host controller performing most of the low-level
   5  * handling.
   6  *
   7  * Copyright (C) 2017, Intel Corporation
   8  * Authors: Michael Jamet <michael.jamet@intel.com>
   9  *          Mika Westerberg <mika.westerberg@linux.intel.com>
  10  */
  11
  12 #include <linux/delay.h>
  13 #include <linux/mutex.h>
  14 #include <linux/moduleparam.h>
  15 #include <linux/pci.h>
  16 #include <linux/pm_runtime.h>
  17 #include <linux/platform_data/x86/apple.h>
  18 #include <linux/sizes.h>
  19 #include <linux/slab.h>
  20 #include <linux/workqueue.h>
  21
  22 #include "ctl.h"
  23 #include "nhi_regs.h"
  24 #include "tb.h"
  25
  26 #define PCIE2CIO_CMD                    0x30
  27 #define PCIE2CIO_CMD_TIMEOUT            BIT(31)
  28 #define PCIE2CIO_CMD_START              BIT(30)
  29 #define PCIE2CIO_CMD_WRITE              BIT(21)
  30 #define PCIE2CIO_CMD_CS_MASK            GENMASK(20, 19)
  31 #define PCIE2CIO_CMD_CS_SHIFT           19
  32 #define PCIE2CIO_CMD_PORT_MASK          GENMASK(18, 13)
  33 #define PCIE2CIO_CMD_PORT_SHIFT         13
  34
  35 #define PCIE2CIO_WRDATA                 0x34
  36 #define PCIE2CIO_RDDATA                 0x38
  37
  38 #define PHY_PORT_CS1                    0x37
  39 #define PHY_PORT_CS1_LINK_DISABLE       BIT(14)
  40 #define PHY_PORT_CS1_LINK_STATE_MASK    GENMASK(29, 26)
  41 #define PHY_PORT_CS1_LINK_STATE_SHIFT   26
  42
  43 #define ICM_TIMEOUT                     5000    /* ms */
  44 #define ICM_APPROVE_TIMEOUT             10000   /* ms */
  45 #define ICM_MAX_LINK                    4
  46
  47 static bool start_icm;
  48 module_param(start_icm, bool, 0444);
  49 MODULE_PARM_DESC(start_icm, "start ICM firmware if it is not running (default: false)");
  50
  51 /**
  52  * struct usb4_switch_nvm_auth - Holds USB4 NVM_AUTH status
  53  * @reply: Reply from ICM firmware is placed here
  54  * @request: Request that is sent to ICM firmware
  55  * @icm: Pointer to ICM private data
  56  */
  57 struct usb4_switch_nvm_auth {
  58         struct icm_usb4_switch_op_response reply;
  59         struct icm_usb4_switch_op request;
  60         struct icm *icm;
  61 };
  62
  63 /**
  64  * struct icm - Internal connection manager private data
  65  * @request_lock: Makes sure only one message is send to ICM at time
  66  * @rescan_work: Work used to rescan the surviving switches after resume
  67  * @upstream_port: Pointer to the PCIe upstream port this host
  68  *                 controller is connected. This is only set for systems
  69  *                 where ICM needs to be started manually
  70  * @vnd_cap: Vendor defined capability where PCIe2CIO mailbox resides
  71  *           (only set when @upstream_port is not %NULL)
  72  * @safe_mode: ICM is in safe mode
  73  * @max_boot_acl: Maximum number of preboot ACL entries (%0 if not supported)
  74  * @rpm: Does the controller support runtime PM (RTD3)
  75  * @can_upgrade_nvm: Can the NVM firmware be upgrade on this controller
  76  * @proto_version: Firmware protocol version
  77  * @last_nvm_auth: Last USB4 router NVM_AUTH result (or %NULL if not set)
  78  * @veto: Is RTD3 veto in effect
  79  * @is_supported: Checks if we can support ICM on this controller
  80  * @cio_reset: Trigger CIO reset
  81  * @get_mode: Read and return the ICM firmware mode (optional)
  82  * @get_route: Find a route string for given switch
  83  * @save_devices: Ask ICM to save devices to ACL when suspending (optional)
  84  * @driver_ready: Send driver ready message to ICM
  85  * @set_uuid: Set UUID for the root switch (optional)
  86  * @device_connected: Handle device connected ICM message
  87  * @device_disconnected: Handle device disconnected ICM message
  88  * @xdomain_connected - Handle XDomain connected ICM message
  89  * @xdomain_disconnected - Handle XDomain disconnected ICM message
  90  * @rtd3_veto: Handle RTD3 veto notification ICM message
  91  */
  92 struct icm {
  93         struct mutex request_lock;
  94         struct delayed_work rescan_work;
  95         struct pci_dev *upstream_port;
  96         int vnd_cap;
  97         bool safe_mode;
  98         size_t max_boot_acl;
  99         bool rpm;
 100         bool can_upgrade_nvm;
 101         u8 proto_version;
 102         struct usb4_switch_nvm_auth *last_nvm_auth;
 103         bool veto;
 104         bool (*is_supported)(struct tb *tb);
 105         int (*cio_reset)(struct tb *tb);
 106         int (*get_mode)(struct tb *tb);
 107         int (*get_route)(struct tb *tb, u8 link, u8 depth, u64 *route);
 108         void (*save_devices)(struct tb *tb);
 109         int (*driver_ready)(struct tb *tb,
 110                             enum tb_security_level *security_level,
 111                             u8 *proto_version, size_t *nboot_acl, bool *rpm);
 112         void (*set_uuid)(struct tb *tb);
 113         void (*device_connected)(struct tb *tb,
 114                                  const struct icm_pkg_header *hdr);
 115         void (*device_disconnected)(struct tb *tb,
 116                                     const struct icm_pkg_header *hdr);
 117         void (*xdomain_connected)(struct tb *tb,
 118                                   const struct icm_pkg_header *hdr);
 119         void (*xdomain_disconnected)(struct tb *tb,
 120                                      const struct icm_pkg_header *hdr);
 121         void (*rtd3_veto)(struct tb *tb, const struct icm_pkg_header *hdr);
 122 };
 123
 124 struct icm_notification {
 125         struct work_struct work;
 126         struct icm_pkg_header *pkg;
 127         struct tb *tb;
 128 };
 129
 130 struct ep_name_entry {
 131         u8 len;
 132         u8 type;
 133         u8 data[];
 134 };
 135
 136 #define EP_NAME_INTEL_VSS       0x10
 137
 138 /* Intel Vendor specific structure */
 139 struct intel_vss {
 140         u16 vendor;
 141         u16 model;
 142         u8 mc;
 143         u8 flags;
 144         u16 pci_devid;
 145         u32 nvm_version;
 146 };
 147
 148 #define INTEL_VSS_FLAGS_RTD3    BIT(0)
 149
 150 static const struct intel_vss *parse_intel_vss(const void *ep_name, size_t size)
 151 {
 152         const void *end = ep_name + size;
 153
 154         while (ep_name < end) {
 155                 const struct ep_name_entry *ep = ep_name;
 156
 157                 if (!ep->len)
 158                         break;
 159                 if (ep_name + ep->len > end)
 160                         break;
 161
 162                 if (ep->type == EP_NAME_INTEL_VSS)
 163                         return (const struct intel_vss *)ep->data;
 164
 165                 ep_name += ep->len;
 166         }
 167
 168         return NULL;
 169 }
 170
 171 static bool intel_vss_is_rtd3(const void *ep_name, size_t size)
 172 {
 173         const struct intel_vss *vss;
 174
 175         vss = parse_intel_vss(ep_name, size);
 176         if (vss)
 177                 return !!(vss->flags & INTEL_VSS_FLAGS_RTD3);
 178
 179         return false;
 180 }
 181
 182 static inline struct tb *icm_to_tb(struct icm *icm)
 183 {
 184         return ((void *)icm - sizeof(struct tb));
 185 }
 186
 187 static inline u8 phy_port_from_route(u64 route, u8 depth)
 188 {
 189         u8 link;
 190
 191         link = depth ? route >> ((depth - 1) * 8) : route;
 192         return tb_phy_port_from_link(link);
 193 }
 194
 195 static inline u8 dual_link_from_link(u8 link)
 196 {
 197         return link ? ((link - 1) ^ 0x01) + 1 : 0;
 198 }
 199
 200 static inline u64 get_route(u32 route_hi, u32 route_lo)
 201 {
 202         return (u64)route_hi << 32 | route_lo;
 203 }
 204
 205 static inline u64 get_parent_route(u64 route)
 206 {
 207         int depth = tb_route_length(route);
 208         return depth ? route & ~(0xffULL << (depth - 1) * TB_ROUTE_SHIFT) : 0;
 209 }
 210
 211 static int pci2cio_wait_completion(struct icm *icm, unsigned long timeout_msec)
 212 {
 213         unsigned long end = jiffies + msecs_to_jiffies(timeout_msec);
 214         u32 cmd;
 215
 216         do {
 217                 pci_read_config_dword(icm->upstream_port,
 218                                       icm->vnd_cap + PCIE2CIO_CMD, &cmd);
 219                 if (!(cmd & PCIE2CIO_CMD_START)) {
 220                         if (cmd & PCIE2CIO_CMD_TIMEOUT)
 221                                 break;
 222                         return 0;
 223                 }
 224
 225                 msleep(50);
 226         } while (time_before(jiffies, end));
 227
 228         return -ETIMEDOUT;
 229 }
 230
 231 static int pcie2cio_read(struct icm *icm, enum tb_cfg_space cs,
 232                          unsigned int port, unsigned int index, u32 *data)
 233 {
 234         struct pci_dev *pdev = icm->upstream_port;
 235         int ret, vnd_cap = icm->vnd_cap;
 236         u32 cmd;
 237
 238         cmd = index;
 239         cmd |= (port << PCIE2CIO_CMD_PORT_SHIFT) & PCIE2CIO_CMD_PORT_MASK;
 240         cmd |= (cs << PCIE2CIO_CMD_CS_SHIFT) & PCIE2CIO_CMD_CS_MASK;
 241         cmd |= PCIE2CIO_CMD_START;
 242         pci_write_config_dword(pdev, vnd_cap + PCIE2CIO_CMD, cmd);
 243
 244         ret = pci2cio_wait_completion(icm, 5000);
 245         if (ret)
 246                 return ret;
 247
 248         pci_read_config_dword(pdev, vnd_cap + PCIE2CIO_RDDATA, data);
 249         return 0;
 250 }
 251
 252 static int pcie2cio_write(struct icm *icm, enum tb_cfg_space cs,
 253                           unsigned int port, unsigned int index, u32 data)
 254 {
 255         struct pci_dev *pdev = icm->upstream_port;
 256         int vnd_cap = icm->vnd_cap;
 257         u32 cmd;
 258
 259         pci_write_config_dword(pdev, vnd_cap + PCIE2CIO_WRDATA, data);
 260
 261         cmd = index;
 262         cmd |= (port << PCIE2CIO_CMD_PORT_SHIFT) & PCIE2CIO_CMD_PORT_MASK;
 263         cmd |= (cs << PCIE2CIO_CMD_CS_SHIFT) & PCIE2CIO_CMD_CS_MASK;
 264         cmd |= PCIE2CIO_CMD_WRITE | PCIE2CIO_CMD_START;
 265         pci_write_config_dword(pdev, vnd_cap + PCIE2CIO_CMD, cmd);
 266
 267         return pci2cio_wait_completion(icm, 5000);
 268 }
 269
 270 static bool icm_match(const struct tb_cfg_request *req,
 271                       const struct ctl_pkg *pkg)
 272 {
 273         const struct icm_pkg_header *res_hdr = pkg->buffer;
 274         const struct icm_pkg_header *req_hdr = req->request;
 275
 276         if (pkg->frame.eof != req->response_type)
 277                 return false;
 278         if (res_hdr->code != req_hdr->code)
 279                 return false;
 280
 281         return true;
 282 }
 283
 284 static bool icm_copy(struct tb_cfg_request *req, const struct ctl_pkg *pkg)
 285 {
 286         const struct icm_pkg_header *hdr = pkg->buffer;
 287
 288         if (hdr->packet_id < req->npackets) {
 289                 size_t offset = hdr->packet_id * req->response_size;
 290
 291                 memcpy(req->response + offset, pkg->buffer, req->response_size);
 292         }
 293
 294         return hdr->packet_id == hdr->total_packets - 1;
 295 }
 296
 297 static int icm_request(struct tb *tb, const void *request, size_t request_size,
 298                        void *response, size_t response_size, size_t npackets,
 299                        unsigned int timeout_msec)
 300 {
 301         struct icm *icm = tb_priv(tb);
 302         int retries = 3;
 303
 304         do {
 305                 struct tb_cfg_request *req;
 306                 struct tb_cfg_result res;
 307
 308                 req = tb_cfg_request_alloc();
 309                 if (!req)
 310                         return -ENOMEM;
 311
 312                 req->match = icm_match;
 313                 req->copy = icm_copy;
 314                 req->request = request;
 315                 req->request_size = request_size;
 316                 req->request_type = TB_CFG_PKG_ICM_CMD;
 317                 req->response = response;
 318                 req->npackets = npackets;
 319                 req->response_size = response_size;
 320                 req->response_type = TB_CFG_PKG_ICM_RESP;
 321
 322                 mutex_lock(&icm->request_lock);
 323                 res = tb_cfg_request_sync(tb->ctl, req, timeout_msec);
 324                 mutex_unlock(&icm->request_lock);
 325
 326                 tb_cfg_request_put(req);
 327
 328                 if (res.err != -ETIMEDOUT)
 329                         return res.err == 1 ? -EIO : res.err;
 330
 331                 usleep_range(20, 50);
 332         } while (retries--);
 333
 334         return -ETIMEDOUT;
 335 }
 336
 337 /*
 338  * If rescan is queued to run (we are resuming), postpone it to give the
 339  * firmware some more time to send device connected notifications for next
 340  * devices in the chain.
 341  */
 342 static void icm_postpone_rescan(struct tb *tb)
 343 {
 344         struct icm *icm = tb_priv(tb);
 345
 346         if (delayed_work_pending(&icm->rescan_work))
 347                 mod_delayed_work(tb->wq, &icm->rescan_work,
 348                                  msecs_to_jiffies(500));
 349 }
 350
 351 static void icm_veto_begin(struct tb *tb)
 352 {
 353         struct icm *icm = tb_priv(tb);
 354
 355         if (!icm->veto) {
 356                 icm->veto = true;
 357                 /* Keep the domain powered while veto is in effect */
 358                 pm_runtime_get(&tb->dev);
 359         }
 360 }
 361
 362 static void icm_veto_end(struct tb *tb)
 363 {
 364         struct icm *icm = tb_priv(tb);
 365
 366         if (icm->veto) {
 367                 icm->veto = false;
 368                 /* Allow the domain suspend now */
 369                 pm_runtime_mark_last_busy(&tb->dev);
 370                 pm_runtime_put_autosuspend(&tb->dev);
 371         }
 372 }
 373
 374 static bool icm_firmware_running(const struct tb_nhi *nhi)
 375 {
 376         u32 val;
 377
 378         val = ioread32(nhi->iobase + REG_FW_STS);
 379         return !!(val & REG_FW_STS_ICM_EN);
 380 }
 381
 382 static bool icm_fr_is_supported(struct tb *tb)
 383 {
 384         return !x86_apple_machine;
 385 }
 386
 387 static inline int icm_fr_get_switch_index(u32 port)
 388 {
 389         int index;
 390
 391         if ((port & ICM_PORT_TYPE_MASK) != TB_TYPE_PORT)
 392                 return 0;
 393
 394         index = port >> ICM_PORT_INDEX_SHIFT;
 395         return index != 0xff ? index : 0;
 396 }
 397
 398 static int icm_fr_get_route(struct tb *tb, u8 link, u8 depth, u64 *route)
 399 {
 400         struct icm_fr_pkg_get_topology_response *switches, *sw;
 401         struct icm_fr_pkg_get_topology request = {
 402                 .hdr = { .code = ICM_GET_TOPOLOGY },
 403         };
 404         size_t npackets = ICM_GET_TOPOLOGY_PACKETS;
 405         int ret, index;
 406         u8 i;
 407
 408         switches = kcalloc(npackets, sizeof(*switches), GFP_KERNEL);
 409         if (!switches)
 410                 return -ENOMEM;
 411
 412         ret = icm_request(tb, &request, sizeof(request), switches,
 413                           sizeof(*switches), npackets, ICM_TIMEOUT);
 414         if (ret)
 415                 goto err_free;
 416
 417         sw = &switches[0];
 418         index = icm_fr_get_switch_index(sw->ports[link]);
 419         if (!index) {
 420                 ret = -ENODEV;
 421                 goto err_free;
 422         }
 423
 424         sw = &switches[index];
 425         for (i = 1; i < depth; i++) {
 426                 unsigned int j;
 427
 428                 if (!(sw->first_data & ICM_SWITCH_USED)) {
 429                         ret = -ENODEV;
 430                         goto err_free;
 431                 }
 432
 433                 for (j = 0; j < ARRAY_SIZE(sw->ports); j++) {
 434                         index = icm_fr_get_switch_index(sw->ports[j]);
 435                         if (index > sw->switch_index) {
 436                                 sw = &switches[index];
 437                                 break;
 438                         }
 439                 }
 440         }
 441
 442         *route = get_route(sw->route_hi, sw->route_lo);
 443
 444 err_free:
 445         kfree(switches);
 446         return ret;
 447 }
 448
 449 static void icm_fr_save_devices(struct tb *tb)
 450 {
 451         nhi_mailbox_cmd(tb->nhi, NHI_MAILBOX_SAVE_DEVS, 0);
 452 }
 453
 454 static int
 455 icm_fr_driver_ready(struct tb *tb, enum tb_security_level *security_level,
 456                     u8 *proto_version, size_t *nboot_acl, bool *rpm)
 457 {
 458         struct icm_fr_pkg_driver_ready_response reply;
 459         struct icm_pkg_driver_ready request = {
 460                 .hdr.code = ICM_DRIVER_READY,
 461         };
 462         int ret;
 463
 464         memset(&reply, 0, sizeof(reply));
 465         ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
 466                           1, ICM_TIMEOUT);
 467         if (ret)
 468                 return ret;
 469
 470         if (security_level)
 471                 *security_level = reply.security_level & ICM_FR_SLEVEL_MASK;
 472
 473         return 0;
 474 }
 475
 476 static int icm_fr_approve_switch(struct tb *tb, struct tb_switch *sw)
 477 {
 478         struct icm_fr_pkg_approve_device request;
 479         struct icm_fr_pkg_approve_device reply;
 480         int ret;
 481
 482         memset(&request, 0, sizeof(request));
 483         memcpy(&request.ep_uuid, sw->uuid, sizeof(request.ep_uuid));
 484         request.hdr.code = ICM_APPROVE_DEVICE;
 485         request.connection_id = sw->connection_id;
 486         request.connection_key = sw->connection_key;
 487
 488         memset(&reply, 0, sizeof(reply));
 489         /* Use larger timeout as establishing tunnels can take some time */
 490         ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
 491                           1, ICM_APPROVE_TIMEOUT);
 492         if (ret)
 493                 return ret;
 494
 495         if (reply.hdr.flags & ICM_FLAGS_ERROR) {
 496                 tb_warn(tb, "PCIe tunnel creation failed\n");
 497                 return -EIO;
 498         }
 499
 500         return 0;
 501 }
 502
 503 static int icm_fr_add_switch_key(struct tb *tb, struct tb_switch *sw)
 504 {
 505         struct icm_fr_pkg_add_device_key request;
 506         struct icm_fr_pkg_add_device_key_response reply;
 507         int ret;
 508
 509         memset(&request, 0, sizeof(request));
 510         memcpy(&request.ep_uuid, sw->uuid, sizeof(request.ep_uuid));
 511         request.hdr.code = ICM_ADD_DEVICE_KEY;
 512         request.connection_id = sw->connection_id;
 513         request.connection_key = sw->connection_key;
 514         memcpy(request.key, sw->key, TB_SWITCH_KEY_SIZE);
 515
 516         memset(&reply, 0, sizeof(reply));
 517         ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
 518                           1, ICM_TIMEOUT);
 519         if (ret)
 520                 return ret;
 521
 522         if (reply.hdr.flags & ICM_FLAGS_ERROR) {
 523                 tb_warn(tb, "Adding key to switch failed\n");
 524                 return -EIO;
 525         }
 526
 527         return 0;
 528 }
 529
 530 static int icm_fr_challenge_switch_key(struct tb *tb, struct tb_switch *sw,
 531                                        const u8 *challenge, u8 *response)
 532 {
 533         struct icm_fr_pkg_challenge_device request;
 534         struct icm_fr_pkg_challenge_device_response reply;
 535         int ret;
 536
 537         memset(&request, 0, sizeof(request));
 538         memcpy(&request.ep_uuid, sw->uuid, sizeof(request.ep_uuid));
 539         request.hdr.code = ICM_CHALLENGE_DEVICE;
 540         request.connection_id = sw->connection_id;
 541         request.connection_key = sw->connection_key;
 542         memcpy(request.challenge, challenge, TB_SWITCH_KEY_SIZE);
 543
 544         memset(&reply, 0, sizeof(reply));
 545         ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
 546                           1, ICM_TIMEOUT);
 547         if (ret)
 548                 return ret;
 549
 550         if (reply.hdr.flags & ICM_FLAGS_ERROR)
 551                 return -EKEYREJECTED;
 552         if (reply.hdr.flags & ICM_FLAGS_NO_KEY)
 553                 return -ENOKEY;
 554
 555         memcpy(response, reply.response, TB_SWITCH_KEY_SIZE);
 556
 557         return 0;
 558 }
 559
 560 static int icm_fr_approve_xdomain_paths(struct tb *tb, struct tb_xdomain *xd)
 561 {
 562         struct icm_fr_pkg_approve_xdomain_response reply;
 563         struct icm_fr_pkg_approve_xdomain request;
 564         int ret;
 565
 566         memset(&request, 0, sizeof(request));
 567         request.hdr.code = ICM_APPROVE_XDOMAIN;
 568         request.link_info = xd->depth << ICM_LINK_INFO_DEPTH_SHIFT | xd->link;
 569         memcpy(&request.remote_uuid, xd->remote_uuid, sizeof(*xd->remote_uuid));
 570
 571         request.transmit_path = xd->transmit_path;
 572         request.transmit_ring = xd->transmit_ring;
 573         request.receive_path = xd->receive_path;
 574         request.receive_ring = xd->receive_ring;
 575
 576         memset(&reply, 0, sizeof(reply));
 577         ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
 578                           1, ICM_TIMEOUT);
 579         if (ret)
 580                 return ret;
 581
 582         if (reply.hdr.flags & ICM_FLAGS_ERROR)
 583                 return -EIO;
 584
 585         return 0;
 586 }
 587
 588 static int icm_fr_disconnect_xdomain_paths(struct tb *tb, struct tb_xdomain *xd)
 589 {
 590         u8 phy_port;
 591         u8 cmd;
 592
 593         phy_port = tb_phy_port_from_link(xd->link);
 594         if (phy_port == 0)
 595                 cmd = NHI_MAILBOX_DISCONNECT_PA;
 596         else
 597                 cmd = NHI_MAILBOX_DISCONNECT_PB;
 598
 599         nhi_mailbox_cmd(tb->nhi, cmd, 1);
 600         usleep_range(10, 50);
 601         nhi_mailbox_cmd(tb->nhi, cmd, 2);
 602         return 0;
 603 }
 604
 605 static struct tb_switch *alloc_switch(struct tb_switch *parent_sw, u64 route,
 606                                       const uuid_t *uuid)
 607 {
 608         struct tb *tb = parent_sw->tb;
 609         struct tb_switch *sw;
 610
 611         sw = tb_switch_alloc(tb, &parent_sw->dev, route);
 612         if (IS_ERR(sw)) {
 613                 tb_warn(tb, "failed to allocate switch at %llx\n", route);
 614                 return sw;
 615         }
 616
 617         sw->uuid = kmemdup(uuid, sizeof(*uuid), GFP_KERNEL);
 618         if (!sw->uuid) {
 619                 tb_switch_put(sw);
 620                 return ERR_PTR(-ENOMEM);
 621         }
 622
 623         init_completion(&sw->rpm_complete);
 624         return sw;
 625 }
 626
 627 static int add_switch(struct tb_switch *parent_sw, struct tb_switch *sw)
 628 {
 629         u64 route = tb_route(sw);
 630         int ret;
 631
 632         /* Link the two switches now */
 633         tb_port_at(route, parent_sw)->remote = tb_upstream_port(sw);
 634         tb_upstream_port(sw)->remote = tb_port_at(route, parent_sw);
 635
 636         ret = tb_switch_add(sw);
 637         if (ret)
 638                 tb_port_at(tb_route(sw), parent_sw)->remote = NULL;
 639
 640         return ret;
 641 }
 642
 643 static void update_switch(struct tb_switch *parent_sw, struct tb_switch *sw,
 644                           u64 route, u8 connection_id, u8 connection_key,
 645                           u8 link, u8 depth, bool boot)
 646 {
 647         /* Disconnect from parent */
 648         tb_port_at(tb_route(sw), parent_sw)->remote = NULL;
 649         /* Re-connect via updated port*/
 650         tb_port_at(route, parent_sw)->remote = tb_upstream_port(sw);
 651
 652         /* Update with the new addressing information */
 653         sw->config.route_hi = upper_32_bits(route);
 654         sw->config.route_lo = lower_32_bits(route);
 655         sw->connection_id = connection_id;
 656         sw->connection_key = connection_key;
 657         sw->link = link;
 658         sw->depth = depth;
 659         sw->boot = boot;
 660
 661         /* This switch still exists */
 662         sw->is_unplugged = false;
 663
 664         /* Runtime resume is now complete */
 665         complete(&sw->rpm_complete);
 666 }
 667
 668 static void remove_switch(struct tb_switch *sw)
 669 {
 670         struct tb_switch *parent_sw;
 671
 672         parent_sw = tb_to_switch(sw->dev.parent);
 673         tb_port_at(tb_route(sw), parent_sw)->remote = NULL;
 674         tb_switch_remove(sw);
 675 }
 676
 677 static void add_xdomain(struct tb_switch *sw, u64 route,
 678                         const uuid_t *local_uuid, const uuid_t *remote_uuid,
 679                         u8 link, u8 depth)
 680 {
 681         struct tb_xdomain *xd;
 682
 683         pm_runtime_get_sync(&sw->dev);
 684
 685         xd = tb_xdomain_alloc(sw->tb, &sw->dev, route, local_uuid, remote_uuid);
 686         if (!xd)
 687                 goto out;
 688
 689         xd->link = link;
 690         xd->depth = depth;
 691
 692         tb_port_at(route, sw)->xdomain = xd;
 693
 694         tb_xdomain_add(xd);
 695
 696 out:
 697         pm_runtime_mark_last_busy(&sw->dev);
 698         pm_runtime_put_autosuspend(&sw->dev);
 699 }
 700
 701 static void update_xdomain(struct tb_xdomain *xd, u64 route, u8 link)
 702 {
 703         xd->link = link;
 704         xd->route = route;
 705         xd->is_unplugged = false;
 706 }
 707
 708 static void remove_xdomain(struct tb_xdomain *xd)
 709 {
 710         struct tb_switch *sw;
 711
 712         sw = tb_to_switch(xd->dev.parent);
 713         tb_port_at(xd->route, sw)->xdomain = NULL;
 714         tb_xdomain_remove(xd);
 715 }
 716
 717 static void
 718 icm_fr_device_connected(struct tb *tb, const struct icm_pkg_header *hdr)
 719 {
 720         const struct icm_fr_event_device_connected *pkg =
 721                 (const struct icm_fr_event_device_connected *)hdr;
 722         enum tb_security_level security_level;
 723         struct tb_switch *sw, *parent_sw;
 724         bool boot, dual_lane, speed_gen3;
 725         struct icm *icm = tb_priv(tb);
 726         bool authorized = false;
 727         struct tb_xdomain *xd;
 728         u8 link, depth;
 729         u64 route;
 730         int ret;
 731
 732         icm_postpone_rescan(tb);
 733
 734         link = pkg->link_info & ICM_LINK_INFO_LINK_MASK;
 735         depth = (pkg->link_info & ICM_LINK_INFO_DEPTH_MASK) >>
 736                 ICM_LINK_INFO_DEPTH_SHIFT;
 737         authorized = pkg->link_info & ICM_LINK_INFO_APPROVED;
 738         security_level = (pkg->hdr.flags & ICM_FLAGS_SLEVEL_MASK) >>
 739                          ICM_FLAGS_SLEVEL_SHIFT;
 740         boot = pkg->link_info & ICM_LINK_INFO_BOOT;
 741         dual_lane = pkg->hdr.flags & ICM_FLAGS_DUAL_LANE;
 742         speed_gen3 = pkg->hdr.flags & ICM_FLAGS_SPEED_GEN3;
 743
 744         if (pkg->link_info & ICM_LINK_INFO_REJECTED) {
 745                 tb_info(tb, "switch at %u.%u was rejected by ICM firmware because topology limit exceeded\n",
 746                         link, depth);
 747                 return;
 748         }
 749
 750         sw = tb_switch_find_by_uuid(tb, &pkg->ep_uuid);
 751         if (sw) {
 752                 u8 phy_port, sw_phy_port;
 753
 754                 parent_sw = tb_to_switch(sw->dev.parent);
 755                 sw_phy_port = tb_phy_port_from_link(sw->link);
 756                 phy_port = tb_phy_port_from_link(link);
 757
 758                 /*
 759                  * On resume ICM will send us connected events for the
 760                  * devices that still are present. However, that
 761                  * information might have changed for example by the
 762                  * fact that a switch on a dual-link connection might
 763                  * have been enumerated using the other link now. Make
 764                  * sure our book keeping matches that.
 765                  */
 766                 if (sw->depth == depth && sw_phy_port == phy_port &&
 767                     !!sw->authorized == authorized) {
 768                         /*
 769                          * It was enumerated through another link so update
 770                          * route string accordingly.
 771                          */
 772                         if (sw->link != link) {
 773                                 ret = icm->get_route(tb, link, depth, &route);
 774                                 if (ret) {
 775                                         tb_err(tb, "failed to update route string for switch at %u.%u\n",
 776                                                link, depth);
 777                                         tb_switch_put(sw);
 778                                         return;
 779                                 }
 780                         } else {
 781                                 route = tb_route(sw);
 782                         }
 783
 784                         update_switch(parent_sw, sw, route, pkg->connection_id,
 785                                       pkg->connection_key, link, depth, boot);
 786                         tb_switch_put(sw);
 787                         return;
 788                 }
 789
 790                 /*
 791                  * User connected the same switch to another physical
 792                  * port or to another part of the topology. Remove the
 793                  * existing switch now before adding the new one.
 794                  */
 795                 remove_switch(sw);
 796                 tb_switch_put(sw);
 797         }
 798
 799         /*
 800          * If the switch was not found by UUID, look for a switch on
 801          * same physical port (taking possible link aggregation into
 802          * account) and depth. If we found one it is definitely a stale
 803          * one so remove it first.
 804          */
 805         sw = tb_switch_find_by_link_depth(tb, link, depth);
 806         if (!sw) {
 807                 u8 dual_link;
 808
 809                 dual_link = dual_link_from_link(link);
 810                 if (dual_link)
 811                         sw = tb_switch_find_by_link_depth(tb, dual_link, depth);
 812         }
 813         if (sw) {
 814                 remove_switch(sw);
 815                 tb_switch_put(sw);
 816         }
 817
 818         /* Remove existing XDomain connection if found */
 819         xd = tb_xdomain_find_by_link_depth(tb, link, depth);
 820         if (xd) {
 821                 remove_xdomain(xd);
 822                 tb_xdomain_put(xd);
 823         }
 824
 825         parent_sw = tb_switch_find_by_link_depth(tb, link, depth - 1);
 826         if (!parent_sw) {
 827                 tb_err(tb, "failed to find parent switch for %u.%u\n",
 828                        link, depth);
 829                 return;
 830         }
 831
 832         ret = icm->get_route(tb, link, depth, &route);
 833         if (ret) {
 834                 tb_err(tb, "failed to find route string for switch at %u.%u\n",
 835                        link, depth);
 836                 tb_switch_put(parent_sw);
 837                 return;
 838         }
 839
 840         pm_runtime_get_sync(&parent_sw->dev);
 841
 842         sw = alloc_switch(parent_sw, route, &pkg->ep_uuid);
 843         if (!IS_ERR(sw)) {
 844                 sw->connection_id = pkg->connection_id;
 845                 sw->connection_key = pkg->connection_key;
 846                 sw->link = link;
 847                 sw->depth = depth;
 848                 sw->authorized = authorized;
 849                 sw->security_level = security_level;
 850                 sw->boot = boot;
 851                 sw->link_speed = speed_gen3 ? 20 : 10;
 852                 sw->link_width = dual_lane ? 2 : 1;
 853                 sw->rpm = intel_vss_is_rtd3(pkg->ep_name, sizeof(pkg->ep_name));
 854
 855                 if (add_switch(parent_sw, sw))
 856                         tb_switch_put(sw);
 857         }
 858
 859         pm_runtime_mark_last_busy(&parent_sw->dev);
 860         pm_runtime_put_autosuspend(&parent_sw->dev);
 861
 862         tb_switch_put(parent_sw);
 863 }
 864
 865 static void
 866 icm_fr_device_disconnected(struct tb *tb, const struct icm_pkg_header *hdr)
 867 {
 868         const struct icm_fr_event_device_disconnected *pkg =
 869                 (const struct icm_fr_event_device_disconnected *)hdr;
 870         struct tb_switch *sw;
 871         u8 link, depth;
 872
 873         link = pkg->link_info & ICM_LINK_INFO_LINK_MASK;
 874         depth = (pkg->link_info & ICM_LINK_INFO_DEPTH_MASK) >>
 875                 ICM_LINK_INFO_DEPTH_SHIFT;
 876
 877         if (link > ICM_MAX_LINK || depth > TB_SWITCH_MAX_DEPTH) {
 878                 tb_warn(tb, "invalid topology %u.%u, ignoring\n", link, depth);
 879                 return;
 880         }
 881
 882         sw = tb_switch_find_by_link_depth(tb, link, depth);
 883         if (!sw) {
 884                 tb_warn(tb, "no switch exists at %u.%u, ignoring\n", link,
 885                         depth);
 886                 return;
 887         }
 888
 889         pm_runtime_get_sync(sw->dev.parent);
 890
 891         remove_switch(sw);
 892
 893         pm_runtime_mark_last_busy(sw->dev.parent);
 894         pm_runtime_put_autosuspend(sw->dev.parent);
 895
 896         tb_switch_put(sw);
 897 }
 898
 899 static void
 900 icm_fr_xdomain_connected(struct tb *tb, const struct icm_pkg_header *hdr)
 901 {
 902         const struct icm_fr_event_xdomain_connected *pkg =
 903                 (const struct icm_fr_event_xdomain_connected *)hdr;
 904         struct tb_xdomain *xd;
 905         struct tb_switch *sw;
 906         u8 link, depth;
 907         u64 route;
 908
 909         link = pkg->link_info & ICM_LINK_INFO_LINK_MASK;
 910         depth = (pkg->link_info & ICM_LINK_INFO_DEPTH_MASK) >>
 911                 ICM_LINK_INFO_DEPTH_SHIFT;
 912
 913         if (link > ICM_MAX_LINK || depth > TB_SWITCH_MAX_DEPTH) {
 914                 tb_warn(tb, "invalid topology %u.%u, ignoring\n", link, depth);
 915                 return;
 916         }
 917
 918         route = get_route(pkg->local_route_hi, pkg->local_route_lo);
 919
 920         xd = tb_xdomain_find_by_uuid(tb, &pkg->remote_uuid);
 921         if (xd) {
 922                 u8 xd_phy_port, phy_port;
 923
 924                 xd_phy_port = phy_port_from_route(xd->route, xd->depth);
 925                 phy_port = phy_port_from_route(route, depth);
 926
 927                 if (xd->depth == depth && xd_phy_port == phy_port) {
 928                         update_xdomain(xd, route, link);
 929                         tb_xdomain_put(xd);
 930                         return;
 931                 }
 932
 933                 /*
 934                  * If we find an existing XDomain connection remove it
 935                  * now. We need to go through login handshake and
 936                  * everything anyway to be able to re-establish the
 937                  * connection.
 938                  */
 939                 remove_xdomain(xd);
 940                 tb_xdomain_put(xd);
 941         }
 942
 943         /*
 944          * Look if there already exists an XDomain in the same place
 945          * than the new one and in that case remove it because it is
 946          * most likely another host that got disconnected.
 947          */
 948         xd = tb_xdomain_find_by_link_depth(tb, link, depth);
 949         if (!xd) {
 950                 u8 dual_link;
 951
 952                 dual_link = dual_link_from_link(link);
 953                 if (dual_link)
 954                         xd = tb_xdomain_find_by_link_depth(tb, dual_link,
 955                                                            depth);
 956         }
 957         if (xd) {
 958                 remove_xdomain(xd);
 959                 tb_xdomain_put(xd);
 960         }
 961
 962         /*
 963          * If the user disconnected a switch during suspend and
 964          * connected another host to the same port, remove the switch
 965          * first.
 966          */
 967         sw = tb_switch_find_by_route(tb, route);
 968         if (sw) {
 969                 remove_switch(sw);
 970                 tb_switch_put(sw);
 971         }
 972
 973         sw = tb_switch_find_by_link_depth(tb, link, depth);
 974         if (!sw) {
 975                 tb_warn(tb, "no switch exists at %u.%u, ignoring\n", link,
 976                         depth);
 977                 return;
 978         }
 979
 980         add_xdomain(sw, route, &pkg->local_uuid, &pkg->remote_uuid, link,
 981                     depth);
 982         tb_switch_put(sw);
 983 }
 984
 985 static void
 986 icm_fr_xdomain_disconnected(struct tb *tb, const struct icm_pkg_header *hdr)
 987 {
 988         const struct icm_fr_event_xdomain_disconnected *pkg =
 989                 (const struct icm_fr_event_xdomain_disconnected *)hdr;
 990         struct tb_xdomain *xd;
 991
 992         /*
 993          * If the connection is through one or multiple devices, the
 994          * XDomain device is removed along with them so it is fine if we
 995          * cannot find it here.
 996          */
 997         xd = tb_xdomain_find_by_uuid(tb, &pkg->remote_uuid);
 998         if (xd) {
 999                 remove_xdomain(xd);
1000                 tb_xdomain_put(xd);
1001         }
1002 }
1003
1004 static int icm_tr_cio_reset(struct tb *tb)
1005 {
1006         return pcie2cio_write(tb_priv(tb), TB_CFG_SWITCH, 0, 0x777, BIT(1));
1007 }
1008
1009 static int
1010 icm_tr_driver_ready(struct tb *tb, enum tb_security_level *security_level,
1011                     u8 *proto_version, size_t *nboot_acl, bool *rpm)
1012 {
1013         struct icm_tr_pkg_driver_ready_response reply;
1014         struct icm_pkg_driver_ready request = {
1015                 .hdr.code = ICM_DRIVER_READY,
1016         };
1017         int ret;
1018
1019         memset(&reply, 0, sizeof(reply));
1020         ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1021                           1, 20000);
1022         if (ret)
1023                 return ret;
1024
1025         if (security_level)
1026                 *security_level = reply.info & ICM_TR_INFO_SLEVEL_MASK;
1027         if (proto_version)
1028                 *proto_version = (reply.info & ICM_TR_INFO_PROTO_VERSION_MASK) >>
1029                                 ICM_TR_INFO_PROTO_VERSION_SHIFT;
1030         if (nboot_acl)
1031                 *nboot_acl = (reply.info & ICM_TR_INFO_BOOT_ACL_MASK) >>
1032                                 ICM_TR_INFO_BOOT_ACL_SHIFT;
1033         if (rpm)
1034                 *rpm = !!(reply.hdr.flags & ICM_TR_FLAGS_RTD3);
1035
1036         return 0;
1037 }
1038
1039 static int icm_tr_approve_switch(struct tb *tb, struct tb_switch *sw)
1040 {
1041         struct icm_tr_pkg_approve_device request;
1042         struct icm_tr_pkg_approve_device reply;
1043         int ret;
1044
1045         memset(&request, 0, sizeof(request));
1046         memcpy(&request.ep_uuid, sw->uuid, sizeof(request.ep_uuid));
1047         request.hdr.code = ICM_APPROVE_DEVICE;
1048         request.route_lo = sw->config.route_lo;
1049         request.route_hi = sw->config.route_hi;
1050         request.connection_id = sw->connection_id;
1051
1052         memset(&reply, 0, sizeof(reply));
1053         ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1054                           1, ICM_APPROVE_TIMEOUT);
1055         if (ret)
1056                 return ret;
1057
1058         if (reply.hdr.flags & ICM_FLAGS_ERROR) {
1059                 tb_warn(tb, "PCIe tunnel creation failed\n");
1060                 return -EIO;
1061         }
1062
1063         return 0;
1064 }
1065
1066 static int icm_tr_add_switch_key(struct tb *tb, struct tb_switch *sw)
1067 {
1068         struct icm_tr_pkg_add_device_key_response reply;
1069         struct icm_tr_pkg_add_device_key request;
1070         int ret;
1071
1072         memset(&request, 0, sizeof(request));
1073         memcpy(&request.ep_uuid, sw->uuid, sizeof(request.ep_uuid));
1074         request.hdr.code = ICM_ADD_DEVICE_KEY;
1075         request.route_lo = sw->config.route_lo;
1076         request.route_hi = sw->config.route_hi;
1077         request.connection_id = sw->connection_id;
1078         memcpy(request.key, sw->key, TB_SWITCH_KEY_SIZE);
1079
1080         memset(&reply, 0, sizeof(reply));
1081         ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1082                           1, ICM_TIMEOUT);
1083         if (ret)
1084                 return ret;
1085
1086         if (reply.hdr.flags & ICM_FLAGS_ERROR) {
1087                 tb_warn(tb, "Adding key to switch failed\n");
1088                 return -EIO;
1089         }
1090
1091         return 0;
1092 }
1093
1094 static int icm_tr_challenge_switch_key(struct tb *tb, struct tb_switch *sw,
1095                                        const u8 *challenge, u8 *response)
1096 {
1097         struct icm_tr_pkg_challenge_device_response reply;
1098         struct icm_tr_pkg_challenge_device request;
1099         int ret;
1100
1101         memset(&request, 0, sizeof(request));
1102         memcpy(&request.ep_uuid, sw->uuid, sizeof(request.ep_uuid));
1103         request.hdr.code = ICM_CHALLENGE_DEVICE;
1104         request.route_lo = sw->config.route_lo;
1105         request.route_hi = sw->config.route_hi;
1106         request.connection_id = sw->connection_id;
1107         memcpy(request.challenge, challenge, TB_SWITCH_KEY_SIZE);
1108
1109         memset(&reply, 0, sizeof(reply));
1110         ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1111                           1, ICM_TIMEOUT);
1112         if (ret)
1113                 return ret;
1114
1115         if (reply.hdr.flags & ICM_FLAGS_ERROR)
1116                 return -EKEYREJECTED;
1117         if (reply.hdr.flags & ICM_FLAGS_NO_KEY)
1118                 return -ENOKEY;
1119
1120         memcpy(response, reply.response, TB_SWITCH_KEY_SIZE);
1121
1122         return 0;
1123 }
1124
1125 static int icm_tr_approve_xdomain_paths(struct tb *tb, struct tb_xdomain *xd)
1126 {
1127         struct icm_tr_pkg_approve_xdomain_response reply;
1128         struct icm_tr_pkg_approve_xdomain request;
1129         int ret;
1130
1131         memset(&request, 0, sizeof(request));
1132         request.hdr.code = ICM_APPROVE_XDOMAIN;
1133         request.route_hi = upper_32_bits(xd->route);
1134         request.route_lo = lower_32_bits(xd->route);
1135         request.transmit_path = xd->transmit_path;
1136         request.transmit_ring = xd->transmit_ring;
1137         request.receive_path = xd->receive_path;
1138         request.receive_ring = xd->receive_ring;
1139         memcpy(&request.remote_uuid, xd->remote_uuid, sizeof(*xd->remote_uuid));
1140
1141         memset(&reply, 0, sizeof(reply));
1142         ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1143                           1, ICM_TIMEOUT);
1144         if (ret)
1145                 return ret;
1146
1147         if (reply.hdr.flags & ICM_FLAGS_ERROR)
1148                 return -EIO;
1149
1150         return 0;
1151 }
1152
1153 static int icm_tr_xdomain_tear_down(struct tb *tb, struct tb_xdomain *xd,
1154                                     int stage)
1155 {
1156         struct icm_tr_pkg_disconnect_xdomain_response reply;
1157         struct icm_tr_pkg_disconnect_xdomain request;
1158         int ret;
1159
1160         memset(&request, 0, sizeof(request));
1161         request.hdr.code = ICM_DISCONNECT_XDOMAIN;
1162         request.stage = stage;
1163         request.route_hi = upper_32_bits(xd->route);
1164         request.route_lo = lower_32_bits(xd->route);
1165         memcpy(&request.remote_uuid, xd->remote_uuid, sizeof(*xd->remote_uuid));
1166
1167         memset(&reply, 0, sizeof(reply));
1168         ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1169                           1, ICM_TIMEOUT);
1170         if (ret)
1171                 return ret;
1172
1173         if (reply.hdr.flags & ICM_FLAGS_ERROR)
1174                 return -EIO;
1175
1176         return 0;
1177 }
1178
1179 static int icm_tr_disconnect_xdomain_paths(struct tb *tb, struct tb_xdomain *xd)
1180 {
1181         int ret;
1182
1183         ret = icm_tr_xdomain_tear_down(tb, xd, 1);
1184         if (ret)
1185                 return ret;
1186
1187         usleep_range(10, 50);
1188         return icm_tr_xdomain_tear_down(tb, xd, 2);
1189 }
1190
1191 static void
1192 __icm_tr_device_connected(struct tb *tb, const struct icm_pkg_header *hdr,
1193                           bool force_rtd3)
1194 {
1195         const struct icm_tr_event_device_connected *pkg =
1196                 (const struct icm_tr_event_device_connected *)hdr;
1197         bool authorized, boot, dual_lane, speed_gen3;
1198         enum tb_security_level security_level;
1199         struct tb_switch *sw, *parent_sw;
1200         struct tb_xdomain *xd;
1201         u64 route;
1202
1203         icm_postpone_rescan(tb);
1204
1205         /*
1206          * Currently we don't use the QoS information coming with the
1207          * device connected message so simply just ignore that extra
1208          * packet for now.
1209          */
1210         if (pkg->hdr.packet_id)
1211                 return;
1212
1213         route = get_route(pkg->route_hi, pkg->route_lo);
1214         authorized = pkg->link_info & ICM_LINK_INFO_APPROVED;
1215         security_level = (pkg->hdr.flags & ICM_FLAGS_SLEVEL_MASK) >>
1216                          ICM_FLAGS_SLEVEL_SHIFT;
1217         boot = pkg->link_info & ICM_LINK_INFO_BOOT;
1218         dual_lane = pkg->hdr.flags & ICM_FLAGS_DUAL_LANE;
1219         speed_gen3 = pkg->hdr.flags & ICM_FLAGS_SPEED_GEN3;
1220
1221         if (pkg->link_info & ICM_LINK_INFO_REJECTED) {
1222                 tb_info(tb, "switch at %llx was rejected by ICM firmware because topology limit exceeded\n",
1223                         route);
1224                 return;
1225         }
1226
1227         sw = tb_switch_find_by_uuid(tb, &pkg->ep_uuid);
1228         if (sw) {
1229                 /* Update the switch if it is still in the same place */
1230                 if (tb_route(sw) == route && !!sw->authorized == authorized) {
1231                         parent_sw = tb_to_switch(sw->dev.parent);
1232                         update_switch(parent_sw, sw, route, pkg->connection_id,
1233                                       0, 0, 0, boot);
1234                         tb_switch_put(sw);
1235                         return;
1236                 }
1237
1238                 remove_switch(sw);
1239                 tb_switch_put(sw);
1240         }
1241
1242         /* Another switch with the same address */
1243         sw = tb_switch_find_by_route(tb, route);
1244         if (sw) {
1245                 remove_switch(sw);
1246                 tb_switch_put(sw);
1247         }
1248
1249         /* XDomain connection with the same address */
1250         xd = tb_xdomain_find_by_route(tb, route);
1251         if (xd) {
1252                 remove_xdomain(xd);
1253                 tb_xdomain_put(xd);
1254         }
1255
1256         parent_sw = tb_switch_find_by_route(tb, get_parent_route(route));
1257         if (!parent_sw) {
1258                 tb_err(tb, "failed to find parent switch for %llx\n", route);
1259                 return;
1260         }
1261
1262         pm_runtime_get_sync(&parent_sw->dev);
1263
1264         sw = alloc_switch(parent_sw, route, &pkg->ep_uuid);
1265         if (!IS_ERR(sw)) {
1266                 sw->connection_id = pkg->connection_id;
1267                 sw->authorized = authorized;
1268                 sw->security_level = security_level;
1269                 sw->boot = boot;
1270                 sw->link_speed = speed_gen3 ? 20 : 10;
1271                 sw->link_width = dual_lane ? 2 : 1;
1272                 sw->rpm = force_rtd3;
1273                 if (!sw->rpm)
1274                         sw->rpm = intel_vss_is_rtd3(pkg->ep_name,
1275                                                     sizeof(pkg->ep_name));
1276
1277                 if (add_switch(parent_sw, sw))
1278                         tb_switch_put(sw);
1279         }
1280
1281         pm_runtime_mark_last_busy(&parent_sw->dev);
1282         pm_runtime_put_autosuspend(&parent_sw->dev);
1283
1284         tb_switch_put(parent_sw);
1285 }
1286
1287 static void
1288 icm_tr_device_connected(struct tb *tb, const struct icm_pkg_header *hdr)
1289 {
1290         __icm_tr_device_connected(tb, hdr, false);
1291 }
1292
1293 static void
1294 icm_tr_device_disconnected(struct tb *tb, const struct icm_pkg_header *hdr)
1295 {
1296         const struct icm_tr_event_device_disconnected *pkg =
1297                 (const struct icm_tr_event_device_disconnected *)hdr;
1298         struct tb_switch *sw;
1299         u64 route;
1300
1301         route = get_route(pkg->route_hi, pkg->route_lo);
1302
1303         sw = tb_switch_find_by_route(tb, route);
1304         if (!sw) {
1305                 tb_warn(tb, "no switch exists at %llx, ignoring\n", route);
1306                 return;
1307         }
1308         pm_runtime_get_sync(sw->dev.parent);
1309
1310         remove_switch(sw);
1311
1312         pm_runtime_mark_last_busy(sw->dev.parent);
1313         pm_runtime_put_autosuspend(sw->dev.parent);
1314
1315         tb_switch_put(sw);
1316 }
1317
1318 static void
1319 icm_tr_xdomain_connected(struct tb *tb, const struct icm_pkg_header *hdr)
1320 {
1321         const struct icm_tr_event_xdomain_connected *pkg =
1322                 (const struct icm_tr_event_xdomain_connected *)hdr;
1323         struct tb_xdomain *xd;
1324         struct tb_switch *sw;
1325         u64 route;
1326
1327         if (!tb->root_switch)
1328                 return;
1329
1330         route = get_route(pkg->local_route_hi, pkg->local_route_lo);
1331
1332         xd = tb_xdomain_find_by_uuid(tb, &pkg->remote_uuid);
1333         if (xd) {
1334                 if (xd->route == route) {
1335                         update_xdomain(xd, route, 0);
1336                         tb_xdomain_put(xd);
1337                         return;
1338                 }
1339
1340                 remove_xdomain(xd);
1341                 tb_xdomain_put(xd);
1342         }
1343
1344         /* An existing xdomain with the same address */
1345         xd = tb_xdomain_find_by_route(tb, route);
1346         if (xd) {
1347                 remove_xdomain(xd);
1348                 tb_xdomain_put(xd);
1349         }
1350
1351         /*
1352          * If the user disconnected a switch during suspend and
1353          * connected another host to the same port, remove the switch
1354          * first.
1355          */
1356         sw = tb_switch_find_by_route(tb, route);
1357         if (sw) {
1358                 remove_switch(sw);
1359                 tb_switch_put(sw);
1360         }
1361
1362         sw = tb_switch_find_by_route(tb, get_parent_route(route));
1363         if (!sw) {
1364                 tb_warn(tb, "no switch exists at %llx, ignoring\n", route);
1365                 return;
1366         }
1367
1368         add_xdomain(sw, route, &pkg->local_uuid, &pkg->remote_uuid, 0, 0);
1369         tb_switch_put(sw);
1370 }
1371
1372 static void
1373 icm_tr_xdomain_disconnected(struct tb *tb, const struct icm_pkg_header *hdr)
1374 {
1375         const struct icm_tr_event_xdomain_disconnected *pkg =
1376                 (const struct icm_tr_event_xdomain_disconnected *)hdr;
1377         struct tb_xdomain *xd;
1378         u64 route;
1379
1380         route = get_route(pkg->route_hi, pkg->route_lo);
1381
1382         xd = tb_xdomain_find_by_route(tb, route);
1383         if (xd) {
1384                 remove_xdomain(xd);
1385                 tb_xdomain_put(xd);
1386         }
1387 }
1388
1389 static struct pci_dev *get_upstream_port(struct pci_dev *pdev)
1390 {
1391         struct pci_dev *parent;
1392
1393         parent = pci_upstream_bridge(pdev);
1394         while (parent) {
1395                 if (!pci_is_pcie(parent))
1396                         return NULL;
1397                 if (pci_pcie_type(parent) == PCI_EXP_TYPE_UPSTREAM)
1398                         break;
1399                 parent = pci_upstream_bridge(parent);
1400         }
1401
1402         if (!parent)
1403                 return NULL;
1404
1405         switch (parent->device) {
1406         case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_2C_BRIDGE:
1407         case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_4C_BRIDGE:
1408         case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_LP_BRIDGE:
1409         case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_C_4C_BRIDGE:
1410         case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_C_2C_BRIDGE:
1411         case PCI_DEVICE_ID_INTEL_TITAN_RIDGE_2C_BRIDGE:
1412         case PCI_DEVICE_ID_INTEL_TITAN_RIDGE_4C_BRIDGE:
1413                 return parent;
1414         }
1415
1416         return NULL;
1417 }
1418
1419 static bool icm_ar_is_supported(struct tb *tb)
1420 {
1421         struct pci_dev *upstream_port;
1422         struct icm *icm = tb_priv(tb);
1423
1424         /*
1425          * Starting from Alpine Ridge we can use ICM on Apple machines
1426          * as well. We just need to reset and re-enable it first.
1427          * However, only start it if explicitly asked by the user.
1428          */
1429         if (icm_firmware_running(tb->nhi))
1430                 return true;
1431         if (!start_icm)
1432                 return false;
1433
1434         /*
1435          * Find the upstream PCIe port in case we need to do reset
1436          * through its vendor specific registers.
1437          */
1438         upstream_port = get_upstream_port(tb->nhi->pdev);
1439         if (upstream_port) {
1440                 int cap;
1441
1442                 cap = pci_find_ext_capability(upstream_port,
1443                                               PCI_EXT_CAP_ID_VNDR);
1444                 if (cap > 0) {
1445                         icm->upstream_port = upstream_port;
1446                         icm->vnd_cap = cap;
1447
1448                         return true;
1449                 }
1450         }
1451
1452         return false;
1453 }
1454
1455 static int icm_ar_cio_reset(struct tb *tb)
1456 {
1457         return pcie2cio_write(tb_priv(tb), TB_CFG_SWITCH, 0, 0x50, BIT(9));
1458 }
1459
1460 static int icm_ar_get_mode(struct tb *tb)
1461 {
1462         struct tb_nhi *nhi = tb->nhi;
1463         int retries = 60;
1464         u32 val;
1465
1466         do {
1467                 val = ioread32(nhi->iobase + REG_FW_STS);
1468                 if (val & REG_FW_STS_NVM_AUTH_DONE)
1469                         break;
1470                 msleep(50);
1471         } while (--retries);
1472
1473         if (!retries) {
1474                 dev_err(&nhi->pdev->dev, "ICM firmware not authenticated\n");
1475                 return -ENODEV;
1476         }
1477
1478         return nhi_mailbox_mode(nhi);
1479 }
1480
1481 static int
1482 icm_ar_driver_ready(struct tb *tb, enum tb_security_level *security_level,
1483                     u8 *proto_version, size_t *nboot_acl, bool *rpm)
1484 {
1485         struct icm_ar_pkg_driver_ready_response reply;
1486         struct icm_pkg_driver_ready request = {
1487                 .hdr.code = ICM_DRIVER_READY,
1488         };
1489         int ret;
1490
1491         memset(&reply, 0, sizeof(reply));
1492         ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1493                           1, ICM_TIMEOUT);
1494         if (ret)
1495                 return ret;
1496
1497         if (security_level)
1498                 *security_level = reply.info & ICM_AR_INFO_SLEVEL_MASK;
1499         if (nboot_acl && (reply.info & ICM_AR_INFO_BOOT_ACL_SUPPORTED))
1500                 *nboot_acl = (reply.info & ICM_AR_INFO_BOOT_ACL_MASK) >>
1501                                 ICM_AR_INFO_BOOT_ACL_SHIFT;
1502         if (rpm)
1503                 *rpm = !!(reply.hdr.flags & ICM_AR_FLAGS_RTD3);
1504
1505         return 0;
1506 }
1507
1508 static int icm_ar_get_route(struct tb *tb, u8 link, u8 depth, u64 *route)
1509 {
1510         struct icm_ar_pkg_get_route_response reply;
1511         struct icm_ar_pkg_get_route request = {
1512                 .hdr = { .code = ICM_GET_ROUTE },
1513                 .link_info = depth << ICM_LINK_INFO_DEPTH_SHIFT | link,
1514         };
1515         int ret;
1516
1517         memset(&reply, 0, sizeof(reply));
1518         ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1519                           1, ICM_TIMEOUT);
1520         if (ret)
1521                 return ret;
1522
1523         if (reply.hdr.flags & ICM_FLAGS_ERROR)
1524                 return -EIO;
1525
1526         *route = get_route(reply.route_hi, reply.route_lo);
1527         return 0;
1528 }
1529
1530 static int icm_ar_get_boot_acl(struct tb *tb, uuid_t *uuids, size_t nuuids)
1531 {
1532         struct icm_ar_pkg_preboot_acl_response reply;
1533         struct icm_ar_pkg_preboot_acl request = {
1534                 .hdr = { .code = ICM_PREBOOT_ACL },
1535         };
1536         int ret, i;
1537
1538         memset(&reply, 0, sizeof(reply));
1539         ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1540                           1, ICM_TIMEOUT);
1541         if (ret)
1542                 return ret;
1543
1544         if (reply.hdr.flags & ICM_FLAGS_ERROR)
1545                 return -EIO;
1546
1547         for (i = 0; i < nuuids; i++) {
1548                 u32 *uuid = (u32 *)&uuids[i];
1549
1550                 uuid[0] = reply.acl[i].uuid_lo;
1551                 uuid[1] = reply.acl[i].uuid_hi;
1552
1553                 if (uuid[0] == 0xffffffff && uuid[1] == 0xffffffff) {
1554                         /* Map empty entries to null UUID */
1555                         uuid[0] = 0;
1556                         uuid[1] = 0;
1557                 } else if (uuid[0] != 0 || uuid[1] != 0) {
1558                         /* Upper two DWs are always one's */
1559                         uuid[2] = 0xffffffff;
1560                         uuid[3] = 0xffffffff;
1561                 }
1562         }
1563
1564         return ret;
1565 }
1566
1567 static int icm_ar_set_boot_acl(struct tb *tb, const uuid_t *uuids,
1568                                size_t nuuids)
1569 {
1570         struct icm_ar_pkg_preboot_acl_response reply;
1571         struct icm_ar_pkg_preboot_acl request = {
1572                 .hdr = {
1573                         .code = ICM_PREBOOT_ACL,
1574                         .flags = ICM_FLAGS_WRITE,
1575                 },
1576         };
1577         int ret, i;
1578
1579         for (i = 0; i < nuuids; i++) {
1580                 const u32 *uuid = (const u32 *)&uuids[i];
1581
1582                 if (uuid_is_null(&uuids[i])) {
1583                         /*
1584                          * Map null UUID to the empty (all one) entries
1585                          * for ICM.
1586                          */
1587                         request.acl[i].uuid_lo = 0xffffffff;
1588                         request.acl[i].uuid_hi = 0xffffffff;
1589                 } else {
1590                         /* Two high DWs need to be set to all one */
1591                         if (uuid[2] != 0xffffffff || uuid[3] != 0xffffffff)
1592                                 return -EINVAL;
1593
1594                         request.acl[i].uuid_lo = uuid[0];
1595                         request.acl[i].uuid_hi = uuid[1];
1596                 }
1597         }
1598
1599         memset(&reply, 0, sizeof(reply));
1600         ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1601                           1, ICM_TIMEOUT);
1602         if (ret)
1603                 return ret;
1604
1605         if (reply.hdr.flags & ICM_FLAGS_ERROR)
1606                 return -EIO;
1607
1608         return 0;
1609 }
1610
1611 static int
1612 icm_icl_driver_ready(struct tb *tb, enum tb_security_level *security_level,
1613                      u8 *proto_version, size_t *nboot_acl, bool *rpm)
1614 {
1615         struct icm_tr_pkg_driver_ready_response reply;
1616         struct icm_pkg_driver_ready request = {
1617                 .hdr.code = ICM_DRIVER_READY,
1618         };
1619         int ret;
1620
1621         memset(&reply, 0, sizeof(reply));
1622         ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1623                           1, 20000);
1624         if (ret)
1625                 return ret;
1626
1627         if (proto_version)
1628                 *proto_version = (reply.info & ICM_TR_INFO_PROTO_VERSION_MASK) >>
1629                                 ICM_TR_INFO_PROTO_VERSION_SHIFT;
1630
1631         /* Ice Lake always supports RTD3 */
1632         if (rpm)
1633                 *rpm = true;
1634
1635         return 0;
1636 }
1637
1638 static void icm_icl_set_uuid(struct tb *tb)
1639 {
1640         struct tb_nhi *nhi = tb->nhi;
1641         u32 uuid[4];
1642
1643         pci_read_config_dword(nhi->pdev, VS_CAP_10, &uuid[0]);
1644         pci_read_config_dword(nhi->pdev, VS_CAP_11, &uuid[1]);
1645         uuid[2] = 0xffffffff;
1646         uuid[3] = 0xffffffff;
1647
1648         tb->root_switch->uuid = kmemdup(uuid, sizeof(uuid), GFP_KERNEL);
1649 }
1650
1651 static void
1652 icm_icl_device_connected(struct tb *tb, const struct icm_pkg_header *hdr)
1653 {
1654         __icm_tr_device_connected(tb, hdr, true);
1655 }
1656
1657 static void icm_icl_rtd3_veto(struct tb *tb, const struct icm_pkg_header *hdr)
1658 {
1659         const struct icm_icl_event_rtd3_veto *pkg =
1660                 (const struct icm_icl_event_rtd3_veto *)hdr;
1661
1662         tb_dbg(tb, "ICM rtd3 veto=0x%08x\n", pkg->veto_reason);
1663
1664         if (pkg->veto_reason)
1665                 icm_veto_begin(tb);
1666         else
1667                 icm_veto_end(tb);
1668 }
1669
1670 static bool icm_tgl_is_supported(struct tb *tb)
1671 {
1672         u32 val;
1673
1674         /*
1675          * If the firmware is not running use software CM. This platform
1676          * should fully support both.
1677          */
1678         val = ioread32(tb->nhi->iobase + REG_FW_STS);
1679         return !!(val & REG_FW_STS_NVM_AUTH_DONE);
1680 }
1681
1682 static void icm_handle_notification(struct work_struct *work)
1683 {
1684         struct icm_notification *n = container_of(work, typeof(*n), work);
1685         struct tb *tb = n->tb;
1686         struct icm *icm = tb_priv(tb);
1687
1688         mutex_lock(&tb->lock);
1689
1690         /*
1691          * When the domain is stopped we flush its workqueue but before
1692          * that the root switch is removed. In that case we should treat
1693          * the queued events as being canceled.
1694          */
1695         if (tb->root_switch) {
1696                 switch (n->pkg->code) {
1697                 case ICM_EVENT_DEVICE_CONNECTED:
1698                         icm->device_connected(tb, n->pkg);
1699                         break;
1700                 case ICM_EVENT_DEVICE_DISCONNECTED:
1701                         icm->device_disconnected(tb, n->pkg);
1702                         break;
1703                 case ICM_EVENT_XDOMAIN_CONNECTED:
1704                         icm->xdomain_connected(tb, n->pkg);
1705                         break;
1706                 case ICM_EVENT_XDOMAIN_DISCONNECTED:
1707                         icm->xdomain_disconnected(tb, n->pkg);
1708                         break;
1709                 case ICM_EVENT_RTD3_VETO:
1710                         icm->rtd3_veto(tb, n->pkg);
1711                         break;
1712                 }
1713         }
1714
1715         mutex_unlock(&tb->lock);
1716
1717         kfree(n->pkg);
1718         kfree(n);
1719 }
1720
1721 static void icm_handle_event(struct tb *tb, enum tb_cfg_pkg_type type,
1722                              const void *buf, size_t size)
1723 {
1724         struct icm_notification *n;
1725
1726         n = kmalloc(sizeof(*n), GFP_KERNEL);
1727         if (!n)
1728                 return;
1729
1730         INIT_WORK(&n->work, icm_handle_notification);
1731         n->pkg = kmemdup(buf, size, GFP_KERNEL);
1732         n->tb = tb;
1733
1734         queue_work(tb->wq, &n->work);
1735 }
1736
1737 static int
1738 __icm_driver_ready(struct tb *tb, enum tb_security_level *security_level,
1739                    u8 *proto_version, size_t *nboot_acl, bool *rpm)
1740 {
1741         struct icm *icm = tb_priv(tb);
1742         unsigned int retries = 50;
1743         int ret;
1744
1745         ret = icm->driver_ready(tb, security_level, proto_version, nboot_acl,
1746                                 rpm);
1747         if (ret) {
1748                 tb_err(tb, "failed to send driver ready to ICM\n");
1749                 return ret;
1750         }
1751
1752         /*
1753          * Hold on here until the switch config space is accessible so
1754          * that we can read root switch config successfully.
1755          */
1756         do {
1757                 struct tb_cfg_result res;
1758                 u32 tmp;
1759
1760                 res = tb_cfg_read_raw(tb->ctl, &tmp, 0, 0, TB_CFG_SWITCH,
1761                                       0, 1, 100);
1762                 if (!res.err)
1763                         return 0;
1764
1765                 msleep(50);
1766         } while (--retries);
1767
1768         tb_err(tb, "failed to read root switch config space, giving up\n");
1769         return -ETIMEDOUT;
1770 }
1771
1772 static int icm_firmware_reset(struct tb *tb, struct tb_nhi *nhi)
1773 {
1774         struct icm *icm = tb_priv(tb);
1775         u32 val;
1776
1777         if (!icm->upstream_port)
1778                 return -ENODEV;
1779
1780         /* Put ARC to wait for CIO reset event to happen */
1781         val = ioread32(nhi->iobase + REG_FW_STS);
1782         val |= REG_FW_STS_CIO_RESET_REQ;
1783         iowrite32(val, nhi->iobase + REG_FW_STS);
1784
1785         /* Re-start ARC */
1786         val = ioread32(nhi->iobase + REG_FW_STS);
1787         val |= REG_FW_STS_ICM_EN_INVERT;
1788         val |= REG_FW_STS_ICM_EN_CPU;
1789         iowrite32(val, nhi->iobase + REG_FW_STS);
1790
1791         /* Trigger CIO reset now */
1792         return icm->cio_reset(tb);
1793 }
1794
1795 static int icm_firmware_start(struct tb *tb, struct tb_nhi *nhi)
1796 {
1797         unsigned int retries = 10;
1798         int ret;
1799         u32 val;
1800
1801         /* Check if the ICM firmware is already running */
1802         if (icm_firmware_running(nhi))
1803                 return 0;
1804
1805         dev_dbg(&nhi->pdev->dev, "starting ICM firmware\n");
1806
1807         ret = icm_firmware_reset(tb, nhi);
1808         if (ret)
1809                 return ret;
1810
1811         /* Wait until the ICM firmware tells us it is up and running */
1812         do {
1813                 /* Check that the ICM firmware is running */
1814                 val = ioread32(nhi->iobase + REG_FW_STS);
1815                 if (val & REG_FW_STS_NVM_AUTH_DONE)
1816                         return 0;
1817
1818                 msleep(300);
1819         } while (--retries);
1820
1821         return -ETIMEDOUT;
1822 }
1823
1824 static int icm_reset_phy_port(struct tb *tb, int phy_port)
1825 {
1826         struct icm *icm = tb_priv(tb);
1827         u32 state0, state1;
1828         int port0, port1;
1829         u32 val0, val1;
1830         int ret;
1831
1832         if (!icm->upstream_port)
1833                 return 0;
1834
1835         if (phy_port) {
1836                 port0 = 3;
1837                 port1 = 4;
1838         } else {
1839                 port0 = 1;
1840                 port1 = 2;
1841         }
1842
1843         /*
1844          * Read link status of both null ports belonging to a single
1845          * physical port.
1846          */
1847         ret = pcie2cio_read(icm, TB_CFG_PORT, port0, PHY_PORT_CS1, &val0);
1848         if (ret)
1849                 return ret;
1850         ret = pcie2cio_read(icm, TB_CFG_PORT, port1, PHY_PORT_CS1, &val1);
1851         if (ret)
1852                 return ret;
1853
1854         state0 = val0 & PHY_PORT_CS1_LINK_STATE_MASK;
1855         state0 >>= PHY_PORT_CS1_LINK_STATE_SHIFT;
1856         state1 = val1 & PHY_PORT_CS1_LINK_STATE_MASK;
1857         state1 >>= PHY_PORT_CS1_LINK_STATE_SHIFT;
1858
1859         /* If they are both up we need to reset them now */
1860         if (state0 != TB_PORT_UP || state1 != TB_PORT_UP)
1861                 return 0;
1862
1863         val0 |= PHY_PORT_CS1_LINK_DISABLE;
1864         ret = pcie2cio_write(icm, TB_CFG_PORT, port0, PHY_PORT_CS1, val0);
1865         if (ret)
1866                 return ret;
1867
1868         val1 |= PHY_PORT_CS1_LINK_DISABLE;
1869         ret = pcie2cio_write(icm, TB_CFG_PORT, port1, PHY_PORT_CS1, val1);
1870         if (ret)
1871                 return ret;
1872
1873         /* Wait a bit and then re-enable both ports */
1874         usleep_range(10, 100);
1875
1876         ret = pcie2cio_read(icm, TB_CFG_PORT, port0, PHY_PORT_CS1, &val0);
1877         if (ret)
1878                 return ret;
1879         ret = pcie2cio_read(icm, TB_CFG_PORT, port1, PHY_PORT_CS1, &val1);
1880         if (ret)
1881                 return ret;
1882
1883         val0 &= ~PHY_PORT_CS1_LINK_DISABLE;
1884         ret = pcie2cio_write(icm, TB_CFG_PORT, port0, PHY_PORT_CS1, val0);
1885         if (ret)
1886                 return ret;
1887
1888         val1 &= ~PHY_PORT_CS1_LINK_DISABLE;
1889         return pcie2cio_write(icm, TB_CFG_PORT, port1, PHY_PORT_CS1, val1);
1890 }
1891
1892 static int icm_firmware_init(struct tb *tb)
1893 {
1894         struct icm *icm = tb_priv(tb);
1895         struct tb_nhi *nhi = tb->nhi;
1896         int ret;
1897
1898         ret = icm_firmware_start(tb, nhi);
1899         if (ret) {
1900                 dev_err(&nhi->pdev->dev, "could not start ICM firmware\n");
1901                 return ret;
1902         }
1903
1904         if (icm->get_mode) {
1905                 ret = icm->get_mode(tb);
1906
1907                 switch (ret) {
1908                 case NHI_FW_SAFE_MODE:
1909                         icm->safe_mode = true;
1910                         break;
1911
1912                 case NHI_FW_CM_MODE:
1913                         /* Ask ICM to accept all Thunderbolt devices */
1914                         nhi_mailbox_cmd(nhi, NHI_MAILBOX_ALLOW_ALL_DEVS, 0);
1915                         break;
1916
1917                 default:
1918                         if (ret < 0)
1919                                 return ret;
1920
1921                         tb_err(tb, "ICM firmware is in wrong mode: %u\n", ret);
1922                         return -ENODEV;
1923                 }
1924         }
1925
1926         /*
1927          * Reset both physical ports if there is anything connected to
1928          * them already.
1929          */
1930         ret = icm_reset_phy_port(tb, 0);
1931         if (ret)
1932                 dev_warn(&nhi->pdev->dev, "failed to reset links on port0\n");
1933         ret = icm_reset_phy_port(tb, 1);
1934         if (ret)
1935                 dev_warn(&nhi->pdev->dev, "failed to reset links on port1\n");
1936
1937         return 0;
1938 }
1939
1940 static int icm_driver_ready(struct tb *tb)
1941 {
1942         struct icm *icm = tb_priv(tb);
1943         int ret;
1944
1945         ret = icm_firmware_init(tb);
1946         if (ret)
1947                 return ret;
1948
1949         if (icm->safe_mode) {
1950                 tb_info(tb, "Thunderbolt host controller is in safe mode.\n");
1951                 tb_info(tb, "You need to update NVM firmware of the controller before it can be used.\n");
1952                 tb_info(tb, "For latest updates check https://thunderbolttechnology.net/updates.\n");
1953                 return 0;
1954         }
1955
1956         ret = __icm_driver_ready(tb, &tb->security_level, &icm->proto_version,
1957                                  &tb->nboot_acl, &icm->rpm);
1958         if (ret)
1959                 return ret;
1960
1961         /*
1962          * Make sure the number of supported preboot ACL matches what we
1963          * expect or disable the whole feature.
1964          */
1965         if (tb->nboot_acl > icm->max_boot_acl)
1966                 tb->nboot_acl = 0;
1967
1968         if (icm->proto_version >= 3)
1969                 tb_dbg(tb, "USB4 proxy operations supported\n");
1970
1971         return 0;
1972 }
1973
1974 static int icm_suspend(struct tb *tb)
1975 {
1976         struct icm *icm = tb_priv(tb);
1977
1978         if (icm->save_devices)
1979                 icm->save_devices(tb);
1980
1981         nhi_mailbox_cmd(tb->nhi, NHI_MAILBOX_DRV_UNLOADS, 0);
1982         return 0;
1983 }
1984
1985 /*
1986  * Mark all switches (except root switch) below this one unplugged. ICM
1987  * firmware will send us an updated list of switches after we have send
1988  * it driver ready command. If a switch is not in that list it will be
1989  * removed when we perform rescan.
1990  */
1991 static void icm_unplug_children(struct tb_switch *sw)
1992 {
1993         struct tb_port *port;
1994
1995         if (tb_route(sw))
1996                 sw->is_unplugged = true;
1997
1998         tb_switch_for_each_port(sw, port) {
1999                 if (port->xdomain)
2000                         port->xdomain->is_unplugged = true;
2001                 else if (tb_port_has_remote(port))
2002                         icm_unplug_children(port->remote->sw);
2003         }
2004 }
2005
2006 static int complete_rpm(struct device *dev, void *data)
2007 {
2008         struct tb_switch *sw = tb_to_switch(dev);
2009
2010         if (sw)
2011                 complete(&sw->rpm_complete);
2012         return 0;
2013 }
2014
2015 static void remove_unplugged_switch(struct tb_switch *sw)
2016 {
2017         struct device *parent = get_device(sw->dev.parent);
2018
2019         pm_runtime_get_sync(parent);
2020
2021         /*
2022          * Signal this and switches below for rpm_complete because
2023          * tb_switch_remove() calls pm_runtime_get_sync() that then waits
2024          * for it.
2025          */
2026         complete_rpm(&sw->dev, NULL);
2027         bus_for_each_dev(&tb_bus_type, &sw->dev, NULL, complete_rpm);
2028         tb_switch_remove(sw);
2029
2030         pm_runtime_mark_last_busy(parent);
2031         pm_runtime_put_autosuspend(parent);
2032
2033         put_device(parent);
2034 }
2035
2036 static void icm_free_unplugged_children(struct tb_switch *sw)
2037 {
2038         struct tb_port *port;
2039
2040         tb_switch_for_each_port(sw, port) {
2041                 if (port->xdomain && port->xdomain->is_unplugged) {
2042                         tb_xdomain_remove(port->xdomain);
2043                         port->xdomain = NULL;
2044                 } else if (tb_port_has_remote(port)) {
2045                         if (port->remote->sw->is_unplugged) {
2046                                 remove_unplugged_switch(port->remote->sw);
2047                                 port->remote = NULL;
2048                         } else {
2049                                 icm_free_unplugged_children(port->remote->sw);
2050                         }
2051                 }
2052         }
2053 }
2054
2055 static void icm_rescan_work(struct work_struct *work)
2056 {
2057         struct icm *icm = container_of(work, struct icm, rescan_work.work);
2058         struct tb *tb = icm_to_tb(icm);
2059
2060         mutex_lock(&tb->lock);
2061         if (tb->root_switch)
2062                 icm_free_unplugged_children(tb->root_switch);
2063         mutex_unlock(&tb->lock);
2064 }
2065
2066 static void icm_complete(struct tb *tb)
2067 {
2068         struct icm *icm = tb_priv(tb);
2069
2070         if (tb->nhi->going_away)
2071                 return;
2072
2073         /*
2074          * If RTD3 was vetoed before we entered system suspend allow it
2075          * again now before driver ready is sent. Firmware sends a new RTD3
2076          * veto if it is still the case after we have sent it driver ready
2077          * command.
2078          */
2079         icm_veto_end(tb);
2080         icm_unplug_children(tb->root_switch);
2081
2082         /*
2083          * Now all existing children should be resumed, start events
2084          * from ICM to get updated status.
2085          */
2086         __icm_driver_ready(tb, NULL, NULL, NULL, NULL);
2087
2088         /*
2089          * We do not get notifications of devices that have been
2090          * unplugged during suspend so schedule rescan to clean them up
2091          * if any.
2092          */
2093         queue_delayed_work(tb->wq, &icm->rescan_work, msecs_to_jiffies(500));
2094 }
2095
2096 static int icm_runtime_suspend(struct tb *tb)
2097 {
2098         nhi_mailbox_cmd(tb->nhi, NHI_MAILBOX_DRV_UNLOADS, 0);
2099         return 0;
2100 }
2101
2102 static int icm_runtime_suspend_switch(struct tb_switch *sw)
2103 {
2104         if (tb_route(sw))
2105                 reinit_completion(&sw->rpm_complete);
2106         return 0;
2107 }
2108
2109 static int icm_runtime_resume_switch(struct tb_switch *sw)
2110 {
2111         if (tb_route(sw)) {
2112                 if (!wait_for_completion_timeout(&sw->rpm_complete,
2113                                                  msecs_to_jiffies(500))) {
2114                         dev_dbg(&sw->dev, "runtime resuming timed out\n");
2115                 }
2116         }
2117         return 0;
2118 }
2119
2120 static int icm_runtime_resume(struct tb *tb)
2121 {
2122         /*
2123          * We can reuse the same resume functionality than with system
2124          * suspend.
2125          */
2126         icm_complete(tb);
2127         return 0;
2128 }
2129
2130 static int icm_start(struct tb *tb)
2131 {
2132         struct icm *icm = tb_priv(tb);
2133         int ret;
2134
2135         if (icm->safe_mode)
2136                 tb->root_switch = tb_switch_alloc_safe_mode(tb, &tb->dev, 0);
2137         else
2138                 tb->root_switch = tb_switch_alloc(tb, &tb->dev, 0);
2139         if (IS_ERR(tb->root_switch))
2140                 return PTR_ERR(tb->root_switch);
2141
2142         tb->root_switch->no_nvm_upgrade = !icm->can_upgrade_nvm;
2143         tb->root_switch->rpm = icm->rpm;
2144
2145         if (icm->set_uuid)
2146                 icm->set_uuid(tb);
2147
2148         ret = tb_switch_add(tb->root_switch);
2149         if (ret) {
2150                 tb_switch_put(tb->root_switch);
2151                 tb->root_switch = NULL;
2152         }
2153
2154         return ret;
2155 }
2156
2157 static void icm_stop(struct tb *tb)
2158 {
2159         struct icm *icm = tb_priv(tb);
2160
2161         cancel_delayed_work(&icm->rescan_work);
2162         tb_switch_remove(tb->root_switch);
2163         tb->root_switch = NULL;
2164         nhi_mailbox_cmd(tb->nhi, NHI_MAILBOX_DRV_UNLOADS, 0);
2165         kfree(icm->last_nvm_auth);
2166         icm->last_nvm_auth = NULL;
2167 }
2168
2169 static int icm_disconnect_pcie_paths(struct tb *tb)
2170 {
2171         return nhi_mailbox_cmd(tb->nhi, NHI_MAILBOX_DISCONNECT_PCIE_PATHS, 0);
2172 }
2173
2174 static void icm_usb4_switch_nvm_auth_complete(void *data)
2175 {
2176         struct usb4_switch_nvm_auth *auth = data;
2177         struct icm *icm = auth->icm;
2178         struct tb *tb = icm_to_tb(icm);
2179
2180         tb_dbg(tb, "NVM_AUTH response for %llx flags %#x status %#x\n",
2181                get_route(auth->reply.route_hi, auth->reply.route_lo),
2182                auth->reply.hdr.flags, auth->reply.status);
2183
2184         mutex_lock(&tb->lock);
2185         if (WARN_ON(icm->last_nvm_auth))
2186                 kfree(icm->last_nvm_auth);
2187         icm->last_nvm_auth = auth;
2188         mutex_unlock(&tb->lock);
2189 }
2190
2191 static int icm_usb4_switch_nvm_authenticate(struct tb *tb, u64 route)
2192 {
2193         struct usb4_switch_nvm_auth *auth;
2194         struct icm *icm = tb_priv(tb);
2195         struct tb_cfg_request *req;
2196         int ret;
2197
2198         auth = kzalloc(sizeof(*auth), GFP_KERNEL);
2199         if (!auth)
2200                 return -ENOMEM;
2201
2202         auth->icm = icm;
2203         auth->request.hdr.code = ICM_USB4_SWITCH_OP;
2204         auth->request.route_hi = upper_32_bits(route);
2205         auth->request.route_lo = lower_32_bits(route);
2206         auth->request.opcode = USB4_SWITCH_OP_NVM_AUTH;
2207
2208         req = tb_cfg_request_alloc();
2209         if (!req) {
2210                 ret = -ENOMEM;
2211                 goto err_free_auth;
2212         }
2213
2214         req->match = icm_match;
2215         req->copy = icm_copy;
2216         req->request = &auth->request;
2217         req->request_size = sizeof(auth->request);
2218         req->request_type = TB_CFG_PKG_ICM_CMD;
2219         req->response = &auth->reply;
2220         req->npackets = 1;
2221         req->response_size = sizeof(auth->reply);
2222         req->response_type = TB_CFG_PKG_ICM_RESP;
2223
2224         tb_dbg(tb, "NVM_AUTH request for %llx\n", route);
2225
2226         mutex_lock(&icm->request_lock);
2227         ret = tb_cfg_request(tb->ctl, req, icm_usb4_switch_nvm_auth_complete,
2228                              auth);
2229         mutex_unlock(&icm->request_lock);
2230
2231         tb_cfg_request_put(req);
2232         if (ret)
2233                 goto err_free_auth;
2234         return 0;
2235
2236 err_free_auth:
2237         kfree(auth);
2238         return ret;
2239 }
2240
2241 static int icm_usb4_switch_op(struct tb_switch *sw, u16 opcode, u32 *metadata,
2242                               u8 *status, const void *tx_data, size_t tx_data_len,
2243                               void *rx_data, size_t rx_data_len)
2244 {
2245         struct icm_usb4_switch_op_response reply;
2246         struct icm_usb4_switch_op request;
2247         struct tb *tb = sw->tb;
2248         struct icm *icm = tb_priv(tb);
2249         u64 route = tb_route(sw);
2250         int ret;
2251
2252         /*
2253          * USB4 router operation proxy is supported in firmware if the
2254          * protocol version is 3 or higher.
2255          */
2256         if (icm->proto_version < 3)
2257                 return -EOPNOTSUPP;
2258
2259         /*
2260          * NVM_AUTH is a special USB4 proxy operation that does not
2261          * return immediately so handle it separately.
2262          */
2263         if (opcode == USB4_SWITCH_OP_NVM_AUTH)
2264                 return icm_usb4_switch_nvm_authenticate(tb, route);
2265
2266         memset(&request, 0, sizeof(request));
2267         request.hdr.code = ICM_USB4_SWITCH_OP;
2268         request.route_hi = upper_32_bits(route);
2269         request.route_lo = lower_32_bits(route);
2270         request.opcode = opcode;
2271         if (metadata)
2272                 request.metadata = *metadata;
2273
2274         if (tx_data_len) {
2275                 request.data_len_valid |= ICM_USB4_SWITCH_DATA_VALID;
2276                 if (tx_data_len < ARRAY_SIZE(request.data))
2277                         request.data_len_valid =
2278                                 tx_data_len & ICM_USB4_SWITCH_DATA_LEN_MASK;
2279                 memcpy(request.data, tx_data, tx_data_len * sizeof(u32));
2280         }
2281
2282         memset(&reply, 0, sizeof(reply));
2283         ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
2284                           1, ICM_TIMEOUT);
2285         if (ret)
2286                 return ret;
2287
2288         if (reply.hdr.flags & ICM_FLAGS_ERROR)
2289                 return -EIO;
2290
2291         if (status)
2292                 *status = reply.status;
2293
2294         if (metadata)
2295                 *metadata = reply.metadata;
2296
2297         if (rx_data_len)
2298                 memcpy(rx_data, reply.data, rx_data_len * sizeof(u32));
2299
2300         return 0;
2301 }
2302
2303 static int icm_usb4_switch_nvm_authenticate_status(struct tb_switch *sw,
2304                                                    u32 *status)
2305 {
2306         struct usb4_switch_nvm_auth *auth;
2307         struct tb *tb = sw->tb;
2308         struct icm *icm = tb_priv(tb);
2309         int ret = 0;
2310
2311         if (icm->proto_version < 3)
2312                 return -EOPNOTSUPP;
2313
2314         auth = icm->last_nvm_auth;
2315         icm->last_nvm_auth = NULL;
2316
2317         if (auth && auth->reply.route_hi == sw->config.route_hi &&
2318             auth->reply.route_lo == sw->config.route_lo) {
2319                 tb_dbg(tb, "NVM_AUTH found for %llx flags 0x%#x status %#x\n",
2320                        tb_route(sw), auth->reply.hdr.flags, auth->reply.status);
2321                 if (auth->reply.hdr.flags & ICM_FLAGS_ERROR)
2322                         ret = -EIO;
2323                 else
2324                         *status = auth->reply.status;
2325         } else {
2326                 *status = 0;
2327         }
2328
2329         kfree(auth);
2330         return ret;
2331 }
2332
2333 /* Falcon Ridge */
2334 static const struct tb_cm_ops icm_fr_ops = {
2335         .driver_ready = icm_driver_ready,
2336         .start = icm_start,
2337         .stop = icm_stop,
2338         .suspend = icm_suspend,
2339         .complete = icm_complete,
2340         .handle_event = icm_handle_event,
2341         .approve_switch = icm_fr_approve_switch,
2342         .add_switch_key = icm_fr_add_switch_key,
2343         .challenge_switch_key = icm_fr_challenge_switch_key,
2344         .disconnect_pcie_paths = icm_disconnect_pcie_paths,
2345         .approve_xdomain_paths = icm_fr_approve_xdomain_paths,
2346         .disconnect_xdomain_paths = icm_fr_disconnect_xdomain_paths,
2347 };
2348
2349 /* Alpine Ridge */
2350 static const struct tb_cm_ops icm_ar_ops = {
2351         .driver_ready = icm_driver_ready,
2352         .start = icm_start,
2353         .stop = icm_stop,
2354         .suspend = icm_suspend,
2355         .complete = icm_complete,
2356         .runtime_suspend = icm_runtime_suspend,
2357         .runtime_resume = icm_runtime_resume,
2358         .runtime_suspend_switch = icm_runtime_suspend_switch,
2359         .runtime_resume_switch = icm_runtime_resume_switch,
2360         .handle_event = icm_handle_event,
2361         .get_boot_acl = icm_ar_get_boot_acl,
2362         .set_boot_acl = icm_ar_set_boot_acl,
2363         .approve_switch = icm_fr_approve_switch,
2364         .add_switch_key = icm_fr_add_switch_key,
2365         .challenge_switch_key = icm_fr_challenge_switch_key,
2366         .disconnect_pcie_paths = icm_disconnect_pcie_paths,
2367         .approve_xdomain_paths = icm_fr_approve_xdomain_paths,
2368         .disconnect_xdomain_paths = icm_fr_disconnect_xdomain_paths,
2369 };
2370
2371 /* Titan Ridge */
2372 static const struct tb_cm_ops icm_tr_ops = {
2373         .driver_ready = icm_driver_ready,
2374         .start = icm_start,
2375         .stop = icm_stop,
2376         .suspend = icm_suspend,
2377         .complete = icm_complete,
2378         .runtime_suspend = icm_runtime_suspend,
2379         .runtime_resume = icm_runtime_resume,
2380         .runtime_suspend_switch = icm_runtime_suspend_switch,
2381         .runtime_resume_switch = icm_runtime_resume_switch,
2382         .handle_event = icm_handle_event,
2383         .get_boot_acl = icm_ar_get_boot_acl,
2384         .set_boot_acl = icm_ar_set_boot_acl,
2385         .approve_switch = icm_tr_approve_switch,
2386         .add_switch_key = icm_tr_add_switch_key,
2387         .challenge_switch_key = icm_tr_challenge_switch_key,
2388         .disconnect_pcie_paths = icm_disconnect_pcie_paths,
2389         .approve_xdomain_paths = icm_tr_approve_xdomain_paths,
2390         .disconnect_xdomain_paths = icm_tr_disconnect_xdomain_paths,
2391         .usb4_switch_op = icm_usb4_switch_op,
2392         .usb4_switch_nvm_authenticate_status =
2393                 icm_usb4_switch_nvm_authenticate_status,
2394 };
2395
2396 /* Ice Lake */
2397 static const struct tb_cm_ops icm_icl_ops = {
2398         .driver_ready = icm_driver_ready,
2399         .start = icm_start,
2400         .stop = icm_stop,
2401         .complete = icm_complete,
2402         .runtime_suspend = icm_runtime_suspend,
2403         .runtime_resume = icm_runtime_resume,
2404         .handle_event = icm_handle_event,
2405         .approve_xdomain_paths = icm_tr_approve_xdomain_paths,
2406         .disconnect_xdomain_paths = icm_tr_disconnect_xdomain_paths,
2407         .usb4_switch_op = icm_usb4_switch_op,
2408         .usb4_switch_nvm_authenticate_status =
2409                 icm_usb4_switch_nvm_authenticate_status,
2410 };
2411
2412 struct tb *icm_probe(struct tb_nhi *nhi)
2413 {
2414         struct icm *icm;
2415         struct tb *tb;
2416
2417         tb = tb_domain_alloc(nhi, sizeof(struct icm));
2418         if (!tb)
2419                 return NULL;
2420
2421         icm = tb_priv(tb);
2422         INIT_DELAYED_WORK(&icm->rescan_work, icm_rescan_work);
2423         mutex_init(&icm->request_lock);
2424
2425         switch (nhi->pdev->device) {
2426         case PCI_DEVICE_ID_INTEL_FALCON_RIDGE_2C_NHI:
2427         case PCI_DEVICE_ID_INTEL_FALCON_RIDGE_4C_NHI:
2428                 icm->can_upgrade_nvm = true;
2429                 icm->is_supported = icm_fr_is_supported;
2430                 icm->get_route = icm_fr_get_route;
2431                 icm->save_devices = icm_fr_save_devices;
2432                 icm->driver_ready = icm_fr_driver_ready;
2433                 icm->device_connected = icm_fr_device_connected;
2434                 icm->device_disconnected = icm_fr_device_disconnected;
2435                 icm->xdomain_connected = icm_fr_xdomain_connected;
2436                 icm->xdomain_disconnected = icm_fr_xdomain_disconnected;
2437                 tb->cm_ops = &icm_fr_ops;
2438                 break;
2439
2440         case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_2C_NHI:
2441         case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_4C_NHI:
2442         case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_LP_NHI:
2443         case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_C_4C_NHI:
2444         case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_C_2C_NHI:
2445                 icm->max_boot_acl = ICM_AR_PREBOOT_ACL_ENTRIES;
2446                 /*
2447                  * NVM upgrade has not been tested on Apple systems and
2448                  * they don't provide images publicly either. To be on
2449                  * the safe side prevent root switch NVM upgrade on Macs
2450                  * for now.
2451                  */
2452                 icm->can_upgrade_nvm = !x86_apple_machine;
2453                 icm->is_supported = icm_ar_is_supported;
2454                 icm->cio_reset = icm_ar_cio_reset;
2455                 icm->get_mode = icm_ar_get_mode;
2456                 icm->get_route = icm_ar_get_route;
2457                 icm->save_devices = icm_fr_save_devices;
2458                 icm->driver_ready = icm_ar_driver_ready;
2459                 icm->device_connected = icm_fr_device_connected;
2460                 icm->device_disconnected = icm_fr_device_disconnected;
2461                 icm->xdomain_connected = icm_fr_xdomain_connected;
2462                 icm->xdomain_disconnected = icm_fr_xdomain_disconnected;
2463                 tb->cm_ops = &icm_ar_ops;
2464                 break;
2465
2466         case PCI_DEVICE_ID_INTEL_TITAN_RIDGE_2C_NHI:
2467         case PCI_DEVICE_ID_INTEL_TITAN_RIDGE_4C_NHI:
2468                 icm->max_boot_acl = ICM_AR_PREBOOT_ACL_ENTRIES;
2469                 icm->can_upgrade_nvm = !x86_apple_machine;
2470                 icm->is_supported = icm_ar_is_supported;
2471                 icm->cio_reset = icm_tr_cio_reset;
2472                 icm->get_mode = icm_ar_get_mode;
2473                 icm->driver_ready = icm_tr_driver_ready;
2474                 icm->device_connected = icm_tr_device_connected;
2475                 icm->device_disconnected = icm_tr_device_disconnected;
2476                 icm->xdomain_connected = icm_tr_xdomain_connected;
2477                 icm->xdomain_disconnected = icm_tr_xdomain_disconnected;
2478                 tb->cm_ops = &icm_tr_ops;
2479                 break;
2480
2481         case PCI_DEVICE_ID_INTEL_ICL_NHI0:
2482         case PCI_DEVICE_ID_INTEL_ICL_NHI1:
2483                 icm->is_supported = icm_fr_is_supported;
2484                 icm->driver_ready = icm_icl_driver_ready;
2485                 icm->set_uuid = icm_icl_set_uuid;
2486                 icm->device_connected = icm_icl_device_connected;
2487                 icm->device_disconnected = icm_tr_device_disconnected;
2488                 icm->xdomain_connected = icm_tr_xdomain_connected;
2489                 icm->xdomain_disconnected = icm_tr_xdomain_disconnected;
2490                 icm->rtd3_veto = icm_icl_rtd3_veto;
2491                 tb->cm_ops = &icm_icl_ops;
2492                 break;
2493
2494         case PCI_DEVICE_ID_INTEL_TGL_NHI0:
2495         case PCI_DEVICE_ID_INTEL_TGL_NHI1:
2496         case PCI_DEVICE_ID_INTEL_TGL_H_NHI0:
2497         case PCI_DEVICE_ID_INTEL_TGL_H_NHI1:
2498                 icm->is_supported = icm_tgl_is_supported;
2499                 icm->driver_ready = icm_icl_driver_ready;
2500                 icm->set_uuid = icm_icl_set_uuid;
2501                 icm->device_connected = icm_icl_device_connected;
2502                 icm->device_disconnected = icm_tr_device_disconnected;
2503                 icm->xdomain_connected = icm_tr_xdomain_connected;
2504                 icm->xdomain_disconnected = icm_tr_xdomain_disconnected;
2505                 icm->rtd3_veto = icm_icl_rtd3_veto;
2506                 tb->cm_ops = &icm_icl_ops;
2507                 break;
2508
2509         case PCI_DEVICE_ID_INTEL_MAPLE_RIDGE_4C_NHI:
2510                 icm->is_supported = icm_tgl_is_supported;
2511                 icm->get_mode = icm_ar_get_mode;
2512                 icm->driver_ready = icm_tr_driver_ready;
2513                 icm->device_connected = icm_tr_device_connected;
2514                 icm->device_disconnected = icm_tr_device_disconnected;
2515                 icm->xdomain_connected = icm_tr_xdomain_connected;
2516                 icm->xdomain_disconnected = icm_tr_xdomain_disconnected;
2517                 tb->cm_ops = &icm_tr_ops;
2518                 break;
2519         }
2520
2521         if (!icm->is_supported || !icm->is_supported(tb)) {
2522                 dev_dbg(&nhi->pdev->dev, "ICM not supported on this controller\n");
2523                 tb_domain_put(tb);
2524                 return NULL;
2525         }
2526
2527         tb_dbg(tb, "using firmware connection manager\n");
2528
2529         return tb;
2530 }