1 // SPDX-License-Identifier: GPL-2.0
3 * Internal Thunderbolt Connection Manager. This is a firmware running on
4 * the Thunderbolt host controller performing most of the low-level
7 * Copyright (C) 2017, Intel Corporation
8 * Authors: Michael Jamet <michael.jamet@intel.com>
9 * Mika Westerberg <mika.westerberg@linux.intel.com>
12 #include <linux/delay.h>
13 #include <linux/mutex.h>
14 #include <linux/moduleparam.h>
15 #include <linux/pci.h>
16 #include <linux/pm_runtime.h>
17 #include <linux/platform_data/x86/apple.h>
18 #include <linux/sizes.h>
19 #include <linux/slab.h>
20 #include <linux/workqueue.h>
26 #define PCIE2CIO_CMD 0x30
27 #define PCIE2CIO_CMD_TIMEOUT BIT(31)
28 #define PCIE2CIO_CMD_START BIT(30)
29 #define PCIE2CIO_CMD_WRITE BIT(21)
30 #define PCIE2CIO_CMD_CS_MASK GENMASK(20, 19)
31 #define PCIE2CIO_CMD_CS_SHIFT 19
32 #define PCIE2CIO_CMD_PORT_MASK GENMASK(18, 13)
33 #define PCIE2CIO_CMD_PORT_SHIFT 13
35 #define PCIE2CIO_WRDATA 0x34
36 #define PCIE2CIO_RDDATA 0x38
38 #define PHY_PORT_CS1 0x37
39 #define PHY_PORT_CS1_LINK_DISABLE BIT(14)
40 #define PHY_PORT_CS1_LINK_STATE_MASK GENMASK(29, 26)
41 #define PHY_PORT_CS1_LINK_STATE_SHIFT 26
43 #define ICM_TIMEOUT 5000 /* ms */
44 #define ICM_APPROVE_TIMEOUT 10000 /* ms */
45 #define ICM_MAX_LINK 4
47 static bool start_icm;
48 module_param(start_icm, bool, 0444);
49 MODULE_PARM_DESC(start_icm, "start ICM firmware if it is not running (default: false)");
52 * struct usb4_switch_nvm_auth - Holds USB4 NVM_AUTH status
53 * @reply: Reply from ICM firmware is placed here
54 * @request: Request that is sent to ICM firmware
55 * @icm: Pointer to ICM private data
57 struct usb4_switch_nvm_auth {
58 struct icm_usb4_switch_op_response reply;
59 struct icm_usb4_switch_op request;
64 * struct icm - Internal connection manager private data
65 * @request_lock: Makes sure only one message is send to ICM at time
66 * @rescan_work: Work used to rescan the surviving switches after resume
67 * @upstream_port: Pointer to the PCIe upstream port this host
68 * controller is connected. This is only set for systems
69 * where ICM needs to be started manually
70 * @vnd_cap: Vendor defined capability where PCIe2CIO mailbox resides
71 * (only set when @upstream_port is not %NULL)
72 * @safe_mode: ICM is in safe mode
73 * @max_boot_acl: Maximum number of preboot ACL entries (%0 if not supported)
74 * @rpm: Does the controller support runtime PM (RTD3)
75 * @can_upgrade_nvm: Can the NVM firmware be upgrade on this controller
76 * @proto_version: Firmware protocol version
77 * @last_nvm_auth: Last USB4 router NVM_AUTH result (or %NULL if not set)
78 * @veto: Is RTD3 veto in effect
79 * @is_supported: Checks if we can support ICM on this controller
80 * @cio_reset: Trigger CIO reset
81 * @get_mode: Read and return the ICM firmware mode (optional)
82 * @get_route: Find a route string for given switch
83 * @save_devices: Ask ICM to save devices to ACL when suspending (optional)
84 * @driver_ready: Send driver ready message to ICM
85 * @set_uuid: Set UUID for the root switch (optional)
86 * @device_connected: Handle device connected ICM message
87 * @device_disconnected: Handle device disconnected ICM message
88 * @xdomain_connected: Handle XDomain connected ICM message
89 * @xdomain_disconnected: Handle XDomain disconnected ICM message
90 * @rtd3_veto: Handle RTD3 veto notification ICM message
93 struct mutex request_lock;
94 struct delayed_work rescan_work;
95 struct pci_dev *upstream_port;
100 bool can_upgrade_nvm;
102 struct usb4_switch_nvm_auth *last_nvm_auth;
104 bool (*is_supported)(struct tb *tb);
105 int (*cio_reset)(struct tb *tb);
106 int (*get_mode)(struct tb *tb);
107 int (*get_route)(struct tb *tb, u8 link, u8 depth, u64 *route);
108 void (*save_devices)(struct tb *tb);
109 int (*driver_ready)(struct tb *tb,
110 enum tb_security_level *security_level,
111 u8 *proto_version, size_t *nboot_acl, bool *rpm);
112 void (*set_uuid)(struct tb *tb);
113 void (*device_connected)(struct tb *tb,
114 const struct icm_pkg_header *hdr);
115 void (*device_disconnected)(struct tb *tb,
116 const struct icm_pkg_header *hdr);
117 void (*xdomain_connected)(struct tb *tb,
118 const struct icm_pkg_header *hdr);
119 void (*xdomain_disconnected)(struct tb *tb,
120 const struct icm_pkg_header *hdr);
121 void (*rtd3_veto)(struct tb *tb, const struct icm_pkg_header *hdr);
124 struct icm_notification {
125 struct work_struct work;
126 struct icm_pkg_header *pkg;
130 struct ep_name_entry {
136 #define EP_NAME_INTEL_VSS 0x10
138 /* Intel Vendor specific structure */
148 #define INTEL_VSS_FLAGS_RTD3 BIT(0)
150 static const struct intel_vss *parse_intel_vss(const void *ep_name, size_t size)
152 const void *end = ep_name + size;
154 while (ep_name < end) {
155 const struct ep_name_entry *ep = ep_name;
159 if (ep_name + ep->len > end)
162 if (ep->type == EP_NAME_INTEL_VSS)
163 return (const struct intel_vss *)ep->data;
171 static bool intel_vss_is_rtd3(const void *ep_name, size_t size)
173 const struct intel_vss *vss;
175 vss = parse_intel_vss(ep_name, size);
177 return !!(vss->flags & INTEL_VSS_FLAGS_RTD3);
182 static inline struct tb *icm_to_tb(struct icm *icm)
184 return ((void *)icm - sizeof(struct tb));
187 static inline u8 phy_port_from_route(u64 route, u8 depth)
191 link = depth ? route >> ((depth - 1) * 8) : route;
192 return tb_phy_port_from_link(link);
195 static inline u8 dual_link_from_link(u8 link)
197 return link ? ((link - 1) ^ 0x01) + 1 : 0;
200 static inline u64 get_route(u32 route_hi, u32 route_lo)
202 return (u64)route_hi << 32 | route_lo;
205 static inline u64 get_parent_route(u64 route)
207 int depth = tb_route_length(route);
208 return depth ? route & ~(0xffULL << (depth - 1) * TB_ROUTE_SHIFT) : 0;
211 static int pci2cio_wait_completion(struct icm *icm, unsigned long timeout_msec)
213 unsigned long end = jiffies + msecs_to_jiffies(timeout_msec);
217 pci_read_config_dword(icm->upstream_port,
218 icm->vnd_cap + PCIE2CIO_CMD, &cmd);
219 if (!(cmd & PCIE2CIO_CMD_START)) {
220 if (cmd & PCIE2CIO_CMD_TIMEOUT)
226 } while (time_before(jiffies, end));
231 static int pcie2cio_read(struct icm *icm, enum tb_cfg_space cs,
232 unsigned int port, unsigned int index, u32 *data)
234 struct pci_dev *pdev = icm->upstream_port;
235 int ret, vnd_cap = icm->vnd_cap;
239 cmd |= (port << PCIE2CIO_CMD_PORT_SHIFT) & PCIE2CIO_CMD_PORT_MASK;
240 cmd |= (cs << PCIE2CIO_CMD_CS_SHIFT) & PCIE2CIO_CMD_CS_MASK;
241 cmd |= PCIE2CIO_CMD_START;
242 pci_write_config_dword(pdev, vnd_cap + PCIE2CIO_CMD, cmd);
244 ret = pci2cio_wait_completion(icm, 5000);
248 pci_read_config_dword(pdev, vnd_cap + PCIE2CIO_RDDATA, data);
252 static int pcie2cio_write(struct icm *icm, enum tb_cfg_space cs,
253 unsigned int port, unsigned int index, u32 data)
255 struct pci_dev *pdev = icm->upstream_port;
256 int vnd_cap = icm->vnd_cap;
259 pci_write_config_dword(pdev, vnd_cap + PCIE2CIO_WRDATA, data);
262 cmd |= (port << PCIE2CIO_CMD_PORT_SHIFT) & PCIE2CIO_CMD_PORT_MASK;
263 cmd |= (cs << PCIE2CIO_CMD_CS_SHIFT) & PCIE2CIO_CMD_CS_MASK;
264 cmd |= PCIE2CIO_CMD_WRITE | PCIE2CIO_CMD_START;
265 pci_write_config_dword(pdev, vnd_cap + PCIE2CIO_CMD, cmd);
267 return pci2cio_wait_completion(icm, 5000);
270 static bool icm_match(const struct tb_cfg_request *req,
271 const struct ctl_pkg *pkg)
273 const struct icm_pkg_header *res_hdr = pkg->buffer;
274 const struct icm_pkg_header *req_hdr = req->request;
276 if (pkg->frame.eof != req->response_type)
278 if (res_hdr->code != req_hdr->code)
284 static bool icm_copy(struct tb_cfg_request *req, const struct ctl_pkg *pkg)
286 const struct icm_pkg_header *hdr = pkg->buffer;
288 if (hdr->packet_id < req->npackets) {
289 size_t offset = hdr->packet_id * req->response_size;
291 memcpy(req->response + offset, pkg->buffer, req->response_size);
294 return hdr->packet_id == hdr->total_packets - 1;
297 static int icm_request(struct tb *tb, const void *request, size_t request_size,
298 void *response, size_t response_size, size_t npackets,
299 unsigned int timeout_msec)
301 struct icm *icm = tb_priv(tb);
305 struct tb_cfg_request *req;
306 struct tb_cfg_result res;
308 req = tb_cfg_request_alloc();
312 req->match = icm_match;
313 req->copy = icm_copy;
314 req->request = request;
315 req->request_size = request_size;
316 req->request_type = TB_CFG_PKG_ICM_CMD;
317 req->response = response;
318 req->npackets = npackets;
319 req->response_size = response_size;
320 req->response_type = TB_CFG_PKG_ICM_RESP;
322 mutex_lock(&icm->request_lock);
323 res = tb_cfg_request_sync(tb->ctl, req, timeout_msec);
324 mutex_unlock(&icm->request_lock);
326 tb_cfg_request_put(req);
328 if (res.err != -ETIMEDOUT)
329 return res.err == 1 ? -EIO : res.err;
331 usleep_range(20, 50);
338 * If rescan is queued to run (we are resuming), postpone it to give the
339 * firmware some more time to send device connected notifications for next
340 * devices in the chain.
342 static void icm_postpone_rescan(struct tb *tb)
344 struct icm *icm = tb_priv(tb);
346 if (delayed_work_pending(&icm->rescan_work))
347 mod_delayed_work(tb->wq, &icm->rescan_work,
348 msecs_to_jiffies(500));
351 static void icm_veto_begin(struct tb *tb)
353 struct icm *icm = tb_priv(tb);
357 /* Keep the domain powered while veto is in effect */
358 pm_runtime_get(&tb->dev);
362 static void icm_veto_end(struct tb *tb)
364 struct icm *icm = tb_priv(tb);
368 /* Allow the domain suspend now */
369 pm_runtime_mark_last_busy(&tb->dev);
370 pm_runtime_put_autosuspend(&tb->dev);
374 static bool icm_firmware_running(const struct tb_nhi *nhi)
378 val = ioread32(nhi->iobase + REG_FW_STS);
379 return !!(val & REG_FW_STS_ICM_EN);
382 static bool icm_fr_is_supported(struct tb *tb)
384 return !x86_apple_machine;
387 static inline int icm_fr_get_switch_index(u32 port)
391 if ((port & ICM_PORT_TYPE_MASK) != TB_TYPE_PORT)
394 index = port >> ICM_PORT_INDEX_SHIFT;
395 return index != 0xff ? index : 0;
398 static int icm_fr_get_route(struct tb *tb, u8 link, u8 depth, u64 *route)
400 struct icm_fr_pkg_get_topology_response *switches, *sw;
401 struct icm_fr_pkg_get_topology request = {
402 .hdr = { .code = ICM_GET_TOPOLOGY },
404 size_t npackets = ICM_GET_TOPOLOGY_PACKETS;
408 switches = kcalloc(npackets, sizeof(*switches), GFP_KERNEL);
412 ret = icm_request(tb, &request, sizeof(request), switches,
413 sizeof(*switches), npackets, ICM_TIMEOUT);
418 index = icm_fr_get_switch_index(sw->ports[link]);
424 sw = &switches[index];
425 for (i = 1; i < depth; i++) {
428 if (!(sw->first_data & ICM_SWITCH_USED)) {
433 for (j = 0; j < ARRAY_SIZE(sw->ports); j++) {
434 index = icm_fr_get_switch_index(sw->ports[j]);
435 if (index > sw->switch_index) {
436 sw = &switches[index];
442 *route = get_route(sw->route_hi, sw->route_lo);
449 static void icm_fr_save_devices(struct tb *tb)
451 nhi_mailbox_cmd(tb->nhi, NHI_MAILBOX_SAVE_DEVS, 0);
455 icm_fr_driver_ready(struct tb *tb, enum tb_security_level *security_level,
456 u8 *proto_version, size_t *nboot_acl, bool *rpm)
458 struct icm_fr_pkg_driver_ready_response reply;
459 struct icm_pkg_driver_ready request = {
460 .hdr.code = ICM_DRIVER_READY,
464 memset(&reply, 0, sizeof(reply));
465 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
471 *security_level = reply.security_level & ICM_FR_SLEVEL_MASK;
476 static int icm_fr_approve_switch(struct tb *tb, struct tb_switch *sw)
478 struct icm_fr_pkg_approve_device request;
479 struct icm_fr_pkg_approve_device reply;
482 memset(&request, 0, sizeof(request));
483 memcpy(&request.ep_uuid, sw->uuid, sizeof(request.ep_uuid));
484 request.hdr.code = ICM_APPROVE_DEVICE;
485 request.connection_id = sw->connection_id;
486 request.connection_key = sw->connection_key;
488 memset(&reply, 0, sizeof(reply));
489 /* Use larger timeout as establishing tunnels can take some time */
490 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
491 1, ICM_APPROVE_TIMEOUT);
495 if (reply.hdr.flags & ICM_FLAGS_ERROR) {
496 tb_warn(tb, "PCIe tunnel creation failed\n");
503 static int icm_fr_add_switch_key(struct tb *tb, struct tb_switch *sw)
505 struct icm_fr_pkg_add_device_key request;
506 struct icm_fr_pkg_add_device_key_response reply;
509 memset(&request, 0, sizeof(request));
510 memcpy(&request.ep_uuid, sw->uuid, sizeof(request.ep_uuid));
511 request.hdr.code = ICM_ADD_DEVICE_KEY;
512 request.connection_id = sw->connection_id;
513 request.connection_key = sw->connection_key;
514 memcpy(request.key, sw->key, TB_SWITCH_KEY_SIZE);
516 memset(&reply, 0, sizeof(reply));
517 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
522 if (reply.hdr.flags & ICM_FLAGS_ERROR) {
523 tb_warn(tb, "Adding key to switch failed\n");
530 static int icm_fr_challenge_switch_key(struct tb *tb, struct tb_switch *sw,
531 const u8 *challenge, u8 *response)
533 struct icm_fr_pkg_challenge_device request;
534 struct icm_fr_pkg_challenge_device_response reply;
537 memset(&request, 0, sizeof(request));
538 memcpy(&request.ep_uuid, sw->uuid, sizeof(request.ep_uuid));
539 request.hdr.code = ICM_CHALLENGE_DEVICE;
540 request.connection_id = sw->connection_id;
541 request.connection_key = sw->connection_key;
542 memcpy(request.challenge, challenge, TB_SWITCH_KEY_SIZE);
544 memset(&reply, 0, sizeof(reply));
545 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
550 if (reply.hdr.flags & ICM_FLAGS_ERROR)
551 return -EKEYREJECTED;
552 if (reply.hdr.flags & ICM_FLAGS_NO_KEY)
555 memcpy(response, reply.response, TB_SWITCH_KEY_SIZE);
560 static int icm_fr_approve_xdomain_paths(struct tb *tb, struct tb_xdomain *xd)
562 struct icm_fr_pkg_approve_xdomain_response reply;
563 struct icm_fr_pkg_approve_xdomain request;
566 memset(&request, 0, sizeof(request));
567 request.hdr.code = ICM_APPROVE_XDOMAIN;
568 request.link_info = xd->depth << ICM_LINK_INFO_DEPTH_SHIFT | xd->link;
569 memcpy(&request.remote_uuid, xd->remote_uuid, sizeof(*xd->remote_uuid));
571 request.transmit_path = xd->transmit_path;
572 request.transmit_ring = xd->transmit_ring;
573 request.receive_path = xd->receive_path;
574 request.receive_ring = xd->receive_ring;
576 memset(&reply, 0, sizeof(reply));
577 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
582 if (reply.hdr.flags & ICM_FLAGS_ERROR)
588 static int icm_fr_disconnect_xdomain_paths(struct tb *tb, struct tb_xdomain *xd)
593 phy_port = tb_phy_port_from_link(xd->link);
595 cmd = NHI_MAILBOX_DISCONNECT_PA;
597 cmd = NHI_MAILBOX_DISCONNECT_PB;
599 nhi_mailbox_cmd(tb->nhi, cmd, 1);
600 usleep_range(10, 50);
601 nhi_mailbox_cmd(tb->nhi, cmd, 2);
605 static struct tb_switch *alloc_switch(struct tb_switch *parent_sw, u64 route,
608 struct tb *tb = parent_sw->tb;
609 struct tb_switch *sw;
611 sw = tb_switch_alloc(tb, &parent_sw->dev, route);
613 tb_warn(tb, "failed to allocate switch at %llx\n", route);
617 sw->uuid = kmemdup(uuid, sizeof(*uuid), GFP_KERNEL);
620 return ERR_PTR(-ENOMEM);
623 init_completion(&sw->rpm_complete);
627 static int add_switch(struct tb_switch *parent_sw, struct tb_switch *sw)
629 u64 route = tb_route(sw);
632 /* Link the two switches now */
633 tb_port_at(route, parent_sw)->remote = tb_upstream_port(sw);
634 tb_upstream_port(sw)->remote = tb_port_at(route, parent_sw);
636 ret = tb_switch_add(sw);
638 tb_port_at(tb_route(sw), parent_sw)->remote = NULL;
643 static void update_switch(struct tb_switch *parent_sw, struct tb_switch *sw,
644 u64 route, u8 connection_id, u8 connection_key,
645 u8 link, u8 depth, bool boot)
647 /* Disconnect from parent */
648 tb_port_at(tb_route(sw), parent_sw)->remote = NULL;
649 /* Re-connect via updated port*/
650 tb_port_at(route, parent_sw)->remote = tb_upstream_port(sw);
652 /* Update with the new addressing information */
653 sw->config.route_hi = upper_32_bits(route);
654 sw->config.route_lo = lower_32_bits(route);
655 sw->connection_id = connection_id;
656 sw->connection_key = connection_key;
661 /* This switch still exists */
662 sw->is_unplugged = false;
664 /* Runtime resume is now complete */
665 complete(&sw->rpm_complete);
668 static void remove_switch(struct tb_switch *sw)
670 struct tb_switch *parent_sw;
672 parent_sw = tb_to_switch(sw->dev.parent);
673 tb_port_at(tb_route(sw), parent_sw)->remote = NULL;
674 tb_switch_remove(sw);
677 static void add_xdomain(struct tb_switch *sw, u64 route,
678 const uuid_t *local_uuid, const uuid_t *remote_uuid,
681 struct tb_xdomain *xd;
683 pm_runtime_get_sync(&sw->dev);
685 xd = tb_xdomain_alloc(sw->tb, &sw->dev, route, local_uuid, remote_uuid);
692 tb_port_at(route, sw)->xdomain = xd;
697 pm_runtime_mark_last_busy(&sw->dev);
698 pm_runtime_put_autosuspend(&sw->dev);
701 static void update_xdomain(struct tb_xdomain *xd, u64 route, u8 link)
705 xd->is_unplugged = false;
708 static void remove_xdomain(struct tb_xdomain *xd)
710 struct tb_switch *sw;
712 sw = tb_to_switch(xd->dev.parent);
713 tb_port_at(xd->route, sw)->xdomain = NULL;
714 tb_xdomain_remove(xd);
718 icm_fr_device_connected(struct tb *tb, const struct icm_pkg_header *hdr)
720 const struct icm_fr_event_device_connected *pkg =
721 (const struct icm_fr_event_device_connected *)hdr;
722 enum tb_security_level security_level;
723 struct tb_switch *sw, *parent_sw;
724 bool boot, dual_lane, speed_gen3;
725 struct icm *icm = tb_priv(tb);
726 bool authorized = false;
727 struct tb_xdomain *xd;
732 icm_postpone_rescan(tb);
734 link = pkg->link_info & ICM_LINK_INFO_LINK_MASK;
735 depth = (pkg->link_info & ICM_LINK_INFO_DEPTH_MASK) >>
736 ICM_LINK_INFO_DEPTH_SHIFT;
737 authorized = pkg->link_info & ICM_LINK_INFO_APPROVED;
738 security_level = (pkg->hdr.flags & ICM_FLAGS_SLEVEL_MASK) >>
739 ICM_FLAGS_SLEVEL_SHIFT;
740 boot = pkg->link_info & ICM_LINK_INFO_BOOT;
741 dual_lane = pkg->hdr.flags & ICM_FLAGS_DUAL_LANE;
742 speed_gen3 = pkg->hdr.flags & ICM_FLAGS_SPEED_GEN3;
744 if (pkg->link_info & ICM_LINK_INFO_REJECTED) {
745 tb_info(tb, "switch at %u.%u was rejected by ICM firmware because topology limit exceeded\n",
750 sw = tb_switch_find_by_uuid(tb, &pkg->ep_uuid);
752 u8 phy_port, sw_phy_port;
754 parent_sw = tb_to_switch(sw->dev.parent);
755 sw_phy_port = tb_phy_port_from_link(sw->link);
756 phy_port = tb_phy_port_from_link(link);
759 * On resume ICM will send us connected events for the
760 * devices that still are present. However, that
761 * information might have changed for example by the
762 * fact that a switch on a dual-link connection might
763 * have been enumerated using the other link now. Make
764 * sure our book keeping matches that.
766 if (sw->depth == depth && sw_phy_port == phy_port &&
767 !!sw->authorized == authorized) {
769 * It was enumerated through another link so update
770 * route string accordingly.
772 if (sw->link != link) {
773 ret = icm->get_route(tb, link, depth, &route);
775 tb_err(tb, "failed to update route string for switch at %u.%u\n",
781 route = tb_route(sw);
784 update_switch(parent_sw, sw, route, pkg->connection_id,
785 pkg->connection_key, link, depth, boot);
791 * User connected the same switch to another physical
792 * port or to another part of the topology. Remove the
793 * existing switch now before adding the new one.
800 * If the switch was not found by UUID, look for a switch on
801 * same physical port (taking possible link aggregation into
802 * account) and depth. If we found one it is definitely a stale
803 * one so remove it first.
805 sw = tb_switch_find_by_link_depth(tb, link, depth);
809 dual_link = dual_link_from_link(link);
811 sw = tb_switch_find_by_link_depth(tb, dual_link, depth);
818 /* Remove existing XDomain connection if found */
819 xd = tb_xdomain_find_by_link_depth(tb, link, depth);
825 parent_sw = tb_switch_find_by_link_depth(tb, link, depth - 1);
827 tb_err(tb, "failed to find parent switch for %u.%u\n",
832 ret = icm->get_route(tb, link, depth, &route);
834 tb_err(tb, "failed to find route string for switch at %u.%u\n",
836 tb_switch_put(parent_sw);
840 pm_runtime_get_sync(&parent_sw->dev);
842 sw = alloc_switch(parent_sw, route, &pkg->ep_uuid);
844 sw->connection_id = pkg->connection_id;
845 sw->connection_key = pkg->connection_key;
848 sw->authorized = authorized;
849 sw->security_level = security_level;
851 sw->link_speed = speed_gen3 ? 20 : 10;
852 sw->link_width = dual_lane ? 2 : 1;
853 sw->rpm = intel_vss_is_rtd3(pkg->ep_name, sizeof(pkg->ep_name));
855 if (add_switch(parent_sw, sw))
859 pm_runtime_mark_last_busy(&parent_sw->dev);
860 pm_runtime_put_autosuspend(&parent_sw->dev);
862 tb_switch_put(parent_sw);
866 icm_fr_device_disconnected(struct tb *tb, const struct icm_pkg_header *hdr)
868 const struct icm_fr_event_device_disconnected *pkg =
869 (const struct icm_fr_event_device_disconnected *)hdr;
870 struct tb_switch *sw;
873 link = pkg->link_info & ICM_LINK_INFO_LINK_MASK;
874 depth = (pkg->link_info & ICM_LINK_INFO_DEPTH_MASK) >>
875 ICM_LINK_INFO_DEPTH_SHIFT;
877 if (link > ICM_MAX_LINK || depth > TB_SWITCH_MAX_DEPTH) {
878 tb_warn(tb, "invalid topology %u.%u, ignoring\n", link, depth);
882 sw = tb_switch_find_by_link_depth(tb, link, depth);
884 tb_warn(tb, "no switch exists at %u.%u, ignoring\n", link,
889 pm_runtime_get_sync(sw->dev.parent);
893 pm_runtime_mark_last_busy(sw->dev.parent);
894 pm_runtime_put_autosuspend(sw->dev.parent);
900 icm_fr_xdomain_connected(struct tb *tb, const struct icm_pkg_header *hdr)
902 const struct icm_fr_event_xdomain_connected *pkg =
903 (const struct icm_fr_event_xdomain_connected *)hdr;
904 struct tb_xdomain *xd;
905 struct tb_switch *sw;
909 link = pkg->link_info & ICM_LINK_INFO_LINK_MASK;
910 depth = (pkg->link_info & ICM_LINK_INFO_DEPTH_MASK) >>
911 ICM_LINK_INFO_DEPTH_SHIFT;
913 if (link > ICM_MAX_LINK || depth > TB_SWITCH_MAX_DEPTH) {
914 tb_warn(tb, "invalid topology %u.%u, ignoring\n", link, depth);
918 route = get_route(pkg->local_route_hi, pkg->local_route_lo);
920 xd = tb_xdomain_find_by_uuid(tb, &pkg->remote_uuid);
922 u8 xd_phy_port, phy_port;
924 xd_phy_port = phy_port_from_route(xd->route, xd->depth);
925 phy_port = phy_port_from_route(route, depth);
927 if (xd->depth == depth && xd_phy_port == phy_port) {
928 update_xdomain(xd, route, link);
934 * If we find an existing XDomain connection remove it
935 * now. We need to go through login handshake and
936 * everything anyway to be able to re-establish the
944 * Look if there already exists an XDomain in the same place
945 * than the new one and in that case remove it because it is
946 * most likely another host that got disconnected.
948 xd = tb_xdomain_find_by_link_depth(tb, link, depth);
952 dual_link = dual_link_from_link(link);
954 xd = tb_xdomain_find_by_link_depth(tb, dual_link,
963 * If the user disconnected a switch during suspend and
964 * connected another host to the same port, remove the switch
967 sw = tb_switch_find_by_route(tb, route);
973 sw = tb_switch_find_by_link_depth(tb, link, depth);
975 tb_warn(tb, "no switch exists at %u.%u, ignoring\n", link,
980 add_xdomain(sw, route, &pkg->local_uuid, &pkg->remote_uuid, link,
986 icm_fr_xdomain_disconnected(struct tb *tb, const struct icm_pkg_header *hdr)
988 const struct icm_fr_event_xdomain_disconnected *pkg =
989 (const struct icm_fr_event_xdomain_disconnected *)hdr;
990 struct tb_xdomain *xd;
993 * If the connection is through one or multiple devices, the
994 * XDomain device is removed along with them so it is fine if we
995 * cannot find it here.
997 xd = tb_xdomain_find_by_uuid(tb, &pkg->remote_uuid);
1004 static int icm_tr_cio_reset(struct tb *tb)
1006 return pcie2cio_write(tb_priv(tb), TB_CFG_SWITCH, 0, 0x777, BIT(1));
1010 icm_tr_driver_ready(struct tb *tb, enum tb_security_level *security_level,
1011 u8 *proto_version, size_t *nboot_acl, bool *rpm)
1013 struct icm_tr_pkg_driver_ready_response reply;
1014 struct icm_pkg_driver_ready request = {
1015 .hdr.code = ICM_DRIVER_READY,
1019 memset(&reply, 0, sizeof(reply));
1020 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1026 *security_level = reply.info & ICM_TR_INFO_SLEVEL_MASK;
1028 *proto_version = (reply.info & ICM_TR_INFO_PROTO_VERSION_MASK) >>
1029 ICM_TR_INFO_PROTO_VERSION_SHIFT;
1031 *nboot_acl = (reply.info & ICM_TR_INFO_BOOT_ACL_MASK) >>
1032 ICM_TR_INFO_BOOT_ACL_SHIFT;
1034 *rpm = !!(reply.hdr.flags & ICM_TR_FLAGS_RTD3);
1039 static int icm_tr_approve_switch(struct tb *tb, struct tb_switch *sw)
1041 struct icm_tr_pkg_approve_device request;
1042 struct icm_tr_pkg_approve_device reply;
1045 memset(&request, 0, sizeof(request));
1046 memcpy(&request.ep_uuid, sw->uuid, sizeof(request.ep_uuid));
1047 request.hdr.code = ICM_APPROVE_DEVICE;
1048 request.route_lo = sw->config.route_lo;
1049 request.route_hi = sw->config.route_hi;
1050 request.connection_id = sw->connection_id;
1052 memset(&reply, 0, sizeof(reply));
1053 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1054 1, ICM_APPROVE_TIMEOUT);
1058 if (reply.hdr.flags & ICM_FLAGS_ERROR) {
1059 tb_warn(tb, "PCIe tunnel creation failed\n");
1066 static int icm_tr_add_switch_key(struct tb *tb, struct tb_switch *sw)
1068 struct icm_tr_pkg_add_device_key_response reply;
1069 struct icm_tr_pkg_add_device_key request;
1072 memset(&request, 0, sizeof(request));
1073 memcpy(&request.ep_uuid, sw->uuid, sizeof(request.ep_uuid));
1074 request.hdr.code = ICM_ADD_DEVICE_KEY;
1075 request.route_lo = sw->config.route_lo;
1076 request.route_hi = sw->config.route_hi;
1077 request.connection_id = sw->connection_id;
1078 memcpy(request.key, sw->key, TB_SWITCH_KEY_SIZE);
1080 memset(&reply, 0, sizeof(reply));
1081 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1086 if (reply.hdr.flags & ICM_FLAGS_ERROR) {
1087 tb_warn(tb, "Adding key to switch failed\n");
1094 static int icm_tr_challenge_switch_key(struct tb *tb, struct tb_switch *sw,
1095 const u8 *challenge, u8 *response)
1097 struct icm_tr_pkg_challenge_device_response reply;
1098 struct icm_tr_pkg_challenge_device request;
1101 memset(&request, 0, sizeof(request));
1102 memcpy(&request.ep_uuid, sw->uuid, sizeof(request.ep_uuid));
1103 request.hdr.code = ICM_CHALLENGE_DEVICE;
1104 request.route_lo = sw->config.route_lo;
1105 request.route_hi = sw->config.route_hi;
1106 request.connection_id = sw->connection_id;
1107 memcpy(request.challenge, challenge, TB_SWITCH_KEY_SIZE);
1109 memset(&reply, 0, sizeof(reply));
1110 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1115 if (reply.hdr.flags & ICM_FLAGS_ERROR)
1116 return -EKEYREJECTED;
1117 if (reply.hdr.flags & ICM_FLAGS_NO_KEY)
1120 memcpy(response, reply.response, TB_SWITCH_KEY_SIZE);
1125 static int icm_tr_approve_xdomain_paths(struct tb *tb, struct tb_xdomain *xd)
1127 struct icm_tr_pkg_approve_xdomain_response reply;
1128 struct icm_tr_pkg_approve_xdomain request;
1131 memset(&request, 0, sizeof(request));
1132 request.hdr.code = ICM_APPROVE_XDOMAIN;
1133 request.route_hi = upper_32_bits(xd->route);
1134 request.route_lo = lower_32_bits(xd->route);
1135 request.transmit_path = xd->transmit_path;
1136 request.transmit_ring = xd->transmit_ring;
1137 request.receive_path = xd->receive_path;
1138 request.receive_ring = xd->receive_ring;
1139 memcpy(&request.remote_uuid, xd->remote_uuid, sizeof(*xd->remote_uuid));
1141 memset(&reply, 0, sizeof(reply));
1142 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1147 if (reply.hdr.flags & ICM_FLAGS_ERROR)
1153 static int icm_tr_xdomain_tear_down(struct tb *tb, struct tb_xdomain *xd,
1156 struct icm_tr_pkg_disconnect_xdomain_response reply;
1157 struct icm_tr_pkg_disconnect_xdomain request;
1160 memset(&request, 0, sizeof(request));
1161 request.hdr.code = ICM_DISCONNECT_XDOMAIN;
1162 request.stage = stage;
1163 request.route_hi = upper_32_bits(xd->route);
1164 request.route_lo = lower_32_bits(xd->route);
1165 memcpy(&request.remote_uuid, xd->remote_uuid, sizeof(*xd->remote_uuid));
1167 memset(&reply, 0, sizeof(reply));
1168 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1173 if (reply.hdr.flags & ICM_FLAGS_ERROR)
1179 static int icm_tr_disconnect_xdomain_paths(struct tb *tb, struct tb_xdomain *xd)
1183 ret = icm_tr_xdomain_tear_down(tb, xd, 1);
1187 usleep_range(10, 50);
1188 return icm_tr_xdomain_tear_down(tb, xd, 2);
1192 __icm_tr_device_connected(struct tb *tb, const struct icm_pkg_header *hdr,
1195 const struct icm_tr_event_device_connected *pkg =
1196 (const struct icm_tr_event_device_connected *)hdr;
1197 bool authorized, boot, dual_lane, speed_gen3;
1198 enum tb_security_level security_level;
1199 struct tb_switch *sw, *parent_sw;
1200 struct tb_xdomain *xd;
1203 icm_postpone_rescan(tb);
1206 * Currently we don't use the QoS information coming with the
1207 * device connected message so simply just ignore that extra
1210 if (pkg->hdr.packet_id)
1213 route = get_route(pkg->route_hi, pkg->route_lo);
1214 authorized = pkg->link_info & ICM_LINK_INFO_APPROVED;
1215 security_level = (pkg->hdr.flags & ICM_FLAGS_SLEVEL_MASK) >>
1216 ICM_FLAGS_SLEVEL_SHIFT;
1217 boot = pkg->link_info & ICM_LINK_INFO_BOOT;
1218 dual_lane = pkg->hdr.flags & ICM_FLAGS_DUAL_LANE;
1219 speed_gen3 = pkg->hdr.flags & ICM_FLAGS_SPEED_GEN3;
1221 if (pkg->link_info & ICM_LINK_INFO_REJECTED) {
1222 tb_info(tb, "switch at %llx was rejected by ICM firmware because topology limit exceeded\n",
1227 sw = tb_switch_find_by_uuid(tb, &pkg->ep_uuid);
1229 /* Update the switch if it is still in the same place */
1230 if (tb_route(sw) == route && !!sw->authorized == authorized) {
1231 parent_sw = tb_to_switch(sw->dev.parent);
1232 update_switch(parent_sw, sw, route, pkg->connection_id,
1242 /* Another switch with the same address */
1243 sw = tb_switch_find_by_route(tb, route);
1249 /* XDomain connection with the same address */
1250 xd = tb_xdomain_find_by_route(tb, route);
1256 parent_sw = tb_switch_find_by_route(tb, get_parent_route(route));
1258 tb_err(tb, "failed to find parent switch for %llx\n", route);
1262 pm_runtime_get_sync(&parent_sw->dev);
1264 sw = alloc_switch(parent_sw, route, &pkg->ep_uuid);
1266 sw->connection_id = pkg->connection_id;
1267 sw->authorized = authorized;
1268 sw->security_level = security_level;
1270 sw->link_speed = speed_gen3 ? 20 : 10;
1271 sw->link_width = dual_lane ? 2 : 1;
1272 sw->rpm = force_rtd3;
1274 sw->rpm = intel_vss_is_rtd3(pkg->ep_name,
1275 sizeof(pkg->ep_name));
1277 if (add_switch(parent_sw, sw))
1281 pm_runtime_mark_last_busy(&parent_sw->dev);
1282 pm_runtime_put_autosuspend(&parent_sw->dev);
1284 tb_switch_put(parent_sw);
1288 icm_tr_device_connected(struct tb *tb, const struct icm_pkg_header *hdr)
1290 __icm_tr_device_connected(tb, hdr, false);
1294 icm_tr_device_disconnected(struct tb *tb, const struct icm_pkg_header *hdr)
1296 const struct icm_tr_event_device_disconnected *pkg =
1297 (const struct icm_tr_event_device_disconnected *)hdr;
1298 struct tb_switch *sw;
1301 route = get_route(pkg->route_hi, pkg->route_lo);
1303 sw = tb_switch_find_by_route(tb, route);
1305 tb_warn(tb, "no switch exists at %llx, ignoring\n", route);
1308 pm_runtime_get_sync(sw->dev.parent);
1312 pm_runtime_mark_last_busy(sw->dev.parent);
1313 pm_runtime_put_autosuspend(sw->dev.parent);
1319 icm_tr_xdomain_connected(struct tb *tb, const struct icm_pkg_header *hdr)
1321 const struct icm_tr_event_xdomain_connected *pkg =
1322 (const struct icm_tr_event_xdomain_connected *)hdr;
1323 struct tb_xdomain *xd;
1324 struct tb_switch *sw;
1327 if (!tb->root_switch)
1330 route = get_route(pkg->local_route_hi, pkg->local_route_lo);
1332 xd = tb_xdomain_find_by_uuid(tb, &pkg->remote_uuid);
1334 if (xd->route == route) {
1335 update_xdomain(xd, route, 0);
1344 /* An existing xdomain with the same address */
1345 xd = tb_xdomain_find_by_route(tb, route);
1352 * If the user disconnected a switch during suspend and
1353 * connected another host to the same port, remove the switch
1356 sw = tb_switch_find_by_route(tb, route);
1362 sw = tb_switch_find_by_route(tb, get_parent_route(route));
1364 tb_warn(tb, "no switch exists at %llx, ignoring\n", route);
1368 add_xdomain(sw, route, &pkg->local_uuid, &pkg->remote_uuid, 0, 0);
1373 icm_tr_xdomain_disconnected(struct tb *tb, const struct icm_pkg_header *hdr)
1375 const struct icm_tr_event_xdomain_disconnected *pkg =
1376 (const struct icm_tr_event_xdomain_disconnected *)hdr;
1377 struct tb_xdomain *xd;
1380 route = get_route(pkg->route_hi, pkg->route_lo);
1382 xd = tb_xdomain_find_by_route(tb, route);
1389 static struct pci_dev *get_upstream_port(struct pci_dev *pdev)
1391 struct pci_dev *parent;
1393 parent = pci_upstream_bridge(pdev);
1395 if (!pci_is_pcie(parent))
1397 if (pci_pcie_type(parent) == PCI_EXP_TYPE_UPSTREAM)
1399 parent = pci_upstream_bridge(parent);
1405 switch (parent->device) {
1406 case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_2C_BRIDGE:
1407 case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_4C_BRIDGE:
1408 case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_LP_BRIDGE:
1409 case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_C_4C_BRIDGE:
1410 case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_C_2C_BRIDGE:
1411 case PCI_DEVICE_ID_INTEL_TITAN_RIDGE_2C_BRIDGE:
1412 case PCI_DEVICE_ID_INTEL_TITAN_RIDGE_4C_BRIDGE:
1419 static bool icm_ar_is_supported(struct tb *tb)
1421 struct pci_dev *upstream_port;
1422 struct icm *icm = tb_priv(tb);
1425 * Starting from Alpine Ridge we can use ICM on Apple machines
1426 * as well. We just need to reset and re-enable it first.
1427 * However, only start it if explicitly asked by the user.
1429 if (icm_firmware_running(tb->nhi))
1435 * Find the upstream PCIe port in case we need to do reset
1436 * through its vendor specific registers.
1438 upstream_port = get_upstream_port(tb->nhi->pdev);
1439 if (upstream_port) {
1442 cap = pci_find_ext_capability(upstream_port,
1443 PCI_EXT_CAP_ID_VNDR);
1445 icm->upstream_port = upstream_port;
1455 static int icm_ar_cio_reset(struct tb *tb)
1457 return pcie2cio_write(tb_priv(tb), TB_CFG_SWITCH, 0, 0x50, BIT(9));
1460 static int icm_ar_get_mode(struct tb *tb)
1462 struct tb_nhi *nhi = tb->nhi;
1467 val = ioread32(nhi->iobase + REG_FW_STS);
1468 if (val & REG_FW_STS_NVM_AUTH_DONE)
1471 } while (--retries);
1474 dev_err(&nhi->pdev->dev, "ICM firmware not authenticated\n");
1478 return nhi_mailbox_mode(nhi);
1482 icm_ar_driver_ready(struct tb *tb, enum tb_security_level *security_level,
1483 u8 *proto_version, size_t *nboot_acl, bool *rpm)
1485 struct icm_ar_pkg_driver_ready_response reply;
1486 struct icm_pkg_driver_ready request = {
1487 .hdr.code = ICM_DRIVER_READY,
1491 memset(&reply, 0, sizeof(reply));
1492 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1498 *security_level = reply.info & ICM_AR_INFO_SLEVEL_MASK;
1499 if (nboot_acl && (reply.info & ICM_AR_INFO_BOOT_ACL_SUPPORTED))
1500 *nboot_acl = (reply.info & ICM_AR_INFO_BOOT_ACL_MASK) >>
1501 ICM_AR_INFO_BOOT_ACL_SHIFT;
1503 *rpm = !!(reply.hdr.flags & ICM_AR_FLAGS_RTD3);
1508 static int icm_ar_get_route(struct tb *tb, u8 link, u8 depth, u64 *route)
1510 struct icm_ar_pkg_get_route_response reply;
1511 struct icm_ar_pkg_get_route request = {
1512 .hdr = { .code = ICM_GET_ROUTE },
1513 .link_info = depth << ICM_LINK_INFO_DEPTH_SHIFT | link,
1517 memset(&reply, 0, sizeof(reply));
1518 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1523 if (reply.hdr.flags & ICM_FLAGS_ERROR)
1526 *route = get_route(reply.route_hi, reply.route_lo);
1530 static int icm_ar_get_boot_acl(struct tb *tb, uuid_t *uuids, size_t nuuids)
1532 struct icm_ar_pkg_preboot_acl_response reply;
1533 struct icm_ar_pkg_preboot_acl request = {
1534 .hdr = { .code = ICM_PREBOOT_ACL },
1538 memset(&reply, 0, sizeof(reply));
1539 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1544 if (reply.hdr.flags & ICM_FLAGS_ERROR)
1547 for (i = 0; i < nuuids; i++) {
1548 u32 *uuid = (u32 *)&uuids[i];
1550 uuid[0] = reply.acl[i].uuid_lo;
1551 uuid[1] = reply.acl[i].uuid_hi;
1553 if (uuid[0] == 0xffffffff && uuid[1] == 0xffffffff) {
1554 /* Map empty entries to null UUID */
1557 } else if (uuid[0] != 0 || uuid[1] != 0) {
1558 /* Upper two DWs are always one's */
1559 uuid[2] = 0xffffffff;
1560 uuid[3] = 0xffffffff;
1567 static int icm_ar_set_boot_acl(struct tb *tb, const uuid_t *uuids,
1570 struct icm_ar_pkg_preboot_acl_response reply;
1571 struct icm_ar_pkg_preboot_acl request = {
1573 .code = ICM_PREBOOT_ACL,
1574 .flags = ICM_FLAGS_WRITE,
1579 for (i = 0; i < nuuids; i++) {
1580 const u32 *uuid = (const u32 *)&uuids[i];
1582 if (uuid_is_null(&uuids[i])) {
1584 * Map null UUID to the empty (all one) entries
1587 request.acl[i].uuid_lo = 0xffffffff;
1588 request.acl[i].uuid_hi = 0xffffffff;
1590 /* Two high DWs need to be set to all one */
1591 if (uuid[2] != 0xffffffff || uuid[3] != 0xffffffff)
1594 request.acl[i].uuid_lo = uuid[0];
1595 request.acl[i].uuid_hi = uuid[1];
1599 memset(&reply, 0, sizeof(reply));
1600 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1605 if (reply.hdr.flags & ICM_FLAGS_ERROR)
1612 icm_icl_driver_ready(struct tb *tb, enum tb_security_level *security_level,
1613 u8 *proto_version, size_t *nboot_acl, bool *rpm)
1615 struct icm_tr_pkg_driver_ready_response reply;
1616 struct icm_pkg_driver_ready request = {
1617 .hdr.code = ICM_DRIVER_READY,
1621 memset(&reply, 0, sizeof(reply));
1622 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1628 *proto_version = (reply.info & ICM_TR_INFO_PROTO_VERSION_MASK) >>
1629 ICM_TR_INFO_PROTO_VERSION_SHIFT;
1631 /* Ice Lake always supports RTD3 */
1638 static void icm_icl_set_uuid(struct tb *tb)
1640 struct tb_nhi *nhi = tb->nhi;
1643 pci_read_config_dword(nhi->pdev, VS_CAP_10, &uuid[0]);
1644 pci_read_config_dword(nhi->pdev, VS_CAP_11, &uuid[1]);
1645 uuid[2] = 0xffffffff;
1646 uuid[3] = 0xffffffff;
1648 tb->root_switch->uuid = kmemdup(uuid, sizeof(uuid), GFP_KERNEL);
1652 icm_icl_device_connected(struct tb *tb, const struct icm_pkg_header *hdr)
1654 __icm_tr_device_connected(tb, hdr, true);
1657 static void icm_icl_rtd3_veto(struct tb *tb, const struct icm_pkg_header *hdr)
1659 const struct icm_icl_event_rtd3_veto *pkg =
1660 (const struct icm_icl_event_rtd3_veto *)hdr;
1662 tb_dbg(tb, "ICM rtd3 veto=0x%08x\n", pkg->veto_reason);
1664 if (pkg->veto_reason)
1670 static bool icm_tgl_is_supported(struct tb *tb)
1675 * If the firmware is not running use software CM. This platform
1676 * should fully support both.
1678 val = ioread32(tb->nhi->iobase + REG_FW_STS);
1679 return !!(val & REG_FW_STS_NVM_AUTH_DONE);
1682 static void icm_handle_notification(struct work_struct *work)
1684 struct icm_notification *n = container_of(work, typeof(*n), work);
1685 struct tb *tb = n->tb;
1686 struct icm *icm = tb_priv(tb);
1688 mutex_lock(&tb->lock);
1691 * When the domain is stopped we flush its workqueue but before
1692 * that the root switch is removed. In that case we should treat
1693 * the queued events as being canceled.
1695 if (tb->root_switch) {
1696 switch (n->pkg->code) {
1697 case ICM_EVENT_DEVICE_CONNECTED:
1698 icm->device_connected(tb, n->pkg);
1700 case ICM_EVENT_DEVICE_DISCONNECTED:
1701 icm->device_disconnected(tb, n->pkg);
1703 case ICM_EVENT_XDOMAIN_CONNECTED:
1704 if (tb_is_xdomain_enabled())
1705 icm->xdomain_connected(tb, n->pkg);
1707 case ICM_EVENT_XDOMAIN_DISCONNECTED:
1708 if (tb_is_xdomain_enabled())
1709 icm->xdomain_disconnected(tb, n->pkg);
1711 case ICM_EVENT_RTD3_VETO:
1712 icm->rtd3_veto(tb, n->pkg);
1717 mutex_unlock(&tb->lock);
1723 static void icm_handle_event(struct tb *tb, enum tb_cfg_pkg_type type,
1724 const void *buf, size_t size)
1726 struct icm_notification *n;
1728 n = kmalloc(sizeof(*n), GFP_KERNEL);
1732 INIT_WORK(&n->work, icm_handle_notification);
1733 n->pkg = kmemdup(buf, size, GFP_KERNEL);
1736 queue_work(tb->wq, &n->work);
1740 __icm_driver_ready(struct tb *tb, enum tb_security_level *security_level,
1741 u8 *proto_version, size_t *nboot_acl, bool *rpm)
1743 struct icm *icm = tb_priv(tb);
1744 unsigned int retries = 50;
1747 ret = icm->driver_ready(tb, security_level, proto_version, nboot_acl,
1750 tb_err(tb, "failed to send driver ready to ICM\n");
1755 * Hold on here until the switch config space is accessible so
1756 * that we can read root switch config successfully.
1759 struct tb_cfg_result res;
1762 res = tb_cfg_read_raw(tb->ctl, &tmp, 0, 0, TB_CFG_SWITCH,
1768 } while (--retries);
1770 tb_err(tb, "failed to read root switch config space, giving up\n");
1774 static int icm_firmware_reset(struct tb *tb, struct tb_nhi *nhi)
1776 struct icm *icm = tb_priv(tb);
1779 if (!icm->upstream_port)
1782 /* Put ARC to wait for CIO reset event to happen */
1783 val = ioread32(nhi->iobase + REG_FW_STS);
1784 val |= REG_FW_STS_CIO_RESET_REQ;
1785 iowrite32(val, nhi->iobase + REG_FW_STS);
1788 val = ioread32(nhi->iobase + REG_FW_STS);
1789 val |= REG_FW_STS_ICM_EN_INVERT;
1790 val |= REG_FW_STS_ICM_EN_CPU;
1791 iowrite32(val, nhi->iobase + REG_FW_STS);
1793 /* Trigger CIO reset now */
1794 return icm->cio_reset(tb);
1797 static int icm_firmware_start(struct tb *tb, struct tb_nhi *nhi)
1799 unsigned int retries = 10;
1803 /* Check if the ICM firmware is already running */
1804 if (icm_firmware_running(nhi))
1807 dev_dbg(&nhi->pdev->dev, "starting ICM firmware\n");
1809 ret = icm_firmware_reset(tb, nhi);
1813 /* Wait until the ICM firmware tells us it is up and running */
1815 /* Check that the ICM firmware is running */
1816 val = ioread32(nhi->iobase + REG_FW_STS);
1817 if (val & REG_FW_STS_NVM_AUTH_DONE)
1821 } while (--retries);
1826 static int icm_reset_phy_port(struct tb *tb, int phy_port)
1828 struct icm *icm = tb_priv(tb);
1834 if (!icm->upstream_port)
1846 * Read link status of both null ports belonging to a single
1849 ret = pcie2cio_read(icm, TB_CFG_PORT, port0, PHY_PORT_CS1, &val0);
1852 ret = pcie2cio_read(icm, TB_CFG_PORT, port1, PHY_PORT_CS1, &val1);
1856 state0 = val0 & PHY_PORT_CS1_LINK_STATE_MASK;
1857 state0 >>= PHY_PORT_CS1_LINK_STATE_SHIFT;
1858 state1 = val1 & PHY_PORT_CS1_LINK_STATE_MASK;
1859 state1 >>= PHY_PORT_CS1_LINK_STATE_SHIFT;
1861 /* If they are both up we need to reset them now */
1862 if (state0 != TB_PORT_UP || state1 != TB_PORT_UP)
1865 val0 |= PHY_PORT_CS1_LINK_DISABLE;
1866 ret = pcie2cio_write(icm, TB_CFG_PORT, port0, PHY_PORT_CS1, val0);
1870 val1 |= PHY_PORT_CS1_LINK_DISABLE;
1871 ret = pcie2cio_write(icm, TB_CFG_PORT, port1, PHY_PORT_CS1, val1);
1875 /* Wait a bit and then re-enable both ports */
1876 usleep_range(10, 100);
1878 ret = pcie2cio_read(icm, TB_CFG_PORT, port0, PHY_PORT_CS1, &val0);
1881 ret = pcie2cio_read(icm, TB_CFG_PORT, port1, PHY_PORT_CS1, &val1);
1885 val0 &= ~PHY_PORT_CS1_LINK_DISABLE;
1886 ret = pcie2cio_write(icm, TB_CFG_PORT, port0, PHY_PORT_CS1, val0);
1890 val1 &= ~PHY_PORT_CS1_LINK_DISABLE;
1891 return pcie2cio_write(icm, TB_CFG_PORT, port1, PHY_PORT_CS1, val1);
1894 static int icm_firmware_init(struct tb *tb)
1896 struct icm *icm = tb_priv(tb);
1897 struct tb_nhi *nhi = tb->nhi;
1900 ret = icm_firmware_start(tb, nhi);
1902 dev_err(&nhi->pdev->dev, "could not start ICM firmware\n");
1906 if (icm->get_mode) {
1907 ret = icm->get_mode(tb);
1910 case NHI_FW_SAFE_MODE:
1911 icm->safe_mode = true;
1914 case NHI_FW_CM_MODE:
1915 /* Ask ICM to accept all Thunderbolt devices */
1916 nhi_mailbox_cmd(nhi, NHI_MAILBOX_ALLOW_ALL_DEVS, 0);
1923 tb_err(tb, "ICM firmware is in wrong mode: %u\n", ret);
1929 * Reset both physical ports if there is anything connected to
1932 ret = icm_reset_phy_port(tb, 0);
1934 dev_warn(&nhi->pdev->dev, "failed to reset links on port0\n");
1935 ret = icm_reset_phy_port(tb, 1);
1937 dev_warn(&nhi->pdev->dev, "failed to reset links on port1\n");
1942 static int icm_driver_ready(struct tb *tb)
1944 struct icm *icm = tb_priv(tb);
1947 ret = icm_firmware_init(tb);
1951 if (icm->safe_mode) {
1952 tb_info(tb, "Thunderbolt host controller is in safe mode.\n");
1953 tb_info(tb, "You need to update NVM firmware of the controller before it can be used.\n");
1954 tb_info(tb, "For latest updates check https://thunderbolttechnology.net/updates.\n");
1958 ret = __icm_driver_ready(tb, &tb->security_level, &icm->proto_version,
1959 &tb->nboot_acl, &icm->rpm);
1964 * Make sure the number of supported preboot ACL matches what we
1965 * expect or disable the whole feature.
1967 if (tb->nboot_acl > icm->max_boot_acl)
1970 if (icm->proto_version >= 3)
1971 tb_dbg(tb, "USB4 proxy operations supported\n");
1976 static int icm_suspend(struct tb *tb)
1978 struct icm *icm = tb_priv(tb);
1980 if (icm->save_devices)
1981 icm->save_devices(tb);
1983 nhi_mailbox_cmd(tb->nhi, NHI_MAILBOX_DRV_UNLOADS, 0);
1988 * Mark all switches (except root switch) below this one unplugged. ICM
1989 * firmware will send us an updated list of switches after we have send
1990 * it driver ready command. If a switch is not in that list it will be
1991 * removed when we perform rescan.
1993 static void icm_unplug_children(struct tb_switch *sw)
1995 struct tb_port *port;
1998 sw->is_unplugged = true;
2000 tb_switch_for_each_port(sw, port) {
2002 port->xdomain->is_unplugged = true;
2003 else if (tb_port_has_remote(port))
2004 icm_unplug_children(port->remote->sw);
2008 static int complete_rpm(struct device *dev, void *data)
2010 struct tb_switch *sw = tb_to_switch(dev);
2013 complete(&sw->rpm_complete);
2017 static void remove_unplugged_switch(struct tb_switch *sw)
2019 struct device *parent = get_device(sw->dev.parent);
2021 pm_runtime_get_sync(parent);
2024 * Signal this and switches below for rpm_complete because
2025 * tb_switch_remove() calls pm_runtime_get_sync() that then waits
2028 complete_rpm(&sw->dev, NULL);
2029 bus_for_each_dev(&tb_bus_type, &sw->dev, NULL, complete_rpm);
2030 tb_switch_remove(sw);
2032 pm_runtime_mark_last_busy(parent);
2033 pm_runtime_put_autosuspend(parent);
2038 static void icm_free_unplugged_children(struct tb_switch *sw)
2040 struct tb_port *port;
2042 tb_switch_for_each_port(sw, port) {
2043 if (port->xdomain && port->xdomain->is_unplugged) {
2044 tb_xdomain_remove(port->xdomain);
2045 port->xdomain = NULL;
2046 } else if (tb_port_has_remote(port)) {
2047 if (port->remote->sw->is_unplugged) {
2048 remove_unplugged_switch(port->remote->sw);
2049 port->remote = NULL;
2051 icm_free_unplugged_children(port->remote->sw);
2057 static void icm_rescan_work(struct work_struct *work)
2059 struct icm *icm = container_of(work, struct icm, rescan_work.work);
2060 struct tb *tb = icm_to_tb(icm);
2062 mutex_lock(&tb->lock);
2063 if (tb->root_switch)
2064 icm_free_unplugged_children(tb->root_switch);
2065 mutex_unlock(&tb->lock);
2068 static void icm_complete(struct tb *tb)
2070 struct icm *icm = tb_priv(tb);
2072 if (tb->nhi->going_away)
2076 * If RTD3 was vetoed before we entered system suspend allow it
2077 * again now before driver ready is sent. Firmware sends a new RTD3
2078 * veto if it is still the case after we have sent it driver ready
2082 icm_unplug_children(tb->root_switch);
2085 * Now all existing children should be resumed, start events
2086 * from ICM to get updated status.
2088 __icm_driver_ready(tb, NULL, NULL, NULL, NULL);
2091 * We do not get notifications of devices that have been
2092 * unplugged during suspend so schedule rescan to clean them up
2095 queue_delayed_work(tb->wq, &icm->rescan_work, msecs_to_jiffies(500));
2098 static int icm_runtime_suspend(struct tb *tb)
2100 nhi_mailbox_cmd(tb->nhi, NHI_MAILBOX_DRV_UNLOADS, 0);
2104 static int icm_runtime_suspend_switch(struct tb_switch *sw)
2107 reinit_completion(&sw->rpm_complete);
2111 static int icm_runtime_resume_switch(struct tb_switch *sw)
2114 if (!wait_for_completion_timeout(&sw->rpm_complete,
2115 msecs_to_jiffies(500))) {
2116 dev_dbg(&sw->dev, "runtime resuming timed out\n");
2122 static int icm_runtime_resume(struct tb *tb)
2125 * We can reuse the same resume functionality than with system
2132 static int icm_start(struct tb *tb)
2134 struct icm *icm = tb_priv(tb);
2138 tb->root_switch = tb_switch_alloc_safe_mode(tb, &tb->dev, 0);
2140 tb->root_switch = tb_switch_alloc(tb, &tb->dev, 0);
2141 if (IS_ERR(tb->root_switch))
2142 return PTR_ERR(tb->root_switch);
2144 tb->root_switch->no_nvm_upgrade = !icm->can_upgrade_nvm;
2145 tb->root_switch->rpm = icm->rpm;
2150 ret = tb_switch_add(tb->root_switch);
2152 tb_switch_put(tb->root_switch);
2153 tb->root_switch = NULL;
2159 static void icm_stop(struct tb *tb)
2161 struct icm *icm = tb_priv(tb);
2163 cancel_delayed_work(&icm->rescan_work);
2164 tb_switch_remove(tb->root_switch);
2165 tb->root_switch = NULL;
2166 nhi_mailbox_cmd(tb->nhi, NHI_MAILBOX_DRV_UNLOADS, 0);
2167 kfree(icm->last_nvm_auth);
2168 icm->last_nvm_auth = NULL;
2171 static int icm_disconnect_pcie_paths(struct tb *tb)
2173 return nhi_mailbox_cmd(tb->nhi, NHI_MAILBOX_DISCONNECT_PCIE_PATHS, 0);
2176 static void icm_usb4_switch_nvm_auth_complete(void *data)
2178 struct usb4_switch_nvm_auth *auth = data;
2179 struct icm *icm = auth->icm;
2180 struct tb *tb = icm_to_tb(icm);
2182 tb_dbg(tb, "NVM_AUTH response for %llx flags %#x status %#x\n",
2183 get_route(auth->reply.route_hi, auth->reply.route_lo),
2184 auth->reply.hdr.flags, auth->reply.status);
2186 mutex_lock(&tb->lock);
2187 if (WARN_ON(icm->last_nvm_auth))
2188 kfree(icm->last_nvm_auth);
2189 icm->last_nvm_auth = auth;
2190 mutex_unlock(&tb->lock);
2193 static int icm_usb4_switch_nvm_authenticate(struct tb *tb, u64 route)
2195 struct usb4_switch_nvm_auth *auth;
2196 struct icm *icm = tb_priv(tb);
2197 struct tb_cfg_request *req;
2200 auth = kzalloc(sizeof(*auth), GFP_KERNEL);
2205 auth->request.hdr.code = ICM_USB4_SWITCH_OP;
2206 auth->request.route_hi = upper_32_bits(route);
2207 auth->request.route_lo = lower_32_bits(route);
2208 auth->request.opcode = USB4_SWITCH_OP_NVM_AUTH;
2210 req = tb_cfg_request_alloc();
2216 req->match = icm_match;
2217 req->copy = icm_copy;
2218 req->request = &auth->request;
2219 req->request_size = sizeof(auth->request);
2220 req->request_type = TB_CFG_PKG_ICM_CMD;
2221 req->response = &auth->reply;
2223 req->response_size = sizeof(auth->reply);
2224 req->response_type = TB_CFG_PKG_ICM_RESP;
2226 tb_dbg(tb, "NVM_AUTH request for %llx\n", route);
2228 mutex_lock(&icm->request_lock);
2229 ret = tb_cfg_request(tb->ctl, req, icm_usb4_switch_nvm_auth_complete,
2231 mutex_unlock(&icm->request_lock);
2233 tb_cfg_request_put(req);
2243 static int icm_usb4_switch_op(struct tb_switch *sw, u16 opcode, u32 *metadata,
2244 u8 *status, const void *tx_data, size_t tx_data_len,
2245 void *rx_data, size_t rx_data_len)
2247 struct icm_usb4_switch_op_response reply;
2248 struct icm_usb4_switch_op request;
2249 struct tb *tb = sw->tb;
2250 struct icm *icm = tb_priv(tb);
2251 u64 route = tb_route(sw);
2255 * USB4 router operation proxy is supported in firmware if the
2256 * protocol version is 3 or higher.
2258 if (icm->proto_version < 3)
2262 * NVM_AUTH is a special USB4 proxy operation that does not
2263 * return immediately so handle it separately.
2265 if (opcode == USB4_SWITCH_OP_NVM_AUTH)
2266 return icm_usb4_switch_nvm_authenticate(tb, route);
2268 memset(&request, 0, sizeof(request));
2269 request.hdr.code = ICM_USB4_SWITCH_OP;
2270 request.route_hi = upper_32_bits(route);
2271 request.route_lo = lower_32_bits(route);
2272 request.opcode = opcode;
2274 request.metadata = *metadata;
2277 request.data_len_valid |= ICM_USB4_SWITCH_DATA_VALID;
2278 if (tx_data_len < ARRAY_SIZE(request.data))
2279 request.data_len_valid =
2280 tx_data_len & ICM_USB4_SWITCH_DATA_LEN_MASK;
2281 memcpy(request.data, tx_data, tx_data_len * sizeof(u32));
2284 memset(&reply, 0, sizeof(reply));
2285 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
2290 if (reply.hdr.flags & ICM_FLAGS_ERROR)
2294 *status = reply.status;
2297 *metadata = reply.metadata;
2300 memcpy(rx_data, reply.data, rx_data_len * sizeof(u32));
2305 static int icm_usb4_switch_nvm_authenticate_status(struct tb_switch *sw,
2308 struct usb4_switch_nvm_auth *auth;
2309 struct tb *tb = sw->tb;
2310 struct icm *icm = tb_priv(tb);
2313 if (icm->proto_version < 3)
2316 auth = icm->last_nvm_auth;
2317 icm->last_nvm_auth = NULL;
2319 if (auth && auth->reply.route_hi == sw->config.route_hi &&
2320 auth->reply.route_lo == sw->config.route_lo) {
2321 tb_dbg(tb, "NVM_AUTH found for %llx flags %#x status %#x\n",
2322 tb_route(sw), auth->reply.hdr.flags, auth->reply.status);
2323 if (auth->reply.hdr.flags & ICM_FLAGS_ERROR)
2326 *status = auth->reply.status;
2336 static const struct tb_cm_ops icm_fr_ops = {
2337 .driver_ready = icm_driver_ready,
2340 .suspend = icm_suspend,
2341 .complete = icm_complete,
2342 .handle_event = icm_handle_event,
2343 .approve_switch = icm_fr_approve_switch,
2344 .add_switch_key = icm_fr_add_switch_key,
2345 .challenge_switch_key = icm_fr_challenge_switch_key,
2346 .disconnect_pcie_paths = icm_disconnect_pcie_paths,
2347 .approve_xdomain_paths = icm_fr_approve_xdomain_paths,
2348 .disconnect_xdomain_paths = icm_fr_disconnect_xdomain_paths,
2352 static const struct tb_cm_ops icm_ar_ops = {
2353 .driver_ready = icm_driver_ready,
2356 .suspend = icm_suspend,
2357 .complete = icm_complete,
2358 .runtime_suspend = icm_runtime_suspend,
2359 .runtime_resume = icm_runtime_resume,
2360 .runtime_suspend_switch = icm_runtime_suspend_switch,
2361 .runtime_resume_switch = icm_runtime_resume_switch,
2362 .handle_event = icm_handle_event,
2363 .get_boot_acl = icm_ar_get_boot_acl,
2364 .set_boot_acl = icm_ar_set_boot_acl,
2365 .approve_switch = icm_fr_approve_switch,
2366 .add_switch_key = icm_fr_add_switch_key,
2367 .challenge_switch_key = icm_fr_challenge_switch_key,
2368 .disconnect_pcie_paths = icm_disconnect_pcie_paths,
2369 .approve_xdomain_paths = icm_fr_approve_xdomain_paths,
2370 .disconnect_xdomain_paths = icm_fr_disconnect_xdomain_paths,
2374 static const struct tb_cm_ops icm_tr_ops = {
2375 .driver_ready = icm_driver_ready,
2378 .suspend = icm_suspend,
2379 .complete = icm_complete,
2380 .runtime_suspend = icm_runtime_suspend,
2381 .runtime_resume = icm_runtime_resume,
2382 .runtime_suspend_switch = icm_runtime_suspend_switch,
2383 .runtime_resume_switch = icm_runtime_resume_switch,
2384 .handle_event = icm_handle_event,
2385 .get_boot_acl = icm_ar_get_boot_acl,
2386 .set_boot_acl = icm_ar_set_boot_acl,
2387 .approve_switch = icm_tr_approve_switch,
2388 .add_switch_key = icm_tr_add_switch_key,
2389 .challenge_switch_key = icm_tr_challenge_switch_key,
2390 .disconnect_pcie_paths = icm_disconnect_pcie_paths,
2391 .approve_xdomain_paths = icm_tr_approve_xdomain_paths,
2392 .disconnect_xdomain_paths = icm_tr_disconnect_xdomain_paths,
2393 .usb4_switch_op = icm_usb4_switch_op,
2394 .usb4_switch_nvm_authenticate_status =
2395 icm_usb4_switch_nvm_authenticate_status,
2399 static const struct tb_cm_ops icm_icl_ops = {
2400 .driver_ready = icm_driver_ready,
2403 .complete = icm_complete,
2404 .runtime_suspend = icm_runtime_suspend,
2405 .runtime_resume = icm_runtime_resume,
2406 .handle_event = icm_handle_event,
2407 .approve_xdomain_paths = icm_tr_approve_xdomain_paths,
2408 .disconnect_xdomain_paths = icm_tr_disconnect_xdomain_paths,
2409 .usb4_switch_op = icm_usb4_switch_op,
2410 .usb4_switch_nvm_authenticate_status =
2411 icm_usb4_switch_nvm_authenticate_status,
2414 struct tb *icm_probe(struct tb_nhi *nhi)
2419 tb = tb_domain_alloc(nhi, ICM_TIMEOUT, sizeof(struct icm));
2424 INIT_DELAYED_WORK(&icm->rescan_work, icm_rescan_work);
2425 mutex_init(&icm->request_lock);
2427 switch (nhi->pdev->device) {
2428 case PCI_DEVICE_ID_INTEL_FALCON_RIDGE_2C_NHI:
2429 case PCI_DEVICE_ID_INTEL_FALCON_RIDGE_4C_NHI:
2430 icm->can_upgrade_nvm = true;
2431 icm->is_supported = icm_fr_is_supported;
2432 icm->get_route = icm_fr_get_route;
2433 icm->save_devices = icm_fr_save_devices;
2434 icm->driver_ready = icm_fr_driver_ready;
2435 icm->device_connected = icm_fr_device_connected;
2436 icm->device_disconnected = icm_fr_device_disconnected;
2437 icm->xdomain_connected = icm_fr_xdomain_connected;
2438 icm->xdomain_disconnected = icm_fr_xdomain_disconnected;
2439 tb->cm_ops = &icm_fr_ops;
2442 case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_2C_NHI:
2443 case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_4C_NHI:
2444 case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_LP_NHI:
2445 case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_C_4C_NHI:
2446 case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_C_2C_NHI:
2447 icm->max_boot_acl = ICM_AR_PREBOOT_ACL_ENTRIES;
2449 * NVM upgrade has not been tested on Apple systems and
2450 * they don't provide images publicly either. To be on
2451 * the safe side prevent root switch NVM upgrade on Macs
2454 icm->can_upgrade_nvm = !x86_apple_machine;
2455 icm->is_supported = icm_ar_is_supported;
2456 icm->cio_reset = icm_ar_cio_reset;
2457 icm->get_mode = icm_ar_get_mode;
2458 icm->get_route = icm_ar_get_route;
2459 icm->save_devices = icm_fr_save_devices;
2460 icm->driver_ready = icm_ar_driver_ready;
2461 icm->device_connected = icm_fr_device_connected;
2462 icm->device_disconnected = icm_fr_device_disconnected;
2463 icm->xdomain_connected = icm_fr_xdomain_connected;
2464 icm->xdomain_disconnected = icm_fr_xdomain_disconnected;
2465 tb->cm_ops = &icm_ar_ops;
2468 case PCI_DEVICE_ID_INTEL_TITAN_RIDGE_2C_NHI:
2469 case PCI_DEVICE_ID_INTEL_TITAN_RIDGE_4C_NHI:
2470 icm->max_boot_acl = ICM_AR_PREBOOT_ACL_ENTRIES;
2471 icm->can_upgrade_nvm = !x86_apple_machine;
2472 icm->is_supported = icm_ar_is_supported;
2473 icm->cio_reset = icm_tr_cio_reset;
2474 icm->get_mode = icm_ar_get_mode;
2475 icm->driver_ready = icm_tr_driver_ready;
2476 icm->device_connected = icm_tr_device_connected;
2477 icm->device_disconnected = icm_tr_device_disconnected;
2478 icm->xdomain_connected = icm_tr_xdomain_connected;
2479 icm->xdomain_disconnected = icm_tr_xdomain_disconnected;
2480 tb->cm_ops = &icm_tr_ops;
2483 case PCI_DEVICE_ID_INTEL_ICL_NHI0:
2484 case PCI_DEVICE_ID_INTEL_ICL_NHI1:
2485 icm->is_supported = icm_fr_is_supported;
2486 icm->driver_ready = icm_icl_driver_ready;
2487 icm->set_uuid = icm_icl_set_uuid;
2488 icm->device_connected = icm_icl_device_connected;
2489 icm->device_disconnected = icm_tr_device_disconnected;
2490 icm->xdomain_connected = icm_tr_xdomain_connected;
2491 icm->xdomain_disconnected = icm_tr_xdomain_disconnected;
2492 icm->rtd3_veto = icm_icl_rtd3_veto;
2493 tb->cm_ops = &icm_icl_ops;
2496 case PCI_DEVICE_ID_INTEL_TGL_NHI0:
2497 case PCI_DEVICE_ID_INTEL_TGL_NHI1:
2498 case PCI_DEVICE_ID_INTEL_TGL_H_NHI0:
2499 case PCI_DEVICE_ID_INTEL_TGL_H_NHI1:
2500 icm->is_supported = icm_tgl_is_supported;
2501 icm->driver_ready = icm_icl_driver_ready;
2502 icm->set_uuid = icm_icl_set_uuid;
2503 icm->device_connected = icm_icl_device_connected;
2504 icm->device_disconnected = icm_tr_device_disconnected;
2505 icm->xdomain_connected = icm_tr_xdomain_connected;
2506 icm->xdomain_disconnected = icm_tr_xdomain_disconnected;
2507 icm->rtd3_veto = icm_icl_rtd3_veto;
2508 tb->cm_ops = &icm_icl_ops;
2511 case PCI_DEVICE_ID_INTEL_MAPLE_RIDGE_4C_NHI:
2512 icm->is_supported = icm_tgl_is_supported;
2513 icm->get_mode = icm_ar_get_mode;
2514 icm->driver_ready = icm_tr_driver_ready;
2515 icm->device_connected = icm_tr_device_connected;
2516 icm->device_disconnected = icm_tr_device_disconnected;
2517 icm->xdomain_connected = icm_tr_xdomain_connected;
2518 icm->xdomain_disconnected = icm_tr_xdomain_disconnected;
2519 tb->cm_ops = &icm_tr_ops;
2523 if (!icm->is_supported || !icm->is_supported(tb)) {
2524 dev_dbg(&nhi->pdev->dev, "ICM not supported on this controller\n");
2529 tb_dbg(tb, "using firmware connection manager\n");