drivers/thunderbolt/tb.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Thunderbolt driver - bus logic (NHI independent)
   4  *
   5  * Copyright (c) 2014 Andreas Noever <andreas.noever@gmail.com>
   6  * Copyright (C) 2019, Intel Corporation
   7  */
   8
   9 #include <linux/slab.h>
  10 #include <linux/errno.h>
  11 #include <linux/delay.h>
  12 #include <linux/pm_runtime.h>
  13
  14 #include "tb.h"
  15 #include "tb_regs.h"
  16 #include "tunnel.h"
  17
  18 #define TB_TIMEOUT      100 /* ms */
  19
  20 /**
  21  * struct tb_cm - Simple Thunderbolt connection manager
  22  * @tunnel_list: List of active tunnels
  23  * @dp_resources: List of available DP resources for DP tunneling
  24  * @hotplug_active: tb_handle_hotplug will stop progressing plug
  25  *                  events and exit if this is not set (it needs to
  26  *                  acquire the lock one more time). Used to drain wq
  27  *                  after cfg has been paused.
  28  * @remove_work: Work used to remove any unplugged routers after
  29  *               runtime resume
  30  */
  31 struct tb_cm {
  32         struct list_head tunnel_list;
  33         struct list_head dp_resources;
  34         bool hotplug_active;
  35         struct delayed_work remove_work;
  36 };
  37
  38 static inline struct tb *tcm_to_tb(struct tb_cm *tcm)
  39 {
  40         return ((void *)tcm - sizeof(struct tb));
  41 }
  42
  43 struct tb_hotplug_event {
  44         struct work_struct work;
  45         struct tb *tb;
  46         u64 route;
  47         u8 port;
  48         bool unplug;
  49 };
  50
  51 static void tb_handle_hotplug(struct work_struct *work);
  52
  53 static void tb_queue_hotplug(struct tb *tb, u64 route, u8 port, bool unplug)
  54 {
  55         struct tb_hotplug_event *ev;
  56
  57         ev = kmalloc(sizeof(*ev), GFP_KERNEL);
  58         if (!ev)
  59                 return;
  60
  61         ev->tb = tb;
  62         ev->route = route;
  63         ev->port = port;
  64         ev->unplug = unplug;
  65         INIT_WORK(&ev->work, tb_handle_hotplug);
  66         queue_work(tb->wq, &ev->work);
  67 }
  68
  69 /* enumeration & hot plug handling */
  70
  71 static void tb_add_dp_resources(struct tb_switch *sw)
  72 {
  73         struct tb_cm *tcm = tb_priv(sw->tb);
  74         struct tb_port *port;
  75
  76         tb_switch_for_each_port(sw, port) {
  77                 if (!tb_port_is_dpin(port))
  78                         continue;
  79
  80                 if (!tb_switch_query_dp_resource(sw, port))
  81                         continue;
  82
  83                 list_add_tail(&port->list, &tcm->dp_resources);
  84                 tb_port_dbg(port, "DP IN resource available\n");
  85         }
  86 }
  87
  88 static void tb_remove_dp_resources(struct tb_switch *sw)
  89 {
  90         struct tb_cm *tcm = tb_priv(sw->tb);
  91         struct tb_port *port, *tmp;
  92
  93         /* Clear children resources first */
  94         tb_switch_for_each_port(sw, port) {
  95                 if (tb_port_has_remote(port))
  96                         tb_remove_dp_resources(port->remote->sw);
  97         }
  98
  99         list_for_each_entry_safe(port, tmp, &tcm->dp_resources, list) {
 100                 if (port->sw == sw) {
 101                         tb_port_dbg(port, "DP OUT resource unavailable\n");
 102                         list_del_init(&port->list);
 103                 }
 104         }
 105 }
 106
 107 static void tb_discover_tunnels(struct tb_switch *sw)
 108 {
 109         struct tb *tb = sw->tb;
 110         struct tb_cm *tcm = tb_priv(tb);
 111         struct tb_port *port;
 112
 113         tb_switch_for_each_port(sw, port) {
 114                 struct tb_tunnel *tunnel = NULL;
 115
 116                 switch (port->config.type) {
 117                 case TB_TYPE_DP_HDMI_IN:
 118                         tunnel = tb_tunnel_discover_dp(tb, port);
 119                         break;
 120
 121                 case TB_TYPE_PCIE_DOWN:
 122                         tunnel = tb_tunnel_discover_pci(tb, port);
 123                         break;
 124
 125                 case TB_TYPE_USB3_DOWN:
 126                         tunnel = tb_tunnel_discover_usb3(tb, port);
 127                         break;
 128
 129                 default:
 130                         break;
 131                 }
 132
 133                 if (!tunnel)
 134                         continue;
 135
 136                 if (tb_tunnel_is_pci(tunnel)) {
 137                         struct tb_switch *parent = tunnel->dst_port->sw;
 138
 139                         while (parent != tunnel->src_port->sw) {
 140                                 parent->boot = true;
 141                                 parent = tb_switch_parent(parent);
 142                         }
 143                 } else if (tb_tunnel_is_dp(tunnel)) {
 144                         /* Keep the domain from powering down */
 145                         pm_runtime_get_sync(&tunnel->src_port->sw->dev);
 146                         pm_runtime_get_sync(&tunnel->dst_port->sw->dev);
 147                 }
 148
 149                 list_add_tail(&tunnel->list, &tcm->tunnel_list);
 150         }
 151
 152         tb_switch_for_each_port(sw, port) {
 153                 if (tb_port_has_remote(port))
 154                         tb_discover_tunnels(port->remote->sw);
 155         }
 156 }
 157
 158 static int tb_port_configure_xdomain(struct tb_port *port)
 159 {
 160         /*
 161          * XDomain paths currently only support single lane so we must
 162          * disable the other lane according to USB4 spec.
 163          */
 164         tb_port_disable(port->dual_link_port);
 165
 166         if (tb_switch_is_usb4(port->sw))
 167                 return usb4_port_configure_xdomain(port);
 168         return tb_lc_configure_xdomain(port);
 169 }
 170
 171 static void tb_port_unconfigure_xdomain(struct tb_port *port)
 172 {
 173         if (tb_switch_is_usb4(port->sw))
 174                 usb4_port_unconfigure_xdomain(port);
 175         else
 176                 tb_lc_unconfigure_xdomain(port);
 177
 178         tb_port_enable(port->dual_link_port);
 179 }
 180
 181 static void tb_scan_xdomain(struct tb_port *port)
 182 {
 183         struct tb_switch *sw = port->sw;
 184         struct tb *tb = sw->tb;
 185         struct tb_xdomain *xd;
 186         u64 route;
 187
 188         if (!tb_is_xdomain_enabled())
 189                 return;
 190
 191         route = tb_downstream_route(port);
 192         xd = tb_xdomain_find_by_route(tb, route);
 193         if (xd) {
 194                 tb_xdomain_put(xd);
 195                 return;
 196         }
 197
 198         xd = tb_xdomain_alloc(tb, &sw->dev, route, tb->root_switch->uuid,
 199                               NULL);
 200         if (xd) {
 201                 tb_port_at(route, sw)->xdomain = xd;
 202                 tb_port_configure_xdomain(port);
 203                 tb_xdomain_add(xd);
 204         }
 205 }
 206
 207 static int tb_enable_tmu(struct tb_switch *sw)
 208 {
 209         int ret;
 210
 211         /* If it is already enabled in correct mode, don't touch it */
 212         if (tb_switch_tmu_is_enabled(sw))
 213                 return 0;
 214
 215         ret = tb_switch_tmu_disable(sw);
 216         if (ret)
 217                 return ret;
 218
 219         ret = tb_switch_tmu_post_time(sw);
 220         if (ret)
 221                 return ret;
 222
 223         return tb_switch_tmu_enable(sw);
 224 }
 225
 226 /**
 227  * tb_find_unused_port() - return the first inactive port on @sw
 228  * @sw: Switch to find the port on
 229  * @type: Port type to look for
 230  */
 231 static struct tb_port *tb_find_unused_port(struct tb_switch *sw,
 232                                            enum tb_port_type type)
 233 {
 234         struct tb_port *port;
 235
 236         tb_switch_for_each_port(sw, port) {
 237                 if (tb_is_upstream_port(port))
 238                         continue;
 239                 if (port->config.type != type)
 240                         continue;
 241                 if (!port->cap_adap)
 242                         continue;
 243                 if (tb_port_is_enabled(port))
 244                         continue;
 245                 return port;
 246         }
 247         return NULL;
 248 }
 249
 250 static struct tb_port *tb_find_usb3_down(struct tb_switch *sw,
 251                                          const struct tb_port *port)
 252 {
 253         struct tb_port *down;
 254
 255         down = usb4_switch_map_usb3_down(sw, port);
 256         if (down && !tb_usb3_port_is_enabled(down))
 257                 return down;
 258         return NULL;
 259 }
 260
 261 static struct tb_tunnel *tb_find_tunnel(struct tb *tb, enum tb_tunnel_type type,
 262                                         struct tb_port *src_port,
 263                                         struct tb_port *dst_port)
 264 {
 265         struct tb_cm *tcm = tb_priv(tb);
 266         struct tb_tunnel *tunnel;
 267
 268         list_for_each_entry(tunnel, &tcm->tunnel_list, list) {
 269                 if (tunnel->type == type &&
 270                     ((src_port && src_port == tunnel->src_port) ||
 271                      (dst_port && dst_port == tunnel->dst_port))) {
 272                         return tunnel;
 273                 }
 274         }
 275
 276         return NULL;
 277 }
 278
 279 static struct tb_tunnel *tb_find_first_usb3_tunnel(struct tb *tb,
 280                                                    struct tb_port *src_port,
 281                                                    struct tb_port *dst_port)
 282 {
 283         struct tb_port *port, *usb3_down;
 284         struct tb_switch *sw;
 285
 286         /* Pick the router that is deepest in the topology */
 287         if (dst_port->sw->config.depth > src_port->sw->config.depth)
 288                 sw = dst_port->sw;
 289         else
 290                 sw = src_port->sw;
 291
 292         /* Can't be the host router */
 293         if (sw == tb->root_switch)
 294                 return NULL;
 295
 296         /* Find the downstream USB4 port that leads to this router */
 297         port = tb_port_at(tb_route(sw), tb->root_switch);
 298         /* Find the corresponding host router USB3 downstream port */
 299         usb3_down = usb4_switch_map_usb3_down(tb->root_switch, port);
 300         if (!usb3_down)
 301                 return NULL;
 302
 303         return tb_find_tunnel(tb, TB_TUNNEL_USB3, usb3_down, NULL);
 304 }
 305
 306 static int tb_available_bandwidth(struct tb *tb, struct tb_port *src_port,
 307         struct tb_port *dst_port, int *available_up, int *available_down)
 308 {
 309         int usb3_consumed_up, usb3_consumed_down, ret;
 310         struct tb_cm *tcm = tb_priv(tb);
 311         struct tb_tunnel *tunnel;
 312         struct tb_port *port;
 313
 314         tb_port_dbg(dst_port, "calculating available bandwidth\n");
 315
 316         tunnel = tb_find_first_usb3_tunnel(tb, src_port, dst_port);
 317         if (tunnel) {
 318                 ret = tb_tunnel_consumed_bandwidth(tunnel, &usb3_consumed_up,
 319                                                    &usb3_consumed_down);
 320                 if (ret)
 321                         return ret;
 322         } else {
 323                 usb3_consumed_up = 0;
 324                 usb3_consumed_down = 0;
 325         }
 326
 327         *available_up = *available_down = 40000;
 328
 329         /* Find the minimum available bandwidth over all links */
 330         tb_for_each_port_on_path(src_port, dst_port, port) {
 331                 int link_speed, link_width, up_bw, down_bw;
 332
 333                 if (!tb_port_is_null(port))
 334                         continue;
 335
 336                 if (tb_is_upstream_port(port)) {
 337                         link_speed = port->sw->link_speed;
 338                 } else {
 339                         link_speed = tb_port_get_link_speed(port);
 340                         if (link_speed < 0)
 341                                 return link_speed;
 342                 }
 343
 344                 link_width = port->bonded ? 2 : 1;
 345
 346                 up_bw = link_speed * link_width * 1000; /* Mb/s */
 347                 /* Leave 10% guard band */
 348                 up_bw -= up_bw / 10;
 349                 down_bw = up_bw;
 350
 351                 tb_port_dbg(port, "link total bandwidth %d Mb/s\n", up_bw);
 352
 353                 /*
 354                  * Find all DP tunnels that cross the port and reduce
 355                  * their consumed bandwidth from the available.
 356                  */
 357                 list_for_each_entry(tunnel, &tcm->tunnel_list, list) {
 358                         int dp_consumed_up, dp_consumed_down;
 359
 360                         if (!tb_tunnel_is_dp(tunnel))
 361                                 continue;
 362
 363                         if (!tb_tunnel_port_on_path(tunnel, port))
 364                                 continue;
 365
 366                         ret = tb_tunnel_consumed_bandwidth(tunnel,
 367                                                            &dp_consumed_up,
 368                                                            &dp_consumed_down);
 369                         if (ret)
 370                                 return ret;
 371
 372                         up_bw -= dp_consumed_up;
 373                         down_bw -= dp_consumed_down;
 374                 }
 375
 376                 /*
 377                  * If USB3 is tunneled from the host router down to the
 378                  * branch leading to port we need to take USB3 consumed
 379                  * bandwidth into account regardless whether it actually
 380                  * crosses the port.
 381                  */
 382                 up_bw -= usb3_consumed_up;
 383                 down_bw -= usb3_consumed_down;
 384
 385                 if (up_bw < *available_up)
 386                         *available_up = up_bw;
 387                 if (down_bw < *available_down)
 388                         *available_down = down_bw;
 389         }
 390
 391         if (*available_up < 0)
 392                 *available_up = 0;
 393         if (*available_down < 0)
 394                 *available_down = 0;
 395
 396         return 0;
 397 }
 398
 399 static int tb_release_unused_usb3_bandwidth(struct tb *tb,
 400                                             struct tb_port *src_port,
 401                                             struct tb_port *dst_port)
 402 {
 403         struct tb_tunnel *tunnel;
 404
 405         tunnel = tb_find_first_usb3_tunnel(tb, src_port, dst_port);
 406         return tunnel ? tb_tunnel_release_unused_bandwidth(tunnel) : 0;
 407 }
 408
 409 static void tb_reclaim_usb3_bandwidth(struct tb *tb, struct tb_port *src_port,
 410                                       struct tb_port *dst_port)
 411 {
 412         int ret, available_up, available_down;
 413         struct tb_tunnel *tunnel;
 414
 415         tunnel = tb_find_first_usb3_tunnel(tb, src_port, dst_port);
 416         if (!tunnel)
 417                 return;
 418
 419         tb_dbg(tb, "reclaiming unused bandwidth for USB3\n");
 420
 421         /*
 422          * Calculate available bandwidth for the first hop USB3 tunnel.
 423          * That determines the whole USB3 bandwidth for this branch.
 424          */
 425         ret = tb_available_bandwidth(tb, tunnel->src_port, tunnel->dst_port,
 426                                      &available_up, &available_down);
 427         if (ret) {
 428                 tb_warn(tb, "failed to calculate available bandwidth\n");
 429                 return;
 430         }
 431
 432         tb_dbg(tb, "available bandwidth for USB3 %d/%d Mb/s\n",
 433                available_up, available_down);
 434
 435         tb_tunnel_reclaim_available_bandwidth(tunnel, &available_up, &available_down);
 436 }
 437
 438 static int tb_tunnel_usb3(struct tb *tb, struct tb_switch *sw)
 439 {
 440         struct tb_switch *parent = tb_switch_parent(sw);
 441         int ret, available_up, available_down;
 442         struct tb_port *up, *down, *port;
 443         struct tb_cm *tcm = tb_priv(tb);
 444         struct tb_tunnel *tunnel;
 445
 446         if (!tb_acpi_may_tunnel_usb3()) {
 447                 tb_dbg(tb, "USB3 tunneling disabled, not creating tunnel\n");
 448                 return 0;
 449         }
 450
 451         up = tb_switch_find_port(sw, TB_TYPE_USB3_UP);
 452         if (!up)
 453                 return 0;
 454
 455         if (!sw->link_usb4)
 456                 return 0;
 457
 458         /*
 459          * Look up available down port. Since we are chaining it should
 460          * be found right above this switch.
 461          */
 462         port = tb_port_at(tb_route(sw), parent);
 463         down = tb_find_usb3_down(parent, port);
 464         if (!down)
 465                 return 0;
 466
 467         if (tb_route(parent)) {
 468                 struct tb_port *parent_up;
 469                 /*
 470                  * Check first that the parent switch has its upstream USB3
 471                  * port enabled. Otherwise the chain is not complete and
 472                  * there is no point setting up a new tunnel.
 473                  */
 474                 parent_up = tb_switch_find_port(parent, TB_TYPE_USB3_UP);
 475                 if (!parent_up || !tb_port_is_enabled(parent_up))
 476                         return 0;
 477
 478                 /* Make all unused bandwidth available for the new tunnel */
 479                 ret = tb_release_unused_usb3_bandwidth(tb, down, up);
 480                 if (ret)
 481                         return ret;
 482         }
 483
 484         ret = tb_available_bandwidth(tb, down, up, &available_up,
 485                                      &available_down);
 486         if (ret)
 487                 goto err_reclaim;
 488
 489         tb_port_dbg(up, "available bandwidth for new USB3 tunnel %d/%d Mb/s\n",
 490                     available_up, available_down);
 491
 492         tunnel = tb_tunnel_alloc_usb3(tb, up, down, available_up,
 493                                       available_down);
 494         if (!tunnel) {
 495                 ret = -ENOMEM;
 496                 goto err_reclaim;
 497         }
 498
 499         if (tb_tunnel_activate(tunnel)) {
 500                 tb_port_info(up,
 501                              "USB3 tunnel activation failed, aborting\n");
 502                 ret = -EIO;
 503                 goto err_free;
 504         }
 505
 506         list_add_tail(&tunnel->list, &tcm->tunnel_list);
 507         if (tb_route(parent))
 508                 tb_reclaim_usb3_bandwidth(tb, down, up);
 509
 510         return 0;
 511
 512 err_free:
 513         tb_tunnel_free(tunnel);
 514 err_reclaim:
 515         if (tb_route(parent))
 516                 tb_reclaim_usb3_bandwidth(tb, down, up);
 517
 518         return ret;
 519 }
 520
 521 static int tb_create_usb3_tunnels(struct tb_switch *sw)
 522 {
 523         struct tb_port *port;
 524         int ret;
 525
 526         if (!tb_acpi_may_tunnel_usb3())
 527                 return 0;
 528
 529         if (tb_route(sw)) {
 530                 ret = tb_tunnel_usb3(sw->tb, sw);
 531                 if (ret)
 532                         return ret;
 533         }
 534
 535         tb_switch_for_each_port(sw, port) {
 536                 if (!tb_port_has_remote(port))
 537                         continue;
 538                 ret = tb_create_usb3_tunnels(port->remote->sw);
 539                 if (ret)
 540                         return ret;
 541         }
 542
 543         return 0;
 544 }
 545
 546 static void tb_scan_port(struct tb_port *port);
 547
 548 /*
 549  * tb_scan_switch() - scan for and initialize downstream switches
 550  */
 551 static void tb_scan_switch(struct tb_switch *sw)
 552 {
 553         struct tb_port *port;
 554
 555         pm_runtime_get_sync(&sw->dev);
 556
 557         tb_switch_for_each_port(sw, port)
 558                 tb_scan_port(port);
 559
 560         pm_runtime_mark_last_busy(&sw->dev);
 561         pm_runtime_put_autosuspend(&sw->dev);
 562 }
 563
 564 /*
 565  * tb_scan_port() - check for and initialize switches below port
 566  */
 567 static void tb_scan_port(struct tb_port *port)
 568 {
 569         struct tb_cm *tcm = tb_priv(port->sw->tb);
 570         struct tb_port *upstream_port;
 571         struct tb_switch *sw;
 572
 573         if (tb_is_upstream_port(port))
 574                 return;
 575
 576         if (tb_port_is_dpout(port) && tb_dp_port_hpd_is_active(port) == 1 &&
 577             !tb_dp_port_is_enabled(port)) {
 578                 tb_port_dbg(port, "DP adapter HPD set, queuing hotplug\n");
 579                 tb_queue_hotplug(port->sw->tb, tb_route(port->sw), port->port,
 580                                  false);
 581                 return;
 582         }
 583
 584         if (port->config.type != TB_TYPE_PORT)
 585                 return;
 586         if (port->dual_link_port && port->link_nr)
 587                 return; /*
 588                          * Downstream switch is reachable through two ports.
 589                          * Only scan on the primary port (link_nr == 0).
 590                          */
 591         if (tb_wait_for_port(port, false) <= 0)
 592                 return;
 593         if (port->remote) {
 594                 tb_port_dbg(port, "port already has a remote\n");
 595                 return;
 596         }
 597
 598         tb_retimer_scan(port);
 599
 600         sw = tb_switch_alloc(port->sw->tb, &port->sw->dev,
 601                              tb_downstream_route(port));
 602         if (IS_ERR(sw)) {
 603                 /*
 604                  * If there is an error accessing the connected switch
 605                  * it may be connected to another domain. Also we allow
 606                  * the other domain to be connected to a max depth switch.
 607                  */
 608                 if (PTR_ERR(sw) == -EIO || PTR_ERR(sw) == -EADDRNOTAVAIL)
 609                         tb_scan_xdomain(port);
 610                 return;
 611         }
 612
 613         if (tb_switch_configure(sw)) {
 614                 tb_switch_put(sw);
 615                 return;
 616         }
 617
 618         /*
 619          * If there was previously another domain connected remove it
 620          * first.
 621          */
 622         if (port->xdomain) {
 623                 tb_xdomain_remove(port->xdomain);
 624                 tb_port_unconfigure_xdomain(port);
 625                 port->xdomain = NULL;
 626         }
 627
 628         /*
 629          * Do not send uevents until we have discovered all existing
 630          * tunnels and know which switches were authorized already by
 631          * the boot firmware.
 632          */
 633         if (!tcm->hotplug_active)
 634                 dev_set_uevent_suppress(&sw->dev, true);
 635
 636         /*
 637          * At the moment Thunderbolt 2 and beyond (devices with LC) we
 638          * can support runtime PM.
 639          */
 640         sw->rpm = sw->generation > 1;
 641
 642         if (tb_switch_add(sw)) {
 643                 tb_switch_put(sw);
 644                 return;
 645         }
 646
 647         /* Link the switches using both links if available */
 648         upstream_port = tb_upstream_port(sw);
 649         port->remote = upstream_port;
 650         upstream_port->remote = port;
 651         if (port->dual_link_port && upstream_port->dual_link_port) {
 652                 port->dual_link_port->remote = upstream_port->dual_link_port;
 653                 upstream_port->dual_link_port->remote = port->dual_link_port;
 654         }
 655
 656         /* Enable lane bonding if supported */
 657         tb_switch_lane_bonding_enable(sw);
 658         /* Set the link configured */
 659         tb_switch_configure_link(sw);
 660
 661         if (tb_enable_tmu(sw))
 662                 tb_sw_warn(sw, "failed to enable TMU\n");
 663
 664         /* Scan upstream retimers */
 665         tb_retimer_scan(upstream_port);
 666
 667         /*
 668          * Create USB 3.x tunnels only when the switch is plugged to the
 669          * domain. This is because we scan the domain also during discovery
 670          * and want to discover existing USB 3.x tunnels before we create
 671          * any new.
 672          */
 673         if (tcm->hotplug_active && tb_tunnel_usb3(sw->tb, sw))
 674                 tb_sw_warn(sw, "USB3 tunnel creation failed\n");
 675
 676         tb_add_dp_resources(sw);
 677         tb_scan_switch(sw);
 678 }
 679
 680 static void tb_deactivate_and_free_tunnel(struct tb_tunnel *tunnel)
 681 {
 682         struct tb_port *src_port, *dst_port;
 683         struct tb *tb;
 684
 685         if (!tunnel)
 686                 return;
 687
 688         tb_tunnel_deactivate(tunnel);
 689         list_del(&tunnel->list);
 690
 691         tb = tunnel->tb;
 692         src_port = tunnel->src_port;
 693         dst_port = tunnel->dst_port;
 694
 695         switch (tunnel->type) {
 696         case TB_TUNNEL_DP:
 697                 /*
 698                  * In case of DP tunnel make sure the DP IN resource is
 699                  * deallocated properly.
 700                  */
 701                 tb_switch_dealloc_dp_resource(src_port->sw, src_port);
 702                 /* Now we can allow the domain to runtime suspend again */
 703                 pm_runtime_mark_last_busy(&dst_port->sw->dev);
 704                 pm_runtime_put_autosuspend(&dst_port->sw->dev);
 705                 pm_runtime_mark_last_busy(&src_port->sw->dev);
 706                 pm_runtime_put_autosuspend(&src_port->sw->dev);
 707                 fallthrough;
 708
 709         case TB_TUNNEL_USB3:
 710                 tb_reclaim_usb3_bandwidth(tb, src_port, dst_port);
 711                 break;
 712
 713         default:
 714                 /*
 715                  * PCIe and DMA tunnels do not consume guaranteed
 716                  * bandwidth.
 717                  */
 718                 break;
 719         }
 720
 721         tb_tunnel_free(tunnel);
 722 }
 723
 724 /*
 725  * tb_free_invalid_tunnels() - destroy tunnels of devices that have gone away
 726  */
 727 static void tb_free_invalid_tunnels(struct tb *tb)
 728 {
 729         struct tb_cm *tcm = tb_priv(tb);
 730         struct tb_tunnel *tunnel;
 731         struct tb_tunnel *n;
 732
 733         list_for_each_entry_safe(tunnel, n, &tcm->tunnel_list, list) {
 734                 if (tb_tunnel_is_invalid(tunnel))
 735                         tb_deactivate_and_free_tunnel(tunnel);
 736         }
 737 }
 738
 739 /*
 740  * tb_free_unplugged_children() - traverse hierarchy and free unplugged switches
 741  */
 742 static void tb_free_unplugged_children(struct tb_switch *sw)
 743 {
 744         struct tb_port *port;
 745
 746         tb_switch_for_each_port(sw, port) {
 747                 if (!tb_port_has_remote(port))
 748                         continue;
 749
 750                 if (port->remote->sw->is_unplugged) {
 751                         tb_retimer_remove_all(port);
 752                         tb_remove_dp_resources(port->remote->sw);
 753                         tb_switch_unconfigure_link(port->remote->sw);
 754                         tb_switch_lane_bonding_disable(port->remote->sw);
 755                         tb_switch_remove(port->remote->sw);
 756                         port->remote = NULL;
 757                         if (port->dual_link_port)
 758                                 port->dual_link_port->remote = NULL;
 759                 } else {
 760                         tb_free_unplugged_children(port->remote->sw);
 761                 }
 762         }
 763 }
 764
 765 static struct tb_port *tb_find_pcie_down(struct tb_switch *sw,
 766                                          const struct tb_port *port)
 767 {
 768         struct tb_port *down = NULL;
 769
 770         /*
 771          * To keep plugging devices consistently in the same PCIe
 772          * hierarchy, do mapping here for switch downstream PCIe ports.
 773          */
 774         if (tb_switch_is_usb4(sw)) {
 775                 down = usb4_switch_map_pcie_down(sw, port);
 776         } else if (!tb_route(sw)) {
 777                 int phy_port = tb_phy_port_from_link(port->port);
 778                 int index;
 779
 780                 /*
 781                  * Hard-coded Thunderbolt port to PCIe down port mapping
 782                  * per controller.
 783                  */
 784                 if (tb_switch_is_cactus_ridge(sw) ||
 785                     tb_switch_is_alpine_ridge(sw))
 786                         index = !phy_port ? 6 : 7;
 787                 else if (tb_switch_is_falcon_ridge(sw))
 788                         index = !phy_port ? 6 : 8;
 789                 else if (tb_switch_is_titan_ridge(sw))
 790                         index = !phy_port ? 8 : 9;
 791                 else
 792                         goto out;
 793
 794                 /* Validate the hard-coding */
 795                 if (WARN_ON(index > sw->config.max_port_number))
 796                         goto out;
 797
 798                 down = &sw->ports[index];
 799         }
 800
 801         if (down) {
 802                 if (WARN_ON(!tb_port_is_pcie_down(down)))
 803                         goto out;
 804                 if (tb_pci_port_is_enabled(down))
 805                         goto out;
 806
 807                 return down;
 808         }
 809
 810 out:
 811         return tb_find_unused_port(sw, TB_TYPE_PCIE_DOWN);
 812 }
 813
 814 static struct tb_port *tb_find_dp_out(struct tb *tb, struct tb_port *in)
 815 {
 816         struct tb_port *host_port, *port;
 817         struct tb_cm *tcm = tb_priv(tb);
 818
 819         host_port = tb_route(in->sw) ?
 820                 tb_port_at(tb_route(in->sw), tb->root_switch) : NULL;
 821
 822         list_for_each_entry(port, &tcm->dp_resources, list) {
 823                 if (!tb_port_is_dpout(port))
 824                         continue;
 825
 826                 if (tb_port_is_enabled(port)) {
 827                         tb_port_dbg(port, "in use\n");
 828                         continue;
 829                 }
 830
 831                 tb_port_dbg(port, "DP OUT available\n");
 832
 833                 /*
 834                  * Keep the DP tunnel under the topology starting from
 835                  * the same host router downstream port.
 836                  */
 837                 if (host_port && tb_route(port->sw)) {
 838                         struct tb_port *p;
 839
 840                         p = tb_port_at(tb_route(port->sw), tb->root_switch);
 841                         if (p != host_port)
 842                                 continue;
 843                 }
 844
 845                 return port;
 846         }
 847
 848         return NULL;
 849 }
 850
 851 static void tb_tunnel_dp(struct tb *tb)
 852 {
 853         int available_up, available_down, ret;
 854         struct tb_cm *tcm = tb_priv(tb);
 855         struct tb_port *port, *in, *out;
 856         struct tb_tunnel *tunnel;
 857
 858         if (!tb_acpi_may_tunnel_dp()) {
 859                 tb_dbg(tb, "DP tunneling disabled, not creating tunnel\n");
 860                 return;
 861         }
 862
 863         /*
 864          * Find pair of inactive DP IN and DP OUT adapters and then
 865          * establish a DP tunnel between them.
 866          */
 867         tb_dbg(tb, "looking for DP IN <-> DP OUT pairs:\n");
 868
 869         in = NULL;
 870         out = NULL;
 871         list_for_each_entry(port, &tcm->dp_resources, list) {
 872                 if (!tb_port_is_dpin(port))
 873                         continue;
 874
 875                 if (tb_port_is_enabled(port)) {
 876                         tb_port_dbg(port, "in use\n");
 877                         continue;
 878                 }
 879
 880                 tb_port_dbg(port, "DP IN available\n");
 881
 882                 out = tb_find_dp_out(tb, port);
 883                 if (out) {
 884                         in = port;
 885                         break;
 886                 }
 887         }
 888
 889         if (!in) {
 890                 tb_dbg(tb, "no suitable DP IN adapter available, not tunneling\n");
 891                 return;
 892         }
 893         if (!out) {
 894                 tb_dbg(tb, "no suitable DP OUT adapter available, not tunneling\n");
 895                 return;
 896         }
 897
 898         /*
 899          * DP stream needs the domain to be active so runtime resume
 900          * both ends of the tunnel.
 901          *
 902          * This should bring the routers in the middle active as well
 903          * and keeps the domain from runtime suspending while the DP
 904          * tunnel is active.
 905          */
 906         pm_runtime_get_sync(&in->sw->dev);
 907         pm_runtime_get_sync(&out->sw->dev);
 908
 909         if (tb_switch_alloc_dp_resource(in->sw, in)) {
 910                 tb_port_dbg(in, "no resource available for DP IN, not tunneling\n");
 911                 goto err_rpm_put;
 912         }
 913
 914         /* Make all unused USB3 bandwidth available for the new DP tunnel */
 915         ret = tb_release_unused_usb3_bandwidth(tb, in, out);
 916         if (ret) {
 917                 tb_warn(tb, "failed to release unused bandwidth\n");
 918                 goto err_dealloc_dp;
 919         }
 920
 921         ret = tb_available_bandwidth(tb, in, out, &available_up,
 922                                      &available_down);
 923         if (ret)
 924                 goto err_reclaim;
 925
 926         tb_dbg(tb, "available bandwidth for new DP tunnel %u/%u Mb/s\n",
 927                available_up, available_down);
 928
 929         tunnel = tb_tunnel_alloc_dp(tb, in, out, available_up, available_down);
 930         if (!tunnel) {
 931                 tb_port_dbg(out, "could not allocate DP tunnel\n");
 932                 goto err_reclaim;
 933         }
 934
 935         if (tb_tunnel_activate(tunnel)) {
 936                 tb_port_info(out, "DP tunnel activation failed, aborting\n");
 937                 goto err_free;
 938         }
 939
 940         list_add_tail(&tunnel->list, &tcm->tunnel_list);
 941         tb_reclaim_usb3_bandwidth(tb, in, out);
 942         return;
 943
 944 err_free:
 945         tb_tunnel_free(tunnel);
 946 err_reclaim:
 947         tb_reclaim_usb3_bandwidth(tb, in, out);
 948 err_dealloc_dp:
 949         tb_switch_dealloc_dp_resource(in->sw, in);
 950 err_rpm_put:
 951         pm_runtime_mark_last_busy(&out->sw->dev);
 952         pm_runtime_put_autosuspend(&out->sw->dev);
 953         pm_runtime_mark_last_busy(&in->sw->dev);
 954         pm_runtime_put_autosuspend(&in->sw->dev);
 955 }
 956
 957 static void tb_dp_resource_unavailable(struct tb *tb, struct tb_port *port)
 958 {
 959         struct tb_port *in, *out;
 960         struct tb_tunnel *tunnel;
 961
 962         if (tb_port_is_dpin(port)) {
 963                 tb_port_dbg(port, "DP IN resource unavailable\n");
 964                 in = port;
 965                 out = NULL;
 966         } else {
 967                 tb_port_dbg(port, "DP OUT resource unavailable\n");
 968                 in = NULL;
 969                 out = port;
 970         }
 971
 972         tunnel = tb_find_tunnel(tb, TB_TUNNEL_DP, in, out);
 973         tb_deactivate_and_free_tunnel(tunnel);
 974         list_del_init(&port->list);
 975
 976         /*
 977          * See if there is another DP OUT port that can be used for
 978          * to create another tunnel.
 979          */
 980         tb_tunnel_dp(tb);
 981 }
 982
 983 static void tb_dp_resource_available(struct tb *tb, struct tb_port *port)
 984 {
 985         struct tb_cm *tcm = tb_priv(tb);
 986         struct tb_port *p;
 987
 988         if (tb_port_is_enabled(port))
 989                 return;
 990
 991         list_for_each_entry(p, &tcm->dp_resources, list) {
 992                 if (p == port)
 993                         return;
 994         }
 995
 996         tb_port_dbg(port, "DP %s resource available\n",
 997                     tb_port_is_dpin(port) ? "IN" : "OUT");
 998         list_add_tail(&port->list, &tcm->dp_resources);
 999
1000         /* Look for suitable DP IN <-> DP OUT pairs now */
1001         tb_tunnel_dp(tb);
1002 }
1003
1004 static void tb_disconnect_and_release_dp(struct tb *tb)
1005 {
1006         struct tb_cm *tcm = tb_priv(tb);
1007         struct tb_tunnel *tunnel, *n;
1008
1009         /*
1010          * Tear down all DP tunnels and release their resources. They
1011          * will be re-established after resume based on plug events.
1012          */
1013         list_for_each_entry_safe_reverse(tunnel, n, &tcm->tunnel_list, list) {
1014                 if (tb_tunnel_is_dp(tunnel))
1015                         tb_deactivate_and_free_tunnel(tunnel);
1016         }
1017
1018         while (!list_empty(&tcm->dp_resources)) {
1019                 struct tb_port *port;
1020
1021                 port = list_first_entry(&tcm->dp_resources,
1022                                         struct tb_port, list);
1023                 list_del_init(&port->list);
1024         }
1025 }
1026
1027 static int tb_disconnect_pci(struct tb *tb, struct tb_switch *sw)
1028 {
1029         struct tb_tunnel *tunnel;
1030         struct tb_port *up;
1031
1032         up = tb_switch_find_port(sw, TB_TYPE_PCIE_UP);
1033         if (WARN_ON(!up))
1034                 return -ENODEV;
1035
1036         tunnel = tb_find_tunnel(tb, TB_TUNNEL_PCI, NULL, up);
1037         if (WARN_ON(!tunnel))
1038                 return -ENODEV;
1039
1040         tb_tunnel_deactivate(tunnel);
1041         list_del(&tunnel->list);
1042         tb_tunnel_free(tunnel);
1043         return 0;
1044 }
1045
1046 static int tb_tunnel_pci(struct tb *tb, struct tb_switch *sw)
1047 {
1048         struct tb_port *up, *down, *port;
1049         struct tb_cm *tcm = tb_priv(tb);
1050         struct tb_switch *parent_sw;
1051         struct tb_tunnel *tunnel;
1052
1053         up = tb_switch_find_port(sw, TB_TYPE_PCIE_UP);
1054         if (!up)
1055                 return 0;
1056
1057         /*
1058          * Look up available down port. Since we are chaining it should
1059          * be found right above this switch.
1060          */
1061         parent_sw = tb_to_switch(sw->dev.parent);
1062         port = tb_port_at(tb_route(sw), parent_sw);
1063         down = tb_find_pcie_down(parent_sw, port);
1064         if (!down)
1065                 return 0;
1066
1067         tunnel = tb_tunnel_alloc_pci(tb, up, down);
1068         if (!tunnel)
1069                 return -ENOMEM;
1070
1071         if (tb_tunnel_activate(tunnel)) {
1072                 tb_port_info(up,
1073                              "PCIe tunnel activation failed, aborting\n");
1074                 tb_tunnel_free(tunnel);
1075                 return -EIO;
1076         }
1077
1078         list_add_tail(&tunnel->list, &tcm->tunnel_list);
1079         return 0;
1080 }
1081
1082 static int tb_approve_xdomain_paths(struct tb *tb, struct tb_xdomain *xd,
1083                                     int transmit_path, int transmit_ring,
1084                                     int receive_path, int receive_ring)
1085 {
1086         struct tb_cm *tcm = tb_priv(tb);
1087         struct tb_port *nhi_port, *dst_port;
1088         struct tb_tunnel *tunnel;
1089         struct tb_switch *sw;
1090
1091         sw = tb_to_switch(xd->dev.parent);
1092         dst_port = tb_port_at(xd->route, sw);
1093         nhi_port = tb_switch_find_port(tb->root_switch, TB_TYPE_NHI);
1094
1095         mutex_lock(&tb->lock);
1096         tunnel = tb_tunnel_alloc_dma(tb, nhi_port, dst_port, transmit_path,
1097                                      transmit_ring, receive_path, receive_ring);
1098         if (!tunnel) {
1099                 mutex_unlock(&tb->lock);
1100                 return -ENOMEM;
1101         }
1102
1103         if (tb_tunnel_activate(tunnel)) {
1104                 tb_port_info(nhi_port,
1105                              "DMA tunnel activation failed, aborting\n");
1106                 tb_tunnel_free(tunnel);
1107                 mutex_unlock(&tb->lock);
1108                 return -EIO;
1109         }
1110
1111         list_add_tail(&tunnel->list, &tcm->tunnel_list);
1112         mutex_unlock(&tb->lock);
1113         return 0;
1114 }
1115
1116 static void __tb_disconnect_xdomain_paths(struct tb *tb, struct tb_xdomain *xd,
1117                                           int transmit_path, int transmit_ring,
1118                                           int receive_path, int receive_ring)
1119 {
1120         struct tb_cm *tcm = tb_priv(tb);
1121         struct tb_port *nhi_port, *dst_port;
1122         struct tb_tunnel *tunnel, *n;
1123         struct tb_switch *sw;
1124
1125         sw = tb_to_switch(xd->dev.parent);
1126         dst_port = tb_port_at(xd->route, sw);
1127         nhi_port = tb_switch_find_port(tb->root_switch, TB_TYPE_NHI);
1128
1129         list_for_each_entry_safe(tunnel, n, &tcm->tunnel_list, list) {
1130                 if (!tb_tunnel_is_dma(tunnel))
1131                         continue;
1132                 if (tunnel->src_port != nhi_port || tunnel->dst_port != dst_port)
1133                         continue;
1134
1135                 if (tb_tunnel_match_dma(tunnel, transmit_path, transmit_ring,
1136                                         receive_path, receive_ring))
1137                         tb_deactivate_and_free_tunnel(tunnel);
1138         }
1139 }
1140
1141 static int tb_disconnect_xdomain_paths(struct tb *tb, struct tb_xdomain *xd,
1142                                        int transmit_path, int transmit_ring,
1143                                        int receive_path, int receive_ring)
1144 {
1145         if (!xd->is_unplugged) {
1146                 mutex_lock(&tb->lock);
1147                 __tb_disconnect_xdomain_paths(tb, xd, transmit_path,
1148                                               transmit_ring, receive_path,
1149                                               receive_ring);
1150                 mutex_unlock(&tb->lock);
1151         }
1152         return 0;
1153 }
1154
1155 /* hotplug handling */
1156
1157 /*
1158  * tb_handle_hotplug() - handle hotplug event
1159  *
1160  * Executes on tb->wq.
1161  */
1162 static void tb_handle_hotplug(struct work_struct *work)
1163 {
1164         struct tb_hotplug_event *ev = container_of(work, typeof(*ev), work);
1165         struct tb *tb = ev->tb;
1166         struct tb_cm *tcm = tb_priv(tb);
1167         struct tb_switch *sw;
1168         struct tb_port *port;
1169
1170         /* Bring the domain back from sleep if it was suspended */
1171         pm_runtime_get_sync(&tb->dev);
1172
1173         mutex_lock(&tb->lock);
1174         if (!tcm->hotplug_active)
1175                 goto out; /* during init, suspend or shutdown */
1176
1177         sw = tb_switch_find_by_route(tb, ev->route);
1178         if (!sw) {
1179                 tb_warn(tb,
1180                         "hotplug event from non existent switch %llx:%x (unplug: %d)\n",
1181                         ev->route, ev->port, ev->unplug);
1182                 goto out;
1183         }
1184         if (ev->port > sw->config.max_port_number) {
1185                 tb_warn(tb,
1186                         "hotplug event from non existent port %llx:%x (unplug: %d)\n",
1187                         ev->route, ev->port, ev->unplug);
1188                 goto put_sw;
1189         }
1190         port = &sw->ports[ev->port];
1191         if (tb_is_upstream_port(port)) {
1192                 tb_dbg(tb, "hotplug event for upstream port %llx:%x (unplug: %d)\n",
1193                        ev->route, ev->port, ev->unplug);
1194                 goto put_sw;
1195         }
1196
1197         pm_runtime_get_sync(&sw->dev);
1198
1199         if (ev->unplug) {
1200                 tb_retimer_remove_all(port);
1201
1202                 if (tb_port_has_remote(port)) {
1203                         tb_port_dbg(port, "switch unplugged\n");
1204                         tb_sw_set_unplugged(port->remote->sw);
1205                         tb_free_invalid_tunnels(tb);
1206                         tb_remove_dp_resources(port->remote->sw);
1207                         tb_switch_tmu_disable(port->remote->sw);
1208                         tb_switch_unconfigure_link(port->remote->sw);
1209                         tb_switch_lane_bonding_disable(port->remote->sw);
1210                         tb_switch_remove(port->remote->sw);
1211                         port->remote = NULL;
1212                         if (port->dual_link_port)
1213                                 port->dual_link_port->remote = NULL;
1214                         /* Maybe we can create another DP tunnel */
1215                         tb_tunnel_dp(tb);
1216                 } else if (port->xdomain) {
1217                         struct tb_xdomain *xd = tb_xdomain_get(port->xdomain);
1218
1219                         tb_port_dbg(port, "xdomain unplugged\n");
1220                         /*
1221                          * Service drivers are unbound during
1222                          * tb_xdomain_remove() so setting XDomain as
1223                          * unplugged here prevents deadlock if they call
1224                          * tb_xdomain_disable_paths(). We will tear down
1225                          * all the tunnels below.
1226                          */
1227                         xd->is_unplugged = true;
1228                         tb_xdomain_remove(xd);
1229                         port->xdomain = NULL;
1230                         __tb_disconnect_xdomain_paths(tb, xd, -1, -1, -1, -1);
1231                         tb_xdomain_put(xd);
1232                         tb_port_unconfigure_xdomain(port);
1233                 } else if (tb_port_is_dpout(port) || tb_port_is_dpin(port)) {
1234                         tb_dp_resource_unavailable(tb, port);
1235                 } else {
1236                         tb_port_dbg(port,
1237                                    "got unplug event for disconnected port, ignoring\n");
1238                 }
1239         } else if (port->remote) {
1240                 tb_port_dbg(port, "got plug event for connected port, ignoring\n");
1241         } else {
1242                 if (tb_port_is_null(port)) {
1243                         tb_port_dbg(port, "hotplug: scanning\n");
1244                         tb_scan_port(port);
1245                         if (!port->remote)
1246                                 tb_port_dbg(port, "hotplug: no switch found\n");
1247                 } else if (tb_port_is_dpout(port) || tb_port_is_dpin(port)) {
1248                         tb_dp_resource_available(tb, port);
1249                 }
1250         }
1251
1252         pm_runtime_mark_last_busy(&sw->dev);
1253         pm_runtime_put_autosuspend(&sw->dev);
1254
1255 put_sw:
1256         tb_switch_put(sw);
1257 out:
1258         mutex_unlock(&tb->lock);
1259
1260         pm_runtime_mark_last_busy(&tb->dev);
1261         pm_runtime_put_autosuspend(&tb->dev);
1262
1263         kfree(ev);
1264 }
1265
1266 /*
1267  * tb_schedule_hotplug_handler() - callback function for the control channel
1268  *
1269  * Delegates to tb_handle_hotplug.
1270  */
1271 static void tb_handle_event(struct tb *tb, enum tb_cfg_pkg_type type,
1272                             const void *buf, size_t size)
1273 {
1274         const struct cfg_event_pkg *pkg = buf;
1275         u64 route;
1276
1277         if (type != TB_CFG_PKG_EVENT) {
1278                 tb_warn(tb, "unexpected event %#x, ignoring\n", type);
1279                 return;
1280         }
1281
1282         route = tb_cfg_get_route(&pkg->header);
1283
1284         if (tb_cfg_ack_plug(tb->ctl, route, pkg->port, pkg->unplug)) {
1285                 tb_warn(tb, "could not ack plug event on %llx:%x\n", route,
1286                         pkg->port);
1287         }
1288
1289         tb_queue_hotplug(tb, route, pkg->port, pkg->unplug);
1290 }
1291
1292 static void tb_stop(struct tb *tb)
1293 {
1294         struct tb_cm *tcm = tb_priv(tb);
1295         struct tb_tunnel *tunnel;
1296         struct tb_tunnel *n;
1297
1298         cancel_delayed_work(&tcm->remove_work);
1299         /* tunnels are only present after everything has been initialized */
1300         list_for_each_entry_safe(tunnel, n, &tcm->tunnel_list, list) {
1301                 /*
1302                  * DMA tunnels require the driver to be functional so we
1303                  * tear them down. Other protocol tunnels can be left
1304                  * intact.
1305                  */
1306                 if (tb_tunnel_is_dma(tunnel))
1307                         tb_tunnel_deactivate(tunnel);
1308                 tb_tunnel_free(tunnel);
1309         }
1310         tb_switch_remove(tb->root_switch);
1311         tcm->hotplug_active = false; /* signal tb_handle_hotplug to quit */
1312 }
1313
1314 static int tb_scan_finalize_switch(struct device *dev, void *data)
1315 {
1316         if (tb_is_switch(dev)) {
1317                 struct tb_switch *sw = tb_to_switch(dev);
1318
1319                 /*
1320                  * If we found that the switch was already setup by the
1321                  * boot firmware, mark it as authorized now before we
1322                  * send uevent to userspace.
1323                  */
1324                 if (sw->boot)
1325                         sw->authorized = 1;
1326
1327                 dev_set_uevent_suppress(dev, false);
1328                 kobject_uevent(&dev->kobj, KOBJ_ADD);
1329                 device_for_each_child(dev, NULL, tb_scan_finalize_switch);
1330         }
1331
1332         return 0;
1333 }
1334
1335 static int tb_start(struct tb *tb)
1336 {
1337         struct tb_cm *tcm = tb_priv(tb);
1338         int ret;
1339
1340         tb->root_switch = tb_switch_alloc(tb, &tb->dev, 0);
1341         if (IS_ERR(tb->root_switch))
1342                 return PTR_ERR(tb->root_switch);
1343
1344         /*
1345          * ICM firmware upgrade needs running firmware and in native
1346          * mode that is not available so disable firmware upgrade of the
1347          * root switch.
1348          */
1349         tb->root_switch->no_nvm_upgrade = true;
1350         /* All USB4 routers support runtime PM */
1351         tb->root_switch->rpm = tb_switch_is_usb4(tb->root_switch);
1352
1353         ret = tb_switch_configure(tb->root_switch);
1354         if (ret) {
1355                 tb_switch_put(tb->root_switch);
1356                 return ret;
1357         }
1358
1359         /* Announce the switch to the world */
1360         ret = tb_switch_add(tb->root_switch);
1361         if (ret) {
1362                 tb_switch_put(tb->root_switch);
1363                 return ret;
1364         }
1365
1366         /* Enable TMU if it is off */
1367         tb_switch_tmu_enable(tb->root_switch);
1368         /* Full scan to discover devices added before the driver was loaded. */
1369         tb_scan_switch(tb->root_switch);
1370         /* Find out tunnels created by the boot firmware */
1371         tb_discover_tunnels(tb->root_switch);
1372         /*
1373          * If the boot firmware did not create USB 3.x tunnels create them
1374          * now for the whole topology.
1375          */
1376         tb_create_usb3_tunnels(tb->root_switch);
1377         /* Add DP IN resources for the root switch */
1378         tb_add_dp_resources(tb->root_switch);
1379         /* Make the discovered switches available to the userspace */
1380         device_for_each_child(&tb->root_switch->dev, NULL,
1381                               tb_scan_finalize_switch);
1382
1383         /* Allow tb_handle_hotplug to progress events */
1384         tcm->hotplug_active = true;
1385         return 0;
1386 }
1387
1388 static int tb_suspend_noirq(struct tb *tb)
1389 {
1390         struct tb_cm *tcm = tb_priv(tb);
1391
1392         tb_dbg(tb, "suspending...\n");
1393         tb_disconnect_and_release_dp(tb);
1394         tb_switch_suspend(tb->root_switch, false);
1395         tcm->hotplug_active = false; /* signal tb_handle_hotplug to quit */
1396         tb_dbg(tb, "suspend finished\n");
1397
1398         return 0;
1399 }
1400
1401 static void tb_restore_children(struct tb_switch *sw)
1402 {
1403         struct tb_port *port;
1404
1405         /* No need to restore if the router is already unplugged */
1406         if (sw->is_unplugged)
1407                 return;
1408
1409         if (tb_enable_tmu(sw))
1410                 tb_sw_warn(sw, "failed to restore TMU configuration\n");
1411
1412         tb_switch_for_each_port(sw, port) {
1413                 if (!tb_port_has_remote(port) && !port->xdomain)
1414                         continue;
1415
1416                 if (port->remote) {
1417                         tb_switch_lane_bonding_enable(port->remote->sw);
1418                         tb_switch_configure_link(port->remote->sw);
1419
1420                         tb_restore_children(port->remote->sw);
1421                 } else if (port->xdomain) {
1422                         tb_port_configure_xdomain(port);
1423                 }
1424         }
1425 }
1426
1427 static int tb_resume_noirq(struct tb *tb)
1428 {
1429         struct tb_cm *tcm = tb_priv(tb);
1430         struct tb_tunnel *tunnel, *n;
1431
1432         tb_dbg(tb, "resuming...\n");
1433
1434         /* remove any pci devices the firmware might have setup */
1435         tb_switch_reset(tb->root_switch);
1436
1437         tb_switch_resume(tb->root_switch);
1438         tb_free_invalid_tunnels(tb);
1439         tb_free_unplugged_children(tb->root_switch);
1440         tb_restore_children(tb->root_switch);
1441         list_for_each_entry_safe(tunnel, n, &tcm->tunnel_list, list)
1442                 tb_tunnel_restart(tunnel);
1443         if (!list_empty(&tcm->tunnel_list)) {
1444                 /*
1445                  * the pcie links need some time to get going.
1446                  * 100ms works for me...
1447                  */
1448                 tb_dbg(tb, "tunnels restarted, sleeping for 100ms\n");
1449                 msleep(100);
1450         }
1451          /* Allow tb_handle_hotplug to progress events */
1452         tcm->hotplug_active = true;
1453         tb_dbg(tb, "resume finished\n");
1454
1455         return 0;
1456 }
1457
1458 static int tb_free_unplugged_xdomains(struct tb_switch *sw)
1459 {
1460         struct tb_port *port;
1461         int ret = 0;
1462
1463         tb_switch_for_each_port(sw, port) {
1464                 if (tb_is_upstream_port(port))
1465                         continue;
1466                 if (port->xdomain && port->xdomain->is_unplugged) {
1467                         tb_retimer_remove_all(port);
1468                         tb_xdomain_remove(port->xdomain);
1469                         tb_port_unconfigure_xdomain(port);
1470                         port->xdomain = NULL;
1471                         ret++;
1472                 } else if (port->remote) {
1473                         ret += tb_free_unplugged_xdomains(port->remote->sw);
1474                 }
1475         }
1476
1477         return ret;
1478 }
1479
1480 static int tb_freeze_noirq(struct tb *tb)
1481 {
1482         struct tb_cm *tcm = tb_priv(tb);
1483
1484         tcm->hotplug_active = false;
1485         return 0;
1486 }
1487
1488 static int tb_thaw_noirq(struct tb *tb)
1489 {
1490         struct tb_cm *tcm = tb_priv(tb);
1491
1492         tcm->hotplug_active = true;
1493         return 0;
1494 }
1495
1496 static void tb_complete(struct tb *tb)
1497 {
1498         /*
1499          * Release any unplugged XDomains and if there is a case where
1500          * another domain is swapped in place of unplugged XDomain we
1501          * need to run another rescan.
1502          */
1503         mutex_lock(&tb->lock);
1504         if (tb_free_unplugged_xdomains(tb->root_switch))
1505                 tb_scan_switch(tb->root_switch);
1506         mutex_unlock(&tb->lock);
1507 }
1508
1509 static int tb_runtime_suspend(struct tb *tb)
1510 {
1511         struct tb_cm *tcm = tb_priv(tb);
1512
1513         mutex_lock(&tb->lock);
1514         tb_switch_suspend(tb->root_switch, true);
1515         tcm->hotplug_active = false;
1516         mutex_unlock(&tb->lock);
1517
1518         return 0;
1519 }
1520
1521 static void tb_remove_work(struct work_struct *work)
1522 {
1523         struct tb_cm *tcm = container_of(work, struct tb_cm, remove_work.work);
1524         struct tb *tb = tcm_to_tb(tcm);
1525
1526         mutex_lock(&tb->lock);
1527         if (tb->root_switch) {
1528                 tb_free_unplugged_children(tb->root_switch);
1529                 tb_free_unplugged_xdomains(tb->root_switch);
1530         }
1531         mutex_unlock(&tb->lock);
1532 }
1533
1534 static int tb_runtime_resume(struct tb *tb)
1535 {
1536         struct tb_cm *tcm = tb_priv(tb);
1537         struct tb_tunnel *tunnel, *n;
1538
1539         mutex_lock(&tb->lock);
1540         tb_switch_resume(tb->root_switch);
1541         tb_free_invalid_tunnels(tb);
1542         tb_restore_children(tb->root_switch);
1543         list_for_each_entry_safe(tunnel, n, &tcm->tunnel_list, list)
1544                 tb_tunnel_restart(tunnel);
1545         tcm->hotplug_active = true;
1546         mutex_unlock(&tb->lock);
1547
1548         /*
1549          * Schedule cleanup of any unplugged devices. Run this in a
1550          * separate thread to avoid possible deadlock if the device
1551          * removal runtime resumes the unplugged device.
1552          */
1553         queue_delayed_work(tb->wq, &tcm->remove_work, msecs_to_jiffies(50));
1554         return 0;
1555 }
1556
1557 static const struct tb_cm_ops tb_cm_ops = {
1558         .start = tb_start,
1559         .stop = tb_stop,
1560         .suspend_noirq = tb_suspend_noirq,
1561         .resume_noirq = tb_resume_noirq,
1562         .freeze_noirq = tb_freeze_noirq,
1563         .thaw_noirq = tb_thaw_noirq,
1564         .complete = tb_complete,
1565         .runtime_suspend = tb_runtime_suspend,
1566         .runtime_resume = tb_runtime_resume,
1567         .handle_event = tb_handle_event,
1568         .disapprove_switch = tb_disconnect_pci,
1569         .approve_switch = tb_tunnel_pci,
1570         .approve_xdomain_paths = tb_approve_xdomain_paths,
1571         .disconnect_xdomain_paths = tb_disconnect_xdomain_paths,
1572 };
1573
1574 struct tb *tb_probe(struct tb_nhi *nhi)
1575 {
1576         struct tb_cm *tcm;
1577         struct tb *tb;
1578
1579         tb = tb_domain_alloc(nhi, TB_TIMEOUT, sizeof(*tcm));
1580         if (!tb)
1581                 return NULL;
1582
1583         if (tb_acpi_may_tunnel_pcie())
1584                 tb->security_level = TB_SECURITY_USER;
1585         else
1586                 tb->security_level = TB_SECURITY_NOPCIE;
1587
1588         tb->cm_ops = &tb_cm_ops;
1589
1590         tcm = tb_priv(tb);
1591         INIT_LIST_HEAD(&tcm->tunnel_list);
1592         INIT_LIST_HEAD(&tcm->dp_resources);
1593         INIT_DELAYED_WORK(&tcm->remove_work, tb_remove_work);
1594
1595         tb_dbg(tb, "using software connection manager\n");
1596
1597         return tb;
1598 }