drivers/thunderbolt/tb.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Thunderbolt driver - bus logic (NHI independent)
   4  *
   5  * Copyright (c) 2014 Andreas Noever <andreas.noever@gmail.com>
   6  * Copyright (C) 2019, Intel Corporation
   7  */
   8
   9 #include <linux/slab.h>
  10 #include <linux/errno.h>
  11 #include <linux/delay.h>
  12 #include <linux/pm_runtime.h>
  13
  14 #include "tb.h"
  15 #include "tb_regs.h"
  16 #include "tunnel.h"
  17
  18 #define TB_TIMEOUT      100 /* ms */
  19
  20 /**
  21  * struct tb_cm - Simple Thunderbolt connection manager
  22  * @tunnel_list: List of active tunnels
  23  * @dp_resources: List of available DP resources for DP tunneling
  24  * @hotplug_active: tb_handle_hotplug will stop progressing plug
  25  *                  events and exit if this is not set (it needs to
  26  *                  acquire the lock one more time). Used to drain wq
  27  *                  after cfg has been paused.
  28  * @remove_work: Work used to remove any unplugged routers after
  29  *               runtime resume
  30  */
  31 struct tb_cm {
  32         struct list_head tunnel_list;
  33         struct list_head dp_resources;
  34         bool hotplug_active;
  35         struct delayed_work remove_work;
  36 };
  37
  38 static inline struct tb *tcm_to_tb(struct tb_cm *tcm)
  39 {
  40         return ((void *)tcm - sizeof(struct tb));
  41 }
  42
  43 struct tb_hotplug_event {
  44         struct work_struct work;
  45         struct tb *tb;
  46         u64 route;
  47         u8 port;
  48         bool unplug;
  49 };
  50
  51 static void tb_handle_hotplug(struct work_struct *work);
  52
  53 static void tb_queue_hotplug(struct tb *tb, u64 route, u8 port, bool unplug)
  54 {
  55         struct tb_hotplug_event *ev;
  56
  57         ev = kmalloc(sizeof(*ev), GFP_KERNEL);
  58         if (!ev)
  59                 return;
  60
  61         ev->tb = tb;
  62         ev->route = route;
  63         ev->port = port;
  64         ev->unplug = unplug;
  65         INIT_WORK(&ev->work, tb_handle_hotplug);
  66         queue_work(tb->wq, &ev->work);
  67 }
  68
  69 /* enumeration & hot plug handling */
  70
  71 static void tb_add_dp_resources(struct tb_switch *sw)
  72 {
  73         struct tb_cm *tcm = tb_priv(sw->tb);
  74         struct tb_port *port;
  75
  76         tb_switch_for_each_port(sw, port) {
  77                 if (!tb_port_is_dpin(port))
  78                         continue;
  79
  80                 if (!tb_switch_query_dp_resource(sw, port))
  81                         continue;
  82
  83                 list_add_tail(&port->list, &tcm->dp_resources);
  84                 tb_port_dbg(port, "DP IN resource available\n");
  85         }
  86 }
  87
  88 static void tb_remove_dp_resources(struct tb_switch *sw)
  89 {
  90         struct tb_cm *tcm = tb_priv(sw->tb);
  91         struct tb_port *port, *tmp;
  92
  93         /* Clear children resources first */
  94         tb_switch_for_each_port(sw, port) {
  95                 if (tb_port_has_remote(port))
  96                         tb_remove_dp_resources(port->remote->sw);
  97         }
  98
  99         list_for_each_entry_safe(port, tmp, &tcm->dp_resources, list) {
 100                 if (port->sw == sw) {
 101                         tb_port_dbg(port, "DP OUT resource unavailable\n");
 102                         list_del_init(&port->list);
 103                 }
 104         }
 105 }
 106
 107 static void tb_discover_tunnels(struct tb_switch *sw)
 108 {
 109         struct tb *tb = sw->tb;
 110         struct tb_cm *tcm = tb_priv(tb);
 111         struct tb_port *port;
 112
 113         tb_switch_for_each_port(sw, port) {
 114                 struct tb_tunnel *tunnel = NULL;
 115
 116                 switch (port->config.type) {
 117                 case TB_TYPE_DP_HDMI_IN:
 118                         tunnel = tb_tunnel_discover_dp(tb, port);
 119                         break;
 120
 121                 case TB_TYPE_PCIE_DOWN:
 122                         tunnel = tb_tunnel_discover_pci(tb, port);
 123                         break;
 124
 125                 case TB_TYPE_USB3_DOWN:
 126                         tunnel = tb_tunnel_discover_usb3(tb, port);
 127                         break;
 128
 129                 default:
 130                         break;
 131                 }
 132
 133                 if (!tunnel)
 134                         continue;
 135
 136                 if (tb_tunnel_is_pci(tunnel)) {
 137                         struct tb_switch *parent = tunnel->dst_port->sw;
 138
 139                         while (parent != tunnel->src_port->sw) {
 140                                 parent->boot = true;
 141                                 parent = tb_switch_parent(parent);
 142                         }
 143                 }
 144
 145                 list_add_tail(&tunnel->list, &tcm->tunnel_list);
 146         }
 147
 148         tb_switch_for_each_port(sw, port) {
 149                 if (tb_port_has_remote(port))
 150                         tb_discover_tunnels(port->remote->sw);
 151         }
 152 }
 153
 154 static int tb_port_configure_xdomain(struct tb_port *port)
 155 {
 156         /*
 157          * XDomain paths currently only support single lane so we must
 158          * disable the other lane according to USB4 spec.
 159          */
 160         tb_port_disable(port->dual_link_port);
 161
 162         if (tb_switch_is_usb4(port->sw))
 163                 return usb4_port_configure_xdomain(port);
 164         return tb_lc_configure_xdomain(port);
 165 }
 166
 167 static void tb_port_unconfigure_xdomain(struct tb_port *port)
 168 {
 169         if (tb_switch_is_usb4(port->sw))
 170                 usb4_port_unconfigure_xdomain(port);
 171         else
 172                 tb_lc_unconfigure_xdomain(port);
 173
 174         tb_port_enable(port->dual_link_port);
 175 }
 176
 177 static void tb_scan_xdomain(struct tb_port *port)
 178 {
 179         struct tb_switch *sw = port->sw;
 180         struct tb *tb = sw->tb;
 181         struct tb_xdomain *xd;
 182         u64 route;
 183
 184         if (!tb_is_xdomain_enabled())
 185                 return;
 186
 187         route = tb_downstream_route(port);
 188         xd = tb_xdomain_find_by_route(tb, route);
 189         if (xd) {
 190                 tb_xdomain_put(xd);
 191                 return;
 192         }
 193
 194         xd = tb_xdomain_alloc(tb, &sw->dev, route, tb->root_switch->uuid,
 195                               NULL);
 196         if (xd) {
 197                 tb_port_at(route, sw)->xdomain = xd;
 198                 tb_port_configure_xdomain(port);
 199                 tb_xdomain_add(xd);
 200         }
 201 }
 202
 203 static int tb_enable_tmu(struct tb_switch *sw)
 204 {
 205         int ret;
 206
 207         /* If it is already enabled in correct mode, don't touch it */
 208         if (tb_switch_tmu_is_enabled(sw))
 209                 return 0;
 210
 211         ret = tb_switch_tmu_disable(sw);
 212         if (ret)
 213                 return ret;
 214
 215         ret = tb_switch_tmu_post_time(sw);
 216         if (ret)
 217                 return ret;
 218
 219         return tb_switch_tmu_enable(sw);
 220 }
 221
 222 /**
 223  * tb_find_unused_port() - return the first inactive port on @sw
 224  * @sw: Switch to find the port on
 225  * @type: Port type to look for
 226  */
 227 static struct tb_port *tb_find_unused_port(struct tb_switch *sw,
 228                                            enum tb_port_type type)
 229 {
 230         struct tb_port *port;
 231
 232         tb_switch_for_each_port(sw, port) {
 233                 if (tb_is_upstream_port(port))
 234                         continue;
 235                 if (port->config.type != type)
 236                         continue;
 237                 if (!port->cap_adap)
 238                         continue;
 239                 if (tb_port_is_enabled(port))
 240                         continue;
 241                 return port;
 242         }
 243         return NULL;
 244 }
 245
 246 static struct tb_port *tb_find_usb3_down(struct tb_switch *sw,
 247                                          const struct tb_port *port)
 248 {
 249         struct tb_port *down;
 250
 251         down = usb4_switch_map_usb3_down(sw, port);
 252         if (down && !tb_usb3_port_is_enabled(down))
 253                 return down;
 254         return NULL;
 255 }
 256
 257 static struct tb_tunnel *tb_find_tunnel(struct tb *tb, enum tb_tunnel_type type,
 258                                         struct tb_port *src_port,
 259                                         struct tb_port *dst_port)
 260 {
 261         struct tb_cm *tcm = tb_priv(tb);
 262         struct tb_tunnel *tunnel;
 263
 264         list_for_each_entry(tunnel, &tcm->tunnel_list, list) {
 265                 if (tunnel->type == type &&
 266                     ((src_port && src_port == tunnel->src_port) ||
 267                      (dst_port && dst_port == tunnel->dst_port))) {
 268                         return tunnel;
 269                 }
 270         }
 271
 272         return NULL;
 273 }
 274
 275 static struct tb_tunnel *tb_find_first_usb3_tunnel(struct tb *tb,
 276                                                    struct tb_port *src_port,
 277                                                    struct tb_port *dst_port)
 278 {
 279         struct tb_port *port, *usb3_down;
 280         struct tb_switch *sw;
 281
 282         /* Pick the router that is deepest in the topology */
 283         if (dst_port->sw->config.depth > src_port->sw->config.depth)
 284                 sw = dst_port->sw;
 285         else
 286                 sw = src_port->sw;
 287
 288         /* Can't be the host router */
 289         if (sw == tb->root_switch)
 290                 return NULL;
 291
 292         /* Find the downstream USB4 port that leads to this router */
 293         port = tb_port_at(tb_route(sw), tb->root_switch);
 294         /* Find the corresponding host router USB3 downstream port */
 295         usb3_down = usb4_switch_map_usb3_down(tb->root_switch, port);
 296         if (!usb3_down)
 297                 return NULL;
 298
 299         return tb_find_tunnel(tb, TB_TUNNEL_USB3, usb3_down, NULL);
 300 }
 301
 302 static int tb_available_bandwidth(struct tb *tb, struct tb_port *src_port,
 303         struct tb_port *dst_port, int *available_up, int *available_down)
 304 {
 305         int usb3_consumed_up, usb3_consumed_down, ret;
 306         struct tb_cm *tcm = tb_priv(tb);
 307         struct tb_tunnel *tunnel;
 308         struct tb_port *port;
 309
 310         tb_port_dbg(dst_port, "calculating available bandwidth\n");
 311
 312         tunnel = tb_find_first_usb3_tunnel(tb, src_port, dst_port);
 313         if (tunnel) {
 314                 ret = tb_tunnel_consumed_bandwidth(tunnel, &usb3_consumed_up,
 315                                                    &usb3_consumed_down);
 316                 if (ret)
 317                         return ret;
 318         } else {
 319                 usb3_consumed_up = 0;
 320                 usb3_consumed_down = 0;
 321         }
 322
 323         *available_up = *available_down = 40000;
 324
 325         /* Find the minimum available bandwidth over all links */
 326         tb_for_each_port_on_path(src_port, dst_port, port) {
 327                 int link_speed, link_width, up_bw, down_bw;
 328
 329                 if (!tb_port_is_null(port))
 330                         continue;
 331
 332                 if (tb_is_upstream_port(port)) {
 333                         link_speed = port->sw->link_speed;
 334                 } else {
 335                         link_speed = tb_port_get_link_speed(port);
 336                         if (link_speed < 0)
 337                                 return link_speed;
 338                 }
 339
 340                 link_width = port->bonded ? 2 : 1;
 341
 342                 up_bw = link_speed * link_width * 1000; /* Mb/s */
 343                 /* Leave 10% guard band */
 344                 up_bw -= up_bw / 10;
 345                 down_bw = up_bw;
 346
 347                 tb_port_dbg(port, "link total bandwidth %d Mb/s\n", up_bw);
 348
 349                 /*
 350                  * Find all DP tunnels that cross the port and reduce
 351                  * their consumed bandwidth from the available.
 352                  */
 353                 list_for_each_entry(tunnel, &tcm->tunnel_list, list) {
 354                         int dp_consumed_up, dp_consumed_down;
 355
 356                         if (!tb_tunnel_is_dp(tunnel))
 357                                 continue;
 358
 359                         if (!tb_tunnel_port_on_path(tunnel, port))
 360                                 continue;
 361
 362                         ret = tb_tunnel_consumed_bandwidth(tunnel,
 363                                                            &dp_consumed_up,
 364                                                            &dp_consumed_down);
 365                         if (ret)
 366                                 return ret;
 367
 368                         up_bw -= dp_consumed_up;
 369                         down_bw -= dp_consumed_down;
 370                 }
 371
 372                 /*
 373                  * If USB3 is tunneled from the host router down to the
 374                  * branch leading to port we need to take USB3 consumed
 375                  * bandwidth into account regardless whether it actually
 376                  * crosses the port.
 377                  */
 378                 up_bw -= usb3_consumed_up;
 379                 down_bw -= usb3_consumed_down;
 380
 381                 if (up_bw < *available_up)
 382                         *available_up = up_bw;
 383                 if (down_bw < *available_down)
 384                         *available_down = down_bw;
 385         }
 386
 387         if (*available_up < 0)
 388                 *available_up = 0;
 389         if (*available_down < 0)
 390                 *available_down = 0;
 391
 392         return 0;
 393 }
 394
 395 static int tb_release_unused_usb3_bandwidth(struct tb *tb,
 396                                             struct tb_port *src_port,
 397                                             struct tb_port *dst_port)
 398 {
 399         struct tb_tunnel *tunnel;
 400
 401         tunnel = tb_find_first_usb3_tunnel(tb, src_port, dst_port);
 402         return tunnel ? tb_tunnel_release_unused_bandwidth(tunnel) : 0;
 403 }
 404
 405 static void tb_reclaim_usb3_bandwidth(struct tb *tb, struct tb_port *src_port,
 406                                       struct tb_port *dst_port)
 407 {
 408         int ret, available_up, available_down;
 409         struct tb_tunnel *tunnel;
 410
 411         tunnel = tb_find_first_usb3_tunnel(tb, src_port, dst_port);
 412         if (!tunnel)
 413                 return;
 414
 415         tb_dbg(tb, "reclaiming unused bandwidth for USB3\n");
 416
 417         /*
 418          * Calculate available bandwidth for the first hop USB3 tunnel.
 419          * That determines the whole USB3 bandwidth for this branch.
 420          */
 421         ret = tb_available_bandwidth(tb, tunnel->src_port, tunnel->dst_port,
 422                                      &available_up, &available_down);
 423         if (ret) {
 424                 tb_warn(tb, "failed to calculate available bandwidth\n");
 425                 return;
 426         }
 427
 428         tb_dbg(tb, "available bandwidth for USB3 %d/%d Mb/s\n",
 429                available_up, available_down);
 430
 431         tb_tunnel_reclaim_available_bandwidth(tunnel, &available_up, &available_down);
 432 }
 433
 434 static int tb_tunnel_usb3(struct tb *tb, struct tb_switch *sw)
 435 {
 436         struct tb_switch *parent = tb_switch_parent(sw);
 437         int ret, available_up, available_down;
 438         struct tb_port *up, *down, *port;
 439         struct tb_cm *tcm = tb_priv(tb);
 440         struct tb_tunnel *tunnel;
 441
 442         if (!tb_acpi_may_tunnel_usb3()) {
 443                 tb_dbg(tb, "USB3 tunneling disabled, not creating tunnel\n");
 444                 return 0;
 445         }
 446
 447         up = tb_switch_find_port(sw, TB_TYPE_USB3_UP);
 448         if (!up)
 449                 return 0;
 450
 451         if (!sw->link_usb4)
 452                 return 0;
 453
 454         /*
 455          * Look up available down port. Since we are chaining it should
 456          * be found right above this switch.
 457          */
 458         port = tb_port_at(tb_route(sw), parent);
 459         down = tb_find_usb3_down(parent, port);
 460         if (!down)
 461                 return 0;
 462
 463         if (tb_route(parent)) {
 464                 struct tb_port *parent_up;
 465                 /*
 466                  * Check first that the parent switch has its upstream USB3
 467                  * port enabled. Otherwise the chain is not complete and
 468                  * there is no point setting up a new tunnel.
 469                  */
 470                 parent_up = tb_switch_find_port(parent, TB_TYPE_USB3_UP);
 471                 if (!parent_up || !tb_port_is_enabled(parent_up))
 472                         return 0;
 473
 474                 /* Make all unused bandwidth available for the new tunnel */
 475                 ret = tb_release_unused_usb3_bandwidth(tb, down, up);
 476                 if (ret)
 477                         return ret;
 478         }
 479
 480         ret = tb_available_bandwidth(tb, down, up, &available_up,
 481                                      &available_down);
 482         if (ret)
 483                 goto err_reclaim;
 484
 485         tb_port_dbg(up, "available bandwidth for new USB3 tunnel %d/%d Mb/s\n",
 486                     available_up, available_down);
 487
 488         tunnel = tb_tunnel_alloc_usb3(tb, up, down, available_up,
 489                                       available_down);
 490         if (!tunnel) {
 491                 ret = -ENOMEM;
 492                 goto err_reclaim;
 493         }
 494
 495         if (tb_tunnel_activate(tunnel)) {
 496                 tb_port_info(up,
 497                              "USB3 tunnel activation failed, aborting\n");
 498                 ret = -EIO;
 499                 goto err_free;
 500         }
 501
 502         list_add_tail(&tunnel->list, &tcm->tunnel_list);
 503         if (tb_route(parent))
 504                 tb_reclaim_usb3_bandwidth(tb, down, up);
 505
 506         return 0;
 507
 508 err_free:
 509         tb_tunnel_free(tunnel);
 510 err_reclaim:
 511         if (tb_route(parent))
 512                 tb_reclaim_usb3_bandwidth(tb, down, up);
 513
 514         return ret;
 515 }
 516
 517 static int tb_create_usb3_tunnels(struct tb_switch *sw)
 518 {
 519         struct tb_port *port;
 520         int ret;
 521
 522         if (!tb_acpi_may_tunnel_usb3())
 523                 return 0;
 524
 525         if (tb_route(sw)) {
 526                 ret = tb_tunnel_usb3(sw->tb, sw);
 527                 if (ret)
 528                         return ret;
 529         }
 530
 531         tb_switch_for_each_port(sw, port) {
 532                 if (!tb_port_has_remote(port))
 533                         continue;
 534                 ret = tb_create_usb3_tunnels(port->remote->sw);
 535                 if (ret)
 536                         return ret;
 537         }
 538
 539         return 0;
 540 }
 541
 542 static void tb_scan_port(struct tb_port *port);
 543
 544 /*
 545  * tb_scan_switch() - scan for and initialize downstream switches
 546  */
 547 static void tb_scan_switch(struct tb_switch *sw)
 548 {
 549         struct tb_port *port;
 550
 551         pm_runtime_get_sync(&sw->dev);
 552
 553         tb_switch_for_each_port(sw, port)
 554                 tb_scan_port(port);
 555
 556         pm_runtime_mark_last_busy(&sw->dev);
 557         pm_runtime_put_autosuspend(&sw->dev);
 558 }
 559
 560 /*
 561  * tb_scan_port() - check for and initialize switches below port
 562  */
 563 static void tb_scan_port(struct tb_port *port)
 564 {
 565         struct tb_cm *tcm = tb_priv(port->sw->tb);
 566         struct tb_port *upstream_port;
 567         struct tb_switch *sw;
 568
 569         if (tb_is_upstream_port(port))
 570                 return;
 571
 572         if (tb_port_is_dpout(port) && tb_dp_port_hpd_is_active(port) == 1 &&
 573             !tb_dp_port_is_enabled(port)) {
 574                 tb_port_dbg(port, "DP adapter HPD set, queuing hotplug\n");
 575                 tb_queue_hotplug(port->sw->tb, tb_route(port->sw), port->port,
 576                                  false);
 577                 return;
 578         }
 579
 580         if (port->config.type != TB_TYPE_PORT)
 581                 return;
 582         if (port->dual_link_port && port->link_nr)
 583                 return; /*
 584                          * Downstream switch is reachable through two ports.
 585                          * Only scan on the primary port (link_nr == 0).
 586                          */
 587         if (tb_wait_for_port(port, false) <= 0)
 588                 return;
 589         if (port->remote) {
 590                 tb_port_dbg(port, "port already has a remote\n");
 591                 return;
 592         }
 593
 594         tb_retimer_scan(port);
 595
 596         sw = tb_switch_alloc(port->sw->tb, &port->sw->dev,
 597                              tb_downstream_route(port));
 598         if (IS_ERR(sw)) {
 599                 /*
 600                  * If there is an error accessing the connected switch
 601                  * it may be connected to another domain. Also we allow
 602                  * the other domain to be connected to a max depth switch.
 603                  */
 604                 if (PTR_ERR(sw) == -EIO || PTR_ERR(sw) == -EADDRNOTAVAIL)
 605                         tb_scan_xdomain(port);
 606                 return;
 607         }
 608
 609         if (tb_switch_configure(sw)) {
 610                 tb_switch_put(sw);
 611                 return;
 612         }
 613
 614         /*
 615          * If there was previously another domain connected remove it
 616          * first.
 617          */
 618         if (port->xdomain) {
 619                 tb_xdomain_remove(port->xdomain);
 620                 tb_port_unconfigure_xdomain(port);
 621                 port->xdomain = NULL;
 622         }
 623
 624         /*
 625          * Do not send uevents until we have discovered all existing
 626          * tunnels and know which switches were authorized already by
 627          * the boot firmware.
 628          */
 629         if (!tcm->hotplug_active)
 630                 dev_set_uevent_suppress(&sw->dev, true);
 631
 632         /*
 633          * At the moment Thunderbolt 2 and beyond (devices with LC) we
 634          * can support runtime PM.
 635          */
 636         sw->rpm = sw->generation > 1;
 637
 638         if (tb_switch_add(sw)) {
 639                 tb_switch_put(sw);
 640                 return;
 641         }
 642
 643         /* Link the switches using both links if available */
 644         upstream_port = tb_upstream_port(sw);
 645         port->remote = upstream_port;
 646         upstream_port->remote = port;
 647         if (port->dual_link_port && upstream_port->dual_link_port) {
 648                 port->dual_link_port->remote = upstream_port->dual_link_port;
 649                 upstream_port->dual_link_port->remote = port->dual_link_port;
 650         }
 651
 652         /* Enable lane bonding if supported */
 653         tb_switch_lane_bonding_enable(sw);
 654         /* Set the link configured */
 655         tb_switch_configure_link(sw);
 656
 657         if (tb_enable_tmu(sw))
 658                 tb_sw_warn(sw, "failed to enable TMU\n");
 659
 660         /* Scan upstream retimers */
 661         tb_retimer_scan(upstream_port);
 662
 663         /*
 664          * Create USB 3.x tunnels only when the switch is plugged to the
 665          * domain. This is because we scan the domain also during discovery
 666          * and want to discover existing USB 3.x tunnels before we create
 667          * any new.
 668          */
 669         if (tcm->hotplug_active && tb_tunnel_usb3(sw->tb, sw))
 670                 tb_sw_warn(sw, "USB3 tunnel creation failed\n");
 671
 672         tb_add_dp_resources(sw);
 673         tb_scan_switch(sw);
 674 }
 675
 676 static void tb_deactivate_and_free_tunnel(struct tb_tunnel *tunnel)
 677 {
 678         struct tb_port *src_port, *dst_port;
 679         struct tb *tb;
 680
 681         if (!tunnel)
 682                 return;
 683
 684         tb_tunnel_deactivate(tunnel);
 685         list_del(&tunnel->list);
 686
 687         tb = tunnel->tb;
 688         src_port = tunnel->src_port;
 689         dst_port = tunnel->dst_port;
 690
 691         switch (tunnel->type) {
 692         case TB_TUNNEL_DP:
 693                 /*
 694                  * In case of DP tunnel make sure the DP IN resource is
 695                  * deallocated properly.
 696                  */
 697                 tb_switch_dealloc_dp_resource(src_port->sw, src_port);
 698                 /* Now we can allow the domain to runtime suspend again */
 699                 pm_runtime_mark_last_busy(&dst_port->sw->dev);
 700                 pm_runtime_put_autosuspend(&dst_port->sw->dev);
 701                 pm_runtime_mark_last_busy(&src_port->sw->dev);
 702                 pm_runtime_put_autosuspend(&src_port->sw->dev);
 703                 fallthrough;
 704
 705         case TB_TUNNEL_USB3:
 706                 tb_reclaim_usb3_bandwidth(tb, src_port, dst_port);
 707                 break;
 708
 709         default:
 710                 /*
 711                  * PCIe and DMA tunnels do not consume guaranteed
 712                  * bandwidth.
 713                  */
 714                 break;
 715         }
 716
 717         tb_tunnel_free(tunnel);
 718 }
 719
 720 /*
 721  * tb_free_invalid_tunnels() - destroy tunnels of devices that have gone away
 722  */
 723 static void tb_free_invalid_tunnels(struct tb *tb)
 724 {
 725         struct tb_cm *tcm = tb_priv(tb);
 726         struct tb_tunnel *tunnel;
 727         struct tb_tunnel *n;
 728
 729         list_for_each_entry_safe(tunnel, n, &tcm->tunnel_list, list) {
 730                 if (tb_tunnel_is_invalid(tunnel))
 731                         tb_deactivate_and_free_tunnel(tunnel);
 732         }
 733 }
 734
 735 /*
 736  * tb_free_unplugged_children() - traverse hierarchy and free unplugged switches
 737  */
 738 static void tb_free_unplugged_children(struct tb_switch *sw)
 739 {
 740         struct tb_port *port;
 741
 742         tb_switch_for_each_port(sw, port) {
 743                 if (!tb_port_has_remote(port))
 744                         continue;
 745
 746                 if (port->remote->sw->is_unplugged) {
 747                         tb_retimer_remove_all(port);
 748                         tb_remove_dp_resources(port->remote->sw);
 749                         tb_switch_unconfigure_link(port->remote->sw);
 750                         tb_switch_lane_bonding_disable(port->remote->sw);
 751                         tb_switch_remove(port->remote->sw);
 752                         port->remote = NULL;
 753                         if (port->dual_link_port)
 754                                 port->dual_link_port->remote = NULL;
 755                 } else {
 756                         tb_free_unplugged_children(port->remote->sw);
 757                 }
 758         }
 759 }
 760
 761 static struct tb_port *tb_find_pcie_down(struct tb_switch *sw,
 762                                          const struct tb_port *port)
 763 {
 764         struct tb_port *down = NULL;
 765
 766         /*
 767          * To keep plugging devices consistently in the same PCIe
 768          * hierarchy, do mapping here for switch downstream PCIe ports.
 769          */
 770         if (tb_switch_is_usb4(sw)) {
 771                 down = usb4_switch_map_pcie_down(sw, port);
 772         } else if (!tb_route(sw)) {
 773                 int phy_port = tb_phy_port_from_link(port->port);
 774                 int index;
 775
 776                 /*
 777                  * Hard-coded Thunderbolt port to PCIe down port mapping
 778                  * per controller.
 779                  */
 780                 if (tb_switch_is_cactus_ridge(sw) ||
 781                     tb_switch_is_alpine_ridge(sw))
 782                         index = !phy_port ? 6 : 7;
 783                 else if (tb_switch_is_falcon_ridge(sw))
 784                         index = !phy_port ? 6 : 8;
 785                 else if (tb_switch_is_titan_ridge(sw))
 786                         index = !phy_port ? 8 : 9;
 787                 else
 788                         goto out;
 789
 790                 /* Validate the hard-coding */
 791                 if (WARN_ON(index > sw->config.max_port_number))
 792                         goto out;
 793
 794                 down = &sw->ports[index];
 795         }
 796
 797         if (down) {
 798                 if (WARN_ON(!tb_port_is_pcie_down(down)))
 799                         goto out;
 800                 if (tb_pci_port_is_enabled(down))
 801                         goto out;
 802
 803                 return down;
 804         }
 805
 806 out:
 807         return tb_find_unused_port(sw, TB_TYPE_PCIE_DOWN);
 808 }
 809
 810 static struct tb_port *tb_find_dp_out(struct tb *tb, struct tb_port *in)
 811 {
 812         struct tb_port *host_port, *port;
 813         struct tb_cm *tcm = tb_priv(tb);
 814
 815         host_port = tb_route(in->sw) ?
 816                 tb_port_at(tb_route(in->sw), tb->root_switch) : NULL;
 817
 818         list_for_each_entry(port, &tcm->dp_resources, list) {
 819                 if (!tb_port_is_dpout(port))
 820                         continue;
 821
 822                 if (tb_port_is_enabled(port)) {
 823                         tb_port_dbg(port, "in use\n");
 824                         continue;
 825                 }
 826
 827                 tb_port_dbg(port, "DP OUT available\n");
 828
 829                 /*
 830                  * Keep the DP tunnel under the topology starting from
 831                  * the same host router downstream port.
 832                  */
 833                 if (host_port && tb_route(port->sw)) {
 834                         struct tb_port *p;
 835
 836                         p = tb_port_at(tb_route(port->sw), tb->root_switch);
 837                         if (p != host_port)
 838                                 continue;
 839                 }
 840
 841                 return port;
 842         }
 843
 844         return NULL;
 845 }
 846
 847 static void tb_tunnel_dp(struct tb *tb)
 848 {
 849         int available_up, available_down, ret;
 850         struct tb_cm *tcm = tb_priv(tb);
 851         struct tb_port *port, *in, *out;
 852         struct tb_tunnel *tunnel;
 853
 854         if (!tb_acpi_may_tunnel_dp()) {
 855                 tb_dbg(tb, "DP tunneling disabled, not creating tunnel\n");
 856                 return;
 857         }
 858
 859         /*
 860          * Find pair of inactive DP IN and DP OUT adapters and then
 861          * establish a DP tunnel between them.
 862          */
 863         tb_dbg(tb, "looking for DP IN <-> DP OUT pairs:\n");
 864
 865         in = NULL;
 866         out = NULL;
 867         list_for_each_entry(port, &tcm->dp_resources, list) {
 868                 if (!tb_port_is_dpin(port))
 869                         continue;
 870
 871                 if (tb_port_is_enabled(port)) {
 872                         tb_port_dbg(port, "in use\n");
 873                         continue;
 874                 }
 875
 876                 tb_port_dbg(port, "DP IN available\n");
 877
 878                 out = tb_find_dp_out(tb, port);
 879                 if (out) {
 880                         in = port;
 881                         break;
 882                 }
 883         }
 884
 885         if (!in) {
 886                 tb_dbg(tb, "no suitable DP IN adapter available, not tunneling\n");
 887                 return;
 888         }
 889         if (!out) {
 890                 tb_dbg(tb, "no suitable DP OUT adapter available, not tunneling\n");
 891                 return;
 892         }
 893
 894         /*
 895          * DP stream needs the domain to be active so runtime resume
 896          * both ends of the tunnel.
 897          *
 898          * This should bring the routers in the middle active as well
 899          * and keeps the domain from runtime suspending while the DP
 900          * tunnel is active.
 901          */
 902         pm_runtime_get_sync(&in->sw->dev);
 903         pm_runtime_get_sync(&out->sw->dev);
 904
 905         if (tb_switch_alloc_dp_resource(in->sw, in)) {
 906                 tb_port_dbg(in, "no resource available for DP IN, not tunneling\n");
 907                 goto err_rpm_put;
 908         }
 909
 910         /* Make all unused USB3 bandwidth available for the new DP tunnel */
 911         ret = tb_release_unused_usb3_bandwidth(tb, in, out);
 912         if (ret) {
 913                 tb_warn(tb, "failed to release unused bandwidth\n");
 914                 goto err_dealloc_dp;
 915         }
 916
 917         ret = tb_available_bandwidth(tb, in, out, &available_up,
 918                                      &available_down);
 919         if (ret)
 920                 goto err_reclaim;
 921
 922         tb_dbg(tb, "available bandwidth for new DP tunnel %u/%u Mb/s\n",
 923                available_up, available_down);
 924
 925         tunnel = tb_tunnel_alloc_dp(tb, in, out, available_up, available_down);
 926         if (!tunnel) {
 927                 tb_port_dbg(out, "could not allocate DP tunnel\n");
 928                 goto err_reclaim;
 929         }
 930
 931         if (tb_tunnel_activate(tunnel)) {
 932                 tb_port_info(out, "DP tunnel activation failed, aborting\n");
 933                 goto err_free;
 934         }
 935
 936         list_add_tail(&tunnel->list, &tcm->tunnel_list);
 937         tb_reclaim_usb3_bandwidth(tb, in, out);
 938         return;
 939
 940 err_free:
 941         tb_tunnel_free(tunnel);
 942 err_reclaim:
 943         tb_reclaim_usb3_bandwidth(tb, in, out);
 944 err_dealloc_dp:
 945         tb_switch_dealloc_dp_resource(in->sw, in);
 946 err_rpm_put:
 947         pm_runtime_mark_last_busy(&out->sw->dev);
 948         pm_runtime_put_autosuspend(&out->sw->dev);
 949         pm_runtime_mark_last_busy(&in->sw->dev);
 950         pm_runtime_put_autosuspend(&in->sw->dev);
 951 }
 952
 953 static void tb_dp_resource_unavailable(struct tb *tb, struct tb_port *port)
 954 {
 955         struct tb_port *in, *out;
 956         struct tb_tunnel *tunnel;
 957
 958         if (tb_port_is_dpin(port)) {
 959                 tb_port_dbg(port, "DP IN resource unavailable\n");
 960                 in = port;
 961                 out = NULL;
 962         } else {
 963                 tb_port_dbg(port, "DP OUT resource unavailable\n");
 964                 in = NULL;
 965                 out = port;
 966         }
 967
 968         tunnel = tb_find_tunnel(tb, TB_TUNNEL_DP, in, out);
 969         tb_deactivate_and_free_tunnel(tunnel);
 970         list_del_init(&port->list);
 971
 972         /*
 973          * See if there is another DP OUT port that can be used for
 974          * to create another tunnel.
 975          */
 976         tb_tunnel_dp(tb);
 977 }
 978
 979 static void tb_dp_resource_available(struct tb *tb, struct tb_port *port)
 980 {
 981         struct tb_cm *tcm = tb_priv(tb);
 982         struct tb_port *p;
 983
 984         if (tb_port_is_enabled(port))
 985                 return;
 986
 987         list_for_each_entry(p, &tcm->dp_resources, list) {
 988                 if (p == port)
 989                         return;
 990         }
 991
 992         tb_port_dbg(port, "DP %s resource available\n",
 993                     tb_port_is_dpin(port) ? "IN" : "OUT");
 994         list_add_tail(&port->list, &tcm->dp_resources);
 995
 996         /* Look for suitable DP IN <-> DP OUT pairs now */
 997         tb_tunnel_dp(tb);
 998 }
 999
1000 static void tb_disconnect_and_release_dp(struct tb *tb)
1001 {
1002         struct tb_cm *tcm = tb_priv(tb);
1003         struct tb_tunnel *tunnel, *n;
1004
1005         /*
1006          * Tear down all DP tunnels and release their resources. They
1007          * will be re-established after resume based on plug events.
1008          */
1009         list_for_each_entry_safe_reverse(tunnel, n, &tcm->tunnel_list, list) {
1010                 if (tb_tunnel_is_dp(tunnel))
1011                         tb_deactivate_and_free_tunnel(tunnel);
1012         }
1013
1014         while (!list_empty(&tcm->dp_resources)) {
1015                 struct tb_port *port;
1016
1017                 port = list_first_entry(&tcm->dp_resources,
1018                                         struct tb_port, list);
1019                 list_del_init(&port->list);
1020         }
1021 }
1022
1023 static int tb_disconnect_pci(struct tb *tb, struct tb_switch *sw)
1024 {
1025         struct tb_tunnel *tunnel;
1026         struct tb_port *up;
1027
1028         up = tb_switch_find_port(sw, TB_TYPE_PCIE_UP);
1029         if (WARN_ON(!up))
1030                 return -ENODEV;
1031
1032         tunnel = tb_find_tunnel(tb, TB_TUNNEL_PCI, NULL, up);
1033         if (WARN_ON(!tunnel))
1034                 return -ENODEV;
1035
1036         tb_tunnel_deactivate(tunnel);
1037         list_del(&tunnel->list);
1038         tb_tunnel_free(tunnel);
1039         return 0;
1040 }
1041
1042 static int tb_tunnel_pci(struct tb *tb, struct tb_switch *sw)
1043 {
1044         struct tb_port *up, *down, *port;
1045         struct tb_cm *tcm = tb_priv(tb);
1046         struct tb_switch *parent_sw;
1047         struct tb_tunnel *tunnel;
1048
1049         up = tb_switch_find_port(sw, TB_TYPE_PCIE_UP);
1050         if (!up)
1051                 return 0;
1052
1053         /*
1054          * Look up available down port. Since we are chaining it should
1055          * be found right above this switch.
1056          */
1057         parent_sw = tb_to_switch(sw->dev.parent);
1058         port = tb_port_at(tb_route(sw), parent_sw);
1059         down = tb_find_pcie_down(parent_sw, port);
1060         if (!down)
1061                 return 0;
1062
1063         tunnel = tb_tunnel_alloc_pci(tb, up, down);
1064         if (!tunnel)
1065                 return -ENOMEM;
1066
1067         if (tb_tunnel_activate(tunnel)) {
1068                 tb_port_info(up,
1069                              "PCIe tunnel activation failed, aborting\n");
1070                 tb_tunnel_free(tunnel);
1071                 return -EIO;
1072         }
1073
1074         list_add_tail(&tunnel->list, &tcm->tunnel_list);
1075         return 0;
1076 }
1077
1078 static int tb_approve_xdomain_paths(struct tb *tb, struct tb_xdomain *xd)
1079 {
1080         struct tb_cm *tcm = tb_priv(tb);
1081         struct tb_port *nhi_port, *dst_port;
1082         struct tb_tunnel *tunnel;
1083         struct tb_switch *sw;
1084
1085         sw = tb_to_switch(xd->dev.parent);
1086         dst_port = tb_port_at(xd->route, sw);
1087         nhi_port = tb_switch_find_port(tb->root_switch, TB_TYPE_NHI);
1088
1089         mutex_lock(&tb->lock);
1090         tunnel = tb_tunnel_alloc_dma(tb, nhi_port, dst_port, xd->transmit_ring,
1091                                      xd->transmit_path, xd->receive_ring,
1092                                      xd->receive_path);
1093         if (!tunnel) {
1094                 mutex_unlock(&tb->lock);
1095                 return -ENOMEM;
1096         }
1097
1098         if (tb_tunnel_activate(tunnel)) {
1099                 tb_port_info(nhi_port,
1100                              "DMA tunnel activation failed, aborting\n");
1101                 tb_tunnel_free(tunnel);
1102                 mutex_unlock(&tb->lock);
1103                 return -EIO;
1104         }
1105
1106         list_add_tail(&tunnel->list, &tcm->tunnel_list);
1107         mutex_unlock(&tb->lock);
1108         return 0;
1109 }
1110
1111 static void __tb_disconnect_xdomain_paths(struct tb *tb, struct tb_xdomain *xd)
1112 {
1113         struct tb_port *dst_port;
1114         struct tb_tunnel *tunnel;
1115         struct tb_switch *sw;
1116
1117         sw = tb_to_switch(xd->dev.parent);
1118         dst_port = tb_port_at(xd->route, sw);
1119
1120         /*
1121          * It is possible that the tunnel was already teared down (in
1122          * case of cable disconnect) so it is fine if we cannot find it
1123          * here anymore.
1124          */
1125         tunnel = tb_find_tunnel(tb, TB_TUNNEL_DMA, NULL, dst_port);
1126         tb_deactivate_and_free_tunnel(tunnel);
1127 }
1128
1129 static int tb_disconnect_xdomain_paths(struct tb *tb, struct tb_xdomain *xd)
1130 {
1131         if (!xd->is_unplugged) {
1132                 mutex_lock(&tb->lock);
1133                 __tb_disconnect_xdomain_paths(tb, xd);
1134                 mutex_unlock(&tb->lock);
1135         }
1136         return 0;
1137 }
1138
1139 /* hotplug handling */
1140
1141 /*
1142  * tb_handle_hotplug() - handle hotplug event
1143  *
1144  * Executes on tb->wq.
1145  */
1146 static void tb_handle_hotplug(struct work_struct *work)
1147 {
1148         struct tb_hotplug_event *ev = container_of(work, typeof(*ev), work);
1149         struct tb *tb = ev->tb;
1150         struct tb_cm *tcm = tb_priv(tb);
1151         struct tb_switch *sw;
1152         struct tb_port *port;
1153
1154         /* Bring the domain back from sleep if it was suspended */
1155         pm_runtime_get_sync(&tb->dev);
1156
1157         mutex_lock(&tb->lock);
1158         if (!tcm->hotplug_active)
1159                 goto out; /* during init, suspend or shutdown */
1160
1161         sw = tb_switch_find_by_route(tb, ev->route);
1162         if (!sw) {
1163                 tb_warn(tb,
1164                         "hotplug event from non existent switch %llx:%x (unplug: %d)\n",
1165                         ev->route, ev->port, ev->unplug);
1166                 goto out;
1167         }
1168         if (ev->port > sw->config.max_port_number) {
1169                 tb_warn(tb,
1170                         "hotplug event from non existent port %llx:%x (unplug: %d)\n",
1171                         ev->route, ev->port, ev->unplug);
1172                 goto put_sw;
1173         }
1174         port = &sw->ports[ev->port];
1175         if (tb_is_upstream_port(port)) {
1176                 tb_dbg(tb, "hotplug event for upstream port %llx:%x (unplug: %d)\n",
1177                        ev->route, ev->port, ev->unplug);
1178                 goto put_sw;
1179         }
1180
1181         pm_runtime_get_sync(&sw->dev);
1182
1183         if (ev->unplug) {
1184                 tb_retimer_remove_all(port);
1185
1186                 if (tb_port_has_remote(port)) {
1187                         tb_port_dbg(port, "switch unplugged\n");
1188                         tb_sw_set_unplugged(port->remote->sw);
1189                         tb_free_invalid_tunnels(tb);
1190                         tb_remove_dp_resources(port->remote->sw);
1191                         tb_switch_tmu_disable(port->remote->sw);
1192                         tb_switch_unconfigure_link(port->remote->sw);
1193                         tb_switch_lane_bonding_disable(port->remote->sw);
1194                         tb_switch_remove(port->remote->sw);
1195                         port->remote = NULL;
1196                         if (port->dual_link_port)
1197                                 port->dual_link_port->remote = NULL;
1198                         /* Maybe we can create another DP tunnel */
1199                         tb_tunnel_dp(tb);
1200                 } else if (port->xdomain) {
1201                         struct tb_xdomain *xd = tb_xdomain_get(port->xdomain);
1202
1203                         tb_port_dbg(port, "xdomain unplugged\n");
1204                         /*
1205                          * Service drivers are unbound during
1206                          * tb_xdomain_remove() so setting XDomain as
1207                          * unplugged here prevents deadlock if they call
1208                          * tb_xdomain_disable_paths(). We will tear down
1209                          * the path below.
1210                          */
1211                         xd->is_unplugged = true;
1212                         tb_xdomain_remove(xd);
1213                         port->xdomain = NULL;
1214                         __tb_disconnect_xdomain_paths(tb, xd);
1215                         tb_xdomain_put(xd);
1216                         tb_port_unconfigure_xdomain(port);
1217                 } else if (tb_port_is_dpout(port) || tb_port_is_dpin(port)) {
1218                         tb_dp_resource_unavailable(tb, port);
1219                 } else {
1220                         tb_port_dbg(port,
1221                                    "got unplug event for disconnected port, ignoring\n");
1222                 }
1223         } else if (port->remote) {
1224                 tb_port_dbg(port, "got plug event for connected port, ignoring\n");
1225         } else {
1226                 if (tb_port_is_null(port)) {
1227                         tb_port_dbg(port, "hotplug: scanning\n");
1228                         tb_scan_port(port);
1229                         if (!port->remote)
1230                                 tb_port_dbg(port, "hotplug: no switch found\n");
1231                 } else if (tb_port_is_dpout(port) || tb_port_is_dpin(port)) {
1232                         tb_dp_resource_available(tb, port);
1233                 }
1234         }
1235
1236         pm_runtime_mark_last_busy(&sw->dev);
1237         pm_runtime_put_autosuspend(&sw->dev);
1238
1239 put_sw:
1240         tb_switch_put(sw);
1241 out:
1242         mutex_unlock(&tb->lock);
1243
1244         pm_runtime_mark_last_busy(&tb->dev);
1245         pm_runtime_put_autosuspend(&tb->dev);
1246
1247         kfree(ev);
1248 }
1249
1250 /*
1251  * tb_schedule_hotplug_handler() - callback function for the control channel
1252  *
1253  * Delegates to tb_handle_hotplug.
1254  */
1255 static void tb_handle_event(struct tb *tb, enum tb_cfg_pkg_type type,
1256                             const void *buf, size_t size)
1257 {
1258         const struct cfg_event_pkg *pkg = buf;
1259         u64 route;
1260
1261         if (type != TB_CFG_PKG_EVENT) {
1262                 tb_warn(tb, "unexpected event %#x, ignoring\n", type);
1263                 return;
1264         }
1265
1266         route = tb_cfg_get_route(&pkg->header);
1267
1268         if (tb_cfg_ack_plug(tb->ctl, route, pkg->port, pkg->unplug)) {
1269                 tb_warn(tb, "could not ack plug event on %llx:%x\n", route,
1270                         pkg->port);
1271         }
1272
1273         tb_queue_hotplug(tb, route, pkg->port, pkg->unplug);
1274 }
1275
1276 static void tb_stop(struct tb *tb)
1277 {
1278         struct tb_cm *tcm = tb_priv(tb);
1279         struct tb_tunnel *tunnel;
1280         struct tb_tunnel *n;
1281
1282         cancel_delayed_work(&tcm->remove_work);
1283         /* tunnels are only present after everything has been initialized */
1284         list_for_each_entry_safe(tunnel, n, &tcm->tunnel_list, list) {
1285                 /*
1286                  * DMA tunnels require the driver to be functional so we
1287                  * tear them down. Other protocol tunnels can be left
1288                  * intact.
1289                  */
1290                 if (tb_tunnel_is_dma(tunnel))
1291                         tb_tunnel_deactivate(tunnel);
1292                 tb_tunnel_free(tunnel);
1293         }
1294         tb_switch_remove(tb->root_switch);
1295         tcm->hotplug_active = false; /* signal tb_handle_hotplug to quit */
1296 }
1297
1298 static int tb_scan_finalize_switch(struct device *dev, void *data)
1299 {
1300         if (tb_is_switch(dev)) {
1301                 struct tb_switch *sw = tb_to_switch(dev);
1302
1303                 /*
1304                  * If we found that the switch was already setup by the
1305                  * boot firmware, mark it as authorized now before we
1306                  * send uevent to userspace.
1307                  */
1308                 if (sw->boot)
1309                         sw->authorized = 1;
1310
1311                 dev_set_uevent_suppress(dev, false);
1312                 kobject_uevent(&dev->kobj, KOBJ_ADD);
1313                 device_for_each_child(dev, NULL, tb_scan_finalize_switch);
1314         }
1315
1316         return 0;
1317 }
1318
1319 static int tb_start(struct tb *tb)
1320 {
1321         struct tb_cm *tcm = tb_priv(tb);
1322         int ret;
1323
1324         tb->root_switch = tb_switch_alloc(tb, &tb->dev, 0);
1325         if (IS_ERR(tb->root_switch))
1326                 return PTR_ERR(tb->root_switch);
1327
1328         /*
1329          * ICM firmware upgrade needs running firmware and in native
1330          * mode that is not available so disable firmware upgrade of the
1331          * root switch.
1332          */
1333         tb->root_switch->no_nvm_upgrade = true;
1334         /* All USB4 routers support runtime PM */
1335         tb->root_switch->rpm = tb_switch_is_usb4(tb->root_switch);
1336
1337         ret = tb_switch_configure(tb->root_switch);
1338         if (ret) {
1339                 tb_switch_put(tb->root_switch);
1340                 return ret;
1341         }
1342
1343         /* Announce the switch to the world */
1344         ret = tb_switch_add(tb->root_switch);
1345         if (ret) {
1346                 tb_switch_put(tb->root_switch);
1347                 return ret;
1348         }
1349
1350         /* Enable TMU if it is off */
1351         tb_switch_tmu_enable(tb->root_switch);
1352         /* Full scan to discover devices added before the driver was loaded. */
1353         tb_scan_switch(tb->root_switch);
1354         /* Find out tunnels created by the boot firmware */
1355         tb_discover_tunnels(tb->root_switch);
1356         /*
1357          * If the boot firmware did not create USB 3.x tunnels create them
1358          * now for the whole topology.
1359          */
1360         tb_create_usb3_tunnels(tb->root_switch);
1361         /* Add DP IN resources for the root switch */
1362         tb_add_dp_resources(tb->root_switch);
1363         /* Make the discovered switches available to the userspace */
1364         device_for_each_child(&tb->root_switch->dev, NULL,
1365                               tb_scan_finalize_switch);
1366
1367         /* Allow tb_handle_hotplug to progress events */
1368         tcm->hotplug_active = true;
1369         return 0;
1370 }
1371
1372 static int tb_suspend_noirq(struct tb *tb)
1373 {
1374         struct tb_cm *tcm = tb_priv(tb);
1375
1376         tb_dbg(tb, "suspending...\n");
1377         tb_disconnect_and_release_dp(tb);
1378         tb_switch_suspend(tb->root_switch, false);
1379         tcm->hotplug_active = false; /* signal tb_handle_hotplug to quit */
1380         tb_dbg(tb, "suspend finished\n");
1381
1382         return 0;
1383 }
1384
1385 static void tb_restore_children(struct tb_switch *sw)
1386 {
1387         struct tb_port *port;
1388
1389         /* No need to restore if the router is already unplugged */
1390         if (sw->is_unplugged)
1391                 return;
1392
1393         if (tb_enable_tmu(sw))
1394                 tb_sw_warn(sw, "failed to restore TMU configuration\n");
1395
1396         tb_switch_for_each_port(sw, port) {
1397                 if (!tb_port_has_remote(port) && !port->xdomain)
1398                         continue;
1399
1400                 if (port->remote) {
1401                         tb_switch_lane_bonding_enable(port->remote->sw);
1402                         tb_switch_configure_link(port->remote->sw);
1403
1404                         tb_restore_children(port->remote->sw);
1405                 } else if (port->xdomain) {
1406                         tb_port_configure_xdomain(port);
1407                 }
1408         }
1409 }
1410
1411 static int tb_resume_noirq(struct tb *tb)
1412 {
1413         struct tb_cm *tcm = tb_priv(tb);
1414         struct tb_tunnel *tunnel, *n;
1415
1416         tb_dbg(tb, "resuming...\n");
1417
1418         /* remove any pci devices the firmware might have setup */
1419         tb_switch_reset(tb->root_switch);
1420
1421         tb_switch_resume(tb->root_switch);
1422         tb_free_invalid_tunnels(tb);
1423         tb_free_unplugged_children(tb->root_switch);
1424         tb_restore_children(tb->root_switch);
1425         list_for_each_entry_safe(tunnel, n, &tcm->tunnel_list, list)
1426                 tb_tunnel_restart(tunnel);
1427         if (!list_empty(&tcm->tunnel_list)) {
1428                 /*
1429                  * the pcie links need some time to get going.
1430                  * 100ms works for me...
1431                  */
1432                 tb_dbg(tb, "tunnels restarted, sleeping for 100ms\n");
1433                 msleep(100);
1434         }
1435          /* Allow tb_handle_hotplug to progress events */
1436         tcm->hotplug_active = true;
1437         tb_dbg(tb, "resume finished\n");
1438
1439         return 0;
1440 }
1441
1442 static int tb_free_unplugged_xdomains(struct tb_switch *sw)
1443 {
1444         struct tb_port *port;
1445         int ret = 0;
1446
1447         tb_switch_for_each_port(sw, port) {
1448                 if (tb_is_upstream_port(port))
1449                         continue;
1450                 if (port->xdomain && port->xdomain->is_unplugged) {
1451                         tb_retimer_remove_all(port);
1452                         tb_xdomain_remove(port->xdomain);
1453                         tb_port_unconfigure_xdomain(port);
1454                         port->xdomain = NULL;
1455                         ret++;
1456                 } else if (port->remote) {
1457                         ret += tb_free_unplugged_xdomains(port->remote->sw);
1458                 }
1459         }
1460
1461         return ret;
1462 }
1463
1464 static int tb_freeze_noirq(struct tb *tb)
1465 {
1466         struct tb_cm *tcm = tb_priv(tb);
1467
1468         tcm->hotplug_active = false;
1469         return 0;
1470 }
1471
1472 static int tb_thaw_noirq(struct tb *tb)
1473 {
1474         struct tb_cm *tcm = tb_priv(tb);
1475
1476         tcm->hotplug_active = true;
1477         return 0;
1478 }
1479
1480 static void tb_complete(struct tb *tb)
1481 {
1482         /*
1483          * Release any unplugged XDomains and if there is a case where
1484          * another domain is swapped in place of unplugged XDomain we
1485          * need to run another rescan.
1486          */
1487         mutex_lock(&tb->lock);
1488         if (tb_free_unplugged_xdomains(tb->root_switch))
1489                 tb_scan_switch(tb->root_switch);
1490         mutex_unlock(&tb->lock);
1491 }
1492
1493 static int tb_runtime_suspend(struct tb *tb)
1494 {
1495         struct tb_cm *tcm = tb_priv(tb);
1496
1497         mutex_lock(&tb->lock);
1498         tb_switch_suspend(tb->root_switch, true);
1499         tcm->hotplug_active = false;
1500         mutex_unlock(&tb->lock);
1501
1502         return 0;
1503 }
1504
1505 static void tb_remove_work(struct work_struct *work)
1506 {
1507         struct tb_cm *tcm = container_of(work, struct tb_cm, remove_work.work);
1508         struct tb *tb = tcm_to_tb(tcm);
1509
1510         mutex_lock(&tb->lock);
1511         if (tb->root_switch) {
1512                 tb_free_unplugged_children(tb->root_switch);
1513                 tb_free_unplugged_xdomains(tb->root_switch);
1514         }
1515         mutex_unlock(&tb->lock);
1516 }
1517
1518 static int tb_runtime_resume(struct tb *tb)
1519 {
1520         struct tb_cm *tcm = tb_priv(tb);
1521         struct tb_tunnel *tunnel, *n;
1522
1523         mutex_lock(&tb->lock);
1524         tb_switch_resume(tb->root_switch);
1525         tb_free_invalid_tunnels(tb);
1526         tb_restore_children(tb->root_switch);
1527         list_for_each_entry_safe(tunnel, n, &tcm->tunnel_list, list)
1528                 tb_tunnel_restart(tunnel);
1529         tcm->hotplug_active = true;
1530         mutex_unlock(&tb->lock);
1531
1532         /*
1533          * Schedule cleanup of any unplugged devices. Run this in a
1534          * separate thread to avoid possible deadlock if the device
1535          * removal runtime resumes the unplugged device.
1536          */
1537         queue_delayed_work(tb->wq, &tcm->remove_work, msecs_to_jiffies(50));
1538         return 0;
1539 }
1540
1541 static const struct tb_cm_ops tb_cm_ops = {
1542         .start = tb_start,
1543         .stop = tb_stop,
1544         .suspend_noirq = tb_suspend_noirq,
1545         .resume_noirq = tb_resume_noirq,
1546         .freeze_noirq = tb_freeze_noirq,
1547         .thaw_noirq = tb_thaw_noirq,
1548         .complete = tb_complete,
1549         .runtime_suspend = tb_runtime_suspend,
1550         .runtime_resume = tb_runtime_resume,
1551         .handle_event = tb_handle_event,
1552         .disapprove_switch = tb_disconnect_pci,
1553         .approve_switch = tb_tunnel_pci,
1554         .approve_xdomain_paths = tb_approve_xdomain_paths,
1555         .disconnect_xdomain_paths = tb_disconnect_xdomain_paths,
1556 };
1557
1558 struct tb *tb_probe(struct tb_nhi *nhi)
1559 {
1560         struct tb_cm *tcm;
1561         struct tb *tb;
1562
1563         tb = tb_domain_alloc(nhi, TB_TIMEOUT, sizeof(*tcm));
1564         if (!tb)
1565                 return NULL;
1566
1567         if (tb_acpi_may_tunnel_pcie())
1568                 tb->security_level = TB_SECURITY_USER;
1569         else
1570                 tb->security_level = TB_SECURITY_NOPCIE;
1571
1572         tb->cm_ops = &tb_cm_ops;
1573
1574         tcm = tb_priv(tb);
1575         INIT_LIST_HEAD(&tcm->tunnel_list);
1576         INIT_LIST_HEAD(&tcm->dp_resources);
1577         INIT_DELAYED_WORK(&tcm->remove_work, tb_remove_work);
1578
1579         tb_dbg(tb, "using software connection manager\n");
1580
1581         return tb;
1582 }