drivers/thunderbolt/path.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Thunderbolt driver - path/tunnel functionality
   4  *
   5  * Copyright (c) 2014 Andreas Noever <andreas.noever@gmail.com>
   6  * Copyright (C) 2019, Intel Corporation
   7  */
   8
   9 #include <linux/slab.h>
  10 #include <linux/errno.h>
  11 #include <linux/delay.h>
  12 #include <linux/ktime.h>
  13
  14 #include "tb.h"
  15
  16 static void tb_dump_hop(const struct tb_path_hop *hop, const struct tb_regs_hop *regs)
  17 {
  18         const struct tb_port *port = hop->in_port;
  19
  20         tb_port_dbg(port, " In HopID: %d => Out port: %d Out HopID: %d\n",
  21                     hop->in_hop_index, regs->out_port, regs->next_hop);
  22         tb_port_dbg(port, "  Weight: %d Priority: %d Credits: %d Drop: %d\n",
  23                     regs->weight, regs->priority,
  24                     regs->initial_credits, regs->drop_packages);
  25         tb_port_dbg(port, "   Counter enabled: %d Counter index: %d\n",
  26                     regs->counter_enable, regs->counter);
  27         tb_port_dbg(port, "  Flow Control (In/Eg): %d/%d Shared Buffer (In/Eg): %d/%d\n",
  28                     regs->ingress_fc, regs->egress_fc,
  29                     regs->ingress_shared_buffer, regs->egress_shared_buffer);
  30         tb_port_dbg(port, "  Unknown1: %#x Unknown2: %#x Unknown3: %#x\n",
  31                     regs->unknown1, regs->unknown2, regs->unknown3);
  32 }
  33
  34 static struct tb_port *tb_path_find_dst_port(struct tb_port *src, int src_hopid,
  35                                              int dst_hopid)
  36 {
  37         struct tb_port *port, *out_port = NULL;
  38         struct tb_regs_hop hop;
  39         struct tb_switch *sw;
  40         int i, ret, hopid;
  41
  42         hopid = src_hopid;
  43         port = src;
  44
  45         for (i = 0; port && i < TB_PATH_MAX_HOPS; i++) {
  46                 sw = port->sw;
  47
  48                 ret = tb_port_read(port, &hop, TB_CFG_HOPS, 2 * hopid, 2);
  49                 if (ret) {
  50                         tb_port_warn(port, "failed to read path at %d\n", hopid);
  51                         return NULL;
  52                 }
  53
  54                 if (!hop.enable)
  55                         return NULL;
  56
  57                 out_port = &sw->ports[hop.out_port];
  58                 hopid = hop.next_hop;
  59                 port = out_port->remote;
  60         }
  61
  62         return out_port && hopid == dst_hopid ? out_port : NULL;
  63 }
  64
  65 static int tb_path_find_src_hopid(struct tb_port *src,
  66         const struct tb_port *dst, int dst_hopid)
  67 {
  68         struct tb_port *out;
  69         int i;
  70
  71         for (i = TB_PATH_MIN_HOPID; i <= src->config.max_in_hop_id; i++) {
  72                 out = tb_path_find_dst_port(src, i, dst_hopid);
  73                 if (out == dst)
  74                         return i;
  75         }
  76
  77         return 0;
  78 }
  79
  80 /**
  81  * tb_path_discover() - Discover a path
  82  * @src: First input port of a path
  83  * @src_hopid: Starting HopID of a path (%-1 if don't care)
  84  * @dst: Expected destination port of the path (%NULL if don't care)
  85  * @dst_hopid: HopID to the @dst (%-1 if don't care)
  86  * @last: Last port is filled here if not %NULL
  87  * @name: Name of the path
  88  * @alloc_hopid: Allocate HopIDs for the ports
  89  *
  90  * Follows a path starting from @src and @src_hopid to the last output
  91  * port of the path. Allocates HopIDs for the visited ports (if
  92  * @alloc_hopid is true). Call tb_path_free() to release the path and
  93  * allocated HopIDs when the path is not needed anymore.
  94  *
  95  * Note function discovers also incomplete paths so caller should check
  96  * that the @dst port is the expected one. If it is not, the path can be
  97  * cleaned up by calling tb_path_deactivate() before tb_path_free().
  98  *
  99  * Return: Discovered path on success, %NULL in case of failure
 100  */
 101 struct tb_path *tb_path_discover(struct tb_port *src, int src_hopid,
 102                                  struct tb_port *dst, int dst_hopid,
 103                                  struct tb_port **last, const char *name,
 104                                  bool alloc_hopid)
 105 {
 106         struct tb_port *out_port;
 107         struct tb_regs_hop hop;
 108         struct tb_path *path;
 109         struct tb_switch *sw;
 110         struct tb_port *p;
 111         size_t num_hops;
 112         int ret, i, h;
 113
 114         if (src_hopid < 0 && dst) {
 115                 /*
 116                  * For incomplete paths the intermediate HopID can be
 117                  * different from the one used by the protocol adapter
 118                  * so in that case find a path that ends on @dst with
 119                  * matching @dst_hopid. That should give us the correct
 120                  * HopID for the @src.
 121                  */
 122                 src_hopid = tb_path_find_src_hopid(src, dst, dst_hopid);
 123                 if (!src_hopid)
 124                         return NULL;
 125         }
 126
 127         p = src;
 128         h = src_hopid;
 129         num_hops = 0;
 130
 131         for (i = 0; p && i < TB_PATH_MAX_HOPS; i++) {
 132                 sw = p->sw;
 133
 134                 ret = tb_port_read(p, &hop, TB_CFG_HOPS, 2 * h, 2);
 135                 if (ret) {
 136                         tb_port_warn(p, "failed to read path at %d\n", h);
 137                         return NULL;
 138                 }
 139
 140                 /* If the hop is not enabled we got an incomplete path */
 141                 if (!hop.enable)
 142                         break;
 143
 144                 out_port = &sw->ports[hop.out_port];
 145                 if (last)
 146                         *last = out_port;
 147
 148                 h = hop.next_hop;
 149                 p = out_port->remote;
 150                 num_hops++;
 151         }
 152
 153         path = kzalloc(sizeof(*path), GFP_KERNEL);
 154         if (!path)
 155                 return NULL;
 156
 157         path->name = name;
 158         path->tb = src->sw->tb;
 159         path->path_length = num_hops;
 160         path->activated = true;
 161         path->alloc_hopid = alloc_hopid;
 162
 163         path->hops = kcalloc(num_hops, sizeof(*path->hops), GFP_KERNEL);
 164         if (!path->hops) {
 165                 kfree(path);
 166                 return NULL;
 167         }
 168
 169         p = src;
 170         h = src_hopid;
 171
 172         for (i = 0; i < num_hops; i++) {
 173                 int next_hop;
 174
 175                 sw = p->sw;
 176
 177                 ret = tb_port_read(p, &hop, TB_CFG_HOPS, 2 * h, 2);
 178                 if (ret) {
 179                         tb_port_warn(p, "failed to read path at %d\n", h);
 180                         goto err;
 181                 }
 182
 183                 if (alloc_hopid && tb_port_alloc_in_hopid(p, h, h) < 0)
 184                         goto err;
 185
 186                 out_port = &sw->ports[hop.out_port];
 187                 next_hop = hop.next_hop;
 188
 189                 if (alloc_hopid &&
 190                     tb_port_alloc_out_hopid(out_port, next_hop, next_hop) < 0) {
 191                         tb_port_release_in_hopid(p, h);
 192                         goto err;
 193                 }
 194
 195                 path->hops[i].in_port = p;
 196                 path->hops[i].in_hop_index = h;
 197                 path->hops[i].in_counter_index = -1;
 198                 path->hops[i].out_port = out_port;
 199                 path->hops[i].next_hop_index = next_hop;
 200
 201                 h = next_hop;
 202                 p = out_port->remote;
 203         }
 204
 205         return path;
 206
 207 err:
 208         tb_port_warn(src, "failed to discover path starting at HopID %d\n",
 209                      src_hopid);
 210         tb_path_free(path);
 211         return NULL;
 212 }
 213
 214 /**
 215  * tb_path_alloc() - allocate a thunderbolt path between two ports
 216  * @tb: Domain pointer
 217  * @src: Source port of the path
 218  * @src_hopid: HopID used for the first ingress port in the path
 219  * @dst: Destination port of the path
 220  * @dst_hopid: HopID used for the last egress port in the path
 221  * @link_nr: Preferred link if there are dual links on the path
 222  * @name: Name of the path
 223  *
 224  * Creates path between two ports starting with given @src_hopid. Reserves
 225  * HopIDs for each port (they can be different from @src_hopid depending on
 226  * how many HopIDs each port already have reserved). If there are dual
 227  * links on the path, prioritizes using @link_nr but takes into account
 228  * that the lanes may be bonded.
 229  *
 230  * Return: Returns a tb_path on success or NULL on failure.
 231  */
 232 struct tb_path *tb_path_alloc(struct tb *tb, struct tb_port *src, int src_hopid,
 233                               struct tb_port *dst, int dst_hopid, int link_nr,
 234                               const char *name)
 235 {
 236         struct tb_port *in_port, *out_port, *first_port, *last_port;
 237         int in_hopid, out_hopid;
 238         struct tb_path *path;
 239         size_t num_hops;
 240         int i, ret;
 241
 242         path = kzalloc(sizeof(*path), GFP_KERNEL);
 243         if (!path)
 244                 return NULL;
 245
 246         first_port = last_port = NULL;
 247         i = 0;
 248         tb_for_each_port_on_path(src, dst, in_port) {
 249                 if (!first_port)
 250                         first_port = in_port;
 251                 last_port = in_port;
 252                 i++;
 253         }
 254
 255         /* Check that src and dst are reachable */
 256         if (first_port != src || last_port != dst) {
 257                 kfree(path);
 258                 return NULL;
 259         }
 260
 261         /* Each hop takes two ports */
 262         num_hops = i / 2;
 263
 264         path->hops = kcalloc(num_hops, sizeof(*path->hops), GFP_KERNEL);
 265         if (!path->hops) {
 266                 kfree(path);
 267                 return NULL;
 268         }
 269
 270         path->alloc_hopid = true;
 271
 272         in_hopid = src_hopid;
 273         out_port = NULL;
 274
 275         for (i = 0; i < num_hops; i++) {
 276                 in_port = tb_next_port_on_path(src, dst, out_port);
 277                 if (!in_port)
 278                         goto err;
 279
 280                 /* When lanes are bonded primary link must be used */
 281                 if (!in_port->bonded && in_port->dual_link_port &&
 282                     in_port->link_nr != link_nr)
 283                         in_port = in_port->dual_link_port;
 284
 285                 ret = tb_port_alloc_in_hopid(in_port, in_hopid, in_hopid);
 286                 if (ret < 0)
 287                         goto err;
 288                 in_hopid = ret;
 289
 290                 out_port = tb_next_port_on_path(src, dst, in_port);
 291                 if (!out_port)
 292                         goto err;
 293
 294                 /*
 295                  * Pick up right port when going from non-bonded to
 296                  * bonded or from bonded to non-bonded.
 297                  */
 298                 if (out_port->dual_link_port) {
 299                         if (!in_port->bonded && out_port->bonded &&
 300                             out_port->link_nr) {
 301                                 /*
 302                                  * Use primary link when going from
 303                                  * non-bonded to bonded.
 304                                  */
 305                                 out_port = out_port->dual_link_port;
 306                         } else if (!out_port->bonded &&
 307                                    out_port->link_nr != link_nr) {
 308                                 /*
 309                                  * If out port is not bonded follow
 310                                  * link_nr.
 311                                  */
 312                                 out_port = out_port->dual_link_port;
 313                         }
 314                 }
 315
 316                 if (i == num_hops - 1)
 317                         ret = tb_port_alloc_out_hopid(out_port, dst_hopid,
 318                                                       dst_hopid);
 319                 else
 320                         ret = tb_port_alloc_out_hopid(out_port, -1, -1);
 321
 322                 if (ret < 0)
 323                         goto err;
 324                 out_hopid = ret;
 325
 326                 path->hops[i].in_hop_index = in_hopid;
 327                 path->hops[i].in_port = in_port;
 328                 path->hops[i].in_counter_index = -1;
 329                 path->hops[i].out_port = out_port;
 330                 path->hops[i].next_hop_index = out_hopid;
 331
 332                 in_hopid = out_hopid;
 333         }
 334
 335         path->tb = tb;
 336         path->path_length = num_hops;
 337         path->name = name;
 338
 339         return path;
 340
 341 err:
 342         tb_path_free(path);
 343         return NULL;
 344 }
 345
 346 /**
 347  * tb_path_free() - free a path
 348  * @path: Path to free
 349  *
 350  * Frees a path. The path does not need to be deactivated.
 351  */
 352 void tb_path_free(struct tb_path *path)
 353 {
 354         if (path->alloc_hopid) {
 355                 int i;
 356
 357                 for (i = 0; i < path->path_length; i++) {
 358                         const struct tb_path_hop *hop = &path->hops[i];
 359
 360                         if (hop->in_port)
 361                                 tb_port_release_in_hopid(hop->in_port,
 362                                                          hop->in_hop_index);
 363                         if (hop->out_port)
 364                                 tb_port_release_out_hopid(hop->out_port,
 365                                                           hop->next_hop_index);
 366                 }
 367         }
 368
 369         kfree(path->hops);
 370         kfree(path);
 371 }
 372
 373 static void __tb_path_deallocate_nfc(struct tb_path *path, int first_hop)
 374 {
 375         int i, res;
 376         for (i = first_hop; i < path->path_length; i++) {
 377                 res = tb_port_add_nfc_credits(path->hops[i].in_port,
 378                                               -path->hops[i].nfc_credits);
 379                 if (res)
 380                         tb_port_warn(path->hops[i].in_port,
 381                                      "nfc credits deallocation failed for hop %d\n",
 382                                      i);
 383         }
 384 }
 385
 386 static int __tb_path_deactivate_hop(struct tb_port *port, int hop_index,
 387                                     bool clear_fc)
 388 {
 389         struct tb_regs_hop hop;
 390         ktime_t timeout;
 391         int ret;
 392
 393         /* Disable the path */
 394         ret = tb_port_read(port, &hop, TB_CFG_HOPS, 2 * hop_index, 2);
 395         if (ret)
 396                 return ret;
 397
 398         /* Already disabled */
 399         if (!hop.enable)
 400                 return 0;
 401
 402         hop.enable = 0;
 403
 404         ret = tb_port_write(port, &hop, TB_CFG_HOPS, 2 * hop_index, 2);
 405         if (ret)
 406                 return ret;
 407
 408         /* Wait until it is drained */
 409         timeout = ktime_add_ms(ktime_get(), 500);
 410         do {
 411                 ret = tb_port_read(port, &hop, TB_CFG_HOPS, 2 * hop_index, 2);
 412                 if (ret)
 413                         return ret;
 414
 415                 if (!hop.pending) {
 416                         if (clear_fc) {
 417                                 /*
 418                                  * Clear flow control. Protocol adapters
 419                                  * IFC and ISE bits are vendor defined
 420                                  * in the USB4 spec so we clear them
 421                                  * only for pre-USB4 adapters.
 422                                  */
 423                                 if (!tb_switch_is_usb4(port->sw)) {
 424                                         hop.ingress_fc = 0;
 425                                         hop.ingress_shared_buffer = 0;
 426                                 }
 427                                 hop.egress_fc = 0;
 428                                 hop.egress_shared_buffer = 0;
 429
 430                                 return tb_port_write(port, &hop, TB_CFG_HOPS,
 431                                                      2 * hop_index, 2);
 432                         }
 433
 434                         return 0;
 435                 }
 436
 437                 usleep_range(10, 20);
 438         } while (ktime_before(ktime_get(), timeout));
 439
 440         return -ETIMEDOUT;
 441 }
 442
 443 static void __tb_path_deactivate_hops(struct tb_path *path, int first_hop)
 444 {
 445         int i, res;
 446
 447         for (i = first_hop; i < path->path_length; i++) {
 448                 res = __tb_path_deactivate_hop(path->hops[i].in_port,
 449                                                path->hops[i].in_hop_index,
 450                                                path->clear_fc);
 451                 if (res && res != -ENODEV)
 452                         tb_port_warn(path->hops[i].in_port,
 453                                      "hop deactivation failed for hop %d, index %d\n",
 454                                      i, path->hops[i].in_hop_index);
 455         }
 456 }
 457
 458 void tb_path_deactivate(struct tb_path *path)
 459 {
 460         if (!path->activated) {
 461                 tb_WARN(path->tb, "trying to deactivate an inactive path\n");
 462                 return;
 463         }
 464         tb_dbg(path->tb,
 465                "deactivating %s path from %llx:%u to %llx:%u\n",
 466                path->name, tb_route(path->hops[0].in_port->sw),
 467                path->hops[0].in_port->port,
 468                tb_route(path->hops[path->path_length - 1].out_port->sw),
 469                path->hops[path->path_length - 1].out_port->port);
 470         __tb_path_deactivate_hops(path, 0);
 471         __tb_path_deallocate_nfc(path, 0);
 472         path->activated = false;
 473 }
 474
 475 /**
 476  * tb_path_activate() - activate a path
 477  * @path: Path to activate
 478  *
 479  * Activate a path starting with the last hop and iterating backwards. The
 480  * caller must fill path->hops before calling tb_path_activate().
 481  *
 482  * Return: Returns 0 on success or an error code on failure.
 483  */
 484 int tb_path_activate(struct tb_path *path)
 485 {
 486         int i, res;
 487         enum tb_path_port out_mask, in_mask;
 488         if (path->activated) {
 489                 tb_WARN(path->tb, "trying to activate already activated path\n");
 490                 return -EINVAL;
 491         }
 492
 493         tb_dbg(path->tb,
 494                "activating %s path from %llx:%u to %llx:%u\n",
 495                path->name, tb_route(path->hops[0].in_port->sw),
 496                path->hops[0].in_port->port,
 497                tb_route(path->hops[path->path_length - 1].out_port->sw),
 498                path->hops[path->path_length - 1].out_port->port);
 499
 500         /* Clear counters. */
 501         for (i = path->path_length - 1; i >= 0; i--) {
 502                 if (path->hops[i].in_counter_index == -1)
 503                         continue;
 504                 res = tb_port_clear_counter(path->hops[i].in_port,
 505                                             path->hops[i].in_counter_index);
 506                 if (res)
 507                         goto err;
 508         }
 509
 510         /* Add non flow controlled credits. */
 511         for (i = path->path_length - 1; i >= 0; i--) {
 512                 res = tb_port_add_nfc_credits(path->hops[i].in_port,
 513                                               path->hops[i].nfc_credits);
 514                 if (res) {
 515                         __tb_path_deallocate_nfc(path, i);
 516                         goto err;
 517                 }
 518         }
 519
 520         /* Activate hops. */
 521         for (i = path->path_length - 1; i >= 0; i--) {
 522                 struct tb_regs_hop hop = { 0 };
 523
 524                 /* If it is left active deactivate it first */
 525                 __tb_path_deactivate_hop(path->hops[i].in_port,
 526                                 path->hops[i].in_hop_index, path->clear_fc);
 527
 528                 /* dword 0 */
 529                 hop.next_hop = path->hops[i].next_hop_index;
 530                 hop.out_port = path->hops[i].out_port->port;
 531                 hop.initial_credits = path->hops[i].initial_credits;
 532                 hop.unknown1 = 0;
 533                 hop.enable = 1;
 534
 535                 /* dword 1 */
 536                 out_mask = (i == path->path_length - 1) ?
 537                                 TB_PATH_DESTINATION : TB_PATH_INTERNAL;
 538                 in_mask = (i == 0) ? TB_PATH_SOURCE : TB_PATH_INTERNAL;
 539                 hop.weight = path->weight;
 540                 hop.unknown2 = 0;
 541                 hop.priority = path->priority;
 542                 hop.drop_packages = path->drop_packages;
 543                 hop.counter = path->hops[i].in_counter_index;
 544                 hop.counter_enable = path->hops[i].in_counter_index != -1;
 545                 hop.ingress_fc = path->ingress_fc_enable & in_mask;
 546                 hop.egress_fc = path->egress_fc_enable & out_mask;
 547                 hop.ingress_shared_buffer = path->ingress_shared_buffer
 548                                             & in_mask;
 549                 hop.egress_shared_buffer = path->egress_shared_buffer
 550                                             & out_mask;
 551                 hop.unknown3 = 0;
 552
 553                 tb_port_dbg(path->hops[i].in_port, "Writing hop %d\n", i);
 554                 tb_dump_hop(&path->hops[i], &hop);
 555                 res = tb_port_write(path->hops[i].in_port, &hop, TB_CFG_HOPS,
 556                                     2 * path->hops[i].in_hop_index, 2);
 557                 if (res) {
 558                         __tb_path_deactivate_hops(path, i);
 559                         __tb_path_deallocate_nfc(path, 0);
 560                         goto err;
 561                 }
 562         }
 563         path->activated = true;
 564         tb_dbg(path->tb, "path activation complete\n");
 565         return 0;
 566 err:
 567         tb_WARN(path->tb, "path activation failed\n");
 568         return res;
 569 }
 570
 571 /**
 572  * tb_path_is_invalid() - check whether any ports on the path are invalid
 573  * @path: Path to check
 574  *
 575  * Return: Returns true if the path is invalid, false otherwise.
 576  */
 577 bool tb_path_is_invalid(struct tb_path *path)
 578 {
 579         int i = 0;
 580         for (i = 0; i < path->path_length; i++) {
 581                 if (path->hops[i].in_port->sw->is_unplugged)
 582                         return true;
 583                 if (path->hops[i].out_port->sw->is_unplugged)
 584                         return true;
 585         }
 586         return false;
 587 }
 588
 589 /**
 590  * tb_path_port_on_path() - Does the path go through certain port
 591  * @path: Path to check
 592  * @port: Switch to check
 593  *
 594  * Goes over all hops on path and checks if @port is any of them.
 595  * Direction does not matter.
 596  */
 597 bool tb_path_port_on_path(const struct tb_path *path, const struct tb_port *port)
 598 {
 599         int i;
 600
 601         for (i = 0; i < path->path_length; i++) {
 602                 if (path->hops[i].in_port == port ||
 603                     path->hops[i].out_port == port)
 604                         return true;
 605         }
 606
 607         return false;
 608 }