drivers/infiniband/core/counters.c

   1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
   2 /*
   3  * Copyright (c) 2019 Mellanox Technologies. All rights reserved.
   4  */
   5 #include <rdma/ib_verbs.h>
   6 #include <rdma/rdma_counter.h>
   7
   8 #include "core_priv.h"
   9 #include "restrack.h"
  10
  11 #define ALL_AUTO_MODE_MASKS (RDMA_COUNTER_MASK_QP_TYPE)
  12
  13 static int __counter_set_mode(struct rdma_counter_mode *curr,
  14                               enum rdma_nl_counter_mode new_mode,
  15                               enum rdma_nl_counter_mask new_mask)
  16 {
  17         if ((new_mode == RDMA_COUNTER_MODE_AUTO) &&
  18             ((new_mask & (~ALL_AUTO_MODE_MASKS)) ||
  19              (curr->mode != RDMA_COUNTER_MODE_NONE)))
  20                 return -EINVAL;
  21
  22         curr->mode = new_mode;
  23         curr->mask = new_mask;
  24         return 0;
  25 }
  26
  27 /**
  28  * rdma_counter_set_auto_mode() - Turn on/off per-port auto mode
  29  *
  30  * When @on is true, the @mask must be set; When @on is false, it goes
  31  * into manual mode if there's any counter, so that the user is able to
  32  * manually access them.
  33  */
  34 int rdma_counter_set_auto_mode(struct ib_device *dev, u8 port,
  35                                bool on, enum rdma_nl_counter_mask mask)
  36 {
  37         struct rdma_port_counter *port_counter;
  38         int ret;
  39
  40         port_counter = &dev->port_data[port].port_counter;
  41         if (!port_counter->hstats)
  42                 return -EOPNOTSUPP;
  43
  44         mutex_lock(&port_counter->lock);
  45         if (on) {
  46                 ret = __counter_set_mode(&port_counter->mode,
  47                                          RDMA_COUNTER_MODE_AUTO, mask);
  48         } else {
  49                 if (port_counter->mode.mode != RDMA_COUNTER_MODE_AUTO) {
  50                         ret = -EINVAL;
  51                         goto out;
  52                 }
  53
  54                 if (port_counter->num_counters)
  55                         ret = __counter_set_mode(&port_counter->mode,
  56                                                  RDMA_COUNTER_MODE_MANUAL, 0);
  57                 else
  58                         ret = __counter_set_mode(&port_counter->mode,
  59                                                  RDMA_COUNTER_MODE_NONE, 0);
  60         }
  61
  62 out:
  63         mutex_unlock(&port_counter->lock);
  64         return ret;
  65 }
  66
  67 static struct rdma_counter *rdma_counter_alloc(struct ib_device *dev, u8 port,
  68                                                enum rdma_nl_counter_mode mode)
  69 {
  70         struct rdma_port_counter *port_counter;
  71         struct rdma_counter *counter;
  72         int ret;
  73
  74         if (!dev->ops.counter_dealloc || !dev->ops.counter_alloc_stats)
  75                 return NULL;
  76
  77         counter = kzalloc(sizeof(*counter), GFP_KERNEL);
  78         if (!counter)
  79                 return NULL;
  80
  81         counter->device    = dev;
  82         counter->port      = port;
  83         counter->res.type  = RDMA_RESTRACK_COUNTER;
  84         counter->stats     = dev->ops.counter_alloc_stats(counter);
  85         if (!counter->stats)
  86                 goto err_stats;
  87
  88         port_counter = &dev->port_data[port].port_counter;
  89         mutex_lock(&port_counter->lock);
  90         if (mode == RDMA_COUNTER_MODE_MANUAL) {
  91                 ret = __counter_set_mode(&port_counter->mode,
  92                                          RDMA_COUNTER_MODE_MANUAL, 0);
  93                 if (ret)
  94                         goto err_mode;
  95         }
  96
  97         port_counter->num_counters++;
  98         mutex_unlock(&port_counter->lock);
  99
 100         counter->mode.mode = mode;
 101         kref_init(&counter->kref);
 102         mutex_init(&counter->lock);
 103
 104         return counter;
 105
 106 err_mode:
 107         mutex_unlock(&port_counter->lock);
 108         kfree(counter->stats);
 109 err_stats:
 110         kfree(counter);
 111         return NULL;
 112 }
 113
 114 static void rdma_counter_free(struct rdma_counter *counter)
 115 {
 116         struct rdma_port_counter *port_counter;
 117
 118         port_counter = &counter->device->port_data[counter->port].port_counter;
 119         mutex_lock(&port_counter->lock);
 120         port_counter->num_counters--;
 121         if (!port_counter->num_counters &&
 122             (port_counter->mode.mode == RDMA_COUNTER_MODE_MANUAL))
 123                 __counter_set_mode(&port_counter->mode, RDMA_COUNTER_MODE_NONE,
 124                                    0);
 125
 126         mutex_unlock(&port_counter->lock);
 127
 128         rdma_restrack_del(&counter->res);
 129         kfree(counter->stats);
 130         kfree(counter);
 131 }
 132
 133 static void auto_mode_init_counter(struct rdma_counter *counter,
 134                                    const struct ib_qp *qp,
 135                                    enum rdma_nl_counter_mask new_mask)
 136 {
 137         struct auto_mode_param *param = &counter->mode.param;
 138
 139         counter->mode.mode = RDMA_COUNTER_MODE_AUTO;
 140         counter->mode.mask = new_mask;
 141
 142         if (new_mask & RDMA_COUNTER_MASK_QP_TYPE)
 143                 param->qp_type = qp->qp_type;
 144 }
 145
 146 static bool auto_mode_match(struct ib_qp *qp, struct rdma_counter *counter,
 147                             enum rdma_nl_counter_mask auto_mask)
 148 {
 149         struct auto_mode_param *param = &counter->mode.param;
 150         bool match = true;
 151
 152         /*
 153          * Ensure that counter belongs to the right PID.  This operation can
 154          * race with user space which kills the process and leaves QP and
 155          * counters orphans.
 156          *
 157          * It is not a big deal because exitted task will leave both QP and
 158          * counter in the same bucket of zombie process. Just ensure that
 159          * process is still alive before procedding.
 160          *
 161          */
 162         if (task_pid_nr(counter->res.task) != task_pid_nr(qp->res.task) ||
 163             !task_pid_nr(qp->res.task))
 164                 return false;
 165
 166         if (auto_mask & RDMA_COUNTER_MASK_QP_TYPE)
 167                 match &= (param->qp_type == qp->qp_type);
 168
 169         return match;
 170 }
 171
 172 static int __rdma_counter_bind_qp(struct rdma_counter *counter,
 173                                   struct ib_qp *qp)
 174 {
 175         int ret;
 176
 177         if (qp->counter)
 178                 return -EINVAL;
 179
 180         if (!qp->device->ops.counter_bind_qp)
 181                 return -EOPNOTSUPP;
 182
 183         mutex_lock(&counter->lock);
 184         ret = qp->device->ops.counter_bind_qp(counter, qp);
 185         mutex_unlock(&counter->lock);
 186
 187         return ret;
 188 }
 189
 190 static int __rdma_counter_unbind_qp(struct ib_qp *qp)
 191 {
 192         struct rdma_counter *counter = qp->counter;
 193         int ret;
 194
 195         if (!qp->device->ops.counter_unbind_qp)
 196                 return -EOPNOTSUPP;
 197
 198         mutex_lock(&counter->lock);
 199         ret = qp->device->ops.counter_unbind_qp(qp);
 200         mutex_unlock(&counter->lock);
 201
 202         return ret;
 203 }
 204
 205 static void counter_history_stat_update(const struct rdma_counter *counter)
 206 {
 207         struct ib_device *dev = counter->device;
 208         struct rdma_port_counter *port_counter;
 209         int i;
 210
 211         port_counter = &dev->port_data[counter->port].port_counter;
 212         if (!port_counter->hstats)
 213                 return;
 214
 215         for (i = 0; i < counter->stats->num_counters; i++)
 216                 port_counter->hstats->value[i] += counter->stats->value[i];
 217 }
 218
 219 /**
 220  * rdma_get_counter_auto_mode - Find the counter that @qp should be bound
 221  *     with in auto mode
 222  *
 223  * Return: The counter (with ref-count increased) if found
 224  */
 225 static struct rdma_counter *rdma_get_counter_auto_mode(struct ib_qp *qp,
 226                                                        u8 port)
 227 {
 228         struct rdma_port_counter *port_counter;
 229         struct rdma_counter *counter = NULL;
 230         struct ib_device *dev = qp->device;
 231         struct rdma_restrack_entry *res;
 232         struct rdma_restrack_root *rt;
 233         unsigned long id = 0;
 234
 235         port_counter = &dev->port_data[port].port_counter;
 236         rt = &dev->res[RDMA_RESTRACK_COUNTER];
 237         xa_lock(&rt->xa);
 238         xa_for_each(&rt->xa, id, res) {
 239                 counter = container_of(res, struct rdma_counter, res);
 240                 if ((counter->device != qp->device) || (counter->port != port))
 241                         goto next;
 242
 243                 if (auto_mode_match(qp, counter, port_counter->mode.mask))
 244                         break;
 245 next:
 246                 counter = NULL;
 247         }
 248
 249         if (counter && !kref_get_unless_zero(&counter->kref))
 250                 counter = NULL;
 251
 252         xa_unlock(&rt->xa);
 253         return counter;
 254 }
 255
 256 static void rdma_counter_res_add(struct rdma_counter *counter,
 257                                  struct ib_qp *qp)
 258 {
 259         if (rdma_is_kernel_res(&qp->res)) {
 260                 rdma_restrack_set_task(&counter->res, qp->res.kern_name);
 261                 rdma_restrack_kadd(&counter->res);
 262         } else {
 263                 rdma_restrack_attach_task(&counter->res, qp->res.task);
 264                 rdma_restrack_uadd(&counter->res);
 265         }
 266 }
 267
 268 static void counter_release(struct kref *kref)
 269 {
 270         struct rdma_counter *counter;
 271
 272         counter = container_of(kref, struct rdma_counter, kref);
 273         counter_history_stat_update(counter);
 274         counter->device->ops.counter_dealloc(counter);
 275         rdma_counter_free(counter);
 276 }
 277
 278 /**
 279  * rdma_counter_bind_qp_auto - Check and bind the QP to a counter base on
 280  *   the auto-mode rule
 281  */
 282 int rdma_counter_bind_qp_auto(struct ib_qp *qp, u8 port)
 283 {
 284         struct rdma_port_counter *port_counter;
 285         struct ib_device *dev = qp->device;
 286         struct rdma_counter *counter;
 287         int ret;
 288
 289         if (!qp->res.valid)
 290                 return 0;
 291
 292         if (!rdma_is_port_valid(dev, port))
 293                 return -EINVAL;
 294
 295         port_counter = &dev->port_data[port].port_counter;
 296         if (port_counter->mode.mode != RDMA_COUNTER_MODE_AUTO)
 297                 return 0;
 298
 299         counter = rdma_get_counter_auto_mode(qp, port);
 300         if (counter) {
 301                 ret = __rdma_counter_bind_qp(counter, qp);
 302                 if (ret) {
 303                         kref_put(&counter->kref, counter_release);
 304                         return ret;
 305                 }
 306         } else {
 307                 counter = rdma_counter_alloc(dev, port, RDMA_COUNTER_MODE_AUTO);
 308                 if (!counter)
 309                         return -ENOMEM;
 310
 311                 auto_mode_init_counter(counter, qp, port_counter->mode.mask);
 312
 313                 ret = __rdma_counter_bind_qp(counter, qp);
 314                 if (ret) {
 315                         rdma_counter_free(counter);
 316                         return ret;
 317                 }
 318
 319                 rdma_counter_res_add(counter, qp);
 320         }
 321
 322         return 0;
 323 }
 324
 325 /**
 326  * rdma_counter_unbind_qp - Unbind a qp from a counter
 327  * @force:
 328  *   true - Decrease the counter ref-count anyway (e.g., qp destroy)
 329  */
 330 int rdma_counter_unbind_qp(struct ib_qp *qp, bool force)
 331 {
 332         struct rdma_counter *counter = qp->counter;
 333         int ret;
 334
 335         if (!counter)
 336                 return -EINVAL;
 337
 338         ret = __rdma_counter_unbind_qp(qp);
 339         if (ret && !force)
 340                 return ret;
 341
 342         kref_put(&counter->kref, counter_release);
 343         return 0;
 344 }
 345
 346 int rdma_counter_query_stats(struct rdma_counter *counter)
 347 {
 348         struct ib_device *dev = counter->device;
 349         int ret;
 350
 351         if (!dev->ops.counter_update_stats)
 352                 return -EINVAL;
 353
 354         mutex_lock(&counter->lock);
 355         ret = dev->ops.counter_update_stats(counter);
 356         mutex_unlock(&counter->lock);
 357
 358         return ret;
 359 }
 360
 361 static u64 get_running_counters_hwstat_sum(struct ib_device *dev,
 362                                            u8 port, u32 index)
 363 {
 364         struct rdma_restrack_entry *res;
 365         struct rdma_restrack_root *rt;
 366         struct rdma_counter *counter;
 367         unsigned long id = 0;
 368         u64 sum = 0;
 369
 370         rt = &dev->res[RDMA_RESTRACK_COUNTER];
 371         xa_lock(&rt->xa);
 372         xa_for_each(&rt->xa, id, res) {
 373                 if (!rdma_restrack_get(res))
 374                         continue;
 375
 376                 xa_unlock(&rt->xa);
 377
 378                 counter = container_of(res, struct rdma_counter, res);
 379                 if ((counter->device != dev) || (counter->port != port) ||
 380                     rdma_counter_query_stats(counter))
 381                         goto next;
 382
 383                 sum += counter->stats->value[index];
 384
 385 next:
 386                 xa_lock(&rt->xa);
 387                 rdma_restrack_put(res);
 388         }
 389
 390         xa_unlock(&rt->xa);
 391         return sum;
 392 }
 393
 394 /**
 395  * rdma_counter_get_hwstat_value() - Get the sum value of all counters on a
 396  *   specific port, including the running ones and history data
 397  */
 398 u64 rdma_counter_get_hwstat_value(struct ib_device *dev, u8 port, u32 index)
 399 {
 400         struct rdma_port_counter *port_counter;
 401         u64 sum;
 402
 403         port_counter = &dev->port_data[port].port_counter;
 404         if (!port_counter->hstats)
 405                 return 0;
 406
 407         sum = get_running_counters_hwstat_sum(dev, port, index);
 408         sum += port_counter->hstats->value[index];
 409
 410         return sum;
 411 }
 412
 413 static struct ib_qp *rdma_counter_get_qp(struct ib_device *dev, u32 qp_num)
 414 {
 415         struct rdma_restrack_entry *res = NULL;
 416         struct ib_qp *qp = NULL;
 417
 418         res = rdma_restrack_get_byid(dev, RDMA_RESTRACK_QP, qp_num);
 419         if (IS_ERR(res))
 420                 return NULL;
 421
 422         qp = container_of(res, struct ib_qp, res);
 423         if (qp->qp_type == IB_QPT_RAW_PACKET && !capable(CAP_NET_RAW))
 424                 goto err;
 425
 426         return qp;
 427
 428 err:
 429         rdma_restrack_put(res);
 430         return NULL;
 431 }
 432
 433 static int rdma_counter_bind_qp_manual(struct rdma_counter *counter,
 434                                        struct ib_qp *qp)
 435 {
 436         if ((counter->device != qp->device) || (counter->port != qp->port))
 437                 return -EINVAL;
 438
 439         return __rdma_counter_bind_qp(counter, qp);
 440 }
 441
 442 static struct rdma_counter *rdma_get_counter_by_id(struct ib_device *dev,
 443                                                    u32 counter_id)
 444 {
 445         struct rdma_restrack_entry *res;
 446         struct rdma_counter *counter;
 447
 448         res = rdma_restrack_get_byid(dev, RDMA_RESTRACK_COUNTER, counter_id);
 449         if (IS_ERR(res))
 450                 return NULL;
 451
 452         counter = container_of(res, struct rdma_counter, res);
 453         kref_get(&counter->kref);
 454         rdma_restrack_put(res);
 455
 456         return counter;
 457 }
 458
 459 /**
 460  * rdma_counter_bind_qpn() - Bind QP @qp_num to counter @counter_id
 461  */
 462 int rdma_counter_bind_qpn(struct ib_device *dev, u8 port,
 463                           u32 qp_num, u32 counter_id)
 464 {
 465         struct rdma_port_counter *port_counter;
 466         struct rdma_counter *counter;
 467         struct ib_qp *qp;
 468         int ret;
 469
 470         port_counter = &dev->port_data[port].port_counter;
 471         if (port_counter->mode.mode == RDMA_COUNTER_MODE_AUTO)
 472                 return -EINVAL;
 473
 474         qp = rdma_counter_get_qp(dev, qp_num);
 475         if (!qp)
 476                 return -ENOENT;
 477
 478         counter = rdma_get_counter_by_id(dev, counter_id);
 479         if (!counter) {
 480                 ret = -ENOENT;
 481                 goto err;
 482         }
 483
 484         if (counter->res.task != qp->res.task) {
 485                 ret = -EINVAL;
 486                 goto err_task;
 487         }
 488
 489         ret = rdma_counter_bind_qp_manual(counter, qp);
 490         if (ret)
 491                 goto err_task;
 492
 493         rdma_restrack_put(&qp->res);
 494         return 0;
 495
 496 err_task:
 497         kref_put(&counter->kref, counter_release);
 498 err:
 499         rdma_restrack_put(&qp->res);
 500         return ret;
 501 }
 502
 503 /**
 504  * rdma_counter_bind_qpn_alloc() - Alloc a counter and bind QP @qp_num to it
 505  *   The id of new counter is returned in @counter_id
 506  */
 507 int rdma_counter_bind_qpn_alloc(struct ib_device *dev, u8 port,
 508                                 u32 qp_num, u32 *counter_id)
 509 {
 510         struct rdma_port_counter *port_counter;
 511         struct rdma_counter *counter;
 512         struct ib_qp *qp;
 513         int ret;
 514
 515         if (!rdma_is_port_valid(dev, port))
 516                 return -EINVAL;
 517
 518         port_counter = &dev->port_data[port].port_counter;
 519         if (!port_counter->hstats)
 520                 return -EOPNOTSUPP;
 521
 522         if (port_counter->mode.mode == RDMA_COUNTER_MODE_AUTO)
 523                 return -EINVAL;
 524
 525         qp = rdma_counter_get_qp(dev, qp_num);
 526         if (!qp)
 527                 return -ENOENT;
 528
 529         if (rdma_is_port_valid(dev, qp->port) && (qp->port != port)) {
 530                 ret = -EINVAL;
 531                 goto err;
 532         }
 533
 534         counter = rdma_counter_alloc(dev, port, RDMA_COUNTER_MODE_MANUAL);
 535         if (!counter) {
 536                 ret = -ENOMEM;
 537                 goto err;
 538         }
 539
 540         ret = rdma_counter_bind_qp_manual(counter, qp);
 541         if (ret)
 542                 goto err_bind;
 543
 544         if (counter_id)
 545                 *counter_id = counter->id;
 546
 547         rdma_counter_res_add(counter, qp);
 548
 549         rdma_restrack_put(&qp->res);
 550         return ret;
 551
 552 err_bind:
 553         rdma_counter_free(counter);
 554 err:
 555         rdma_restrack_put(&qp->res);
 556         return ret;
 557 }
 558
 559 /**
 560  * rdma_counter_unbind_qpn() - Unbind QP @qp_num from a counter
 561  */
 562 int rdma_counter_unbind_qpn(struct ib_device *dev, u8 port,
 563                             u32 qp_num, u32 counter_id)
 564 {
 565         struct rdma_port_counter *port_counter;
 566         struct ib_qp *qp;
 567         int ret;
 568
 569         if (!rdma_is_port_valid(dev, port))
 570                 return -EINVAL;
 571
 572         qp = rdma_counter_get_qp(dev, qp_num);
 573         if (!qp)
 574                 return -ENOENT;
 575
 576         if (rdma_is_port_valid(dev, qp->port) && (qp->port != port)) {
 577                 ret = -EINVAL;
 578                 goto out;
 579         }
 580
 581         port_counter = &dev->port_data[port].port_counter;
 582         if (!qp->counter || qp->counter->id != counter_id ||
 583             port_counter->mode.mode != RDMA_COUNTER_MODE_MANUAL) {
 584                 ret = -EINVAL;
 585                 goto out;
 586         }
 587
 588         ret = rdma_counter_unbind_qp(qp, false);
 589
 590 out:
 591         rdma_restrack_put(&qp->res);
 592         return ret;
 593 }
 594
 595 int rdma_counter_get_mode(struct ib_device *dev, u8 port,
 596                           enum rdma_nl_counter_mode *mode,
 597                           enum rdma_nl_counter_mask *mask)
 598 {
 599         struct rdma_port_counter *port_counter;
 600
 601         port_counter = &dev->port_data[port].port_counter;
 602         *mode = port_counter->mode.mode;
 603         *mask = port_counter->mode.mask;
 604
 605         return 0;
 606 }
 607
 608 void rdma_counter_init(struct ib_device *dev)
 609 {
 610         struct rdma_port_counter *port_counter;
 611         u32 port, i;
 612
 613         if (!dev->port_data)
 614                 return;
 615
 616         rdma_for_each_port(dev, port) {
 617                 port_counter = &dev->port_data[port].port_counter;
 618                 port_counter->mode.mode = RDMA_COUNTER_MODE_NONE;
 619                 mutex_init(&port_counter->lock);
 620
 621                 if (!dev->ops.alloc_hw_stats)
 622                         continue;
 623
 624                 port_counter->hstats = dev->ops.alloc_hw_stats(dev, port);
 625                 if (!port_counter->hstats)
 626                         goto fail;
 627         }
 628
 629         return;
 630
 631 fail:
 632         for (i = port; i >= rdma_start_port(dev); i--) {
 633                 port_counter = &dev->port_data[port].port_counter;
 634                 kfree(port_counter->hstats);
 635                 port_counter->hstats = NULL;
 636                 mutex_destroy(&port_counter->lock);
 637         }
 638 }
 639
 640 void rdma_counter_release(struct ib_device *dev)
 641 {
 642         struct rdma_port_counter *port_counter;
 643         u32 port;
 644
 645         rdma_for_each_port(dev, port) {
 646                 port_counter = &dev->port_data[port].port_counter;
 647                 kfree(port_counter->hstats);
 648                 mutex_destroy(&port_counter->lock);
 649         }
 650 }