drivers/infiniband/hw/mlx5/counters.c

   1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
   2 /*
   3  * Copyright (c) 2013-2020, Mellanox Technologies inc. All rights reserved.
   4  */
   5
   6 #include "mlx5_ib.h"
   7 #include <linux/mlx5/eswitch.h>
   8 #include "counters.h"
   9 #include "ib_rep.h"
  10 #include "qp.h"
  11
  12 struct mlx5_ib_counter {
  13         const char *name;
  14         size_t offset;
  15 };
  16
  17 #define INIT_Q_COUNTER(_name)           \
  18         { .name = #_name, .offset = MLX5_BYTE_OFF(query_q_counter_out, _name)}
  19
  20 static const struct mlx5_ib_counter basic_q_cnts[] = {
  21         INIT_Q_COUNTER(rx_write_requests),
  22         INIT_Q_COUNTER(rx_read_requests),
  23         INIT_Q_COUNTER(rx_atomic_requests),
  24         INIT_Q_COUNTER(out_of_buffer),
  25 };
  26
  27 static const struct mlx5_ib_counter out_of_seq_q_cnts[] = {
  28         INIT_Q_COUNTER(out_of_sequence),
  29 };
  30
  31 static const struct mlx5_ib_counter retrans_q_cnts[] = {
  32         INIT_Q_COUNTER(duplicate_request),
  33         INIT_Q_COUNTER(rnr_nak_retry_err),
  34         INIT_Q_COUNTER(packet_seq_err),
  35         INIT_Q_COUNTER(implied_nak_seq_err),
  36         INIT_Q_COUNTER(local_ack_timeout_err),
  37 };
  38
  39 #define INIT_CONG_COUNTER(_name)                \
  40         { .name = #_name, .offset =     \
  41                 MLX5_BYTE_OFF(query_cong_statistics_out, _name ## _high)}
  42
  43 static const struct mlx5_ib_counter cong_cnts[] = {
  44         INIT_CONG_COUNTER(rp_cnp_ignored),
  45         INIT_CONG_COUNTER(rp_cnp_handled),
  46         INIT_CONG_COUNTER(np_ecn_marked_roce_packets),
  47         INIT_CONG_COUNTER(np_cnp_sent),
  48 };
  49
  50 static const struct mlx5_ib_counter extended_err_cnts[] = {
  51         INIT_Q_COUNTER(resp_local_length_error),
  52         INIT_Q_COUNTER(resp_cqe_error),
  53         INIT_Q_COUNTER(req_cqe_error),
  54         INIT_Q_COUNTER(req_remote_invalid_request),
  55         INIT_Q_COUNTER(req_remote_access_errors),
  56         INIT_Q_COUNTER(resp_remote_access_errors),
  57         INIT_Q_COUNTER(resp_cqe_flush_error),
  58         INIT_Q_COUNTER(req_cqe_flush_error),
  59 };
  60
  61 static const struct mlx5_ib_counter roce_accl_cnts[] = {
  62         INIT_Q_COUNTER(roce_adp_retrans),
  63         INIT_Q_COUNTER(roce_adp_retrans_to),
  64         INIT_Q_COUNTER(roce_slow_restart),
  65         INIT_Q_COUNTER(roce_slow_restart_cnps),
  66         INIT_Q_COUNTER(roce_slow_restart_trans),
  67 };
  68
  69 #define INIT_EXT_PPCNT_COUNTER(_name)           \
  70         { .name = #_name, .offset =     \
  71         MLX5_BYTE_OFF(ppcnt_reg, \
  72                       counter_set.eth_extended_cntrs_grp_data_layout._name##_high)}
  73
  74 static const struct mlx5_ib_counter ext_ppcnt_cnts[] = {
  75         INIT_EXT_PPCNT_COUNTER(rx_icrc_encapsulated),
  76 };
  77
  78 static int mlx5_ib_read_counters(struct ib_counters *counters,
  79                                  struct ib_counters_read_attr *read_attr,
  80                                  struct uverbs_attr_bundle *attrs)
  81 {
  82         struct mlx5_ib_mcounters *mcounters = to_mcounters(counters);
  83         struct mlx5_read_counters_attr mread_attr = {};
  84         struct mlx5_ib_flow_counters_desc *desc;
  85         int ret, i;
  86
  87         mutex_lock(&mcounters->mcntrs_mutex);
  88         if (mcounters->cntrs_max_index > read_attr->ncounters) {
  89                 ret = -EINVAL;
  90                 goto err_bound;
  91         }
  92
  93         mread_attr.out = kcalloc(mcounters->counters_num, sizeof(u64),
  94                                  GFP_KERNEL);
  95         if (!mread_attr.out) {
  96                 ret = -ENOMEM;
  97                 goto err_bound;
  98         }
  99
 100         mread_attr.hw_cntrs_hndl = mcounters->hw_cntrs_hndl;
 101         mread_attr.flags = read_attr->flags;
 102         ret = mcounters->read_counters(counters->device, &mread_attr);
 103         if (ret)
 104                 goto err_read;
 105
 106         /* do the pass over the counters data array to assign according to the
 107          * descriptions and indexing pairs
 108          */
 109         desc = mcounters->counters_data;
 110         for (i = 0; i < mcounters->ncounters; i++)
 111                 read_attr->counters_buff[desc[i].index] += mread_attr.out[desc[i].description];
 112
 113 err_read:
 114         kfree(mread_attr.out);
 115 err_bound:
 116         mutex_unlock(&mcounters->mcntrs_mutex);
 117         return ret;
 118 }
 119
 120 static int mlx5_ib_destroy_counters(struct ib_counters *counters)
 121 {
 122         struct mlx5_ib_mcounters *mcounters = to_mcounters(counters);
 123
 124         mlx5_ib_counters_clear_description(counters);
 125         if (mcounters->hw_cntrs_hndl)
 126                 mlx5_fc_destroy(to_mdev(counters->device)->mdev,
 127                                 mcounters->hw_cntrs_hndl);
 128         return 0;
 129 }
 130
 131 static int mlx5_ib_create_counters(struct ib_counters *counters,
 132                                    struct uverbs_attr_bundle *attrs)
 133 {
 134         struct mlx5_ib_mcounters *mcounters = to_mcounters(counters);
 135
 136         mutex_init(&mcounters->mcntrs_mutex);
 137         return 0;
 138 }
 139
 140
 141 static const struct mlx5_ib_counters *get_counters(struct mlx5_ib_dev *dev,
 142                                                    u32 port_num)
 143 {
 144         return is_mdev_switchdev_mode(dev->mdev) ? &dev->port[0].cnts :
 145                                                    &dev->port[port_num].cnts;
 146 }
 147
 148 /**
 149  * mlx5_ib_get_counters_id - Returns counters id to use for device+port
 150  * @dev:        Pointer to mlx5 IB device
 151  * @port_num:   Zero based port number
 152  *
 153  * mlx5_ib_get_counters_id() Returns counters set id to use for given
 154  * device port combination in switchdev and non switchdev mode of the
 155  * parent device.
 156  */
 157 u16 mlx5_ib_get_counters_id(struct mlx5_ib_dev *dev, u32 port_num)
 158 {
 159         const struct mlx5_ib_counters *cnts = get_counters(dev, port_num);
 160
 161         return cnts->set_id;
 162 }
 163
 164 static struct rdma_hw_stats *
 165 mlx5_ib_alloc_hw_device_stats(struct ib_device *ibdev)
 166 {
 167         struct mlx5_ib_dev *dev = to_mdev(ibdev);
 168         const struct mlx5_ib_counters *cnts = &dev->port[0].cnts;
 169
 170         return rdma_alloc_hw_stats_struct(cnts->names,
 171                                           cnts->num_q_counters +
 172                                                   cnts->num_cong_counters +
 173                                                   cnts->num_ext_ppcnt_counters,
 174                                           RDMA_HW_STATS_DEFAULT_LIFESPAN);
 175 }
 176
 177 static struct rdma_hw_stats *
 178 mlx5_ib_alloc_hw_port_stats(struct ib_device *ibdev, u32 port_num)
 179 {
 180         struct mlx5_ib_dev *dev = to_mdev(ibdev);
 181         const struct mlx5_ib_counters *cnts = &dev->port[port_num - 1].cnts;
 182
 183         return rdma_alloc_hw_stats_struct(cnts->names,
 184                                           cnts->num_q_counters +
 185                                                   cnts->num_cong_counters +
 186                                                   cnts->num_ext_ppcnt_counters,
 187                                           RDMA_HW_STATS_DEFAULT_LIFESPAN);
 188 }
 189
 190 static int mlx5_ib_query_q_counters(struct mlx5_core_dev *mdev,
 191                                     const struct mlx5_ib_counters *cnts,
 192                                     struct rdma_hw_stats *stats,
 193                                     u16 set_id)
 194 {
 195         u32 out[MLX5_ST_SZ_DW(query_q_counter_out)] = {};
 196         u32 in[MLX5_ST_SZ_DW(query_q_counter_in)] = {};
 197         __be32 val;
 198         int ret, i;
 199
 200         MLX5_SET(query_q_counter_in, in, opcode, MLX5_CMD_OP_QUERY_Q_COUNTER);
 201         MLX5_SET(query_q_counter_in, in, counter_set_id, set_id);
 202         ret = mlx5_cmd_exec_inout(mdev, query_q_counter, in, out);
 203         if (ret)
 204                 return ret;
 205
 206         for (i = 0; i < cnts->num_q_counters; i++) {
 207                 val = *(__be32 *)((void *)out + cnts->offsets[i]);
 208                 stats->value[i] = (u64)be32_to_cpu(val);
 209         }
 210
 211         return 0;
 212 }
 213
 214 static int mlx5_ib_query_ext_ppcnt_counters(struct mlx5_ib_dev *dev,
 215                                             const struct mlx5_ib_counters *cnts,
 216                                             struct rdma_hw_stats *stats)
 217 {
 218         int offset = cnts->num_q_counters + cnts->num_cong_counters;
 219         u32 in[MLX5_ST_SZ_DW(ppcnt_reg)] = {};
 220         int sz = MLX5_ST_SZ_BYTES(ppcnt_reg);
 221         int ret, i;
 222         void *out;
 223
 224         out = kvzalloc(sz, GFP_KERNEL);
 225         if (!out)
 226                 return -ENOMEM;
 227
 228         MLX5_SET(ppcnt_reg, in, local_port, 1);
 229         MLX5_SET(ppcnt_reg, in, grp, MLX5_ETHERNET_EXTENDED_COUNTERS_GROUP);
 230         ret = mlx5_core_access_reg(dev->mdev, in, sz, out, sz, MLX5_REG_PPCNT,
 231                                    0, 0);
 232         if (ret)
 233                 goto free;
 234
 235         for (i = 0; i < cnts->num_ext_ppcnt_counters; i++)
 236                 stats->value[i + offset] =
 237                         be64_to_cpup((__be64 *)(out +
 238                                     cnts->offsets[i + offset]));
 239 free:
 240         kvfree(out);
 241         return ret;
 242 }
 243
 244 static int mlx5_ib_get_hw_stats(struct ib_device *ibdev,
 245                                 struct rdma_hw_stats *stats,
 246                                 u32 port_num, int index)
 247 {
 248         struct mlx5_ib_dev *dev = to_mdev(ibdev);
 249         const struct mlx5_ib_counters *cnts = get_counters(dev, port_num - 1);
 250         struct mlx5_core_dev *mdev;
 251         int ret, num_counters;
 252         u32 mdev_port_num;
 253
 254         if (!stats)
 255                 return -EINVAL;
 256
 257         num_counters = cnts->num_q_counters +
 258                        cnts->num_cong_counters +
 259                        cnts->num_ext_ppcnt_counters;
 260
 261         /* q_counters are per IB device, query the master mdev */
 262         ret = mlx5_ib_query_q_counters(dev->mdev, cnts, stats, cnts->set_id);
 263         if (ret)
 264                 return ret;
 265
 266         if (MLX5_CAP_PCAM_FEATURE(dev->mdev, rx_icrc_encapsulated_counter)) {
 267                 ret =  mlx5_ib_query_ext_ppcnt_counters(dev, cnts, stats);
 268                 if (ret)
 269                         return ret;
 270         }
 271
 272         if (MLX5_CAP_GEN(dev->mdev, cc_query_allowed)) {
 273                 mdev = mlx5_ib_get_native_port_mdev(dev, port_num,
 274                                                     &mdev_port_num);
 275                 if (!mdev) {
 276                         /* If port is not affiliated yet, its in down state
 277                          * which doesn't have any counters yet, so it would be
 278                          * zero. So no need to read from the HCA.
 279                          */
 280                         goto done;
 281                 }
 282                 ret = mlx5_lag_query_cong_counters(dev->mdev,
 283                                                    stats->value +
 284                                                    cnts->num_q_counters,
 285                                                    cnts->num_cong_counters,
 286                                                    cnts->offsets +
 287                                                    cnts->num_q_counters);
 288
 289                 mlx5_ib_put_native_port_mdev(dev, port_num);
 290                 if (ret)
 291                         return ret;
 292         }
 293
 294 done:
 295         return num_counters;
 296 }
 297
 298 static struct rdma_hw_stats *
 299 mlx5_ib_counter_alloc_stats(struct rdma_counter *counter)
 300 {
 301         struct mlx5_ib_dev *dev = to_mdev(counter->device);
 302         const struct mlx5_ib_counters *cnts =
 303                 get_counters(dev, counter->port - 1);
 304
 305         return rdma_alloc_hw_stats_struct(cnts->names,
 306                                           cnts->num_q_counters +
 307                                           cnts->num_cong_counters +
 308                                           cnts->num_ext_ppcnt_counters,
 309                                           RDMA_HW_STATS_DEFAULT_LIFESPAN);
 310 }
 311
 312 static int mlx5_ib_counter_update_stats(struct rdma_counter *counter)
 313 {
 314         struct mlx5_ib_dev *dev = to_mdev(counter->device);
 315         const struct mlx5_ib_counters *cnts =
 316                 get_counters(dev, counter->port - 1);
 317
 318         return mlx5_ib_query_q_counters(dev->mdev, cnts,
 319                                         counter->stats, counter->id);
 320 }
 321
 322 static int mlx5_ib_counter_dealloc(struct rdma_counter *counter)
 323 {
 324         struct mlx5_ib_dev *dev = to_mdev(counter->device);
 325         u32 in[MLX5_ST_SZ_DW(dealloc_q_counter_in)] = {};
 326
 327         if (!counter->id)
 328                 return 0;
 329
 330         MLX5_SET(dealloc_q_counter_in, in, opcode,
 331                  MLX5_CMD_OP_DEALLOC_Q_COUNTER);
 332         MLX5_SET(dealloc_q_counter_in, in, counter_set_id, counter->id);
 333         return mlx5_cmd_exec_in(dev->mdev, dealloc_q_counter, in);
 334 }
 335
 336 static int mlx5_ib_counter_bind_qp(struct rdma_counter *counter,
 337                                    struct ib_qp *qp)
 338 {
 339         struct mlx5_ib_dev *dev = to_mdev(qp->device);
 340         int err;
 341
 342         if (!counter->id) {
 343                 u32 out[MLX5_ST_SZ_DW(alloc_q_counter_out)] = {};
 344                 u32 in[MLX5_ST_SZ_DW(alloc_q_counter_in)] = {};
 345
 346                 MLX5_SET(alloc_q_counter_in, in, opcode,
 347                          MLX5_CMD_OP_ALLOC_Q_COUNTER);
 348                 MLX5_SET(alloc_q_counter_in, in, uid, MLX5_SHARED_RESOURCE_UID);
 349                 err = mlx5_cmd_exec_inout(dev->mdev, alloc_q_counter, in, out);
 350                 if (err)
 351                         return err;
 352                 counter->id =
 353                         MLX5_GET(alloc_q_counter_out, out, counter_set_id);
 354         }
 355
 356         err = mlx5_ib_qp_set_counter(qp, counter);
 357         if (err)
 358                 goto fail_set_counter;
 359
 360         return 0;
 361
 362 fail_set_counter:
 363         mlx5_ib_counter_dealloc(counter);
 364         counter->id = 0;
 365
 366         return err;
 367 }
 368
 369 static int mlx5_ib_counter_unbind_qp(struct ib_qp *qp)
 370 {
 371         return mlx5_ib_qp_set_counter(qp, NULL);
 372 }
 373
 374
 375 static void mlx5_ib_fill_counters(struct mlx5_ib_dev *dev,
 376                                   const char **names,
 377                                   size_t *offsets)
 378 {
 379         int i;
 380         int j = 0;
 381
 382         for (i = 0; i < ARRAY_SIZE(basic_q_cnts); i++, j++) {
 383                 names[j] = basic_q_cnts[i].name;
 384                 offsets[j] = basic_q_cnts[i].offset;
 385         }
 386
 387         if (MLX5_CAP_GEN(dev->mdev, out_of_seq_cnt)) {
 388                 for (i = 0; i < ARRAY_SIZE(out_of_seq_q_cnts); i++, j++) {
 389                         names[j] = out_of_seq_q_cnts[i].name;
 390                         offsets[j] = out_of_seq_q_cnts[i].offset;
 391                 }
 392         }
 393
 394         if (MLX5_CAP_GEN(dev->mdev, retransmission_q_counters)) {
 395                 for (i = 0; i < ARRAY_SIZE(retrans_q_cnts); i++, j++) {
 396                         names[j] = retrans_q_cnts[i].name;
 397                         offsets[j] = retrans_q_cnts[i].offset;
 398                 }
 399         }
 400
 401         if (MLX5_CAP_GEN(dev->mdev, enhanced_error_q_counters)) {
 402                 for (i = 0; i < ARRAY_SIZE(extended_err_cnts); i++, j++) {
 403                         names[j] = extended_err_cnts[i].name;
 404                         offsets[j] = extended_err_cnts[i].offset;
 405                 }
 406         }
 407
 408         if (MLX5_CAP_GEN(dev->mdev, roce_accl)) {
 409                 for (i = 0; i < ARRAY_SIZE(roce_accl_cnts); i++, j++) {
 410                         names[j] = roce_accl_cnts[i].name;
 411                         offsets[j] = roce_accl_cnts[i].offset;
 412                 }
 413         }
 414
 415         if (MLX5_CAP_GEN(dev->mdev, cc_query_allowed)) {
 416                 for (i = 0; i < ARRAY_SIZE(cong_cnts); i++, j++) {
 417                         names[j] = cong_cnts[i].name;
 418                         offsets[j] = cong_cnts[i].offset;
 419                 }
 420         }
 421
 422         if (MLX5_CAP_PCAM_FEATURE(dev->mdev, rx_icrc_encapsulated_counter)) {
 423                 for (i = 0; i < ARRAY_SIZE(ext_ppcnt_cnts); i++, j++) {
 424                         names[j] = ext_ppcnt_cnts[i].name;
 425                         offsets[j] = ext_ppcnt_cnts[i].offset;
 426                 }
 427         }
 428 }
 429
 430
 431 static int __mlx5_ib_alloc_counters(struct mlx5_ib_dev *dev,
 432                                     struct mlx5_ib_counters *cnts)
 433 {
 434         u32 num_counters;
 435
 436         num_counters = ARRAY_SIZE(basic_q_cnts);
 437
 438         if (MLX5_CAP_GEN(dev->mdev, out_of_seq_cnt))
 439                 num_counters += ARRAY_SIZE(out_of_seq_q_cnts);
 440
 441         if (MLX5_CAP_GEN(dev->mdev, retransmission_q_counters))
 442                 num_counters += ARRAY_SIZE(retrans_q_cnts);
 443
 444         if (MLX5_CAP_GEN(dev->mdev, enhanced_error_q_counters))
 445                 num_counters += ARRAY_SIZE(extended_err_cnts);
 446
 447         if (MLX5_CAP_GEN(dev->mdev, roce_accl))
 448                 num_counters += ARRAY_SIZE(roce_accl_cnts);
 449
 450         cnts->num_q_counters = num_counters;
 451
 452         if (MLX5_CAP_GEN(dev->mdev, cc_query_allowed)) {
 453                 cnts->num_cong_counters = ARRAY_SIZE(cong_cnts);
 454                 num_counters += ARRAY_SIZE(cong_cnts);
 455         }
 456         if (MLX5_CAP_PCAM_FEATURE(dev->mdev, rx_icrc_encapsulated_counter)) {
 457                 cnts->num_ext_ppcnt_counters = ARRAY_SIZE(ext_ppcnt_cnts);
 458                 num_counters += ARRAY_SIZE(ext_ppcnt_cnts);
 459         }
 460         cnts->names = kcalloc(num_counters, sizeof(*cnts->names), GFP_KERNEL);
 461         if (!cnts->names)
 462                 return -ENOMEM;
 463
 464         cnts->offsets = kcalloc(num_counters,
 465                                 sizeof(*cnts->offsets), GFP_KERNEL);
 466         if (!cnts->offsets)
 467                 goto err_names;
 468
 469         return 0;
 470
 471 err_names:
 472         kfree(cnts->names);
 473         cnts->names = NULL;
 474         return -ENOMEM;
 475 }
 476
 477 static void mlx5_ib_dealloc_counters(struct mlx5_ib_dev *dev)
 478 {
 479         u32 in[MLX5_ST_SZ_DW(dealloc_q_counter_in)] = {};
 480         int num_cnt_ports;
 481         int i;
 482
 483         num_cnt_ports = is_mdev_switchdev_mode(dev->mdev) ? 1 : dev->num_ports;
 484
 485         MLX5_SET(dealloc_q_counter_in, in, opcode,
 486                  MLX5_CMD_OP_DEALLOC_Q_COUNTER);
 487
 488         for (i = 0; i < num_cnt_ports; i++) {
 489                 if (dev->port[i].cnts.set_id) {
 490                         MLX5_SET(dealloc_q_counter_in, in, counter_set_id,
 491                                  dev->port[i].cnts.set_id);
 492                         mlx5_cmd_exec_in(dev->mdev, dealloc_q_counter, in);
 493                 }
 494                 kfree(dev->port[i].cnts.names);
 495                 kfree(dev->port[i].cnts.offsets);
 496         }
 497 }
 498
 499 static int mlx5_ib_alloc_counters(struct mlx5_ib_dev *dev)
 500 {
 501         u32 out[MLX5_ST_SZ_DW(alloc_q_counter_out)] = {};
 502         u32 in[MLX5_ST_SZ_DW(alloc_q_counter_in)] = {};
 503         int num_cnt_ports;
 504         int err = 0;
 505         int i;
 506         bool is_shared;
 507
 508         MLX5_SET(alloc_q_counter_in, in, opcode, MLX5_CMD_OP_ALLOC_Q_COUNTER);
 509         is_shared = MLX5_CAP_GEN(dev->mdev, log_max_uctx) != 0;
 510         num_cnt_ports = is_mdev_switchdev_mode(dev->mdev) ? 1 : dev->num_ports;
 511
 512         for (i = 0; i < num_cnt_ports; i++) {
 513                 err = __mlx5_ib_alloc_counters(dev, &dev->port[i].cnts);
 514                 if (err)
 515                         goto err_alloc;
 516
 517                 mlx5_ib_fill_counters(dev, dev->port[i].cnts.names,
 518                                       dev->port[i].cnts.offsets);
 519
 520                 MLX5_SET(alloc_q_counter_in, in, uid,
 521                          is_shared ? MLX5_SHARED_RESOURCE_UID : 0);
 522
 523                 err = mlx5_cmd_exec_inout(dev->mdev, alloc_q_counter, in, out);
 524                 if (err) {
 525                         mlx5_ib_warn(dev,
 526                                      "couldn't allocate queue counter for port %d, err %d\n",
 527                                      i + 1, err);
 528                         goto err_alloc;
 529                 }
 530
 531                 dev->port[i].cnts.set_id =
 532                         MLX5_GET(alloc_q_counter_out, out, counter_set_id);
 533         }
 534         return 0;
 535
 536 err_alloc:
 537         mlx5_ib_dealloc_counters(dev);
 538         return err;
 539 }
 540
 541 static int read_flow_counters(struct ib_device *ibdev,
 542                               struct mlx5_read_counters_attr *read_attr)
 543 {
 544         struct mlx5_fc *fc = read_attr->hw_cntrs_hndl;
 545         struct mlx5_ib_dev *dev = to_mdev(ibdev);
 546
 547         return mlx5_fc_query(dev->mdev, fc,
 548                              &read_attr->out[IB_COUNTER_PACKETS],
 549                              &read_attr->out[IB_COUNTER_BYTES]);
 550 }
 551
 552 /* flow counters currently expose two counters packets and bytes */
 553 #define FLOW_COUNTERS_NUM 2
 554 static int counters_set_description(
 555         struct ib_counters *counters, enum mlx5_ib_counters_type counters_type,
 556         struct mlx5_ib_flow_counters_desc *desc_data, u32 ncounters)
 557 {
 558         struct mlx5_ib_mcounters *mcounters = to_mcounters(counters);
 559         u32 cntrs_max_index = 0;
 560         int i;
 561
 562         if (counters_type != MLX5_IB_COUNTERS_FLOW)
 563                 return -EINVAL;
 564
 565         /* init the fields for the object */
 566         mcounters->type = counters_type;
 567         mcounters->read_counters = read_flow_counters;
 568         mcounters->counters_num = FLOW_COUNTERS_NUM;
 569         mcounters->ncounters = ncounters;
 570         /* each counter entry have both description and index pair */
 571         for (i = 0; i < ncounters; i++) {
 572                 if (desc_data[i].description > IB_COUNTER_BYTES)
 573                         return -EINVAL;
 574
 575                 if (cntrs_max_index <= desc_data[i].index)
 576                         cntrs_max_index = desc_data[i].index + 1;
 577         }
 578
 579         mutex_lock(&mcounters->mcntrs_mutex);
 580         mcounters->counters_data = desc_data;
 581         mcounters->cntrs_max_index = cntrs_max_index;
 582         mutex_unlock(&mcounters->mcntrs_mutex);
 583
 584         return 0;
 585 }
 586
 587 #define MAX_COUNTERS_NUM (USHRT_MAX / (sizeof(u32) * 2))
 588 int mlx5_ib_flow_counters_set_data(struct ib_counters *ibcounters,
 589                                    struct mlx5_ib_create_flow *ucmd)
 590 {
 591         struct mlx5_ib_mcounters *mcounters = to_mcounters(ibcounters);
 592         struct mlx5_ib_flow_counters_data *cntrs_data = NULL;
 593         struct mlx5_ib_flow_counters_desc *desc_data = NULL;
 594         bool hw_hndl = false;
 595         int ret = 0;
 596
 597         if (ucmd && ucmd->ncounters_data != 0) {
 598                 cntrs_data = ucmd->data;
 599                 if (cntrs_data->ncounters > MAX_COUNTERS_NUM)
 600                         return -EINVAL;
 601
 602                 desc_data = kcalloc(cntrs_data->ncounters,
 603                                     sizeof(*desc_data),
 604                                     GFP_KERNEL);
 605                 if (!desc_data)
 606                         return  -ENOMEM;
 607
 608                 if (copy_from_user(desc_data,
 609                                    u64_to_user_ptr(cntrs_data->counters_data),
 610                                    sizeof(*desc_data) * cntrs_data->ncounters)) {
 611                         ret = -EFAULT;
 612                         goto free;
 613                 }
 614         }
 615
 616         if (!mcounters->hw_cntrs_hndl) {
 617                 mcounters->hw_cntrs_hndl = mlx5_fc_create(
 618                         to_mdev(ibcounters->device)->mdev, false);
 619                 if (IS_ERR(mcounters->hw_cntrs_hndl)) {
 620                         ret = PTR_ERR(mcounters->hw_cntrs_hndl);
 621                         goto free;
 622                 }
 623                 hw_hndl = true;
 624         }
 625
 626         if (desc_data) {
 627                 /* counters already bound to at least one flow */
 628                 if (mcounters->cntrs_max_index) {
 629                         ret = -EINVAL;
 630                         goto free_hndl;
 631                 }
 632
 633                 ret = counters_set_description(ibcounters,
 634                                                MLX5_IB_COUNTERS_FLOW,
 635                                                desc_data,
 636                                                cntrs_data->ncounters);
 637                 if (ret)
 638                         goto free_hndl;
 639
 640         } else if (!mcounters->cntrs_max_index) {
 641                 /* counters not bound yet, must have udata passed */
 642                 ret = -EINVAL;
 643                 goto free_hndl;
 644         }
 645
 646         return 0;
 647
 648 free_hndl:
 649         if (hw_hndl) {
 650                 mlx5_fc_destroy(to_mdev(ibcounters->device)->mdev,
 651                                 mcounters->hw_cntrs_hndl);
 652                 mcounters->hw_cntrs_hndl = NULL;
 653         }
 654 free:
 655         kfree(desc_data);
 656         return ret;
 657 }
 658
 659 void mlx5_ib_counters_clear_description(struct ib_counters *counters)
 660 {
 661         struct mlx5_ib_mcounters *mcounters;
 662
 663         if (!counters || atomic_read(&counters->usecnt) != 1)
 664                 return;
 665
 666         mcounters = to_mcounters(counters);
 667
 668         mutex_lock(&mcounters->mcntrs_mutex);
 669         kfree(mcounters->counters_data);
 670         mcounters->counters_data = NULL;
 671         mcounters->cntrs_max_index = 0;
 672         mutex_unlock(&mcounters->mcntrs_mutex);
 673 }
 674
 675 static const struct ib_device_ops hw_stats_ops = {
 676         .alloc_hw_port_stats = mlx5_ib_alloc_hw_port_stats,
 677         .get_hw_stats = mlx5_ib_get_hw_stats,
 678         .counter_bind_qp = mlx5_ib_counter_bind_qp,
 679         .counter_unbind_qp = mlx5_ib_counter_unbind_qp,
 680         .counter_dealloc = mlx5_ib_counter_dealloc,
 681         .counter_alloc_stats = mlx5_ib_counter_alloc_stats,
 682         .counter_update_stats = mlx5_ib_counter_update_stats,
 683 };
 684
 685 static const struct ib_device_ops hw_switchdev_stats_ops = {
 686         .alloc_hw_device_stats = mlx5_ib_alloc_hw_device_stats,
 687         .get_hw_stats = mlx5_ib_get_hw_stats,
 688         .counter_bind_qp = mlx5_ib_counter_bind_qp,
 689         .counter_unbind_qp = mlx5_ib_counter_unbind_qp,
 690         .counter_dealloc = mlx5_ib_counter_dealloc,
 691         .counter_alloc_stats = mlx5_ib_counter_alloc_stats,
 692         .counter_update_stats = mlx5_ib_counter_update_stats,
 693 };
 694
 695 static const struct ib_device_ops counters_ops = {
 696         .create_counters = mlx5_ib_create_counters,
 697         .destroy_counters = mlx5_ib_destroy_counters,
 698         .read_counters = mlx5_ib_read_counters,
 699
 700         INIT_RDMA_OBJ_SIZE(ib_counters, mlx5_ib_mcounters, ibcntrs),
 701 };
 702
 703 int mlx5_ib_counters_init(struct mlx5_ib_dev *dev)
 704 {
 705         ib_set_device_ops(&dev->ib_dev, &counters_ops);
 706
 707         if (!MLX5_CAP_GEN(dev->mdev, max_qp_cnt))
 708                 return 0;
 709
 710         if (is_mdev_switchdev_mode(dev->mdev))
 711                 ib_set_device_ops(&dev->ib_dev, &hw_switchdev_stats_ops);
 712         else
 713                 ib_set_device_ops(&dev->ib_dev, &hw_stats_ops);
 714         return mlx5_ib_alloc_counters(dev);
 715 }
 716
 717 void mlx5_ib_counters_cleanup(struct mlx5_ib_dev *dev)
 718 {
 719         if (!MLX5_CAP_GEN(dev->mdev, max_qp_cnt))
 720                 return;
 721
 722         mlx5_ib_dealloc_counters(dev);
 723 }