drivers/net/ethernet/marvell/octeontx2/nic/otx2_tc.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Marvell OcteonTx2 RVU Physcial Function ethernet driver
   3  *
   4  * Copyright (C) 2021 Marvell.
   5  */
   6 #include <linux/netdevice.h>
   7 #include <linux/etherdevice.h>
   8 #include <linux/inetdevice.h>
   9 #include <linux/rhashtable.h>
  10 #include <linux/bitfield.h>
  11 #include <net/flow_dissector.h>
  12 #include <net/pkt_cls.h>
  13 #include <net/tc_act/tc_gact.h>
  14 #include <net/tc_act/tc_mirred.h>
  15 #include <net/tc_act/tc_vlan.h>
  16 #include <net/ipv6.h>
  17
  18 #include "cn10k.h"
  19 #include "otx2_common.h"
  20
  21 /* Egress rate limiting definitions */
  22 #define MAX_BURST_EXPONENT              0x0FULL
  23 #define MAX_BURST_MANTISSA              0xFFULL
  24 #define MAX_BURST_SIZE                  130816ULL
  25 #define MAX_RATE_DIVIDER_EXPONENT       12ULL
  26 #define MAX_RATE_EXPONENT               0x0FULL
  27 #define MAX_RATE_MANTISSA               0xFFULL
  28
  29 /* Bitfields in NIX_TLX_PIR register */
  30 #define TLX_RATE_MANTISSA               GENMASK_ULL(8, 1)
  31 #define TLX_RATE_EXPONENT               GENMASK_ULL(12, 9)
  32 #define TLX_RATE_DIVIDER_EXPONENT       GENMASK_ULL(16, 13)
  33 #define TLX_BURST_MANTISSA              GENMASK_ULL(36, 29)
  34 #define TLX_BURST_EXPONENT              GENMASK_ULL(40, 37)
  35
  36 struct otx2_tc_flow_stats {
  37         u64 bytes;
  38         u64 pkts;
  39         u64 used;
  40 };
  41
  42 struct otx2_tc_flow {
  43         struct rhash_head               node;
  44         unsigned long                   cookie;
  45         unsigned int                    bitpos;
  46         struct rcu_head                 rcu;
  47         struct otx2_tc_flow_stats       stats;
  48         spinlock_t                      lock; /* lock for stats */
  49         u16                             rq;
  50         u16                             entry;
  51         u16                             leaf_profile;
  52         bool                            is_act_police;
  53 };
  54
  55 static void otx2_get_egress_burst_cfg(u32 burst, u32 *burst_exp,
  56                                       u32 *burst_mantissa)
  57 {
  58         unsigned int tmp;
  59
  60         /* Burst is calculated as
  61          * ((256 + BURST_MANTISSA) << (1 + BURST_EXPONENT)) / 256
  62          * Max supported burst size is 130,816 bytes.
  63          */
  64         burst = min_t(u32, burst, MAX_BURST_SIZE);
  65         if (burst) {
  66                 *burst_exp = ilog2(burst) ? ilog2(burst) - 1 : 0;
  67                 tmp = burst - rounddown_pow_of_two(burst);
  68                 if (burst < MAX_BURST_MANTISSA)
  69                         *burst_mantissa = tmp * 2;
  70                 else
  71                         *burst_mantissa = tmp / (1ULL << (*burst_exp - 7));
  72         } else {
  73                 *burst_exp = MAX_BURST_EXPONENT;
  74                 *burst_mantissa = MAX_BURST_MANTISSA;
  75         }
  76 }
  77
  78 static void otx2_get_egress_rate_cfg(u32 maxrate, u32 *exp,
  79                                      u32 *mantissa, u32 *div_exp)
  80 {
  81         unsigned int tmp;
  82
  83         /* Rate calculation by hardware
  84          *
  85          * PIR_ADD = ((256 + mantissa) << exp) / 256
  86          * rate = (2 * PIR_ADD) / ( 1 << div_exp)
  87          * The resultant rate is in Mbps.
  88          */
  89
  90         /* 2Mbps to 100Gbps can be expressed with div_exp = 0.
  91          * Setting this to '0' will ease the calculation of
  92          * exponent and mantissa.
  93          */
  94         *div_exp = 0;
  95
  96         if (maxrate) {
  97                 *exp = ilog2(maxrate) ? ilog2(maxrate) - 1 : 0;
  98                 tmp = maxrate - rounddown_pow_of_two(maxrate);
  99                 if (maxrate < MAX_RATE_MANTISSA)
 100                         *mantissa = tmp * 2;
 101                 else
 102                         *mantissa = tmp / (1ULL << (*exp - 7));
 103         } else {
 104                 /* Instead of disabling rate limiting, set all values to max */
 105                 *exp = MAX_RATE_EXPONENT;
 106                 *mantissa = MAX_RATE_MANTISSA;
 107         }
 108 }
 109
 110 static int otx2_set_matchall_egress_rate(struct otx2_nic *nic, u32 burst, u32 maxrate)
 111 {
 112         struct otx2_hw *hw = &nic->hw;
 113         struct nix_txschq_config *req;
 114         u32 burst_exp, burst_mantissa;
 115         u32 exp, mantissa, div_exp;
 116         int txschq, err;
 117
 118         /* All SQs share the same TL4, so pick the first scheduler */
 119         txschq = hw->txschq_list[NIX_TXSCH_LVL_TL4][0];
 120
 121         /* Get exponent and mantissa values from the desired rate */
 122         otx2_get_egress_burst_cfg(burst, &burst_exp, &burst_mantissa);
 123         otx2_get_egress_rate_cfg(maxrate, &exp, &mantissa, &div_exp);
 124
 125         mutex_lock(&nic->mbox.lock);
 126         req = otx2_mbox_alloc_msg_nix_txschq_cfg(&nic->mbox);
 127         if (!req) {
 128                 mutex_unlock(&nic->mbox.lock);
 129                 return -ENOMEM;
 130         }
 131
 132         req->lvl = NIX_TXSCH_LVL_TL4;
 133         req->num_regs = 1;
 134         req->reg[0] = NIX_AF_TL4X_PIR(txschq);
 135         req->regval[0] = FIELD_PREP(TLX_BURST_EXPONENT, burst_exp) |
 136                          FIELD_PREP(TLX_BURST_MANTISSA, burst_mantissa) |
 137                          FIELD_PREP(TLX_RATE_DIVIDER_EXPONENT, div_exp) |
 138                          FIELD_PREP(TLX_RATE_EXPONENT, exp) |
 139                          FIELD_PREP(TLX_RATE_MANTISSA, mantissa) | BIT_ULL(0);
 140
 141         err = otx2_sync_mbox_msg(&nic->mbox);
 142         mutex_unlock(&nic->mbox.lock);
 143         return err;
 144 }
 145
 146 static int otx2_tc_validate_flow(struct otx2_nic *nic,
 147                                  struct flow_action *actions,
 148                                  struct netlink_ext_ack *extack)
 149 {
 150         if (nic->flags & OTX2_FLAG_INTF_DOWN) {
 151                 NL_SET_ERR_MSG_MOD(extack, "Interface not initialized");
 152                 return -EINVAL;
 153         }
 154
 155         if (!flow_action_has_entries(actions)) {
 156                 NL_SET_ERR_MSG_MOD(extack, "MATCHALL offload called with no action");
 157                 return -EINVAL;
 158         }
 159
 160         if (!flow_offload_has_one_action(actions)) {
 161                 NL_SET_ERR_MSG_MOD(extack,
 162                                    "Egress MATCHALL offload supports only 1 policing action");
 163                 return -EINVAL;
 164         }
 165         return 0;
 166 }
 167
 168 static int otx2_tc_egress_matchall_install(struct otx2_nic *nic,
 169                                            struct tc_cls_matchall_offload *cls)
 170 {
 171         struct netlink_ext_ack *extack = cls->common.extack;
 172         struct flow_action *actions = &cls->rule->action;
 173         struct flow_action_entry *entry;
 174         u32 rate;
 175         int err;
 176
 177         err = otx2_tc_validate_flow(nic, actions, extack);
 178         if (err)
 179                 return err;
 180
 181         if (nic->flags & OTX2_FLAG_TC_MATCHALL_EGRESS_ENABLED) {
 182                 NL_SET_ERR_MSG_MOD(extack,
 183                                    "Only one Egress MATCHALL ratelimiter can be offloaded");
 184                 return -ENOMEM;
 185         }
 186
 187         entry = &cls->rule->action.entries[0];
 188         switch (entry->id) {
 189         case FLOW_ACTION_POLICE:
 190                 if (entry->police.rate_pkt_ps) {
 191                         NL_SET_ERR_MSG_MOD(extack, "QoS offload not support packets per second");
 192                         return -EOPNOTSUPP;
 193                 }
 194                 /* Convert bytes per second to Mbps */
 195                 rate = entry->police.rate_bytes_ps * 8;
 196                 rate = max_t(u32, rate / 1000000, 1);
 197                 err = otx2_set_matchall_egress_rate(nic, entry->police.burst, rate);
 198                 if (err)
 199                         return err;
 200                 nic->flags |= OTX2_FLAG_TC_MATCHALL_EGRESS_ENABLED;
 201                 break;
 202         default:
 203                 NL_SET_ERR_MSG_MOD(extack,
 204                                    "Only police action is supported with Egress MATCHALL offload");
 205                 return -EOPNOTSUPP;
 206         }
 207
 208         return 0;
 209 }
 210
 211 static int otx2_tc_egress_matchall_delete(struct otx2_nic *nic,
 212                                           struct tc_cls_matchall_offload *cls)
 213 {
 214         struct netlink_ext_ack *extack = cls->common.extack;
 215         int err;
 216
 217         if (nic->flags & OTX2_FLAG_INTF_DOWN) {
 218                 NL_SET_ERR_MSG_MOD(extack, "Interface not initialized");
 219                 return -EINVAL;
 220         }
 221
 222         err = otx2_set_matchall_egress_rate(nic, 0, 0);
 223         nic->flags &= ~OTX2_FLAG_TC_MATCHALL_EGRESS_ENABLED;
 224         return err;
 225 }
 226
 227 static int otx2_tc_act_set_police(struct otx2_nic *nic,
 228                                   struct otx2_tc_flow *node,
 229                                   struct flow_cls_offload *f,
 230                                   u64 rate, u32 burst, u32 mark,
 231                                   struct npc_install_flow_req *req, bool pps)
 232 {
 233         struct netlink_ext_ack *extack = f->common.extack;
 234         struct otx2_hw *hw = &nic->hw;
 235         int rq_idx, rc;
 236
 237         rq_idx = find_first_zero_bit(&nic->rq_bmap, hw->rx_queues);
 238         if (rq_idx >= hw->rx_queues) {
 239                 NL_SET_ERR_MSG_MOD(extack, "Police action rules exceeded");
 240                 return -EINVAL;
 241         }
 242
 243         mutex_lock(&nic->mbox.lock);
 244
 245         rc = cn10k_alloc_leaf_profile(nic, &node->leaf_profile);
 246         if (rc) {
 247                 mutex_unlock(&nic->mbox.lock);
 248                 return rc;
 249         }
 250
 251         rc = cn10k_set_ipolicer_rate(nic, node->leaf_profile, burst, rate, pps);
 252         if (rc)
 253                 goto free_leaf;
 254
 255         rc = cn10k_map_unmap_rq_policer(nic, rq_idx, node->leaf_profile, true);
 256         if (rc)
 257                 goto free_leaf;
 258
 259         mutex_unlock(&nic->mbox.lock);
 260
 261         req->match_id = mark & 0xFFFFULL;
 262         req->index = rq_idx;
 263         req->op = NIX_RX_ACTIONOP_UCAST;
 264         set_bit(rq_idx, &nic->rq_bmap);
 265         node->is_act_police = true;
 266         node->rq = rq_idx;
 267
 268         return 0;
 269
 270 free_leaf:
 271         if (cn10k_free_leaf_profile(nic, node->leaf_profile))
 272                 netdev_err(nic->netdev,
 273                            "Unable to free leaf bandwidth profile(%d)\n",
 274                            node->leaf_profile);
 275         mutex_unlock(&nic->mbox.lock);
 276         return rc;
 277 }
 278
 279 static int otx2_tc_parse_actions(struct otx2_nic *nic,
 280                                  struct flow_action *flow_action,
 281                                  struct npc_install_flow_req *req,
 282                                  struct flow_cls_offload *f,
 283                                  struct otx2_tc_flow *node)
 284 {
 285         struct netlink_ext_ack *extack = f->common.extack;
 286         struct flow_action_entry *act;
 287         struct net_device *target;
 288         struct otx2_nic *priv;
 289         u32 burst, mark = 0;
 290         u8 nr_police = 0;
 291         bool pps = false;
 292         u64 rate;
 293         int i;
 294
 295         if (!flow_action_has_entries(flow_action)) {
 296                 NL_SET_ERR_MSG_MOD(extack, "no tc actions specified");
 297                 return -EINVAL;
 298         }
 299
 300         flow_action_for_each(i, act, flow_action) {
 301                 switch (act->id) {
 302                 case FLOW_ACTION_DROP:
 303                         req->op = NIX_RX_ACTIONOP_DROP;
 304                         return 0;
 305                 case FLOW_ACTION_ACCEPT:
 306                         req->op = NIX_RX_ACTION_DEFAULT;
 307                         return 0;
 308                 case FLOW_ACTION_REDIRECT_INGRESS:
 309                         target = act->dev;
 310                         priv = netdev_priv(target);
 311                         /* npc_install_flow_req doesn't support passing a target pcifunc */
 312                         if (rvu_get_pf(nic->pcifunc) != rvu_get_pf(priv->pcifunc)) {
 313                                 NL_SET_ERR_MSG_MOD(extack,
 314                                                    "can't redirect to other pf/vf");
 315                                 return -EOPNOTSUPP;
 316                         }
 317                         req->vf = priv->pcifunc & RVU_PFVF_FUNC_MASK;
 318                         req->op = NIX_RX_ACTION_DEFAULT;
 319                         return 0;
 320                 case FLOW_ACTION_VLAN_POP:
 321                         req->vtag0_valid = true;
 322                         /* use RX_VTAG_TYPE7 which is initialized to strip vlan tag */
 323                         req->vtag0_type = NIX_AF_LFX_RX_VTAG_TYPE7;
 324                         break;
 325                 case FLOW_ACTION_POLICE:
 326                         /* Ingress ratelimiting is not supported on OcteonTx2 */
 327                         if (is_dev_otx2(nic->pdev)) {
 328                                 NL_SET_ERR_MSG_MOD(extack,
 329                                         "Ingress policing not supported on this platform");
 330                                 return -EOPNOTSUPP;
 331                         }
 332
 333                         if (act->police.rate_bytes_ps > 0) {
 334                                 rate = act->police.rate_bytes_ps * 8;
 335                                 burst = act->police.burst;
 336                         } else if (act->police.rate_pkt_ps > 0) {
 337                                 /* The algorithm used to calculate rate
 338                                  * mantissa, exponent values for a given token
 339                                  * rate (token can be byte or packet) requires
 340                                  * token rate to be mutiplied by 8.
 341                                  */
 342                                 rate = act->police.rate_pkt_ps * 8;
 343                                 burst = act->police.burst_pkt;
 344                                 pps = true;
 345                         }
 346                         nr_police++;
 347                         break;
 348                 case FLOW_ACTION_MARK:
 349                         mark = act->mark;
 350                         break;
 351                 default:
 352                         return -EOPNOTSUPP;
 353                 }
 354         }
 355
 356         if (nr_police > 1) {
 357                 NL_SET_ERR_MSG_MOD(extack,
 358                                    "rate limit police offload requires a single action");
 359                 return -EOPNOTSUPP;
 360         }
 361
 362         if (nr_police)
 363                 return otx2_tc_act_set_police(nic, node, f, rate, burst,
 364                                               mark, req, pps);
 365
 366         return 0;
 367 }
 368
 369 static int otx2_tc_prepare_flow(struct otx2_nic *nic, struct otx2_tc_flow *node,
 370                                 struct flow_cls_offload *f,
 371                                 struct npc_install_flow_req *req)
 372 {
 373         struct netlink_ext_ack *extack = f->common.extack;
 374         struct flow_msg *flow_spec = &req->packet;
 375         struct flow_msg *flow_mask = &req->mask;
 376         struct flow_dissector *dissector;
 377         struct flow_rule *rule;
 378         u8 ip_proto = 0;
 379
 380         rule = flow_cls_offload_flow_rule(f);
 381         dissector = rule->match.dissector;
 382
 383         if ((dissector->used_keys &
 384             ~(BIT(FLOW_DISSECTOR_KEY_CONTROL) |
 385               BIT(FLOW_DISSECTOR_KEY_BASIC) |
 386               BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS) |
 387               BIT(FLOW_DISSECTOR_KEY_VLAN) |
 388               BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS) |
 389               BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS) |
 390               BIT(FLOW_DISSECTOR_KEY_PORTS) |
 391               BIT(FLOW_DISSECTOR_KEY_IP))))  {
 392                 netdev_info(nic->netdev, "unsupported flow used key 0x%x",
 393                             dissector->used_keys);
 394                 return -EOPNOTSUPP;
 395         }
 396
 397         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_BASIC)) {
 398                 struct flow_match_basic match;
 399
 400                 flow_rule_match_basic(rule, &match);
 401
 402                 /* All EtherTypes can be matched, no hw limitation */
 403                 flow_spec->etype = match.key->n_proto;
 404                 flow_mask->etype = match.mask->n_proto;
 405                 req->features |= BIT_ULL(NPC_ETYPE);
 406
 407                 if (match.mask->ip_proto &&
 408                     (match.key->ip_proto != IPPROTO_TCP &&
 409                      match.key->ip_proto != IPPROTO_UDP &&
 410                      match.key->ip_proto != IPPROTO_SCTP &&
 411                      match.key->ip_proto != IPPROTO_ICMP &&
 412                      match.key->ip_proto != IPPROTO_ICMPV6)) {
 413                         netdev_info(nic->netdev,
 414                                     "ip_proto=0x%x not supported\n",
 415                                     match.key->ip_proto);
 416                         return -EOPNOTSUPP;
 417                 }
 418                 if (match.mask->ip_proto)
 419                         ip_proto = match.key->ip_proto;
 420
 421                 if (ip_proto == IPPROTO_UDP)
 422                         req->features |= BIT_ULL(NPC_IPPROTO_UDP);
 423                 else if (ip_proto == IPPROTO_TCP)
 424                         req->features |= BIT_ULL(NPC_IPPROTO_TCP);
 425                 else if (ip_proto == IPPROTO_SCTP)
 426                         req->features |= BIT_ULL(NPC_IPPROTO_SCTP);
 427                 else if (ip_proto == IPPROTO_ICMP)
 428                         req->features |= BIT_ULL(NPC_IPPROTO_ICMP);
 429                 else if (ip_proto == IPPROTO_ICMPV6)
 430                         req->features |= BIT_ULL(NPC_IPPROTO_ICMP6);
 431         }
 432
 433         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS)) {
 434                 struct flow_match_eth_addrs match;
 435
 436                 flow_rule_match_eth_addrs(rule, &match);
 437                 if (!is_zero_ether_addr(match.mask->src)) {
 438                         NL_SET_ERR_MSG_MOD(extack, "src mac match not supported");
 439                         return -EOPNOTSUPP;
 440                 }
 441
 442                 if (!is_zero_ether_addr(match.mask->dst)) {
 443                         ether_addr_copy(flow_spec->dmac, (u8 *)&match.key->dst);
 444                         ether_addr_copy(flow_mask->dmac,
 445                                         (u8 *)&match.mask->dst);
 446                         req->features |= BIT_ULL(NPC_DMAC);
 447                 }
 448         }
 449
 450         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_IP)) {
 451                 struct flow_match_ip match;
 452
 453                 flow_rule_match_ip(rule, &match);
 454                 if ((ntohs(flow_spec->etype) != ETH_P_IP) &&
 455                     match.mask->tos) {
 456                         NL_SET_ERR_MSG_MOD(extack, "tos not supported");
 457                         return -EOPNOTSUPP;
 458                 }
 459                 if (match.mask->ttl) {
 460                         NL_SET_ERR_MSG_MOD(extack, "ttl not supported");
 461                         return -EOPNOTSUPP;
 462                 }
 463                 flow_spec->tos = match.key->tos;
 464                 flow_mask->tos = match.mask->tos;
 465                 req->features |= BIT_ULL(NPC_TOS);
 466         }
 467
 468         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_VLAN)) {
 469                 struct flow_match_vlan match;
 470                 u16 vlan_tci, vlan_tci_mask;
 471
 472                 flow_rule_match_vlan(rule, &match);
 473
 474                 if (ntohs(match.key->vlan_tpid) != ETH_P_8021Q) {
 475                         netdev_err(nic->netdev, "vlan tpid 0x%x not supported\n",
 476                                    ntohs(match.key->vlan_tpid));
 477                         return -EOPNOTSUPP;
 478                 }
 479
 480                 if (match.mask->vlan_id ||
 481                     match.mask->vlan_dei ||
 482                     match.mask->vlan_priority) {
 483                         vlan_tci = match.key->vlan_id |
 484                                    match.key->vlan_dei << 12 |
 485                                    match.key->vlan_priority << 13;
 486
 487                         vlan_tci_mask = match.mask->vlan_id |
 488                                         match.key->vlan_dei << 12 |
 489                                         match.key->vlan_priority << 13;
 490
 491                         flow_spec->vlan_tci = htons(vlan_tci);
 492                         flow_mask->vlan_tci = htons(vlan_tci_mask);
 493                         req->features |= BIT_ULL(NPC_OUTER_VID);
 494                 }
 495         }
 496
 497         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_IPV4_ADDRS)) {
 498                 struct flow_match_ipv4_addrs match;
 499
 500                 flow_rule_match_ipv4_addrs(rule, &match);
 501
 502                 flow_spec->ip4dst = match.key->dst;
 503                 flow_mask->ip4dst = match.mask->dst;
 504                 req->features |= BIT_ULL(NPC_DIP_IPV4);
 505
 506                 flow_spec->ip4src = match.key->src;
 507                 flow_mask->ip4src = match.mask->src;
 508                 req->features |= BIT_ULL(NPC_SIP_IPV4);
 509         } else if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_IPV6_ADDRS)) {
 510                 struct flow_match_ipv6_addrs match;
 511
 512                 flow_rule_match_ipv6_addrs(rule, &match);
 513
 514                 if (ipv6_addr_loopback(&match.key->dst) ||
 515                     ipv6_addr_loopback(&match.key->src)) {
 516                         NL_SET_ERR_MSG_MOD(extack,
 517                                            "Flow matching IPv6 loopback addr not supported");
 518                         return -EOPNOTSUPP;
 519                 }
 520
 521                 if (!ipv6_addr_any(&match.mask->dst)) {
 522                         memcpy(&flow_spec->ip6dst,
 523                                (struct in6_addr *)&match.key->dst,
 524                                sizeof(flow_spec->ip6dst));
 525                         memcpy(&flow_mask->ip6dst,
 526                                (struct in6_addr *)&match.mask->dst,
 527                                sizeof(flow_spec->ip6dst));
 528                         req->features |= BIT_ULL(NPC_DIP_IPV6);
 529                 }
 530
 531                 if (!ipv6_addr_any(&match.mask->src)) {
 532                         memcpy(&flow_spec->ip6src,
 533                                (struct in6_addr *)&match.key->src,
 534                                sizeof(flow_spec->ip6src));
 535                         memcpy(&flow_mask->ip6src,
 536                                (struct in6_addr *)&match.mask->src,
 537                                sizeof(flow_spec->ip6src));
 538                         req->features |= BIT_ULL(NPC_SIP_IPV6);
 539                 }
 540         }
 541
 542         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS)) {
 543                 struct flow_match_ports match;
 544
 545                 flow_rule_match_ports(rule, &match);
 546
 547                 flow_spec->dport = match.key->dst;
 548                 flow_mask->dport = match.mask->dst;
 549                 if (ip_proto == IPPROTO_UDP)
 550                         req->features |= BIT_ULL(NPC_DPORT_UDP);
 551                 else if (ip_proto == IPPROTO_TCP)
 552                         req->features |= BIT_ULL(NPC_DPORT_TCP);
 553                 else if (ip_proto == IPPROTO_SCTP)
 554                         req->features |= BIT_ULL(NPC_DPORT_SCTP);
 555
 556                 flow_spec->sport = match.key->src;
 557                 flow_mask->sport = match.mask->src;
 558                 if (ip_proto == IPPROTO_UDP)
 559                         req->features |= BIT_ULL(NPC_SPORT_UDP);
 560                 else if (ip_proto == IPPROTO_TCP)
 561                         req->features |= BIT_ULL(NPC_SPORT_TCP);
 562                 else if (ip_proto == IPPROTO_SCTP)
 563                         req->features |= BIT_ULL(NPC_SPORT_SCTP);
 564         }
 565
 566         return otx2_tc_parse_actions(nic, &rule->action, req, f, node);
 567 }
 568
 569 static int otx2_del_mcam_flow_entry(struct otx2_nic *nic, u16 entry)
 570 {
 571         struct npc_delete_flow_req *req;
 572         int err;
 573
 574         mutex_lock(&nic->mbox.lock);
 575         req = otx2_mbox_alloc_msg_npc_delete_flow(&nic->mbox);
 576         if (!req) {
 577                 mutex_unlock(&nic->mbox.lock);
 578                 return -ENOMEM;
 579         }
 580
 581         req->entry = entry;
 582
 583         /* Send message to AF */
 584         err = otx2_sync_mbox_msg(&nic->mbox);
 585         if (err) {
 586                 netdev_err(nic->netdev, "Failed to delete MCAM flow entry %d\n",
 587                            entry);
 588                 mutex_unlock(&nic->mbox.lock);
 589                 return -EFAULT;
 590         }
 591         mutex_unlock(&nic->mbox.lock);
 592
 593         return 0;
 594 }
 595
 596 static int otx2_tc_del_flow(struct otx2_nic *nic,
 597                             struct flow_cls_offload *tc_flow_cmd)
 598 {
 599         struct otx2_tc_info *tc_info = &nic->tc_info;
 600         struct otx2_tc_flow *flow_node;
 601         int err;
 602
 603         flow_node = rhashtable_lookup_fast(&tc_info->flow_table,
 604                                            &tc_flow_cmd->cookie,
 605                                            tc_info->flow_ht_params);
 606         if (!flow_node) {
 607                 netdev_err(nic->netdev, "tc flow not found for cookie 0x%lx\n",
 608                            tc_flow_cmd->cookie);
 609                 return -EINVAL;
 610         }
 611
 612         if (flow_node->is_act_police) {
 613                 mutex_lock(&nic->mbox.lock);
 614
 615                 err = cn10k_map_unmap_rq_policer(nic, flow_node->rq,
 616                                                  flow_node->leaf_profile, false);
 617                 if (err)
 618                         netdev_err(nic->netdev,
 619                                    "Unmapping RQ %d & profile %d failed\n",
 620                                    flow_node->rq, flow_node->leaf_profile);
 621
 622                 err = cn10k_free_leaf_profile(nic, flow_node->leaf_profile);
 623                 if (err)
 624                         netdev_err(nic->netdev,
 625                                    "Unable to free leaf bandwidth profile(%d)\n",
 626                                    flow_node->leaf_profile);
 627
 628                 __clear_bit(flow_node->rq, &nic->rq_bmap);
 629
 630                 mutex_unlock(&nic->mbox.lock);
 631         }
 632
 633         otx2_del_mcam_flow_entry(nic, flow_node->entry);
 634
 635         WARN_ON(rhashtable_remove_fast(&nic->tc_info.flow_table,
 636                                        &flow_node->node,
 637                                        nic->tc_info.flow_ht_params));
 638         kfree_rcu(flow_node, rcu);
 639
 640         clear_bit(flow_node->bitpos, tc_info->tc_entries_bitmap);
 641         tc_info->num_entries--;
 642
 643         return 0;
 644 }
 645
 646 static int otx2_tc_add_flow(struct otx2_nic *nic,
 647                             struct flow_cls_offload *tc_flow_cmd)
 648 {
 649         struct netlink_ext_ack *extack = tc_flow_cmd->common.extack;
 650         struct otx2_tc_info *tc_info = &nic->tc_info;
 651         struct otx2_tc_flow *new_node, *old_node;
 652         struct npc_install_flow_req *req, dummy;
 653         int rc, err;
 654
 655         if (!(nic->flags & OTX2_FLAG_TC_FLOWER_SUPPORT))
 656                 return -ENOMEM;
 657
 658         if (bitmap_full(tc_info->tc_entries_bitmap, nic->flow_cfg->tc_max_flows)) {
 659                 NL_SET_ERR_MSG_MOD(extack,
 660                                    "Not enough MCAM space to add the flow");
 661                 return -ENOMEM;
 662         }
 663
 664         /* allocate memory for the new flow and it's node */
 665         new_node = kzalloc(sizeof(*new_node), GFP_KERNEL);
 666         if (!new_node)
 667                 return -ENOMEM;
 668         spin_lock_init(&new_node->lock);
 669         new_node->cookie = tc_flow_cmd->cookie;
 670
 671         memset(&dummy, 0, sizeof(struct npc_install_flow_req));
 672
 673         rc = otx2_tc_prepare_flow(nic, new_node, tc_flow_cmd, &dummy);
 674         if (rc) {
 675                 kfree_rcu(new_node, rcu);
 676                 return rc;
 677         }
 678
 679         /* If a flow exists with the same cookie, delete it */
 680         old_node = rhashtable_lookup_fast(&tc_info->flow_table,
 681                                           &tc_flow_cmd->cookie,
 682                                           tc_info->flow_ht_params);
 683         if (old_node)
 684                 otx2_tc_del_flow(nic, tc_flow_cmd);
 685
 686         mutex_lock(&nic->mbox.lock);
 687         req = otx2_mbox_alloc_msg_npc_install_flow(&nic->mbox);
 688         if (!req) {
 689                 mutex_unlock(&nic->mbox.lock);
 690                 rc = -ENOMEM;
 691                 goto free_leaf;
 692         }
 693
 694         memcpy(&dummy.hdr, &req->hdr, sizeof(struct mbox_msghdr));
 695         memcpy(req, &dummy, sizeof(struct npc_install_flow_req));
 696
 697         new_node->bitpos = find_first_zero_bit(tc_info->tc_entries_bitmap,
 698                                                nic->flow_cfg->tc_max_flows);
 699         req->channel = nic->hw.rx_chan_base;
 700         req->entry = nic->flow_cfg->flow_ent[nic->flow_cfg->tc_flower_offset +
 701                                 nic->flow_cfg->tc_max_flows - new_node->bitpos];
 702         req->intf = NIX_INTF_RX;
 703         req->set_cntr = 1;
 704         new_node->entry = req->entry;
 705
 706         /* Send message to AF */
 707         rc = otx2_sync_mbox_msg(&nic->mbox);
 708         if (rc) {
 709                 NL_SET_ERR_MSG_MOD(extack, "Failed to install MCAM flow entry");
 710                 mutex_unlock(&nic->mbox.lock);
 711                 kfree_rcu(new_node, rcu);
 712                 goto free_leaf;
 713         }
 714         mutex_unlock(&nic->mbox.lock);
 715
 716         /* add new flow to flow-table */
 717         rc = rhashtable_insert_fast(&nic->tc_info.flow_table, &new_node->node,
 718                                     nic->tc_info.flow_ht_params);
 719         if (rc) {
 720                 otx2_del_mcam_flow_entry(nic, req->entry);
 721                 kfree_rcu(new_node, rcu);
 722                 goto free_leaf;
 723         }
 724
 725         set_bit(new_node->bitpos, tc_info->tc_entries_bitmap);
 726         tc_info->num_entries++;
 727
 728         return 0;
 729
 730 free_leaf:
 731         if (new_node->is_act_police) {
 732                 mutex_lock(&nic->mbox.lock);
 733
 734                 err = cn10k_map_unmap_rq_policer(nic, new_node->rq,
 735                                                  new_node->leaf_profile, false);
 736                 if (err)
 737                         netdev_err(nic->netdev,
 738                                    "Unmapping RQ %d & profile %d failed\n",
 739                                    new_node->rq, new_node->leaf_profile);
 740                 err = cn10k_free_leaf_profile(nic, new_node->leaf_profile);
 741                 if (err)
 742                         netdev_err(nic->netdev,
 743                                    "Unable to free leaf bandwidth profile(%d)\n",
 744                                    new_node->leaf_profile);
 745
 746                 __clear_bit(new_node->rq, &nic->rq_bmap);
 747
 748                 mutex_unlock(&nic->mbox.lock);
 749         }
 750
 751         return rc;
 752 }
 753
 754 static int otx2_tc_get_flow_stats(struct otx2_nic *nic,
 755                                   struct flow_cls_offload *tc_flow_cmd)
 756 {
 757         struct otx2_tc_info *tc_info = &nic->tc_info;
 758         struct npc_mcam_get_stats_req *req;
 759         struct npc_mcam_get_stats_rsp *rsp;
 760         struct otx2_tc_flow_stats *stats;
 761         struct otx2_tc_flow *flow_node;
 762         int err;
 763
 764         flow_node = rhashtable_lookup_fast(&tc_info->flow_table,
 765                                            &tc_flow_cmd->cookie,
 766                                            tc_info->flow_ht_params);
 767         if (!flow_node) {
 768                 netdev_info(nic->netdev, "tc flow not found for cookie %lx",
 769                             tc_flow_cmd->cookie);
 770                 return -EINVAL;
 771         }
 772
 773         mutex_lock(&nic->mbox.lock);
 774
 775         req = otx2_mbox_alloc_msg_npc_mcam_entry_stats(&nic->mbox);
 776         if (!req) {
 777                 mutex_unlock(&nic->mbox.lock);
 778                 return -ENOMEM;
 779         }
 780
 781         req->entry = flow_node->entry;
 782
 783         err = otx2_sync_mbox_msg(&nic->mbox);
 784         if (err) {
 785                 netdev_err(nic->netdev, "Failed to get stats for MCAM flow entry %d\n",
 786                            req->entry);
 787                 mutex_unlock(&nic->mbox.lock);
 788                 return -EFAULT;
 789         }
 790
 791         rsp = (struct npc_mcam_get_stats_rsp *)otx2_mbox_get_rsp
 792                 (&nic->mbox.mbox, 0, &req->hdr);
 793         if (IS_ERR(rsp)) {
 794                 mutex_unlock(&nic->mbox.lock);
 795                 return PTR_ERR(rsp);
 796         }
 797
 798         mutex_unlock(&nic->mbox.lock);
 799
 800         if (!rsp->stat_ena)
 801                 return -EINVAL;
 802
 803         stats = &flow_node->stats;
 804
 805         spin_lock(&flow_node->lock);
 806         flow_stats_update(&tc_flow_cmd->stats, 0x0, rsp->stat - stats->pkts, 0x0, 0x0,
 807                           FLOW_ACTION_HW_STATS_IMMEDIATE);
 808         stats->pkts = rsp->stat;
 809         spin_unlock(&flow_node->lock);
 810
 811         return 0;
 812 }
 813
 814 static int otx2_setup_tc_cls_flower(struct otx2_nic *nic,
 815                                     struct flow_cls_offload *cls_flower)
 816 {
 817         switch (cls_flower->command) {
 818         case FLOW_CLS_REPLACE:
 819                 return otx2_tc_add_flow(nic, cls_flower);
 820         case FLOW_CLS_DESTROY:
 821                 return otx2_tc_del_flow(nic, cls_flower);
 822         case FLOW_CLS_STATS:
 823                 return otx2_tc_get_flow_stats(nic, cls_flower);
 824         default:
 825                 return -EOPNOTSUPP;
 826         }
 827 }
 828
 829 static int otx2_tc_ingress_matchall_install(struct otx2_nic *nic,
 830                                             struct tc_cls_matchall_offload *cls)
 831 {
 832         struct netlink_ext_ack *extack = cls->common.extack;
 833         struct flow_action *actions = &cls->rule->action;
 834         struct flow_action_entry *entry;
 835         u64 rate;
 836         int err;
 837
 838         err = otx2_tc_validate_flow(nic, actions, extack);
 839         if (err)
 840                 return err;
 841
 842         if (nic->flags & OTX2_FLAG_TC_MATCHALL_INGRESS_ENABLED) {
 843                 NL_SET_ERR_MSG_MOD(extack,
 844                                    "Only one ingress MATCHALL ratelimitter can be offloaded");
 845                 return -ENOMEM;
 846         }
 847
 848         entry = &cls->rule->action.entries[0];
 849         switch (entry->id) {
 850         case FLOW_ACTION_POLICE:
 851                 /* Ingress ratelimiting is not supported on OcteonTx2 */
 852                 if (is_dev_otx2(nic->pdev)) {
 853                         NL_SET_ERR_MSG_MOD(extack,
 854                                            "Ingress policing not supported on this platform");
 855                         return -EOPNOTSUPP;
 856                 }
 857
 858                 err = cn10k_alloc_matchall_ipolicer(nic);
 859                 if (err)
 860                         return err;
 861
 862                 /* Convert to bits per second */
 863                 rate = entry->police.rate_bytes_ps * 8;
 864                 err = cn10k_set_matchall_ipolicer_rate(nic, entry->police.burst, rate);
 865                 if (err)
 866                         return err;
 867                 nic->flags |= OTX2_FLAG_TC_MATCHALL_INGRESS_ENABLED;
 868                 break;
 869         default:
 870                 NL_SET_ERR_MSG_MOD(extack,
 871                                    "Only police action supported with Ingress MATCHALL offload");
 872                 return -EOPNOTSUPP;
 873         }
 874
 875         return 0;
 876 }
 877
 878 static int otx2_tc_ingress_matchall_delete(struct otx2_nic *nic,
 879                                            struct tc_cls_matchall_offload *cls)
 880 {
 881         struct netlink_ext_ack *extack = cls->common.extack;
 882         int err;
 883
 884         if (nic->flags & OTX2_FLAG_INTF_DOWN) {
 885                 NL_SET_ERR_MSG_MOD(extack, "Interface not initialized");
 886                 return -EINVAL;
 887         }
 888
 889         err = cn10k_free_matchall_ipolicer(nic);
 890         nic->flags &= ~OTX2_FLAG_TC_MATCHALL_INGRESS_ENABLED;
 891         return err;
 892 }
 893
 894 static int otx2_setup_tc_ingress_matchall(struct otx2_nic *nic,
 895                                           struct tc_cls_matchall_offload *cls_matchall)
 896 {
 897         switch (cls_matchall->command) {
 898         case TC_CLSMATCHALL_REPLACE:
 899                 return otx2_tc_ingress_matchall_install(nic, cls_matchall);
 900         case TC_CLSMATCHALL_DESTROY:
 901                 return otx2_tc_ingress_matchall_delete(nic, cls_matchall);
 902         case TC_CLSMATCHALL_STATS:
 903         default:
 904                 break;
 905         }
 906
 907         return -EOPNOTSUPP;
 908 }
 909
 910 static int otx2_setup_tc_block_ingress_cb(enum tc_setup_type type,
 911                                           void *type_data, void *cb_priv)
 912 {
 913         struct otx2_nic *nic = cb_priv;
 914
 915         if (!tc_cls_can_offload_and_chain0(nic->netdev, type_data))
 916                 return -EOPNOTSUPP;
 917
 918         switch (type) {
 919         case TC_SETUP_CLSFLOWER:
 920                 return otx2_setup_tc_cls_flower(nic, type_data);
 921         case TC_SETUP_CLSMATCHALL:
 922                 return otx2_setup_tc_ingress_matchall(nic, type_data);
 923         default:
 924                 break;
 925         }
 926
 927         return -EOPNOTSUPP;
 928 }
 929
 930 static int otx2_setup_tc_egress_matchall(struct otx2_nic *nic,
 931                                          struct tc_cls_matchall_offload *cls_matchall)
 932 {
 933         switch (cls_matchall->command) {
 934         case TC_CLSMATCHALL_REPLACE:
 935                 return otx2_tc_egress_matchall_install(nic, cls_matchall);
 936         case TC_CLSMATCHALL_DESTROY:
 937                 return otx2_tc_egress_matchall_delete(nic, cls_matchall);
 938         case TC_CLSMATCHALL_STATS:
 939         default:
 940                 break;
 941         }
 942
 943         return -EOPNOTSUPP;
 944 }
 945
 946 static int otx2_setup_tc_block_egress_cb(enum tc_setup_type type,
 947                                          void *type_data, void *cb_priv)
 948 {
 949         struct otx2_nic *nic = cb_priv;
 950
 951         if (!tc_cls_can_offload_and_chain0(nic->netdev, type_data))
 952                 return -EOPNOTSUPP;
 953
 954         switch (type) {
 955         case TC_SETUP_CLSMATCHALL:
 956                 return otx2_setup_tc_egress_matchall(nic, type_data);
 957         default:
 958                 break;
 959         }
 960
 961         return -EOPNOTSUPP;
 962 }
 963
 964 static LIST_HEAD(otx2_block_cb_list);
 965
 966 static int otx2_setup_tc_block(struct net_device *netdev,
 967                                struct flow_block_offload *f)
 968 {
 969         struct otx2_nic *nic = netdev_priv(netdev);
 970         flow_setup_cb_t *cb;
 971         bool ingress;
 972
 973         if (f->block_shared)
 974                 return -EOPNOTSUPP;
 975
 976         if (f->binder_type == FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS) {
 977                 cb = otx2_setup_tc_block_ingress_cb;
 978                 ingress = true;
 979         } else if (f->binder_type == FLOW_BLOCK_BINDER_TYPE_CLSACT_EGRESS) {
 980                 cb = otx2_setup_tc_block_egress_cb;
 981                 ingress = false;
 982         } else {
 983                 return -EOPNOTSUPP;
 984         }
 985
 986         return flow_block_cb_setup_simple(f, &otx2_block_cb_list, cb,
 987                                           nic, nic, ingress);
 988 }
 989
 990 int otx2_setup_tc(struct net_device *netdev, enum tc_setup_type type,
 991                   void *type_data)
 992 {
 993         switch (type) {
 994         case TC_SETUP_BLOCK:
 995                 return otx2_setup_tc_block(netdev, type_data);
 996         default:
 997                 return -EOPNOTSUPP;
 998         }
 999 }
1000
1001 static const struct rhashtable_params tc_flow_ht_params = {
1002         .head_offset = offsetof(struct otx2_tc_flow, node),
1003         .key_offset = offsetof(struct otx2_tc_flow, cookie),
1004         .key_len = sizeof(((struct otx2_tc_flow *)0)->cookie),
1005         .automatic_shrinking = true,
1006 };
1007
1008 int otx2_init_tc(struct otx2_nic *nic)
1009 {
1010         struct otx2_tc_info *tc = &nic->tc_info;
1011
1012         /* Exclude receive queue 0 being used for police action */
1013         set_bit(0, &nic->rq_bmap);
1014
1015         tc->flow_ht_params = tc_flow_ht_params;
1016         return rhashtable_init(&tc->flow_table, &tc->flow_ht_params);
1017 }
1018
1019 void otx2_shutdown_tc(struct otx2_nic *nic)
1020 {
1021         struct otx2_tc_info *tc = &nic->tc_info;
1022
1023         rhashtable_destroy(&tc->flow_table);
1024 }