drivers/net/ethernet/chelsio/cxgb4/cxgb4_tc_flower.c

   1 /*
   2  * This file is part of the Chelsio T4/T5/T6 Ethernet driver for Linux.
   3  *
   4  * Copyright (c) 2017 Chelsio Communications, Inc. All rights reserved.
   5  *
   6  * This software is available to you under a choice of one of two
   7  * licenses.  You may choose to be licensed under the terms of the GNU
   8  * General Public License (GPL) Version 2, available from the file
   9  * COPYING in the main directory of this source tree, or the
  10  * OpenIB.org BSD license below:
  11  *
  12  *     Redistribution and use in source and binary forms, with or
  13  *     without modification, are permitted provided that the following
  14  *     conditions are met:
  15  *
  16  *      - Redistributions of source code must retain the above
  17  *        copyright notice, this list of conditions and the following
  18  *        disclaimer.
  19  *
  20  *      - Redistributions in binary form must reproduce the above
  21  *        copyright notice, this list of conditions and the following
  22  *        disclaimer in the documentation and/or other materials
  23  *        provided with the distribution.
  24  *
  25  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  26  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  27  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  28  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  29  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  30  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  31  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  32  * SOFTWARE.
  33  */
  34
  35 #include <net/tc_act/tc_mirred.h>
  36 #include <net/tc_act/tc_pedit.h>
  37 #include <net/tc_act/tc_gact.h>
  38 #include <net/tc_act/tc_vlan.h>
  39
  40 #include "cxgb4.h"
  41 #include "cxgb4_filter.h"
  42 #include "cxgb4_tc_flower.h"
  43
  44 #define STATS_CHECK_PERIOD (HZ / 2)
  45
  46 static struct ch_tc_pedit_fields pedits[] = {
  47         PEDIT_FIELDS(ETH_, DMAC_31_0, 4, dmac, 0),
  48         PEDIT_FIELDS(ETH_, DMAC_47_32, 2, dmac, 4),
  49         PEDIT_FIELDS(ETH_, SMAC_15_0, 2, smac, 0),
  50         PEDIT_FIELDS(ETH_, SMAC_47_16, 4, smac, 2),
  51         PEDIT_FIELDS(IP4_, SRC, 4, nat_fip, 0),
  52         PEDIT_FIELDS(IP4_, DST, 4, nat_lip, 0),
  53         PEDIT_FIELDS(IP6_, SRC_31_0, 4, nat_fip, 0),
  54         PEDIT_FIELDS(IP6_, SRC_63_32, 4, nat_fip, 4),
  55         PEDIT_FIELDS(IP6_, SRC_95_64, 4, nat_fip, 8),
  56         PEDIT_FIELDS(IP6_, SRC_127_96, 4, nat_fip, 12),
  57         PEDIT_FIELDS(IP6_, DST_31_0, 4, nat_lip, 0),
  58         PEDIT_FIELDS(IP6_, DST_63_32, 4, nat_lip, 4),
  59         PEDIT_FIELDS(IP6_, DST_95_64, 4, nat_lip, 8),
  60         PEDIT_FIELDS(IP6_, DST_127_96, 4, nat_lip, 12),
  61 };
  62
  63 static const struct cxgb4_natmode_config cxgb4_natmode_config_array[] = {
  64         /* Default supported NAT modes */
  65         {
  66                 .chip = CHELSIO_T5,
  67                 .flags = CXGB4_ACTION_NATMODE_NONE,
  68                 .natmode = NAT_MODE_NONE,
  69         },
  70         {
  71                 .chip = CHELSIO_T5,
  72                 .flags = CXGB4_ACTION_NATMODE_DIP,
  73                 .natmode = NAT_MODE_DIP,
  74         },
  75         {
  76                 .chip = CHELSIO_T5,
  77                 .flags = CXGB4_ACTION_NATMODE_DIP | CXGB4_ACTION_NATMODE_DPORT,
  78                 .natmode = NAT_MODE_DIP_DP,
  79         },
  80         {
  81                 .chip = CHELSIO_T5,
  82                 .flags = CXGB4_ACTION_NATMODE_DIP | CXGB4_ACTION_NATMODE_DPORT |
  83                          CXGB4_ACTION_NATMODE_SIP,
  84                 .natmode = NAT_MODE_DIP_DP_SIP,
  85         },
  86         {
  87                 .chip = CHELSIO_T5,
  88                 .flags = CXGB4_ACTION_NATMODE_DIP | CXGB4_ACTION_NATMODE_DPORT |
  89                          CXGB4_ACTION_NATMODE_SPORT,
  90                 .natmode = NAT_MODE_DIP_DP_SP,
  91         },
  92         {
  93                 .chip = CHELSIO_T5,
  94                 .flags = CXGB4_ACTION_NATMODE_SIP | CXGB4_ACTION_NATMODE_SPORT,
  95                 .natmode = NAT_MODE_SIP_SP,
  96         },
  97         {
  98                 .chip = CHELSIO_T5,
  99                 .flags = CXGB4_ACTION_NATMODE_DIP | CXGB4_ACTION_NATMODE_SIP |
 100                          CXGB4_ACTION_NATMODE_SPORT,
 101                 .natmode = NAT_MODE_DIP_SIP_SP,
 102         },
 103         {
 104                 .chip = CHELSIO_T5,
 105                 .flags = CXGB4_ACTION_NATMODE_DIP | CXGB4_ACTION_NATMODE_SIP |
 106                          CXGB4_ACTION_NATMODE_DPORT |
 107                          CXGB4_ACTION_NATMODE_SPORT,
 108                 .natmode = NAT_MODE_ALL,
 109         },
 110         /* T6+ can ignore L4 ports when they're disabled. */
 111         {
 112                 .chip = CHELSIO_T6,
 113                 .flags = CXGB4_ACTION_NATMODE_SIP,
 114                 .natmode = NAT_MODE_SIP_SP,
 115         },
 116         {
 117                 .chip = CHELSIO_T6,
 118                 .flags = CXGB4_ACTION_NATMODE_DIP | CXGB4_ACTION_NATMODE_SPORT,
 119                 .natmode = NAT_MODE_DIP_DP_SP,
 120         },
 121         {
 122                 .chip = CHELSIO_T6,
 123                 .flags = CXGB4_ACTION_NATMODE_DIP | CXGB4_ACTION_NATMODE_SIP,
 124                 .natmode = NAT_MODE_ALL,
 125         },
 126 };
 127
 128 static void cxgb4_action_natmode_tweak(struct ch_filter_specification *fs,
 129                                        u8 natmode_flags)
 130 {
 131         u8 i = 0;
 132
 133         /* Translate the enabled NAT 4-tuple fields to one of the
 134          * hardware supported NAT mode configurations. This ensures
 135          * that we pick a valid combination, where the disabled fields
 136          * do not get overwritten to 0.
 137          */
 138         for (i = 0; i < ARRAY_SIZE(cxgb4_natmode_config_array); i++) {
 139                 if (cxgb4_natmode_config_array[i].flags == natmode_flags) {
 140                         fs->nat_mode = cxgb4_natmode_config_array[i].natmode;
 141                         return;
 142                 }
 143         }
 144 }
 145
 146 static struct ch_tc_flower_entry *allocate_flower_entry(void)
 147 {
 148         struct ch_tc_flower_entry *new = kzalloc(sizeof(*new), GFP_KERNEL);
 149         if (new)
 150                 spin_lock_init(&new->lock);
 151         return new;
 152 }
 153
 154 /* Must be called with either RTNL or rcu_read_lock */
 155 static struct ch_tc_flower_entry *ch_flower_lookup(struct adapter *adap,
 156                                                    unsigned long flower_cookie)
 157 {
 158         return rhashtable_lookup_fast(&adap->flower_tbl, &flower_cookie,
 159                                       adap->flower_ht_params);
 160 }
 161
 162 static void cxgb4_process_flow_match(struct net_device *dev,
 163                                      struct flow_rule *rule,
 164                                      struct ch_filter_specification *fs)
 165 {
 166         u16 addr_type = 0;
 167
 168         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CONTROL)) {
 169                 struct flow_match_control match;
 170
 171                 flow_rule_match_control(rule, &match);
 172                 addr_type = match.key->addr_type;
 173         } else if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_IPV4_ADDRS)) {
 174                 addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
 175         } else if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_IPV6_ADDRS)) {
 176                 addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
 177         }
 178
 179         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_BASIC)) {
 180                 struct flow_match_basic match;
 181                 u16 ethtype_key, ethtype_mask;
 182
 183                 flow_rule_match_basic(rule, &match);
 184                 ethtype_key = ntohs(match.key->n_proto);
 185                 ethtype_mask = ntohs(match.mask->n_proto);
 186
 187                 if (ethtype_key == ETH_P_ALL) {
 188                         ethtype_key = 0;
 189                         ethtype_mask = 0;
 190                 }
 191
 192                 if (ethtype_key == ETH_P_IPV6)
 193                         fs->type = 1;
 194
 195                 fs->val.ethtype = ethtype_key;
 196                 fs->mask.ethtype = ethtype_mask;
 197                 fs->val.proto = match.key->ip_proto;
 198                 fs->mask.proto = match.mask->ip_proto;
 199         }
 200
 201         if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) {
 202                 struct flow_match_ipv4_addrs match;
 203
 204                 flow_rule_match_ipv4_addrs(rule, &match);
 205                 fs->type = 0;
 206                 memcpy(&fs->val.lip[0], &match.key->dst, sizeof(match.key->dst));
 207                 memcpy(&fs->val.fip[0], &match.key->src, sizeof(match.key->src));
 208                 memcpy(&fs->mask.lip[0], &match.mask->dst, sizeof(match.mask->dst));
 209                 memcpy(&fs->mask.fip[0], &match.mask->src, sizeof(match.mask->src));
 210
 211                 /* also initialize nat_lip/fip to same values */
 212                 memcpy(&fs->nat_lip[0], &match.key->dst, sizeof(match.key->dst));
 213                 memcpy(&fs->nat_fip[0], &match.key->src, sizeof(match.key->src));
 214         }
 215
 216         if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) {
 217                 struct flow_match_ipv6_addrs match;
 218
 219                 flow_rule_match_ipv6_addrs(rule, &match);
 220                 fs->type = 1;
 221                 memcpy(&fs->val.lip[0], match.key->dst.s6_addr,
 222                        sizeof(match.key->dst));
 223                 memcpy(&fs->val.fip[0], match.key->src.s6_addr,
 224                        sizeof(match.key->src));
 225                 memcpy(&fs->mask.lip[0], match.mask->dst.s6_addr,
 226                        sizeof(match.mask->dst));
 227                 memcpy(&fs->mask.fip[0], match.mask->src.s6_addr,
 228                        sizeof(match.mask->src));
 229
 230                 /* also initialize nat_lip/fip to same values */
 231                 memcpy(&fs->nat_lip[0], match.key->dst.s6_addr,
 232                        sizeof(match.key->dst));
 233                 memcpy(&fs->nat_fip[0], match.key->src.s6_addr,
 234                        sizeof(match.key->src));
 235         }
 236
 237         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS)) {
 238                 struct flow_match_ports match;
 239
 240                 flow_rule_match_ports(rule, &match);
 241                 fs->val.lport = be16_to_cpu(match.key->dst);
 242                 fs->mask.lport = be16_to_cpu(match.mask->dst);
 243                 fs->val.fport = be16_to_cpu(match.key->src);
 244                 fs->mask.fport = be16_to_cpu(match.mask->src);
 245
 246                 /* also initialize nat_lport/fport to same values */
 247                 fs->nat_lport = fs->val.lport;
 248                 fs->nat_fport = fs->val.fport;
 249         }
 250
 251         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_IP)) {
 252                 struct flow_match_ip match;
 253
 254                 flow_rule_match_ip(rule, &match);
 255                 fs->val.tos = match.key->tos;
 256                 fs->mask.tos = match.mask->tos;
 257         }
 258
 259         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_KEYID)) {
 260                 struct flow_match_enc_keyid match;
 261
 262                 flow_rule_match_enc_keyid(rule, &match);
 263                 fs->val.vni = be32_to_cpu(match.key->keyid);
 264                 fs->mask.vni = be32_to_cpu(match.mask->keyid);
 265                 if (fs->mask.vni) {
 266                         fs->val.encap_vld = 1;
 267                         fs->mask.encap_vld = 1;
 268                 }
 269         }
 270
 271         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_VLAN)) {
 272                 struct flow_match_vlan match;
 273                 u16 vlan_tci, vlan_tci_mask;
 274
 275                 flow_rule_match_vlan(rule, &match);
 276                 vlan_tci = match.key->vlan_id | (match.key->vlan_priority <<
 277                                                VLAN_PRIO_SHIFT);
 278                 vlan_tci_mask = match.mask->vlan_id | (match.mask->vlan_priority <<
 279                                                      VLAN_PRIO_SHIFT);
 280                 fs->val.ivlan = vlan_tci;
 281                 fs->mask.ivlan = vlan_tci_mask;
 282
 283                 fs->val.ivlan_vld = 1;
 284                 fs->mask.ivlan_vld = 1;
 285
 286                 /* Chelsio adapters use ivlan_vld bit to match vlan packets
 287                  * as 802.1Q. Also, when vlan tag is present in packets,
 288                  * ethtype match is used then to match on ethtype of inner
 289                  * header ie. the header following the vlan header.
 290                  * So, set the ivlan_vld based on ethtype info supplied by
 291                  * TC for vlan packets if its 802.1Q. And then reset the
 292                  * ethtype value else, hw will try to match the supplied
 293                  * ethtype value with ethtype of inner header.
 294                  */
 295                 if (fs->val.ethtype == ETH_P_8021Q) {
 296                         fs->val.ethtype = 0;
 297                         fs->mask.ethtype = 0;
 298                 }
 299         }
 300
 301         /* Match only packets coming from the ingress port where this
 302          * filter will be created.
 303          */
 304         fs->val.iport = netdev2pinfo(dev)->port_id;
 305         fs->mask.iport = ~0;
 306 }
 307
 308 static int cxgb4_validate_flow_match(struct net_device *dev,
 309                                      struct flow_rule *rule)
 310 {
 311         struct flow_dissector *dissector = rule->match.dissector;
 312         u16 ethtype_mask = 0;
 313         u16 ethtype_key = 0;
 314
 315         if (dissector->used_keys &
 316             ~(BIT(FLOW_DISSECTOR_KEY_CONTROL) |
 317               BIT(FLOW_DISSECTOR_KEY_BASIC) |
 318               BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS) |
 319               BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS) |
 320               BIT(FLOW_DISSECTOR_KEY_PORTS) |
 321               BIT(FLOW_DISSECTOR_KEY_ENC_KEYID) |
 322               BIT(FLOW_DISSECTOR_KEY_VLAN) |
 323               BIT(FLOW_DISSECTOR_KEY_IP))) {
 324                 netdev_warn(dev, "Unsupported key used: 0x%x\n",
 325                             dissector->used_keys);
 326                 return -EOPNOTSUPP;
 327         }
 328
 329         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_BASIC)) {
 330                 struct flow_match_basic match;
 331
 332                 flow_rule_match_basic(rule, &match);
 333                 ethtype_key = ntohs(match.key->n_proto);
 334                 ethtype_mask = ntohs(match.mask->n_proto);
 335         }
 336
 337         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_IP)) {
 338                 u16 eth_ip_type = ethtype_key & ethtype_mask;
 339                 struct flow_match_ip match;
 340
 341                 if (eth_ip_type != ETH_P_IP && eth_ip_type != ETH_P_IPV6) {
 342                         netdev_err(dev, "IP Key supported only with IPv4/v6");
 343                         return -EINVAL;
 344                 }
 345
 346                 flow_rule_match_ip(rule, &match);
 347                 if (match.mask->ttl) {
 348                         netdev_warn(dev, "ttl match unsupported for offload");
 349                         return -EOPNOTSUPP;
 350                 }
 351         }
 352
 353         return 0;
 354 }
 355
 356 static void offload_pedit(struct ch_filter_specification *fs, u32 val, u32 mask,
 357                           u8 field)
 358 {
 359         u32 set_val = val & ~mask;
 360         u32 offset = 0;
 361         u8 size = 1;
 362         int i;
 363
 364         for (i = 0; i < ARRAY_SIZE(pedits); i++) {
 365                 if (pedits[i].field == field) {
 366                         offset = pedits[i].offset;
 367                         size = pedits[i].size;
 368                         break;
 369                 }
 370         }
 371         memcpy((u8 *)fs + offset, &set_val, size);
 372 }
 373
 374 static void process_pedit_field(struct ch_filter_specification *fs, u32 val,
 375                                 u32 mask, u32 offset, u8 htype,
 376                                 u8 *natmode_flags)
 377 {
 378         switch (htype) {
 379         case FLOW_ACT_MANGLE_HDR_TYPE_ETH:
 380                 switch (offset) {
 381                 case PEDIT_ETH_DMAC_31_0:
 382                         fs->newdmac = 1;
 383                         offload_pedit(fs, val, mask, ETH_DMAC_31_0);
 384                         break;
 385                 case PEDIT_ETH_DMAC_47_32_SMAC_15_0:
 386                         if (~mask & PEDIT_ETH_DMAC_MASK)
 387                                 offload_pedit(fs, val, mask, ETH_DMAC_47_32);
 388                         else
 389                                 offload_pedit(fs, val >> 16, mask >> 16,
 390                                               ETH_SMAC_15_0);
 391                         break;
 392                 case PEDIT_ETH_SMAC_47_16:
 393                         fs->newsmac = 1;
 394                         offload_pedit(fs, val, mask, ETH_SMAC_47_16);
 395                 }
 396                 break;
 397         case FLOW_ACT_MANGLE_HDR_TYPE_IP4:
 398                 switch (offset) {
 399                 case PEDIT_IP4_SRC:
 400                         offload_pedit(fs, val, mask, IP4_SRC);
 401                         *natmode_flags |= CXGB4_ACTION_NATMODE_SIP;
 402                         break;
 403                 case PEDIT_IP4_DST:
 404                         offload_pedit(fs, val, mask, IP4_DST);
 405                         *natmode_flags |= CXGB4_ACTION_NATMODE_DIP;
 406                 }
 407                 break;
 408         case FLOW_ACT_MANGLE_HDR_TYPE_IP6:
 409                 switch (offset) {
 410                 case PEDIT_IP6_SRC_31_0:
 411                         offload_pedit(fs, val, mask, IP6_SRC_31_0);
 412                         *natmode_flags |= CXGB4_ACTION_NATMODE_SIP;
 413                         break;
 414                 case PEDIT_IP6_SRC_63_32:
 415                         offload_pedit(fs, val, mask, IP6_SRC_63_32);
 416                         *natmode_flags |=  CXGB4_ACTION_NATMODE_SIP;
 417                         break;
 418                 case PEDIT_IP6_SRC_95_64:
 419                         offload_pedit(fs, val, mask, IP6_SRC_95_64);
 420                         *natmode_flags |= CXGB4_ACTION_NATMODE_SIP;
 421                         break;
 422                 case PEDIT_IP6_SRC_127_96:
 423                         offload_pedit(fs, val, mask, IP6_SRC_127_96);
 424                         *natmode_flags |= CXGB4_ACTION_NATMODE_SIP;
 425                         break;
 426                 case PEDIT_IP6_DST_31_0:
 427                         offload_pedit(fs, val, mask, IP6_DST_31_0);
 428                         *natmode_flags |= CXGB4_ACTION_NATMODE_DIP;
 429                         break;
 430                 case PEDIT_IP6_DST_63_32:
 431                         offload_pedit(fs, val, mask, IP6_DST_63_32);
 432                         *natmode_flags |= CXGB4_ACTION_NATMODE_DIP;
 433                         break;
 434                 case PEDIT_IP6_DST_95_64:
 435                         offload_pedit(fs, val, mask, IP6_DST_95_64);
 436                         *natmode_flags |= CXGB4_ACTION_NATMODE_DIP;
 437                         break;
 438                 case PEDIT_IP6_DST_127_96:
 439                         offload_pedit(fs, val, mask, IP6_DST_127_96);
 440                         *natmode_flags |= CXGB4_ACTION_NATMODE_DIP;
 441                 }
 442                 break;
 443         case FLOW_ACT_MANGLE_HDR_TYPE_TCP:
 444                 switch (offset) {
 445                 case PEDIT_TCP_SPORT_DPORT:
 446                         if (~mask & PEDIT_TCP_UDP_SPORT_MASK) {
 447                                 fs->nat_fport = val;
 448                                 *natmode_flags |= CXGB4_ACTION_NATMODE_SPORT;
 449                         } else {
 450                                 fs->nat_lport = val >> 16;
 451                                 *natmode_flags |= CXGB4_ACTION_NATMODE_DPORT;
 452                         }
 453                 }
 454                 break;
 455         case FLOW_ACT_MANGLE_HDR_TYPE_UDP:
 456                 switch (offset) {
 457                 case PEDIT_UDP_SPORT_DPORT:
 458                         if (~mask & PEDIT_TCP_UDP_SPORT_MASK) {
 459                                 fs->nat_fport = val;
 460                                 *natmode_flags |= CXGB4_ACTION_NATMODE_SPORT;
 461                         } else {
 462                                 fs->nat_lport = val >> 16;
 463                                 *natmode_flags |= CXGB4_ACTION_NATMODE_DPORT;
 464                         }
 465                 }
 466                 break;
 467         }
 468 }
 469
 470 static int cxgb4_action_natmode_validate(struct adapter *adap, u8 natmode_flags,
 471                                          struct netlink_ext_ack *extack)
 472 {
 473         u8 i = 0;
 474
 475         /* Extract the NAT mode to enable based on what 4-tuple fields
 476          * are enabled to be overwritten. This ensures that the
 477          * disabled fields don't get overwritten to 0.
 478          */
 479         for (i = 0; i < ARRAY_SIZE(cxgb4_natmode_config_array); i++) {
 480                 const struct cxgb4_natmode_config *c;
 481
 482                 c = &cxgb4_natmode_config_array[i];
 483                 if (CHELSIO_CHIP_VERSION(adap->params.chip) >= c->chip &&
 484                     natmode_flags == c->flags)
 485                         return 0;
 486         }
 487         NL_SET_ERR_MSG_MOD(extack, "Unsupported NAT mode 4-tuple combination");
 488         return -EOPNOTSUPP;
 489 }
 490
 491 void cxgb4_process_flow_actions(struct net_device *in,
 492                                 struct flow_action *actions,
 493                                 struct ch_filter_specification *fs)
 494 {
 495         struct flow_action_entry *act;
 496         u8 natmode_flags = 0;
 497         int i;
 498
 499         flow_action_for_each(i, act, actions) {
 500                 switch (act->id) {
 501                 case FLOW_ACTION_ACCEPT:
 502                         fs->action = FILTER_PASS;
 503                         break;
 504                 case FLOW_ACTION_DROP:
 505                         fs->action = FILTER_DROP;
 506                         break;
 507                 case FLOW_ACTION_MIRRED:
 508                 case FLOW_ACTION_REDIRECT: {
 509                         struct net_device *out = act->dev;
 510                         struct port_info *pi = netdev_priv(out);
 511
 512                         fs->action = FILTER_SWITCH;
 513                         fs->eport = pi->port_id;
 514                         }
 515                         break;
 516                 case FLOW_ACTION_VLAN_POP:
 517                 case FLOW_ACTION_VLAN_PUSH:
 518                 case FLOW_ACTION_VLAN_MANGLE: {
 519                         u8 prio = act->vlan.prio;
 520                         u16 vid = act->vlan.vid;
 521                         u16 vlan_tci = (prio << VLAN_PRIO_SHIFT) | vid;
 522                         switch (act->id) {
 523                         case FLOW_ACTION_VLAN_POP:
 524                                 fs->newvlan |= VLAN_REMOVE;
 525                                 break;
 526                         case FLOW_ACTION_VLAN_PUSH:
 527                                 fs->newvlan |= VLAN_INSERT;
 528                                 fs->vlan = vlan_tci;
 529                                 break;
 530                         case FLOW_ACTION_VLAN_MANGLE:
 531                                 fs->newvlan |= VLAN_REWRITE;
 532                                 fs->vlan = vlan_tci;
 533                                 break;
 534                         default:
 535                                 break;
 536                         }
 537                         }
 538                         break;
 539                 case FLOW_ACTION_MANGLE: {
 540                         u32 mask, val, offset;
 541                         u8 htype;
 542
 543                         htype = act->mangle.htype;
 544                         mask = act->mangle.mask;
 545                         val = act->mangle.val;
 546                         offset = act->mangle.offset;
 547
 548                         process_pedit_field(fs, val, mask, offset, htype,
 549                                             &natmode_flags);
 550                         }
 551                         break;
 552                 case FLOW_ACTION_QUEUE:
 553                         fs->action = FILTER_PASS;
 554                         fs->dirsteer = 1;
 555                         fs->iq = act->queue.index;
 556                         break;
 557                 default:
 558                         break;
 559                 }
 560         }
 561         if (natmode_flags)
 562                 cxgb4_action_natmode_tweak(fs, natmode_flags);
 563
 564 }
 565
 566 static bool valid_l4_mask(u32 mask)
 567 {
 568         u16 hi, lo;
 569
 570         /* Either the upper 16-bits (SPORT) OR the lower
 571          * 16-bits (DPORT) can be set, but NOT BOTH.
 572          */
 573         hi = (mask >> 16) & 0xFFFF;
 574         lo = mask & 0xFFFF;
 575
 576         return hi && lo ? false : true;
 577 }
 578
 579 static bool valid_pedit_action(struct net_device *dev,
 580                                const struct flow_action_entry *act,
 581                                u8 *natmode_flags)
 582 {
 583         u32 mask, offset;
 584         u8 htype;
 585
 586         htype = act->mangle.htype;
 587         mask = act->mangle.mask;
 588         offset = act->mangle.offset;
 589
 590         switch (htype) {
 591         case FLOW_ACT_MANGLE_HDR_TYPE_ETH:
 592                 switch (offset) {
 593                 case PEDIT_ETH_DMAC_31_0:
 594                 case PEDIT_ETH_DMAC_47_32_SMAC_15_0:
 595                 case PEDIT_ETH_SMAC_47_16:
 596                         break;
 597                 default:
 598                         netdev_err(dev, "%s: Unsupported pedit field\n",
 599                                    __func__);
 600                         return false;
 601                 }
 602                 break;
 603         case FLOW_ACT_MANGLE_HDR_TYPE_IP4:
 604                 switch (offset) {
 605                 case PEDIT_IP4_SRC:
 606                         *natmode_flags |= CXGB4_ACTION_NATMODE_SIP;
 607                         break;
 608                 case PEDIT_IP4_DST:
 609                         *natmode_flags |= CXGB4_ACTION_NATMODE_DIP;
 610                         break;
 611                 default:
 612                         netdev_err(dev, "%s: Unsupported pedit field\n",
 613                                    __func__);
 614                         return false;
 615                 }
 616                 break;
 617         case FLOW_ACT_MANGLE_HDR_TYPE_IP6:
 618                 switch (offset) {
 619                 case PEDIT_IP6_SRC_31_0:
 620                 case PEDIT_IP6_SRC_63_32:
 621                 case PEDIT_IP6_SRC_95_64:
 622                 case PEDIT_IP6_SRC_127_96:
 623                         *natmode_flags |= CXGB4_ACTION_NATMODE_SIP;
 624                         break;
 625                 case PEDIT_IP6_DST_31_0:
 626                 case PEDIT_IP6_DST_63_32:
 627                 case PEDIT_IP6_DST_95_64:
 628                 case PEDIT_IP6_DST_127_96:
 629                         *natmode_flags |= CXGB4_ACTION_NATMODE_DIP;
 630                         break;
 631                 default:
 632                         netdev_err(dev, "%s: Unsupported pedit field\n",
 633                                    __func__);
 634                         return false;
 635                 }
 636                 break;
 637         case FLOW_ACT_MANGLE_HDR_TYPE_TCP:
 638                 switch (offset) {
 639                 case PEDIT_TCP_SPORT_DPORT:
 640                         if (!valid_l4_mask(~mask)) {
 641                                 netdev_err(dev, "%s: Unsupported mask for TCP L4 ports\n",
 642                                            __func__);
 643                                 return false;
 644                         }
 645                         if (~mask & PEDIT_TCP_UDP_SPORT_MASK)
 646                                 *natmode_flags |= CXGB4_ACTION_NATMODE_SPORT;
 647                         else
 648                                 *natmode_flags |= CXGB4_ACTION_NATMODE_DPORT;
 649                         break;
 650                 default:
 651                         netdev_err(dev, "%s: Unsupported pedit field\n",
 652                                    __func__);
 653                         return false;
 654                 }
 655                 break;
 656         case FLOW_ACT_MANGLE_HDR_TYPE_UDP:
 657                 switch (offset) {
 658                 case PEDIT_UDP_SPORT_DPORT:
 659                         if (!valid_l4_mask(~mask)) {
 660                                 netdev_err(dev, "%s: Unsupported mask for UDP L4 ports\n",
 661                                            __func__);
 662                                 return false;
 663                         }
 664                         if (~mask & PEDIT_TCP_UDP_SPORT_MASK)
 665                                 *natmode_flags |= CXGB4_ACTION_NATMODE_SPORT;
 666                         else
 667                                 *natmode_flags |= CXGB4_ACTION_NATMODE_DPORT;
 668                         break;
 669                 default:
 670                         netdev_err(dev, "%s: Unsupported pedit field\n",
 671                                    __func__);
 672                         return false;
 673                 }
 674                 break;
 675         default:
 676                 netdev_err(dev, "%s: Unsupported pedit type\n", __func__);
 677                 return false;
 678         }
 679         return true;
 680 }
 681
 682 int cxgb4_validate_flow_actions(struct net_device *dev,
 683                                 struct flow_action *actions,
 684                                 struct netlink_ext_ack *extack,
 685                                 u8 matchall_filter)
 686 {
 687         struct adapter *adap = netdev2adap(dev);
 688         struct flow_action_entry *act;
 689         bool act_redir = false;
 690         bool act_pedit = false;
 691         bool act_vlan = false;
 692         u8 natmode_flags = 0;
 693         int i;
 694
 695         if (!flow_action_basic_hw_stats_check(actions, extack))
 696                 return -EOPNOTSUPP;
 697
 698         flow_action_for_each(i, act, actions) {
 699                 switch (act->id) {
 700                 case FLOW_ACTION_ACCEPT:
 701                 case FLOW_ACTION_DROP:
 702                         /* Do nothing */
 703                         break;
 704                 case FLOW_ACTION_MIRRED:
 705                 case FLOW_ACTION_REDIRECT: {
 706                         struct net_device *n_dev, *target_dev;
 707                         bool found = false;
 708                         unsigned int i;
 709
 710                         if (act->id == FLOW_ACTION_MIRRED &&
 711                             !matchall_filter) {
 712                                 NL_SET_ERR_MSG_MOD(extack,
 713                                                    "Egress mirror action is only supported for tc-matchall");
 714                                 return -EOPNOTSUPP;
 715                         }
 716
 717                         target_dev = act->dev;
 718                         for_each_port(adap, i) {
 719                                 n_dev = adap->port[i];
 720                                 if (target_dev == n_dev) {
 721                                         found = true;
 722                                         break;
 723                                 }
 724                         }
 725
 726                         /* If interface doesn't belong to our hw, then
 727                          * the provided output port is not valid
 728                          */
 729                         if (!found) {
 730                                 netdev_err(dev, "%s: Out port invalid\n",
 731                                            __func__);
 732                                 return -EINVAL;
 733                         }
 734                         act_redir = true;
 735                         }
 736                         break;
 737                 case FLOW_ACTION_VLAN_POP:
 738                 case FLOW_ACTION_VLAN_PUSH:
 739                 case FLOW_ACTION_VLAN_MANGLE: {
 740                         u16 proto = be16_to_cpu(act->vlan.proto);
 741
 742                         switch (act->id) {
 743                         case FLOW_ACTION_VLAN_POP:
 744                                 break;
 745                         case FLOW_ACTION_VLAN_PUSH:
 746                         case FLOW_ACTION_VLAN_MANGLE:
 747                                 if (proto != ETH_P_8021Q) {
 748                                         netdev_err(dev, "%s: Unsupported vlan proto\n",
 749                                                    __func__);
 750                                         return -EOPNOTSUPP;
 751                                 }
 752                                 break;
 753                         default:
 754                                 netdev_err(dev, "%s: Unsupported vlan action\n",
 755                                            __func__);
 756                                 return -EOPNOTSUPP;
 757                         }
 758                         act_vlan = true;
 759                         }
 760                         break;
 761                 case FLOW_ACTION_MANGLE: {
 762                         bool pedit_valid = valid_pedit_action(dev, act,
 763                                                               &natmode_flags);
 764
 765                         if (!pedit_valid)
 766                                 return -EOPNOTSUPP;
 767                         act_pedit = true;
 768                         }
 769                         break;
 770                 case FLOW_ACTION_QUEUE:
 771                         /* Do nothing. cxgb4_set_filter will validate */
 772                         break;
 773                 default:
 774                         netdev_err(dev, "%s: Unsupported action\n", __func__);
 775                         return -EOPNOTSUPP;
 776                 }
 777         }
 778
 779         if ((act_pedit || act_vlan) && !act_redir) {
 780                 netdev_err(dev, "%s: pedit/vlan rewrite invalid without egress redirect\n",
 781                            __func__);
 782                 return -EINVAL;
 783         }
 784
 785         if (act_pedit) {
 786                 int ret;
 787
 788                 ret = cxgb4_action_natmode_validate(adap, natmode_flags,
 789                                                     extack);
 790                 if (ret)
 791                         return ret;
 792         }
 793
 794         return 0;
 795 }
 796
 797 static void cxgb4_tc_flower_hash_prio_add(struct adapter *adap, u32 tc_prio)
 798 {
 799         spin_lock_bh(&adap->tids.ftid_lock);
 800         if (adap->tids.tc_hash_tids_max_prio < tc_prio)
 801                 adap->tids.tc_hash_tids_max_prio = tc_prio;
 802         spin_unlock_bh(&adap->tids.ftid_lock);
 803 }
 804
 805 static void cxgb4_tc_flower_hash_prio_del(struct adapter *adap, u32 tc_prio)
 806 {
 807         struct tid_info *t = &adap->tids;
 808         struct ch_tc_flower_entry *fe;
 809         struct rhashtable_iter iter;
 810         u32 found = 0;
 811
 812         spin_lock_bh(&t->ftid_lock);
 813         /* Bail if the current rule is not the one with the max
 814          * prio.
 815          */
 816         if (t->tc_hash_tids_max_prio != tc_prio)
 817                 goto out_unlock;
 818
 819         /* Search for the next rule having the same or next lower
 820          * max prio.
 821          */
 822         rhashtable_walk_enter(&adap->flower_tbl, &iter);
 823         do {
 824                 rhashtable_walk_start(&iter);
 825
 826                 fe = rhashtable_walk_next(&iter);
 827                 while (!IS_ERR_OR_NULL(fe)) {
 828                         if (fe->fs.hash &&
 829                             fe->fs.tc_prio <= t->tc_hash_tids_max_prio) {
 830                                 t->tc_hash_tids_max_prio = fe->fs.tc_prio;
 831                                 found++;
 832
 833                                 /* Bail if we found another rule
 834                                  * having the same prio as the
 835                                  * current max one.
 836                                  */
 837                                 if (fe->fs.tc_prio == tc_prio)
 838                                         break;
 839                         }
 840
 841                         fe = rhashtable_walk_next(&iter);
 842                 }
 843
 844                 rhashtable_walk_stop(&iter);
 845         } while (fe == ERR_PTR(-EAGAIN));
 846         rhashtable_walk_exit(&iter);
 847
 848         if (!found)
 849                 t->tc_hash_tids_max_prio = 0;
 850
 851 out_unlock:
 852         spin_unlock_bh(&t->ftid_lock);
 853 }
 854
 855 int cxgb4_flow_rule_replace(struct net_device *dev, struct flow_rule *rule,
 856                             u32 tc_prio, struct netlink_ext_ack *extack,
 857                             struct ch_filter_specification *fs, u32 *tid)
 858 {
 859         struct adapter *adap = netdev2adap(dev);
 860         struct filter_ctx ctx;
 861         u8 inet_family;
 862         int fidx, ret;
 863
 864         if (cxgb4_validate_flow_actions(dev, &rule->action, extack, 0))
 865                 return -EOPNOTSUPP;
 866
 867         if (cxgb4_validate_flow_match(dev, rule))
 868                 return -EOPNOTSUPP;
 869
 870         cxgb4_process_flow_match(dev, rule, fs);
 871         cxgb4_process_flow_actions(dev, &rule->action, fs);
 872
 873         fs->hash = is_filter_exact_match(adap, fs);
 874         inet_family = fs->type ? PF_INET6 : PF_INET;
 875
 876         /* Get a free filter entry TID, where we can insert this new
 877          * rule. Only insert rule if its prio doesn't conflict with
 878          * existing rules.
 879          */
 880         fidx = cxgb4_get_free_ftid(dev, inet_family, fs->hash,
 881                                    tc_prio);
 882         if (fidx < 0) {
 883                 NL_SET_ERR_MSG_MOD(extack,
 884                                    "No free LETCAM index available");
 885                 return -ENOMEM;
 886         }
 887
 888         if (fidx < adap->tids.nhpftids) {
 889                 fs->prio = 1;
 890                 fs->hash = 0;
 891         }
 892
 893         /* If the rule can be inserted into HASH region, then ignore
 894          * the index to normal FILTER region.
 895          */
 896         if (fs->hash)
 897                 fidx = 0;
 898
 899         fs->tc_prio = tc_prio;
 900
 901         init_completion(&ctx.completion);
 902         ret = __cxgb4_set_filter(dev, fidx, fs, &ctx);
 903         if (ret) {
 904                 netdev_err(dev, "%s: filter creation err %d\n",
 905                            __func__, ret);
 906                 return ret;
 907         }
 908
 909         /* Wait for reply */
 910         ret = wait_for_completion_timeout(&ctx.completion, 10 * HZ);
 911         if (!ret)
 912                 return -ETIMEDOUT;
 913
 914         /* Check if hw returned error for filter creation */
 915         if (ctx.result)
 916                 return ctx.result;
 917
 918         *tid = ctx.tid;
 919
 920         if (fs->hash)
 921                 cxgb4_tc_flower_hash_prio_add(adap, tc_prio);
 922
 923         return 0;
 924 }
 925
 926 int cxgb4_tc_flower_replace(struct net_device *dev,
 927                             struct flow_cls_offload *cls)
 928 {
 929         struct flow_rule *rule = flow_cls_offload_flow_rule(cls);
 930         struct netlink_ext_ack *extack = cls->common.extack;
 931         struct adapter *adap = netdev2adap(dev);
 932         struct ch_tc_flower_entry *ch_flower;
 933         struct ch_filter_specification *fs;
 934         int ret;
 935
 936         ch_flower = allocate_flower_entry();
 937         if (!ch_flower) {
 938                 netdev_err(dev, "%s: ch_flower alloc failed.\n", __func__);
 939                 return -ENOMEM;
 940         }
 941
 942         fs = &ch_flower->fs;
 943         fs->hitcnts = 1;
 944         fs->tc_cookie = cls->cookie;
 945
 946         ret = cxgb4_flow_rule_replace(dev, rule, cls->common.prio, extack, fs,
 947                                       &ch_flower->filter_id);
 948         if (ret)
 949                 goto free_entry;
 950
 951         ch_flower->tc_flower_cookie = cls->cookie;
 952         ret = rhashtable_insert_fast(&adap->flower_tbl, &ch_flower->node,
 953                                      adap->flower_ht_params);
 954         if (ret)
 955                 goto del_filter;
 956
 957         return 0;
 958
 959 del_filter:
 960         if (fs->hash)
 961                 cxgb4_tc_flower_hash_prio_del(adap, cls->common.prio);
 962
 963         cxgb4_del_filter(dev, ch_flower->filter_id, &ch_flower->fs);
 964
 965 free_entry:
 966         kfree(ch_flower);
 967         return ret;
 968 }
 969
 970 int cxgb4_flow_rule_destroy(struct net_device *dev, u32 tc_prio,
 971                             struct ch_filter_specification *fs, int tid)
 972 {
 973         struct adapter *adap = netdev2adap(dev);
 974         u8 hash;
 975         int ret;
 976
 977         hash = fs->hash;
 978
 979         ret = cxgb4_del_filter(dev, tid, fs);
 980         if (ret)
 981                 return ret;
 982
 983         if (hash)
 984                 cxgb4_tc_flower_hash_prio_del(adap, tc_prio);
 985
 986         return ret;
 987 }
 988
 989 int cxgb4_tc_flower_destroy(struct net_device *dev,
 990                             struct flow_cls_offload *cls)
 991 {
 992         struct adapter *adap = netdev2adap(dev);
 993         struct ch_tc_flower_entry *ch_flower;
 994         int ret;
 995
 996         ch_flower = ch_flower_lookup(adap, cls->cookie);
 997         if (!ch_flower)
 998                 return -ENOENT;
 999
1000         rhashtable_remove_fast(&adap->flower_tbl, &ch_flower->node,
1001                                adap->flower_ht_params);
1002
1003         ret = cxgb4_flow_rule_destroy(dev, ch_flower->fs.tc_prio,
1004                                       &ch_flower->fs, ch_flower->filter_id);
1005         if (ret)
1006                 netdev_err(dev, "Flow rule destroy failed for tid: %u, ret: %d",
1007                            ch_flower->filter_id, ret);
1008
1009         kfree_rcu(ch_flower, rcu);
1010         return ret;
1011 }
1012
1013 static void ch_flower_stats_handler(struct work_struct *work)
1014 {
1015         struct adapter *adap = container_of(work, struct adapter,
1016                                             flower_stats_work);
1017         struct ch_tc_flower_entry *flower_entry;
1018         struct ch_tc_flower_stats *ofld_stats;
1019         struct rhashtable_iter iter;
1020         u64 packets;
1021         u64 bytes;
1022         int ret;
1023
1024         rhashtable_walk_enter(&adap->flower_tbl, &iter);
1025         do {
1026                 rhashtable_walk_start(&iter);
1027
1028                 while ((flower_entry = rhashtable_walk_next(&iter)) &&
1029                        !IS_ERR(flower_entry)) {
1030                         ret = cxgb4_get_filter_counters(adap->port[0],
1031                                                         flower_entry->filter_id,
1032                                                         &packets, &bytes,
1033                                                         flower_entry->fs.hash);
1034                         if (!ret) {
1035                                 spin_lock(&flower_entry->lock);
1036                                 ofld_stats = &flower_entry->stats;
1037
1038                                 if (ofld_stats->prev_packet_count != packets) {
1039                                         ofld_stats->prev_packet_count = packets;
1040                                         ofld_stats->last_used = jiffies;
1041                                 }
1042                                 spin_unlock(&flower_entry->lock);
1043                         }
1044                 }
1045
1046                 rhashtable_walk_stop(&iter);
1047
1048         } while (flower_entry == ERR_PTR(-EAGAIN));
1049         rhashtable_walk_exit(&iter);
1050         mod_timer(&adap->flower_stats_timer, jiffies + STATS_CHECK_PERIOD);
1051 }
1052
1053 static void ch_flower_stats_cb(struct timer_list *t)
1054 {
1055         struct adapter *adap = from_timer(adap, t, flower_stats_timer);
1056
1057         schedule_work(&adap->flower_stats_work);
1058 }
1059
1060 int cxgb4_tc_flower_stats(struct net_device *dev,
1061                           struct flow_cls_offload *cls)
1062 {
1063         struct adapter *adap = netdev2adap(dev);
1064         struct ch_tc_flower_stats *ofld_stats;
1065         struct ch_tc_flower_entry *ch_flower;
1066         u64 packets;
1067         u64 bytes;
1068         int ret;
1069
1070         ch_flower = ch_flower_lookup(adap, cls->cookie);
1071         if (!ch_flower) {
1072                 ret = -ENOENT;
1073                 goto err;
1074         }
1075
1076         ret = cxgb4_get_filter_counters(dev, ch_flower->filter_id,
1077                                         &packets, &bytes,
1078                                         ch_flower->fs.hash);
1079         if (ret < 0)
1080                 goto err;
1081
1082         spin_lock_bh(&ch_flower->lock);
1083         ofld_stats = &ch_flower->stats;
1084         if (ofld_stats->packet_count != packets) {
1085                 if (ofld_stats->prev_packet_count != packets)
1086                         ofld_stats->last_used = jiffies;
1087                 flow_stats_update(&cls->stats, bytes - ofld_stats->byte_count,
1088                                   packets - ofld_stats->packet_count, 0,
1089                                   ofld_stats->last_used,
1090                                   FLOW_ACTION_HW_STATS_IMMEDIATE);
1091
1092                 ofld_stats->packet_count = packets;
1093                 ofld_stats->byte_count = bytes;
1094                 ofld_stats->prev_packet_count = packets;
1095         }
1096         spin_unlock_bh(&ch_flower->lock);
1097         return 0;
1098
1099 err:
1100         return ret;
1101 }
1102
1103 static const struct rhashtable_params cxgb4_tc_flower_ht_params = {
1104         .nelem_hint = 384,
1105         .head_offset = offsetof(struct ch_tc_flower_entry, node),
1106         .key_offset = offsetof(struct ch_tc_flower_entry, tc_flower_cookie),
1107         .key_len = sizeof(((struct ch_tc_flower_entry *)0)->tc_flower_cookie),
1108         .max_size = 524288,
1109         .min_size = 512,
1110         .automatic_shrinking = true
1111 };
1112
1113 int cxgb4_init_tc_flower(struct adapter *adap)
1114 {
1115         int ret;
1116
1117         if (adap->tc_flower_initialized)
1118                 return -EEXIST;
1119
1120         adap->flower_ht_params = cxgb4_tc_flower_ht_params;
1121         ret = rhashtable_init(&adap->flower_tbl, &adap->flower_ht_params);
1122         if (ret)
1123                 return ret;
1124
1125         INIT_WORK(&adap->flower_stats_work, ch_flower_stats_handler);
1126         timer_setup(&adap->flower_stats_timer, ch_flower_stats_cb, 0);
1127         mod_timer(&adap->flower_stats_timer, jiffies + STATS_CHECK_PERIOD);
1128         adap->tc_flower_initialized = true;
1129         return 0;
1130 }
1131
1132 void cxgb4_cleanup_tc_flower(struct adapter *adap)
1133 {
1134         if (!adap->tc_flower_initialized)
1135                 return;
1136
1137         if (adap->flower_stats_timer.function)
1138                 del_timer_sync(&adap->flower_stats_timer);
1139         cancel_work_sync(&adap->flower_stats_work);
1140         rhashtable_destroy(&adap->flower_tbl);
1141         adap->tc_flower_initialized = false;
1142 }