drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_fs.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Marvell RVU Admin Function driver
   3  *
   4  * Copyright (C) 2020 Marvell.
   5  */
   6
   7 #include <linux/bitfield.h>
   8
   9 #include "rvu_struct.h"
  10 #include "rvu_reg.h"
  11 #include "rvu.h"
  12 #include "npc.h"
  13
  14 #define NPC_BYTESM              GENMASK_ULL(19, 16)
  15 #define NPC_HDR_OFFSET          GENMASK_ULL(15, 8)
  16 #define NPC_KEY_OFFSET          GENMASK_ULL(5, 0)
  17 #define NPC_LDATA_EN            BIT_ULL(7)
  18
  19 static const char * const npc_flow_names[] = {
  20         [NPC_DMAC]      = "dmac",
  21         [NPC_SMAC]      = "smac",
  22         [NPC_ETYPE]     = "ether type",
  23         [NPC_VLAN_ETYPE_CTAG] = "vlan ether type ctag",
  24         [NPC_VLAN_ETYPE_STAG] = "vlan ether type stag",
  25         [NPC_OUTER_VID] = "outer vlan id",
  26         [NPC_TOS]       = "tos",
  27         [NPC_SIP_IPV4]  = "ipv4 source ip",
  28         [NPC_DIP_IPV4]  = "ipv4 destination ip",
  29         [NPC_SIP_IPV6]  = "ipv6 source ip",
  30         [NPC_DIP_IPV6]  = "ipv6 destination ip",
  31         [NPC_IPPROTO_TCP] = "ip proto tcp",
  32         [NPC_IPPROTO_UDP] = "ip proto udp",
  33         [NPC_IPPROTO_SCTP] = "ip proto sctp",
  34         [NPC_IPPROTO_ICMP] = "ip proto icmp",
  35         [NPC_IPPROTO_ICMP6] = "ip proto icmp6",
  36         [NPC_IPPROTO_AH] = "ip proto AH",
  37         [NPC_IPPROTO_ESP] = "ip proto ESP",
  38         [NPC_SPORT_TCP] = "tcp source port",
  39         [NPC_DPORT_TCP] = "tcp destination port",
  40         [NPC_SPORT_UDP] = "udp source port",
  41         [NPC_DPORT_UDP] = "udp destination port",
  42         [NPC_SPORT_SCTP] = "sctp source port",
  43         [NPC_DPORT_SCTP] = "sctp destination port",
  44         [NPC_UNKNOWN]   = "unknown",
  45 };
  46
  47 const char *npc_get_field_name(u8 hdr)
  48 {
  49         if (hdr >= ARRAY_SIZE(npc_flow_names))
  50                 return npc_flow_names[NPC_UNKNOWN];
  51
  52         return npc_flow_names[hdr];
  53 }
  54
  55 /* Compute keyword masks and figure out the number of keywords a field
  56  * spans in the key.
  57  */
  58 static void npc_set_kw_masks(struct npc_mcam *mcam, u8 type,
  59                              u8 nr_bits, int start_kwi, int offset, u8 intf)
  60 {
  61         struct npc_key_field *field = &mcam->rx_key_fields[type];
  62         u8 bits_in_kw;
  63         int max_kwi;
  64
  65         if (mcam->banks_per_entry == 1)
  66                 max_kwi = 1; /* NPC_MCAM_KEY_X1 */
  67         else if (mcam->banks_per_entry == 2)
  68                 max_kwi = 3; /* NPC_MCAM_KEY_X2 */
  69         else
  70                 max_kwi = 6; /* NPC_MCAM_KEY_X4 */
  71
  72         if (is_npc_intf_tx(intf))
  73                 field = &mcam->tx_key_fields[type];
  74
  75         if (offset + nr_bits <= 64) {
  76                 /* one KW only */
  77                 if (start_kwi > max_kwi)
  78                         return;
  79                 field->kw_mask[start_kwi] |= GENMASK_ULL(nr_bits - 1, 0)
  80                                              << offset;
  81                 field->nr_kws = 1;
  82         } else if (offset + nr_bits > 64 &&
  83                    offset + nr_bits <= 128) {
  84                 /* two KWs */
  85                 if (start_kwi + 1 > max_kwi)
  86                         return;
  87                 /* first KW mask */
  88                 bits_in_kw = 64 - offset;
  89                 field->kw_mask[start_kwi] |= GENMASK_ULL(bits_in_kw - 1, 0)
  90                                              << offset;
  91                 /* second KW mask i.e. mask for rest of bits */
  92                 bits_in_kw = nr_bits + offset - 64;
  93                 field->kw_mask[start_kwi + 1] |= GENMASK_ULL(bits_in_kw - 1, 0);
  94                 field->nr_kws = 2;
  95         } else {
  96                 /* three KWs */
  97                 if (start_kwi + 2 > max_kwi)
  98                         return;
  99                 /* first KW mask */
 100                 bits_in_kw = 64 - offset;
 101                 field->kw_mask[start_kwi] |= GENMASK_ULL(bits_in_kw - 1, 0)
 102                                              << offset;
 103                 /* second KW mask */
 104                 field->kw_mask[start_kwi + 1] = ~0ULL;
 105                 /* third KW mask i.e. mask for rest of bits */
 106                 bits_in_kw = nr_bits + offset - 128;
 107                 field->kw_mask[start_kwi + 2] |= GENMASK_ULL(bits_in_kw - 1, 0);
 108                 field->nr_kws = 3;
 109         }
 110 }
 111
 112 /* Helper function to figure out whether field exists in the key */
 113 static bool npc_is_field_present(struct rvu *rvu, enum key_fields type, u8 intf)
 114 {
 115         struct npc_mcam *mcam = &rvu->hw->mcam;
 116         struct npc_key_field *input;
 117
 118         input  = &mcam->rx_key_fields[type];
 119         if (is_npc_intf_tx(intf))
 120                 input  = &mcam->tx_key_fields[type];
 121
 122         return input->nr_kws > 0;
 123 }
 124
 125 static bool npc_is_same(struct npc_key_field *input,
 126                         struct npc_key_field *field)
 127 {
 128         return memcmp(&input->layer_mdata, &field->layer_mdata,
 129                      sizeof(struct npc_layer_mdata)) == 0;
 130 }
 131
 132 static void npc_set_layer_mdata(struct npc_mcam *mcam, enum key_fields type,
 133                                 u64 cfg, u8 lid, u8 lt, u8 intf)
 134 {
 135         struct npc_key_field *input = &mcam->rx_key_fields[type];
 136
 137         if (is_npc_intf_tx(intf))
 138                 input = &mcam->tx_key_fields[type];
 139
 140         input->layer_mdata.hdr = FIELD_GET(NPC_HDR_OFFSET, cfg);
 141         input->layer_mdata.key = FIELD_GET(NPC_KEY_OFFSET, cfg);
 142         input->layer_mdata.len = FIELD_GET(NPC_BYTESM, cfg) + 1;
 143         input->layer_mdata.ltype = lt;
 144         input->layer_mdata.lid = lid;
 145 }
 146
 147 static bool npc_check_overlap_fields(struct npc_key_field *input1,
 148                                      struct npc_key_field *input2)
 149 {
 150         int kwi;
 151
 152         /* Fields with same layer id and different ltypes are mutually
 153          * exclusive hence they can be overlapped
 154          */
 155         if (input1->layer_mdata.lid == input2->layer_mdata.lid &&
 156             input1->layer_mdata.ltype != input2->layer_mdata.ltype)
 157                 return false;
 158
 159         for (kwi = 0; kwi < NPC_MAX_KWS_IN_KEY; kwi++) {
 160                 if (input1->kw_mask[kwi] & input2->kw_mask[kwi])
 161                         return true;
 162         }
 163
 164         return false;
 165 }
 166
 167 /* Helper function to check whether given field overlaps with any other fields
 168  * in the key. Due to limitations on key size and the key extraction profile in
 169  * use higher layers can overwrite lower layer's header fields. Hence overlap
 170  * needs to be checked.
 171  */
 172 static bool npc_check_overlap(struct rvu *rvu, int blkaddr,
 173                               enum key_fields type, u8 start_lid, u8 intf)
 174 {
 175         struct npc_mcam *mcam = &rvu->hw->mcam;
 176         struct npc_key_field *dummy, *input;
 177         int start_kwi, offset;
 178         u8 nr_bits, lid, lt, ld;
 179         u64 cfg;
 180
 181         dummy = &mcam->rx_key_fields[NPC_UNKNOWN];
 182         input = &mcam->rx_key_fields[type];
 183
 184         if (is_npc_intf_tx(intf)) {
 185                 dummy = &mcam->tx_key_fields[NPC_UNKNOWN];
 186                 input = &mcam->tx_key_fields[type];
 187         }
 188
 189         for (lid = start_lid; lid < NPC_MAX_LID; lid++) {
 190                 for (lt = 0; lt < NPC_MAX_LT; lt++) {
 191                         for (ld = 0; ld < NPC_MAX_LD; ld++) {
 192                                 cfg = rvu_read64(rvu, blkaddr,
 193                                                  NPC_AF_INTFX_LIDX_LTX_LDX_CFG
 194                                                  (intf, lid, lt, ld));
 195                                 if (!FIELD_GET(NPC_LDATA_EN, cfg))
 196                                         continue;
 197                                 memset(dummy, 0, sizeof(struct npc_key_field));
 198                                 npc_set_layer_mdata(mcam, NPC_UNKNOWN, cfg,
 199                                                     lid, lt, intf);
 200                                 /* exclude input */
 201                                 if (npc_is_same(input, dummy))
 202                                         continue;
 203                                 start_kwi = dummy->layer_mdata.key / 8;
 204                                 offset = (dummy->layer_mdata.key * 8) % 64;
 205                                 nr_bits = dummy->layer_mdata.len * 8;
 206                                 /* form KW masks */
 207                                 npc_set_kw_masks(mcam, NPC_UNKNOWN, nr_bits,
 208                                                  start_kwi, offset, intf);
 209                                 /* check any input field bits falls in any
 210                                  * other field bits.
 211                                  */
 212                                 if (npc_check_overlap_fields(dummy, input))
 213                                         return true;
 214                         }
 215                 }
 216         }
 217
 218         return false;
 219 }
 220
 221 static bool npc_check_field(struct rvu *rvu, int blkaddr, enum key_fields type,
 222                             u8 intf)
 223 {
 224         if (!npc_is_field_present(rvu, type, intf) ||
 225             npc_check_overlap(rvu, blkaddr, type, 0, intf))
 226                 return false;
 227         return true;
 228 }
 229
 230 static void npc_scan_parse_result(struct npc_mcam *mcam, u8 bit_number,
 231                                   u8 key_nibble, u8 intf)
 232 {
 233         u8 offset = (key_nibble * 4) % 64; /* offset within key word */
 234         u8 kwi = (key_nibble * 4) / 64; /* which word in key */
 235         u8 nr_bits = 4; /* bits in a nibble */
 236         u8 type;
 237
 238         switch (bit_number) {
 239         case 0 ... 2:
 240                 type = NPC_CHAN;
 241                 break;
 242         case 3:
 243                 type = NPC_ERRLEV;
 244                 break;
 245         case 4 ... 5:
 246                 type = NPC_ERRCODE;
 247                 break;
 248         case 6:
 249                 type = NPC_LXMB;
 250                 break;
 251         /* check for LTYPE only as of now */
 252         case 9:
 253                 type = NPC_LA;
 254                 break;
 255         case 12:
 256                 type = NPC_LB;
 257                 break;
 258         case 15:
 259                 type = NPC_LC;
 260                 break;
 261         case 18:
 262                 type = NPC_LD;
 263                 break;
 264         case 21:
 265                 type = NPC_LE;
 266                 break;
 267         case 24:
 268                 type = NPC_LF;
 269                 break;
 270         case 27:
 271                 type = NPC_LG;
 272                 break;
 273         case 30:
 274                 type = NPC_LH;
 275                 break;
 276         default:
 277                 return;
 278         }
 279         npc_set_kw_masks(mcam, type, nr_bits, kwi, offset, intf);
 280 }
 281
 282 static void npc_handle_multi_layer_fields(struct rvu *rvu, int blkaddr, u8 intf)
 283 {
 284         struct npc_mcam *mcam = &rvu->hw->mcam;
 285         struct npc_key_field *key_fields;
 286         /* Ether type can come from three layers
 287          * (ethernet, single tagged, double tagged)
 288          */
 289         struct npc_key_field *etype_ether;
 290         struct npc_key_field *etype_tag1;
 291         struct npc_key_field *etype_tag2;
 292         /* Outer VLAN TCI can come from two layers
 293          * (single tagged, double tagged)
 294          */
 295         struct npc_key_field *vlan_tag1;
 296         struct npc_key_field *vlan_tag2;
 297         u64 *features;
 298         u8 start_lid;
 299         int i;
 300
 301         key_fields = mcam->rx_key_fields;
 302         features = &mcam->rx_features;
 303
 304         if (is_npc_intf_tx(intf)) {
 305                 key_fields = mcam->tx_key_fields;
 306                 features = &mcam->tx_features;
 307         }
 308
 309         /* Handle header fields which can come from multiple layers like
 310          * etype, outer vlan tci. These fields should have same position in
 311          * the key otherwise to install a mcam rule more than one entry is
 312          * needed which complicates mcam space management.
 313          */
 314         etype_ether = &key_fields[NPC_ETYPE_ETHER];
 315         etype_tag1 = &key_fields[NPC_ETYPE_TAG1];
 316         etype_tag2 = &key_fields[NPC_ETYPE_TAG2];
 317         vlan_tag1 = &key_fields[NPC_VLAN_TAG1];
 318         vlan_tag2 = &key_fields[NPC_VLAN_TAG2];
 319
 320         /* if key profile programmed does not extract Ethertype at all */
 321         if (!etype_ether->nr_kws && !etype_tag1->nr_kws && !etype_tag2->nr_kws)
 322                 goto vlan_tci;
 323
 324         /* if key profile programmed extracts Ethertype from one layer */
 325         if (etype_ether->nr_kws && !etype_tag1->nr_kws && !etype_tag2->nr_kws)
 326                 key_fields[NPC_ETYPE] = *etype_ether;
 327         if (!etype_ether->nr_kws && etype_tag1->nr_kws && !etype_tag2->nr_kws)
 328                 key_fields[NPC_ETYPE] = *etype_tag1;
 329         if (!etype_ether->nr_kws && !etype_tag1->nr_kws && etype_tag2->nr_kws)
 330                 key_fields[NPC_ETYPE] = *etype_tag2;
 331
 332         /* if key profile programmed extracts Ethertype from multiple layers */
 333         if (etype_ether->nr_kws && etype_tag1->nr_kws) {
 334                 for (i = 0; i < NPC_MAX_KWS_IN_KEY; i++) {
 335                         if (etype_ether->kw_mask[i] != etype_tag1->kw_mask[i])
 336                                 goto vlan_tci;
 337                 }
 338                 key_fields[NPC_ETYPE] = *etype_tag1;
 339         }
 340         if (etype_ether->nr_kws && etype_tag2->nr_kws) {
 341                 for (i = 0; i < NPC_MAX_KWS_IN_KEY; i++) {
 342                         if (etype_ether->kw_mask[i] != etype_tag2->kw_mask[i])
 343                                 goto vlan_tci;
 344                 }
 345                 key_fields[NPC_ETYPE] = *etype_tag2;
 346         }
 347         if (etype_tag1->nr_kws && etype_tag2->nr_kws) {
 348                 for (i = 0; i < NPC_MAX_KWS_IN_KEY; i++) {
 349                         if (etype_tag1->kw_mask[i] != etype_tag2->kw_mask[i])
 350                                 goto vlan_tci;
 351                 }
 352                 key_fields[NPC_ETYPE] = *etype_tag2;
 353         }
 354
 355         /* check none of higher layers overwrite Ethertype */
 356         start_lid = key_fields[NPC_ETYPE].layer_mdata.lid + 1;
 357         if (npc_check_overlap(rvu, blkaddr, NPC_ETYPE, start_lid, intf))
 358                 goto vlan_tci;
 359         *features |= BIT_ULL(NPC_ETYPE);
 360 vlan_tci:
 361         /* if key profile does not extract outer vlan tci at all */
 362         if (!vlan_tag1->nr_kws && !vlan_tag2->nr_kws)
 363                 goto done;
 364
 365         /* if key profile extracts outer vlan tci from one layer */
 366         if (vlan_tag1->nr_kws && !vlan_tag2->nr_kws)
 367                 key_fields[NPC_OUTER_VID] = *vlan_tag1;
 368         if (!vlan_tag1->nr_kws && vlan_tag2->nr_kws)
 369                 key_fields[NPC_OUTER_VID] = *vlan_tag2;
 370
 371         /* if key profile extracts outer vlan tci from multiple layers */
 372         if (vlan_tag1->nr_kws && vlan_tag2->nr_kws) {
 373                 for (i = 0; i < NPC_MAX_KWS_IN_KEY; i++) {
 374                         if (vlan_tag1->kw_mask[i] != vlan_tag2->kw_mask[i])
 375                                 goto done;
 376                 }
 377                 key_fields[NPC_OUTER_VID] = *vlan_tag2;
 378         }
 379         /* check none of higher layers overwrite outer vlan tci */
 380         start_lid = key_fields[NPC_OUTER_VID].layer_mdata.lid + 1;
 381         if (npc_check_overlap(rvu, blkaddr, NPC_OUTER_VID, start_lid, intf))
 382                 goto done;
 383         *features |= BIT_ULL(NPC_OUTER_VID);
 384 done:
 385         return;
 386 }
 387
 388 static void npc_scan_ldata(struct rvu *rvu, int blkaddr, u8 lid,
 389                            u8 lt, u64 cfg, u8 intf)
 390 {
 391         struct npc_mcam *mcam = &rvu->hw->mcam;
 392         u8 hdr, key, nr_bytes, bit_offset;
 393         u8 la_ltype, la_start;
 394         /* starting KW index and starting bit position */
 395         int start_kwi, offset;
 396
 397         nr_bytes = FIELD_GET(NPC_BYTESM, cfg) + 1;
 398         hdr = FIELD_GET(NPC_HDR_OFFSET, cfg);
 399         key = FIELD_GET(NPC_KEY_OFFSET, cfg);
 400         start_kwi = key / 8;
 401         offset = (key * 8) % 64;
 402
 403         /* For Tx, Layer A has NIX_INST_HDR_S(64 bytes) preceding
 404          * ethernet header.
 405          */
 406         if (is_npc_intf_tx(intf)) {
 407                 la_ltype = NPC_LT_LA_IH_NIX_ETHER;
 408                 la_start = 8;
 409         } else {
 410                 la_ltype = NPC_LT_LA_ETHER;
 411                 la_start = 0;
 412         }
 413
 414 #define NPC_SCAN_HDR(name, hlid, hlt, hstart, hlen)                            \
 415 do {                                                                           \
 416         if (lid == (hlid) && lt == (hlt)) {                                    \
 417                 if ((hstart) >= hdr &&                                         \
 418                     ((hstart) + (hlen)) <= (hdr + nr_bytes)) {                 \
 419                         bit_offset = (hdr + nr_bytes - (hstart) - (hlen)) * 8; \
 420                         npc_set_layer_mdata(mcam, (name), cfg, lid, lt, intf); \
 421                         npc_set_kw_masks(mcam, (name), (hlen) * 8,             \
 422                                          start_kwi, offset + bit_offset, intf);\
 423                 }                                                              \
 424         }                                                                      \
 425 } while (0)
 426
 427         /* List LID, LTYPE, start offset from layer and length(in bytes) of
 428          * packet header fields below.
 429          * Example: Source IP is 4 bytes and starts at 12th byte of IP header
 430          */
 431         NPC_SCAN_HDR(NPC_TOS, NPC_LID_LC, NPC_LT_LC_IP, 1, 1);
 432         NPC_SCAN_HDR(NPC_SIP_IPV4, NPC_LID_LC, NPC_LT_LC_IP, 12, 4);
 433         NPC_SCAN_HDR(NPC_DIP_IPV4, NPC_LID_LC, NPC_LT_LC_IP, 16, 4);
 434         NPC_SCAN_HDR(NPC_SIP_IPV6, NPC_LID_LC, NPC_LT_LC_IP6, 8, 16);
 435         NPC_SCAN_HDR(NPC_DIP_IPV6, NPC_LID_LC, NPC_LT_LC_IP6, 24, 16);
 436         NPC_SCAN_HDR(NPC_SPORT_UDP, NPC_LID_LD, NPC_LT_LD_UDP, 0, 2);
 437         NPC_SCAN_HDR(NPC_DPORT_UDP, NPC_LID_LD, NPC_LT_LD_UDP, 2, 2);
 438         NPC_SCAN_HDR(NPC_SPORT_TCP, NPC_LID_LD, NPC_LT_LD_TCP, 0, 2);
 439         NPC_SCAN_HDR(NPC_DPORT_TCP, NPC_LID_LD, NPC_LT_LD_TCP, 2, 2);
 440         NPC_SCAN_HDR(NPC_SPORT_SCTP, NPC_LID_LD, NPC_LT_LD_SCTP, 0, 2);
 441         NPC_SCAN_HDR(NPC_DPORT_SCTP, NPC_LID_LD, NPC_LT_LD_SCTP, 2, 2);
 442         NPC_SCAN_HDR(NPC_ETYPE_ETHER, NPC_LID_LA, NPC_LT_LA_ETHER, 12, 2);
 443         NPC_SCAN_HDR(NPC_ETYPE_TAG1, NPC_LID_LB, NPC_LT_LB_CTAG, 4, 2);
 444         NPC_SCAN_HDR(NPC_ETYPE_TAG2, NPC_LID_LB, NPC_LT_LB_STAG_QINQ, 8, 2);
 445         NPC_SCAN_HDR(NPC_VLAN_TAG1, NPC_LID_LB, NPC_LT_LB_CTAG, 2, 2);
 446         NPC_SCAN_HDR(NPC_VLAN_TAG2, NPC_LID_LB, NPC_LT_LB_STAG_QINQ, 2, 2);
 447         NPC_SCAN_HDR(NPC_DMAC, NPC_LID_LA, la_ltype, la_start, 6);
 448         NPC_SCAN_HDR(NPC_SMAC, NPC_LID_LA, la_ltype, la_start, 6);
 449         /* PF_FUNC is 2 bytes at 0th byte of NPC_LT_LA_IH_NIX_ETHER */
 450         NPC_SCAN_HDR(NPC_PF_FUNC, NPC_LID_LA, NPC_LT_LA_IH_NIX_ETHER, 0, 2);
 451 }
 452
 453 static void npc_set_features(struct rvu *rvu, int blkaddr, u8 intf)
 454 {
 455         struct npc_mcam *mcam = &rvu->hw->mcam;
 456         u64 *features = &mcam->rx_features;
 457         u64 tcp_udp_sctp;
 458         int hdr;
 459
 460         if (is_npc_intf_tx(intf))
 461                 features = &mcam->tx_features;
 462
 463         for (hdr = NPC_DMAC; hdr < NPC_HEADER_FIELDS_MAX; hdr++) {
 464                 if (npc_check_field(rvu, blkaddr, hdr, intf))
 465                         *features |= BIT_ULL(hdr);
 466         }
 467
 468         tcp_udp_sctp = BIT_ULL(NPC_SPORT_TCP) | BIT_ULL(NPC_SPORT_UDP) |
 469                        BIT_ULL(NPC_DPORT_TCP) | BIT_ULL(NPC_DPORT_UDP) |
 470                        BIT_ULL(NPC_SPORT_SCTP) | BIT_ULL(NPC_DPORT_SCTP);
 471
 472         /* for tcp/udp/sctp corresponding layer type should be in the key */
 473         if (*features & tcp_udp_sctp) {
 474                 if (!npc_check_field(rvu, blkaddr, NPC_LD, intf))
 475                         *features &= ~tcp_udp_sctp;
 476                 else
 477                         *features |= BIT_ULL(NPC_IPPROTO_TCP) |
 478                                      BIT_ULL(NPC_IPPROTO_UDP) |
 479                                      BIT_ULL(NPC_IPPROTO_SCTP);
 480         }
 481
 482         /* for AH/ICMP/ICMPv6/, check if corresponding layer type is present in the key */
 483         if (npc_check_field(rvu, blkaddr, NPC_LD, intf)) {
 484                 *features |= BIT_ULL(NPC_IPPROTO_AH);
 485                 *features |= BIT_ULL(NPC_IPPROTO_ICMP);
 486                 *features |= BIT_ULL(NPC_IPPROTO_ICMP6);
 487         }
 488
 489         /* for ESP, check if corresponding layer type is present in the key */
 490         if (npc_check_field(rvu, blkaddr, NPC_LE, intf))
 491                 *features |= BIT_ULL(NPC_IPPROTO_ESP);
 492
 493         /* for vlan corresponding layer type should be in the key */
 494         if (*features & BIT_ULL(NPC_OUTER_VID))
 495                 if (!npc_check_field(rvu, blkaddr, NPC_LB, intf))
 496                         *features &= ~BIT_ULL(NPC_OUTER_VID);
 497
 498         /* for vlan ethertypes corresponding layer type should be in the key */
 499         if (npc_check_field(rvu, blkaddr, NPC_LB, intf))
 500                 *features |= BIT_ULL(NPC_VLAN_ETYPE_CTAG) |
 501                              BIT_ULL(NPC_VLAN_ETYPE_STAG);
 502 }
 503
 504 /* Scan key extraction profile and record how fields of our interest
 505  * fill the key structure. Also verify Channel and DMAC exists in
 506  * key and not overwritten by other header fields.
 507  */
 508 static int npc_scan_kex(struct rvu *rvu, int blkaddr, u8 intf)
 509 {
 510         struct npc_mcam *mcam = &rvu->hw->mcam;
 511         u8 lid, lt, ld, bitnr;
 512         u8 key_nibble = 0;
 513         u64 cfg;
 514
 515         /* Scan and note how parse result is going to be in key.
 516          * A bit set in PARSE_NIBBLE_ENA corresponds to a nibble from
 517          * parse result in the key. The enabled nibbles from parse result
 518          * will be concatenated in key.
 519          */
 520         cfg = rvu_read64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf));
 521         cfg &= NPC_PARSE_NIBBLE;
 522         for_each_set_bit(bitnr, (unsigned long *)&cfg, 31) {
 523                 npc_scan_parse_result(mcam, bitnr, key_nibble, intf);
 524                 key_nibble++;
 525         }
 526
 527         /* Scan and note how layer data is going to be in key */
 528         for (lid = 0; lid < NPC_MAX_LID; lid++) {
 529                 for (lt = 0; lt < NPC_MAX_LT; lt++) {
 530                         for (ld = 0; ld < NPC_MAX_LD; ld++) {
 531                                 cfg = rvu_read64(rvu, blkaddr,
 532                                                  NPC_AF_INTFX_LIDX_LTX_LDX_CFG
 533                                                  (intf, lid, lt, ld));
 534                                 if (!FIELD_GET(NPC_LDATA_EN, cfg))
 535                                         continue;
 536                                 npc_scan_ldata(rvu, blkaddr, lid, lt, cfg,
 537                                                intf);
 538                         }
 539                 }
 540         }
 541
 542         return 0;
 543 }
 544
 545 static int npc_scan_verify_kex(struct rvu *rvu, int blkaddr)
 546 {
 547         int err;
 548
 549         err = npc_scan_kex(rvu, blkaddr, NIX_INTF_RX);
 550         if (err)
 551                 return err;
 552
 553         err = npc_scan_kex(rvu, blkaddr, NIX_INTF_TX);
 554         if (err)
 555                 return err;
 556
 557         /* Channel is mandatory */
 558         if (!npc_is_field_present(rvu, NPC_CHAN, NIX_INTF_RX)) {
 559                 dev_err(rvu->dev, "Channel not present in Key\n");
 560                 return -EINVAL;
 561         }
 562         /* check that none of the fields overwrite channel */
 563         if (npc_check_overlap(rvu, blkaddr, NPC_CHAN, 0, NIX_INTF_RX)) {
 564                 dev_err(rvu->dev, "Channel cannot be overwritten\n");
 565                 return -EINVAL;
 566         }
 567         /* DMAC should be present in key for unicast filter to work */
 568         if (!npc_is_field_present(rvu, NPC_DMAC, NIX_INTF_RX)) {
 569                 dev_err(rvu->dev, "DMAC not present in Key\n");
 570                 return -EINVAL;
 571         }
 572         /* check that none of the fields overwrite DMAC */
 573         if (npc_check_overlap(rvu, blkaddr, NPC_DMAC, 0, NIX_INTF_RX)) {
 574                 dev_err(rvu->dev, "DMAC cannot be overwritten\n");
 575                 return -EINVAL;
 576         }
 577
 578         npc_set_features(rvu, blkaddr, NIX_INTF_TX);
 579         npc_set_features(rvu, blkaddr, NIX_INTF_RX);
 580         npc_handle_multi_layer_fields(rvu, blkaddr, NIX_INTF_TX);
 581         npc_handle_multi_layer_fields(rvu, blkaddr, NIX_INTF_RX);
 582
 583         return 0;
 584 }
 585
 586 int npc_flow_steering_init(struct rvu *rvu, int blkaddr)
 587 {
 588         struct npc_mcam *mcam = &rvu->hw->mcam;
 589
 590         INIT_LIST_HEAD(&mcam->mcam_rules);
 591
 592         return npc_scan_verify_kex(rvu, blkaddr);
 593 }
 594
 595 static int npc_check_unsupported_flows(struct rvu *rvu, u64 features, u8 intf)
 596 {
 597         struct npc_mcam *mcam = &rvu->hw->mcam;
 598         u64 *mcam_features = &mcam->rx_features;
 599         u64 unsupported;
 600         u8 bit;
 601
 602         if (is_npc_intf_tx(intf))
 603                 mcam_features = &mcam->tx_features;
 604
 605         unsupported = (*mcam_features ^ features) & ~(*mcam_features);
 606         if (unsupported) {
 607                 dev_info(rvu->dev, "Unsupported flow(s):\n");
 608                 for_each_set_bit(bit, (unsigned long *)&unsupported, 64)
 609                         dev_info(rvu->dev, "%s ", npc_get_field_name(bit));
 610                 return -EOPNOTSUPP;
 611         }
 612
 613         return 0;
 614 }
 615
 616 /* npc_update_entry - Based on the masks generated during
 617  * the key scanning, updates the given entry with value and
 618  * masks for the field of interest. Maximum 16 bytes of a packet
 619  * header can be extracted by HW hence lo and hi are sufficient.
 620  * When field bytes are less than or equal to 8 then hi should be
 621  * 0 for value and mask.
 622  *
 623  * If exact match of value is required then mask should be all 1's.
 624  * If any bits in mask are 0 then corresponding bits in value are
 625  * dont care.
 626  */
 627 static void npc_update_entry(struct rvu *rvu, enum key_fields type,
 628                              struct mcam_entry *entry, u64 val_lo,
 629                              u64 val_hi, u64 mask_lo, u64 mask_hi, u8 intf)
 630 {
 631         struct npc_mcam *mcam = &rvu->hw->mcam;
 632         struct mcam_entry dummy = { {0} };
 633         struct npc_key_field *field;
 634         u64 kw1, kw2, kw3;
 635         u8 shift;
 636         int i;
 637
 638         field = &mcam->rx_key_fields[type];
 639         if (is_npc_intf_tx(intf))
 640                 field = &mcam->tx_key_fields[type];
 641
 642         if (!field->nr_kws)
 643                 return;
 644
 645         for (i = 0; i < NPC_MAX_KWS_IN_KEY; i++) {
 646                 if (!field->kw_mask[i])
 647                         continue;
 648                 /* place key value in kw[x] */
 649                 shift = __ffs64(field->kw_mask[i]);
 650                 /* update entry value */
 651                 kw1 = (val_lo << shift) & field->kw_mask[i];
 652                 dummy.kw[i] = kw1;
 653                 /* update entry mask */
 654                 kw1 = (mask_lo << shift) & field->kw_mask[i];
 655                 dummy.kw_mask[i] = kw1;
 656
 657                 if (field->nr_kws == 1)
 658                         break;
 659                 /* place remaining bits of key value in kw[x + 1] */
 660                 if (field->nr_kws == 2) {
 661                         /* update entry value */
 662                         kw2 = shift ? val_lo >> (64 - shift) : 0;
 663                         kw2 |= (val_hi << shift);
 664                         kw2 &= field->kw_mask[i + 1];
 665                         dummy.kw[i + 1] = kw2;
 666                         /* update entry mask */
 667                         kw2 = shift ? mask_lo >> (64 - shift) : 0;
 668                         kw2 |= (mask_hi << shift);
 669                         kw2 &= field->kw_mask[i + 1];
 670                         dummy.kw_mask[i + 1] = kw2;
 671                         break;
 672                 }
 673                 /* place remaining bits of key value in kw[x + 1], kw[x + 2] */
 674                 if (field->nr_kws == 3) {
 675                         /* update entry value */
 676                         kw2 = shift ? val_lo >> (64 - shift) : 0;
 677                         kw2 |= (val_hi << shift);
 678                         kw2 &= field->kw_mask[i + 1];
 679                         kw3 = shift ? val_hi >> (64 - shift) : 0;
 680                         kw3 &= field->kw_mask[i + 2];
 681                         dummy.kw[i + 1] = kw2;
 682                         dummy.kw[i + 2] = kw3;
 683                         /* update entry mask */
 684                         kw2 = shift ? mask_lo >> (64 - shift) : 0;
 685                         kw2 |= (mask_hi << shift);
 686                         kw2 &= field->kw_mask[i + 1];
 687                         kw3 = shift ? mask_hi >> (64 - shift) : 0;
 688                         kw3 &= field->kw_mask[i + 2];
 689                         dummy.kw_mask[i + 1] = kw2;
 690                         dummy.kw_mask[i + 2] = kw3;
 691                         break;
 692                 }
 693         }
 694         /* dummy is ready with values and masks for given key
 695          * field now clear and update input entry with those
 696          */
 697         for (i = 0; i < NPC_MAX_KWS_IN_KEY; i++) {
 698                 if (!field->kw_mask[i])
 699                         continue;
 700                 entry->kw[i] &= ~field->kw_mask[i];
 701                 entry->kw_mask[i] &= ~field->kw_mask[i];
 702
 703                 entry->kw[i] |= dummy.kw[i];
 704                 entry->kw_mask[i] |= dummy.kw_mask[i];
 705         }
 706 }
 707
 708 #define IPV6_WORDS     4
 709
 710 static void npc_update_ipv6_flow(struct rvu *rvu, struct mcam_entry *entry,
 711                                  u64 features, struct flow_msg *pkt,
 712                                  struct flow_msg *mask,
 713                                  struct rvu_npc_mcam_rule *output, u8 intf)
 714 {
 715         u32 src_ip[IPV6_WORDS], src_ip_mask[IPV6_WORDS];
 716         u32 dst_ip[IPV6_WORDS], dst_ip_mask[IPV6_WORDS];
 717         struct flow_msg *opkt = &output->packet;
 718         struct flow_msg *omask = &output->mask;
 719         u64 mask_lo, mask_hi;
 720         u64 val_lo, val_hi;
 721
 722         /* For an ipv6 address fe80::2c68:63ff:fe5e:2d0a the packet
 723          * values to be programmed in MCAM should as below:
 724          * val_high: 0xfe80000000000000
 725          * val_low: 0x2c6863fffe5e2d0a
 726          */
 727         if (features & BIT_ULL(NPC_SIP_IPV6)) {
 728                 be32_to_cpu_array(src_ip_mask, mask->ip6src, IPV6_WORDS);
 729                 be32_to_cpu_array(src_ip, pkt->ip6src, IPV6_WORDS);
 730
 731                 mask_hi = (u64)src_ip_mask[0] << 32 | src_ip_mask[1];
 732                 mask_lo = (u64)src_ip_mask[2] << 32 | src_ip_mask[3];
 733                 val_hi = (u64)src_ip[0] << 32 | src_ip[1];
 734                 val_lo = (u64)src_ip[2] << 32 | src_ip[3];
 735
 736                 npc_update_entry(rvu, NPC_SIP_IPV6, entry, val_lo, val_hi,
 737                                  mask_lo, mask_hi, intf);
 738                 memcpy(opkt->ip6src, pkt->ip6src, sizeof(opkt->ip6src));
 739                 memcpy(omask->ip6src, mask->ip6src, sizeof(omask->ip6src));
 740         }
 741         if (features & BIT_ULL(NPC_DIP_IPV6)) {
 742                 be32_to_cpu_array(dst_ip_mask, mask->ip6dst, IPV6_WORDS);
 743                 be32_to_cpu_array(dst_ip, pkt->ip6dst, IPV6_WORDS);
 744
 745                 mask_hi = (u64)dst_ip_mask[0] << 32 | dst_ip_mask[1];
 746                 mask_lo = (u64)dst_ip_mask[2] << 32 | dst_ip_mask[3];
 747                 val_hi = (u64)dst_ip[0] << 32 | dst_ip[1];
 748                 val_lo = (u64)dst_ip[2] << 32 | dst_ip[3];
 749
 750                 npc_update_entry(rvu, NPC_DIP_IPV6, entry, val_lo, val_hi,
 751                                  mask_lo, mask_hi, intf);
 752                 memcpy(opkt->ip6dst, pkt->ip6dst, sizeof(opkt->ip6dst));
 753                 memcpy(omask->ip6dst, mask->ip6dst, sizeof(omask->ip6dst));
 754         }
 755 }
 756
 757 static void npc_update_vlan_features(struct rvu *rvu, struct mcam_entry *entry,
 758                                      u64 features, u8 intf)
 759 {
 760         bool ctag = !!(features & BIT_ULL(NPC_VLAN_ETYPE_CTAG));
 761         bool stag = !!(features & BIT_ULL(NPC_VLAN_ETYPE_STAG));
 762         bool vid = !!(features & BIT_ULL(NPC_OUTER_VID));
 763
 764         /* If only VLAN id is given then always match outer VLAN id */
 765         if (vid && !ctag && !stag) {
 766                 npc_update_entry(rvu, NPC_LB, entry,
 767                                  NPC_LT_LB_STAG_QINQ | NPC_LT_LB_CTAG, 0,
 768                                  NPC_LT_LB_STAG_QINQ & NPC_LT_LB_CTAG, 0, intf);
 769                 return;
 770         }
 771         if (ctag)
 772                 npc_update_entry(rvu, NPC_LB, entry, NPC_LT_LB_CTAG, 0,
 773                                  ~0ULL, 0, intf);
 774         if (stag)
 775                 npc_update_entry(rvu, NPC_LB, entry, NPC_LT_LB_STAG_QINQ, 0,
 776                                  ~0ULL, 0, intf);
 777 }
 778
 779 static void npc_update_flow(struct rvu *rvu, struct mcam_entry *entry,
 780                             u64 features, struct flow_msg *pkt,
 781                             struct flow_msg *mask,
 782                             struct rvu_npc_mcam_rule *output, u8 intf)
 783 {
 784         u64 dmac_mask = ether_addr_to_u64(mask->dmac);
 785         u64 smac_mask = ether_addr_to_u64(mask->smac);
 786         u64 dmac_val = ether_addr_to_u64(pkt->dmac);
 787         u64 smac_val = ether_addr_to_u64(pkt->smac);
 788         struct flow_msg *opkt = &output->packet;
 789         struct flow_msg *omask = &output->mask;
 790
 791         if (!features)
 792                 return;
 793
 794         /* For tcp/udp/sctp LTYPE should be present in entry */
 795         if (features & BIT_ULL(NPC_IPPROTO_TCP))
 796                 npc_update_entry(rvu, NPC_LD, entry, NPC_LT_LD_TCP,
 797                                  0, ~0ULL, 0, intf);
 798         if (features & BIT_ULL(NPC_IPPROTO_UDP))
 799                 npc_update_entry(rvu, NPC_LD, entry, NPC_LT_LD_UDP,
 800                                  0, ~0ULL, 0, intf);
 801         if (features & BIT_ULL(NPC_IPPROTO_SCTP))
 802                 npc_update_entry(rvu, NPC_LD, entry, NPC_LT_LD_SCTP,
 803                                  0, ~0ULL, 0, intf);
 804         if (features & BIT_ULL(NPC_IPPROTO_ICMP))
 805                 npc_update_entry(rvu, NPC_LD, entry, NPC_LT_LD_ICMP,
 806                                  0, ~0ULL, 0, intf);
 807         if (features & BIT_ULL(NPC_IPPROTO_ICMP6))
 808                 npc_update_entry(rvu, NPC_LD, entry, NPC_LT_LD_ICMP6,
 809                                  0, ~0ULL, 0, intf);
 810
 811         /* For AH, LTYPE should be present in entry */
 812         if (features & BIT_ULL(NPC_IPPROTO_AH))
 813                 npc_update_entry(rvu, NPC_LD, entry, NPC_LT_LD_AH,
 814                                  0, ~0ULL, 0, intf);
 815         /* For ESP, LTYPE should be present in entry */
 816         if (features & BIT_ULL(NPC_IPPROTO_ESP))
 817                 npc_update_entry(rvu, NPC_LE, entry, NPC_LT_LE_ESP,
 818                                  0, ~0ULL, 0, intf);
 819
 820 #define NPC_WRITE_FLOW(field, member, val_lo, val_hi, mask_lo, mask_hi)       \
 821 do {                                                                          \
 822         if (features & BIT_ULL((field))) {                                    \
 823                 npc_update_entry(rvu, (field), entry, (val_lo), (val_hi),     \
 824                                  (mask_lo), (mask_hi), intf);                 \
 825                 memcpy(&opkt->member, &pkt->member, sizeof(pkt->member));     \
 826                 memcpy(&omask->member, &mask->member, sizeof(mask->member));  \
 827         }                                                                     \
 828 } while (0)
 829
 830         NPC_WRITE_FLOW(NPC_DMAC, dmac, dmac_val, 0, dmac_mask, 0);
 831         NPC_WRITE_FLOW(NPC_SMAC, smac, smac_val, 0, smac_mask, 0);
 832         NPC_WRITE_FLOW(NPC_ETYPE, etype, ntohs(pkt->etype), 0,
 833                        ntohs(mask->etype), 0);
 834         NPC_WRITE_FLOW(NPC_TOS, tos, pkt->tos, 0, mask->tos, 0);
 835         NPC_WRITE_FLOW(NPC_SIP_IPV4, ip4src, ntohl(pkt->ip4src), 0,
 836                        ntohl(mask->ip4src), 0);
 837         NPC_WRITE_FLOW(NPC_DIP_IPV4, ip4dst, ntohl(pkt->ip4dst), 0,
 838                        ntohl(mask->ip4dst), 0);
 839         NPC_WRITE_FLOW(NPC_SPORT_TCP, sport, ntohs(pkt->sport), 0,
 840                        ntohs(mask->sport), 0);
 841         NPC_WRITE_FLOW(NPC_SPORT_UDP, sport, ntohs(pkt->sport), 0,
 842                        ntohs(mask->sport), 0);
 843         NPC_WRITE_FLOW(NPC_DPORT_TCP, dport, ntohs(pkt->dport), 0,
 844                        ntohs(mask->dport), 0);
 845         NPC_WRITE_FLOW(NPC_DPORT_UDP, dport, ntohs(pkt->dport), 0,
 846                        ntohs(mask->dport), 0);
 847         NPC_WRITE_FLOW(NPC_SPORT_SCTP, sport, ntohs(pkt->sport), 0,
 848                        ntohs(mask->sport), 0);
 849         NPC_WRITE_FLOW(NPC_DPORT_SCTP, dport, ntohs(pkt->dport), 0,
 850                        ntohs(mask->dport), 0);
 851
 852         NPC_WRITE_FLOW(NPC_OUTER_VID, vlan_tci, ntohs(pkt->vlan_tci), 0,
 853                        ntohs(mask->vlan_tci), 0);
 854
 855         npc_update_ipv6_flow(rvu, entry, features, pkt, mask, output, intf);
 856         npc_update_vlan_features(rvu, entry, features, intf);
 857 }
 858
 859 static struct rvu_npc_mcam_rule *rvu_mcam_find_rule(struct npc_mcam *mcam,
 860                                                     u16 entry)
 861 {
 862         struct rvu_npc_mcam_rule *iter;
 863
 864         mutex_lock(&mcam->lock);
 865         list_for_each_entry(iter, &mcam->mcam_rules, list) {
 866                 if (iter->entry == entry) {
 867                         mutex_unlock(&mcam->lock);
 868                         return iter;
 869                 }
 870         }
 871         mutex_unlock(&mcam->lock);
 872
 873         return NULL;
 874 }
 875
 876 static void rvu_mcam_add_rule(struct npc_mcam *mcam,
 877                               struct rvu_npc_mcam_rule *rule)
 878 {
 879         struct list_head *head = &mcam->mcam_rules;
 880         struct rvu_npc_mcam_rule *iter;
 881
 882         mutex_lock(&mcam->lock);
 883         list_for_each_entry(iter, &mcam->mcam_rules, list) {
 884                 if (iter->entry > rule->entry)
 885                         break;
 886                 head = &iter->list;
 887         }
 888
 889         list_add(&rule->list, head);
 890         mutex_unlock(&mcam->lock);
 891 }
 892
 893 static void rvu_mcam_remove_counter_from_rule(struct rvu *rvu, u16 pcifunc,
 894                                               struct rvu_npc_mcam_rule *rule)
 895 {
 896         struct npc_mcam_oper_counter_req free_req = { 0 };
 897         struct msg_rsp free_rsp;
 898
 899         if (!rule->has_cntr)
 900                 return;
 901
 902         free_req.hdr.pcifunc = pcifunc;
 903         free_req.cntr = rule->cntr;
 904
 905         rvu_mbox_handler_npc_mcam_free_counter(rvu, &free_req, &free_rsp);
 906         rule->has_cntr = false;
 907 }
 908
 909 static void rvu_mcam_add_counter_to_rule(struct rvu *rvu, u16 pcifunc,
 910                                          struct rvu_npc_mcam_rule *rule,
 911                                          struct npc_install_flow_rsp *rsp)
 912 {
 913         struct npc_mcam_alloc_counter_req cntr_req = { 0 };
 914         struct npc_mcam_alloc_counter_rsp cntr_rsp = { 0 };
 915         int err;
 916
 917         cntr_req.hdr.pcifunc = pcifunc;
 918         cntr_req.contig = true;
 919         cntr_req.count = 1;
 920
 921         /* we try to allocate a counter to track the stats of this
 922          * rule. If counter could not be allocated then proceed
 923          * without counter because counters are limited than entries.
 924          */
 925         err = rvu_mbox_handler_npc_mcam_alloc_counter(rvu, &cntr_req,
 926                                                       &cntr_rsp);
 927         if (!err && cntr_rsp.count) {
 928                 rule->cntr = cntr_rsp.cntr;
 929                 rule->has_cntr = true;
 930                 rsp->counter = rule->cntr;
 931         } else {
 932                 rsp->counter = err;
 933         }
 934 }
 935
 936 static void npc_update_rx_entry(struct rvu *rvu, struct rvu_pfvf *pfvf,
 937                                 struct mcam_entry *entry,
 938                                 struct npc_install_flow_req *req,
 939                                 u16 target, bool pf_set_vfs_mac)
 940 {
 941         struct rvu_switch *rswitch = &rvu->rswitch;
 942         struct nix_rx_action action;
 943
 944         if (rswitch->mode == DEVLINK_ESWITCH_MODE_SWITCHDEV && pf_set_vfs_mac)
 945                 req->chan_mask = 0x0; /* Do not care channel */
 946
 947         npc_update_entry(rvu, NPC_CHAN, entry, req->channel, 0, req->chan_mask,
 948                          0, NIX_INTF_RX);
 949
 950         *(u64 *)&action = 0x00;
 951         action.pf_func = target;
 952         action.op = req->op;
 953         action.index = req->index;
 954         action.match_id = req->match_id;
 955         action.flow_key_alg = req->flow_key_alg;
 956
 957         if (req->op == NIX_RX_ACTION_DEFAULT && pfvf->def_ucast_rule)
 958                 action = pfvf->def_ucast_rule->rx_action;
 959
 960         entry->action = *(u64 *)&action;
 961
 962         /* VTAG0 starts at 0th byte of LID_B.
 963          * VTAG1 starts at 4th byte of LID_B.
 964          */
 965         entry->vtag_action = FIELD_PREP(RX_VTAG0_VALID_BIT, req->vtag0_valid) |
 966                              FIELD_PREP(RX_VTAG0_TYPE_MASK, req->vtag0_type) |
 967                              FIELD_PREP(RX_VTAG0_LID_MASK, NPC_LID_LB) |
 968                              FIELD_PREP(RX_VTAG0_RELPTR_MASK, 0) |
 969                              FIELD_PREP(RX_VTAG1_VALID_BIT, req->vtag1_valid) |
 970                              FIELD_PREP(RX_VTAG1_TYPE_MASK, req->vtag1_type) |
 971                              FIELD_PREP(RX_VTAG1_LID_MASK, NPC_LID_LB) |
 972                              FIELD_PREP(RX_VTAG1_RELPTR_MASK, 4);
 973 }
 974
 975 static void npc_update_tx_entry(struct rvu *rvu, struct rvu_pfvf *pfvf,
 976                                 struct mcam_entry *entry,
 977                                 struct npc_install_flow_req *req, u16 target)
 978 {
 979         struct nix_tx_action action;
 980         u64 mask = ~0ULL;
 981
 982         /* If AF is installing then do not care about
 983          * PF_FUNC in Send Descriptor
 984          */
 985         if (is_pffunc_af(req->hdr.pcifunc))
 986                 mask = 0;
 987
 988         npc_update_entry(rvu, NPC_PF_FUNC, entry, (__force u16)htons(target),
 989                          0, mask, 0, NIX_INTF_TX);
 990
 991         *(u64 *)&action = 0x00;
 992         action.op = req->op;
 993         action.index = req->index;
 994         action.match_id = req->match_id;
 995
 996         entry->action = *(u64 *)&action;
 997
 998         /* VTAG0 starts at 0th byte of LID_B.
 999          * VTAG1 starts at 4th byte of LID_B.
1000          */
1001         entry->vtag_action = FIELD_PREP(TX_VTAG0_DEF_MASK, req->vtag0_def) |
1002                              FIELD_PREP(TX_VTAG0_OP_MASK, req->vtag0_op) |
1003                              FIELD_PREP(TX_VTAG0_LID_MASK, NPC_LID_LA) |
1004                              FIELD_PREP(TX_VTAG0_RELPTR_MASK, 20) |
1005                              FIELD_PREP(TX_VTAG1_DEF_MASK, req->vtag1_def) |
1006                              FIELD_PREP(TX_VTAG1_OP_MASK, req->vtag1_op) |
1007                              FIELD_PREP(TX_VTAG1_LID_MASK, NPC_LID_LA) |
1008                              FIELD_PREP(TX_VTAG1_RELPTR_MASK, 24);
1009 }
1010
1011 static int npc_install_flow(struct rvu *rvu, int blkaddr, u16 target,
1012                             int nixlf, struct rvu_pfvf *pfvf,
1013                             struct npc_install_flow_req *req,
1014                             struct npc_install_flow_rsp *rsp, bool enable,
1015                             bool pf_set_vfs_mac)
1016 {
1017         struct rvu_npc_mcam_rule *def_ucast_rule = pfvf->def_ucast_rule;
1018         u64 features, installed_features, missing_features = 0;
1019         struct npc_mcam_write_entry_req write_req = { 0 };
1020         struct npc_mcam *mcam = &rvu->hw->mcam;
1021         struct rvu_npc_mcam_rule dummy = { 0 };
1022         struct rvu_npc_mcam_rule *rule;
1023         u16 owner = req->hdr.pcifunc;
1024         struct msg_rsp write_rsp;
1025         struct mcam_entry *entry;
1026         int entry_index, err;
1027         bool new = false;
1028
1029         installed_features = req->features;
1030         features = req->features;
1031         entry = &write_req.entry_data;
1032         entry_index = req->entry;
1033
1034         npc_update_flow(rvu, entry, features, &req->packet, &req->mask, &dummy,
1035                         req->intf);
1036
1037         if (is_npc_intf_rx(req->intf))
1038                 npc_update_rx_entry(rvu, pfvf, entry, req, target, pf_set_vfs_mac);
1039         else
1040                 npc_update_tx_entry(rvu, pfvf, entry, req, target);
1041
1042         /* Default unicast rules do not exist for TX */
1043         if (is_npc_intf_tx(req->intf))
1044                 goto find_rule;
1045
1046         if (req->default_rule) {
1047                 entry_index = npc_get_nixlf_mcam_index(mcam, target, nixlf,
1048                                                        NIXLF_UCAST_ENTRY);
1049                 enable = is_mcam_entry_enabled(rvu, mcam, blkaddr, entry_index);
1050         }
1051
1052         /* update mcam entry with default unicast rule attributes */
1053         if (def_ucast_rule && (req->default_rule && req->append)) {
1054                 missing_features = (def_ucast_rule->features ^ features) &
1055                                         def_ucast_rule->features;
1056                 if (missing_features)
1057                         npc_update_flow(rvu, entry, missing_features,
1058                                         &def_ucast_rule->packet,
1059                                         &def_ucast_rule->mask,
1060                                         &dummy, req->intf);
1061                 installed_features = req->features | missing_features;
1062         }
1063
1064 find_rule:
1065         rule = rvu_mcam_find_rule(mcam, entry_index);
1066         if (!rule) {
1067                 rule = kzalloc(sizeof(*rule), GFP_KERNEL);
1068                 if (!rule)
1069                         return -ENOMEM;
1070                 new = true;
1071         }
1072
1073         /* allocate new counter if rule has no counter */
1074         if (!req->default_rule && req->set_cntr && !rule->has_cntr)
1075                 rvu_mcam_add_counter_to_rule(rvu, owner, rule, rsp);
1076
1077         /* if user wants to delete an existing counter for a rule then
1078          * free the counter
1079          */
1080         if (!req->set_cntr && rule->has_cntr)
1081                 rvu_mcam_remove_counter_from_rule(rvu, owner, rule);
1082
1083         write_req.hdr.pcifunc = owner;
1084
1085         /* AF owns the default rules so change the owner just to relax
1086          * the checks in rvu_mbox_handler_npc_mcam_write_entry
1087          */
1088         if (req->default_rule)
1089                 write_req.hdr.pcifunc = 0;
1090
1091         write_req.entry = entry_index;
1092         write_req.intf = req->intf;
1093         write_req.enable_entry = (u8)enable;
1094         /* if counter is available then clear and use it */
1095         if (req->set_cntr && rule->has_cntr) {
1096                 rvu_write64(rvu, blkaddr, NPC_AF_MATCH_STATX(rule->cntr), 0x00);
1097                 write_req.set_cntr = 1;
1098                 write_req.cntr = rule->cntr;
1099         }
1100
1101         /* update rule */
1102         memcpy(&rule->packet, &dummy.packet, sizeof(rule->packet));
1103         memcpy(&rule->mask, &dummy.mask, sizeof(rule->mask));
1104         rule->entry = entry_index;
1105         memcpy(&rule->rx_action, &entry->action, sizeof(struct nix_rx_action));
1106         if (is_npc_intf_tx(req->intf))
1107                 memcpy(&rule->tx_action, &entry->action,
1108                        sizeof(struct nix_tx_action));
1109         rule->vtag_action = entry->vtag_action;
1110         rule->features = installed_features;
1111         rule->default_rule = req->default_rule;
1112         rule->owner = owner;
1113         rule->enable = enable;
1114         rule->chan_mask = write_req.entry_data.kw_mask[0] & NPC_KEX_CHAN_MASK;
1115         rule->chan = write_req.entry_data.kw[0] & NPC_KEX_CHAN_MASK;
1116         rule->chan &= rule->chan_mask;
1117         if (is_npc_intf_tx(req->intf))
1118                 rule->intf = pfvf->nix_tx_intf;
1119         else
1120                 rule->intf = pfvf->nix_rx_intf;
1121
1122         if (new)
1123                 rvu_mcam_add_rule(mcam, rule);
1124         if (req->default_rule)
1125                 pfvf->def_ucast_rule = rule;
1126
1127         /* write to mcam entry registers */
1128         err = rvu_mbox_handler_npc_mcam_write_entry(rvu, &write_req,
1129                                                     &write_rsp);
1130         if (err) {
1131                 rvu_mcam_remove_counter_from_rule(rvu, owner, rule);
1132                 if (new) {
1133                         list_del(&rule->list);
1134                         kfree(rule);
1135                 }
1136                 return err;
1137         }
1138
1139         /* VF's MAC address is being changed via PF  */
1140         if (pf_set_vfs_mac) {
1141                 ether_addr_copy(pfvf->default_mac, req->packet.dmac);
1142                 ether_addr_copy(pfvf->mac_addr, req->packet.dmac);
1143                 set_bit(PF_SET_VF_MAC, &pfvf->flags);
1144         }
1145
1146         if (test_bit(PF_SET_VF_CFG, &pfvf->flags) &&
1147             req->vtag0_type == NIX_AF_LFX_RX_VTAG_TYPE7)
1148                 rule->vfvlan_cfg = true;
1149
1150         if (is_npc_intf_rx(req->intf) && req->match_id &&
1151             (req->op == NIX_RX_ACTIONOP_UCAST || req->op == NIX_RX_ACTIONOP_RSS))
1152                 return rvu_nix_setup_ratelimit_aggr(rvu, req->hdr.pcifunc,
1153                                              req->index, req->match_id);
1154
1155         return 0;
1156 }
1157
1158 int rvu_mbox_handler_npc_install_flow(struct rvu *rvu,
1159                                       struct npc_install_flow_req *req,
1160                                       struct npc_install_flow_rsp *rsp)
1161 {
1162         bool from_vf = !!(req->hdr.pcifunc & RVU_PFVF_FUNC_MASK);
1163         struct rvu_switch *rswitch = &rvu->rswitch;
1164         int blkaddr, nixlf, err;
1165         struct rvu_pfvf *pfvf;
1166         bool pf_set_vfs_mac = false;
1167         bool enable = true;
1168         u16 target;
1169
1170         blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
1171         if (blkaddr < 0) {
1172                 dev_err(rvu->dev, "%s: NPC block not implemented\n", __func__);
1173                 return NPC_MCAM_INVALID_REQ;
1174         }
1175
1176         if (!is_npc_interface_valid(rvu, req->intf))
1177                 return NPC_FLOW_INTF_INVALID;
1178
1179         if (from_vf && req->default_rule)
1180                 return NPC_FLOW_VF_PERM_DENIED;
1181
1182         /* Each PF/VF info is maintained in struct rvu_pfvf.
1183          * rvu_pfvf for the target PF/VF needs to be retrieved
1184          * hence modify pcifunc accordingly.
1185          */
1186
1187         /* AF installing for a PF/VF */
1188         if (!req->hdr.pcifunc)
1189                 target = req->vf;
1190         /* PF installing for its VF */
1191         else if (!from_vf && req->vf) {
1192                 target = (req->hdr.pcifunc & ~RVU_PFVF_FUNC_MASK) | req->vf;
1193                 pf_set_vfs_mac = req->default_rule &&
1194                                 (req->features & BIT_ULL(NPC_DMAC));
1195         }
1196         /* msg received from PF/VF */
1197         else
1198                 target = req->hdr.pcifunc;
1199
1200         /* ignore chan_mask in case pf func is not AF, revisit later */
1201         if (!is_pffunc_af(req->hdr.pcifunc))
1202                 req->chan_mask = 0xFFF;
1203
1204         err = npc_check_unsupported_flows(rvu, req->features, req->intf);
1205         if (err)
1206                 return NPC_FLOW_NOT_SUPPORTED;
1207
1208         pfvf = rvu_get_pfvf(rvu, target);
1209
1210         /* PF installing for its VF */
1211         if (req->hdr.pcifunc && !from_vf && req->vf)
1212                 set_bit(PF_SET_VF_CFG, &pfvf->flags);
1213
1214         /* update req destination mac addr */
1215         if ((req->features & BIT_ULL(NPC_DMAC)) && is_npc_intf_rx(req->intf) &&
1216             is_zero_ether_addr(req->packet.dmac)) {
1217                 ether_addr_copy(req->packet.dmac, pfvf->mac_addr);
1218                 eth_broadcast_addr((u8 *)&req->mask.dmac);
1219         }
1220
1221         /* Proceed if NIXLF is attached or not for TX rules */
1222         err = nix_get_nixlf(rvu, target, &nixlf, NULL);
1223         if (err && is_npc_intf_rx(req->intf) && !pf_set_vfs_mac)
1224                 return NPC_FLOW_NO_NIXLF;
1225
1226         /* don't enable rule when nixlf not attached or initialized */
1227         if (!(is_nixlf_attached(rvu, target) &&
1228               test_bit(NIXLF_INITIALIZED, &pfvf->flags)))
1229                 enable = false;
1230
1231         /* Packets reaching NPC in Tx path implies that a
1232          * NIXLF is properly setup and transmitting.
1233          * Hence rules can be enabled for Tx.
1234          */
1235         if (is_npc_intf_tx(req->intf))
1236                 enable = true;
1237
1238         /* Do not allow requests from uninitialized VFs */
1239         if (from_vf && !enable)
1240                 return NPC_FLOW_VF_NOT_INIT;
1241
1242         /* PF sets VF mac & VF NIXLF is not attached, update the mac addr */
1243         if (pf_set_vfs_mac && !enable) {
1244                 ether_addr_copy(pfvf->default_mac, req->packet.dmac);
1245                 ether_addr_copy(pfvf->mac_addr, req->packet.dmac);
1246                 set_bit(PF_SET_VF_MAC, &pfvf->flags);
1247                 return 0;
1248         }
1249
1250         mutex_lock(&rswitch->switch_lock);
1251         err = npc_install_flow(rvu, blkaddr, target, nixlf, pfvf,
1252                                req, rsp, enable, pf_set_vfs_mac);
1253         mutex_unlock(&rswitch->switch_lock);
1254
1255         return err;
1256 }
1257
1258 static int npc_delete_flow(struct rvu *rvu, struct rvu_npc_mcam_rule *rule,
1259                            u16 pcifunc)
1260 {
1261         struct npc_mcam_ena_dis_entry_req dis_req = { 0 };
1262         struct msg_rsp dis_rsp;
1263
1264         if (rule->default_rule)
1265                 return 0;
1266
1267         if (rule->has_cntr)
1268                 rvu_mcam_remove_counter_from_rule(rvu, pcifunc, rule);
1269
1270         dis_req.hdr.pcifunc = pcifunc;
1271         dis_req.entry = rule->entry;
1272
1273         list_del(&rule->list);
1274         kfree(rule);
1275
1276         return rvu_mbox_handler_npc_mcam_dis_entry(rvu, &dis_req, &dis_rsp);
1277 }
1278
1279 int rvu_mbox_handler_npc_delete_flow(struct rvu *rvu,
1280                                      struct npc_delete_flow_req *req,
1281                                      struct msg_rsp *rsp)
1282 {
1283         struct npc_mcam *mcam = &rvu->hw->mcam;
1284         struct rvu_npc_mcam_rule *iter, *tmp;
1285         u16 pcifunc = req->hdr.pcifunc;
1286         struct list_head del_list;
1287
1288         INIT_LIST_HEAD(&del_list);
1289
1290         mutex_lock(&mcam->lock);
1291         list_for_each_entry_safe(iter, tmp, &mcam->mcam_rules, list) {
1292                 if (iter->owner == pcifunc) {
1293                         /* All rules */
1294                         if (req->all) {
1295                                 list_move_tail(&iter->list, &del_list);
1296                         /* Range of rules */
1297                         } else if (req->end && iter->entry >= req->start &&
1298                                    iter->entry <= req->end) {
1299                                 list_move_tail(&iter->list, &del_list);
1300                         /* single rule */
1301                         } else if (req->entry == iter->entry) {
1302                                 list_move_tail(&iter->list, &del_list);
1303                                 break;
1304                         }
1305                 }
1306         }
1307         mutex_unlock(&mcam->lock);
1308
1309         list_for_each_entry_safe(iter, tmp, &del_list, list) {
1310                 u16 entry = iter->entry;
1311
1312                 /* clear the mcam entry target pcifunc */
1313                 mcam->entry2target_pffunc[entry] = 0x0;
1314                 if (npc_delete_flow(rvu, iter, pcifunc))
1315                         dev_err(rvu->dev, "rule deletion failed for entry:%u",
1316                                 entry);
1317         }
1318
1319         return 0;
1320 }
1321
1322 static int npc_update_dmac_value(struct rvu *rvu, int npcblkaddr,
1323                                  struct rvu_npc_mcam_rule *rule,
1324                                  struct rvu_pfvf *pfvf)
1325 {
1326         struct npc_mcam_write_entry_req write_req = { 0 };
1327         struct mcam_entry *entry = &write_req.entry_data;
1328         struct npc_mcam *mcam = &rvu->hw->mcam;
1329         struct msg_rsp rsp;
1330         u8 intf, enable;
1331         int err;
1332
1333         ether_addr_copy(rule->packet.dmac, pfvf->mac_addr);
1334
1335         npc_read_mcam_entry(rvu, mcam, npcblkaddr, rule->entry,
1336                             entry, &intf,  &enable);
1337
1338         npc_update_entry(rvu, NPC_DMAC, entry,
1339                          ether_addr_to_u64(pfvf->mac_addr), 0,
1340                          0xffffffffffffull, 0, intf);
1341
1342         write_req.hdr.pcifunc = rule->owner;
1343         write_req.entry = rule->entry;
1344         write_req.intf = pfvf->nix_rx_intf;
1345
1346         mutex_unlock(&mcam->lock);
1347         err = rvu_mbox_handler_npc_mcam_write_entry(rvu, &write_req, &rsp);
1348         mutex_lock(&mcam->lock);
1349
1350         return err;
1351 }
1352
1353 void npc_mcam_enable_flows(struct rvu *rvu, u16 target)
1354 {
1355         struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, target);
1356         struct rvu_npc_mcam_rule *def_ucast_rule;
1357         struct npc_mcam *mcam = &rvu->hw->mcam;
1358         struct rvu_npc_mcam_rule *rule;
1359         int blkaddr, bank, index;
1360         u64 def_action;
1361
1362         blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
1363         if (blkaddr < 0)
1364                 return;
1365
1366         def_ucast_rule = pfvf->def_ucast_rule;
1367
1368         mutex_lock(&mcam->lock);
1369         list_for_each_entry(rule, &mcam->mcam_rules, list) {
1370                 if (is_npc_intf_rx(rule->intf) &&
1371                     rule->rx_action.pf_func == target && !rule->enable) {
1372                         if (rule->default_rule) {
1373                                 npc_enable_mcam_entry(rvu, mcam, blkaddr,
1374                                                       rule->entry, true);
1375                                 rule->enable = true;
1376                                 continue;
1377                         }
1378
1379                         if (rule->vfvlan_cfg)
1380                                 npc_update_dmac_value(rvu, blkaddr, rule, pfvf);
1381
1382                         if (rule->rx_action.op == NIX_RX_ACTION_DEFAULT) {
1383                                 if (!def_ucast_rule)
1384                                         continue;
1385                                 /* Use default unicast entry action */
1386                                 rule->rx_action = def_ucast_rule->rx_action;
1387                                 def_action = *(u64 *)&def_ucast_rule->rx_action;
1388                                 bank = npc_get_bank(mcam, rule->entry);
1389                                 rvu_write64(rvu, blkaddr,
1390                                             NPC_AF_MCAMEX_BANKX_ACTION
1391                                             (rule->entry, bank), def_action);
1392                         }
1393
1394                         npc_enable_mcam_entry(rvu, mcam, blkaddr,
1395                                               rule->entry, true);
1396                         rule->enable = true;
1397                 }
1398         }
1399
1400         /* Enable MCAM entries installed by PF with target as VF pcifunc */
1401         for (index = 0; index < mcam->bmap_entries; index++) {
1402                 if (mcam->entry2target_pffunc[index] == target)
1403                         npc_enable_mcam_entry(rvu, mcam, blkaddr,
1404                                               index, true);
1405         }
1406         mutex_unlock(&mcam->lock);
1407 }
1408
1409 void npc_mcam_disable_flows(struct rvu *rvu, u16 target)
1410 {
1411         struct npc_mcam *mcam = &rvu->hw->mcam;
1412         int blkaddr, index;
1413
1414         blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
1415         if (blkaddr < 0)
1416                 return;
1417
1418         mutex_lock(&mcam->lock);
1419         /* Disable MCAM entries installed by PF with target as VF pcifunc */
1420         for (index = 0; index < mcam->bmap_entries; index++) {
1421                 if (mcam->entry2target_pffunc[index] == target)
1422                         npc_enable_mcam_entry(rvu, mcam, blkaddr,
1423                                               index, false);
1424         }
1425         mutex_unlock(&mcam->lock);
1426 }