drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_fs.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Marvell OcteonTx2 RVU Admin Function driver
   3  *
   4  * Copyright (C) 2020 Marvell.
   5  */
   6
   7 #include <linux/bitfield.h>
   8
   9 #include "rvu_struct.h"
  10 #include "rvu_reg.h"
  11 #include "rvu.h"
  12 #include "npc.h"
  13
  14 #define NPC_BYTESM              GENMASK_ULL(19, 16)
  15 #define NPC_HDR_OFFSET          GENMASK_ULL(15, 8)
  16 #define NPC_KEY_OFFSET          GENMASK_ULL(5, 0)
  17 #define NPC_LDATA_EN            BIT_ULL(7)
  18
  19 static const char * const npc_flow_names[] = {
  20         [NPC_DMAC]      = "dmac",
  21         [NPC_SMAC]      = "smac",
  22         [NPC_ETYPE]     = "ether type",
  23         [NPC_OUTER_VID] = "outer vlan id",
  24         [NPC_TOS]       = "tos",
  25         [NPC_SIP_IPV4]  = "ipv4 source ip",
  26         [NPC_DIP_IPV4]  = "ipv4 destination ip",
  27         [NPC_SIP_IPV6]  = "ipv6 source ip",
  28         [NPC_DIP_IPV6]  = "ipv6 destination ip",
  29         [NPC_IPPROTO_TCP] = "ip proto tcp",
  30         [NPC_IPPROTO_UDP] = "ip proto udp",
  31         [NPC_IPPROTO_SCTP] = "ip proto sctp",
  32         [NPC_IPPROTO_ICMP] = "ip proto icmp",
  33         [NPC_IPPROTO_ICMP6] = "ip proto icmp6",
  34         [NPC_IPPROTO_AH] = "ip proto AH",
  35         [NPC_IPPROTO_ESP] = "ip proto ESP",
  36         [NPC_SPORT_TCP] = "tcp source port",
  37         [NPC_DPORT_TCP] = "tcp destination port",
  38         [NPC_SPORT_UDP] = "udp source port",
  39         [NPC_DPORT_UDP] = "udp destination port",
  40         [NPC_SPORT_SCTP] = "sctp source port",
  41         [NPC_DPORT_SCTP] = "sctp destination port",
  42         [NPC_UNKNOWN]   = "unknown",
  43 };
  44
  45 const char *npc_get_field_name(u8 hdr)
  46 {
  47         if (hdr >= ARRAY_SIZE(npc_flow_names))
  48                 return npc_flow_names[NPC_UNKNOWN];
  49
  50         return npc_flow_names[hdr];
  51 }
  52
  53 /* Compute keyword masks and figure out the number of keywords a field
  54  * spans in the key.
  55  */
  56 static void npc_set_kw_masks(struct npc_mcam *mcam, u8 type,
  57                              u8 nr_bits, int start_kwi, int offset, u8 intf)
  58 {
  59         struct npc_key_field *field = &mcam->rx_key_fields[type];
  60         u8 bits_in_kw;
  61         int max_kwi;
  62
  63         if (mcam->banks_per_entry == 1)
  64                 max_kwi = 1; /* NPC_MCAM_KEY_X1 */
  65         else if (mcam->banks_per_entry == 2)
  66                 max_kwi = 3; /* NPC_MCAM_KEY_X2 */
  67         else
  68                 max_kwi = 6; /* NPC_MCAM_KEY_X4 */
  69
  70         if (is_npc_intf_tx(intf))
  71                 field = &mcam->tx_key_fields[type];
  72
  73         if (offset + nr_bits <= 64) {
  74                 /* one KW only */
  75                 if (start_kwi > max_kwi)
  76                         return;
  77                 field->kw_mask[start_kwi] |= GENMASK_ULL(nr_bits - 1, 0)
  78                                              << offset;
  79                 field->nr_kws = 1;
  80         } else if (offset + nr_bits > 64 &&
  81                    offset + nr_bits <= 128) {
  82                 /* two KWs */
  83                 if (start_kwi + 1 > max_kwi)
  84                         return;
  85                 /* first KW mask */
  86                 bits_in_kw = 64 - offset;
  87                 field->kw_mask[start_kwi] |= GENMASK_ULL(bits_in_kw - 1, 0)
  88                                              << offset;
  89                 /* second KW mask i.e. mask for rest of bits */
  90                 bits_in_kw = nr_bits + offset - 64;
  91                 field->kw_mask[start_kwi + 1] |= GENMASK_ULL(bits_in_kw - 1, 0);
  92                 field->nr_kws = 2;
  93         } else {
  94                 /* three KWs */
  95                 if (start_kwi + 2 > max_kwi)
  96                         return;
  97                 /* first KW mask */
  98                 bits_in_kw = 64 - offset;
  99                 field->kw_mask[start_kwi] |= GENMASK_ULL(bits_in_kw - 1, 0)
 100                                              << offset;
 101                 /* second KW mask */
 102                 field->kw_mask[start_kwi + 1] = ~0ULL;
 103                 /* third KW mask i.e. mask for rest of bits */
 104                 bits_in_kw = nr_bits + offset - 128;
 105                 field->kw_mask[start_kwi + 2] |= GENMASK_ULL(bits_in_kw - 1, 0);
 106                 field->nr_kws = 3;
 107         }
 108 }
 109
 110 /* Helper function to figure out whether field exists in the key */
 111 static bool npc_is_field_present(struct rvu *rvu, enum key_fields type, u8 intf)
 112 {
 113         struct npc_mcam *mcam = &rvu->hw->mcam;
 114         struct npc_key_field *input;
 115
 116         input  = &mcam->rx_key_fields[type];
 117         if (is_npc_intf_tx(intf))
 118                 input  = &mcam->tx_key_fields[type];
 119
 120         return input->nr_kws > 0;
 121 }
 122
 123 static bool npc_is_same(struct npc_key_field *input,
 124                         struct npc_key_field *field)
 125 {
 126         return memcmp(&input->layer_mdata, &field->layer_mdata,
 127                      sizeof(struct npc_layer_mdata)) == 0;
 128 }
 129
 130 static void npc_set_layer_mdata(struct npc_mcam *mcam, enum key_fields type,
 131                                 u64 cfg, u8 lid, u8 lt, u8 intf)
 132 {
 133         struct npc_key_field *input = &mcam->rx_key_fields[type];
 134
 135         if (is_npc_intf_tx(intf))
 136                 input = &mcam->tx_key_fields[type];
 137
 138         input->layer_mdata.hdr = FIELD_GET(NPC_HDR_OFFSET, cfg);
 139         input->layer_mdata.key = FIELD_GET(NPC_KEY_OFFSET, cfg);
 140         input->layer_mdata.len = FIELD_GET(NPC_BYTESM, cfg) + 1;
 141         input->layer_mdata.ltype = lt;
 142         input->layer_mdata.lid = lid;
 143 }
 144
 145 static bool npc_check_overlap_fields(struct npc_key_field *input1,
 146                                      struct npc_key_field *input2)
 147 {
 148         int kwi;
 149
 150         /* Fields with same layer id and different ltypes are mutually
 151          * exclusive hence they can be overlapped
 152          */
 153         if (input1->layer_mdata.lid == input2->layer_mdata.lid &&
 154             input1->layer_mdata.ltype != input2->layer_mdata.ltype)
 155                 return false;
 156
 157         for (kwi = 0; kwi < NPC_MAX_KWS_IN_KEY; kwi++) {
 158                 if (input1->kw_mask[kwi] & input2->kw_mask[kwi])
 159                         return true;
 160         }
 161
 162         return false;
 163 }
 164
 165 /* Helper function to check whether given field overlaps with any other fields
 166  * in the key. Due to limitations on key size and the key extraction profile in
 167  * use higher layers can overwrite lower layer's header fields. Hence overlap
 168  * needs to be checked.
 169  */
 170 static bool npc_check_overlap(struct rvu *rvu, int blkaddr,
 171                               enum key_fields type, u8 start_lid, u8 intf)
 172 {
 173         struct npc_mcam *mcam = &rvu->hw->mcam;
 174         struct npc_key_field *dummy, *input;
 175         int start_kwi, offset;
 176         u8 nr_bits, lid, lt, ld;
 177         u64 cfg;
 178
 179         dummy = &mcam->rx_key_fields[NPC_UNKNOWN];
 180         input = &mcam->rx_key_fields[type];
 181
 182         if (is_npc_intf_tx(intf)) {
 183                 dummy = &mcam->tx_key_fields[NPC_UNKNOWN];
 184                 input = &mcam->tx_key_fields[type];
 185         }
 186
 187         for (lid = start_lid; lid < NPC_MAX_LID; lid++) {
 188                 for (lt = 0; lt < NPC_MAX_LT; lt++) {
 189                         for (ld = 0; ld < NPC_MAX_LD; ld++) {
 190                                 cfg = rvu_read64(rvu, blkaddr,
 191                                                  NPC_AF_INTFX_LIDX_LTX_LDX_CFG
 192                                                  (intf, lid, lt, ld));
 193                                 if (!FIELD_GET(NPC_LDATA_EN, cfg))
 194                                         continue;
 195                                 memset(dummy, 0, sizeof(struct npc_key_field));
 196                                 npc_set_layer_mdata(mcam, NPC_UNKNOWN, cfg,
 197                                                     lid, lt, intf);
 198                                 /* exclude input */
 199                                 if (npc_is_same(input, dummy))
 200                                         continue;
 201                                 start_kwi = dummy->layer_mdata.key / 8;
 202                                 offset = (dummy->layer_mdata.key * 8) % 64;
 203                                 nr_bits = dummy->layer_mdata.len * 8;
 204                                 /* form KW masks */
 205                                 npc_set_kw_masks(mcam, NPC_UNKNOWN, nr_bits,
 206                                                  start_kwi, offset, intf);
 207                                 /* check any input field bits falls in any
 208                                  * other field bits.
 209                                  */
 210                                 if (npc_check_overlap_fields(dummy, input))
 211                                         return true;
 212                         }
 213                 }
 214         }
 215
 216         return false;
 217 }
 218
 219 static bool npc_check_field(struct rvu *rvu, int blkaddr, enum key_fields type,
 220                             u8 intf)
 221 {
 222         if (!npc_is_field_present(rvu, type, intf) ||
 223             npc_check_overlap(rvu, blkaddr, type, 0, intf))
 224                 return false;
 225         return true;
 226 }
 227
 228 static void npc_scan_parse_result(struct npc_mcam *mcam, u8 bit_number,
 229                                   u8 key_nibble, u8 intf)
 230 {
 231         u8 offset = (key_nibble * 4) % 64; /* offset within key word */
 232         u8 kwi = (key_nibble * 4) / 64; /* which word in key */
 233         u8 nr_bits = 4; /* bits in a nibble */
 234         u8 type;
 235
 236         switch (bit_number) {
 237         case 0 ... 2:
 238                 type = NPC_CHAN;
 239                 break;
 240         case 3:
 241                 type = NPC_ERRLEV;
 242                 break;
 243         case 4 ... 5:
 244                 type = NPC_ERRCODE;
 245                 break;
 246         case 6:
 247                 type = NPC_LXMB;
 248                 break;
 249         /* check for LTYPE only as of now */
 250         case 9:
 251                 type = NPC_LA;
 252                 break;
 253         case 12:
 254                 type = NPC_LB;
 255                 break;
 256         case 15:
 257                 type = NPC_LC;
 258                 break;
 259         case 18:
 260                 type = NPC_LD;
 261                 break;
 262         case 21:
 263                 type = NPC_LE;
 264                 break;
 265         case 24:
 266                 type = NPC_LF;
 267                 break;
 268         case 27:
 269                 type = NPC_LG;
 270                 break;
 271         case 30:
 272                 type = NPC_LH;
 273                 break;
 274         default:
 275                 return;
 276         }
 277         npc_set_kw_masks(mcam, type, nr_bits, kwi, offset, intf);
 278 }
 279
 280 static void npc_handle_multi_layer_fields(struct rvu *rvu, int blkaddr, u8 intf)
 281 {
 282         struct npc_mcam *mcam = &rvu->hw->mcam;
 283         struct npc_key_field *key_fields;
 284         /* Ether type can come from three layers
 285          * (ethernet, single tagged, double tagged)
 286          */
 287         struct npc_key_field *etype_ether;
 288         struct npc_key_field *etype_tag1;
 289         struct npc_key_field *etype_tag2;
 290         /* Outer VLAN TCI can come from two layers
 291          * (single tagged, double tagged)
 292          */
 293         struct npc_key_field *vlan_tag1;
 294         struct npc_key_field *vlan_tag2;
 295         u64 *features;
 296         u8 start_lid;
 297         int i;
 298
 299         key_fields = mcam->rx_key_fields;
 300         features = &mcam->rx_features;
 301
 302         if (is_npc_intf_tx(intf)) {
 303                 key_fields = mcam->tx_key_fields;
 304                 features = &mcam->tx_features;
 305         }
 306
 307         /* Handle header fields which can come from multiple layers like
 308          * etype, outer vlan tci. These fields should have same position in
 309          * the key otherwise to install a mcam rule more than one entry is
 310          * needed which complicates mcam space management.
 311          */
 312         etype_ether = &key_fields[NPC_ETYPE_ETHER];
 313         etype_tag1 = &key_fields[NPC_ETYPE_TAG1];
 314         etype_tag2 = &key_fields[NPC_ETYPE_TAG2];
 315         vlan_tag1 = &key_fields[NPC_VLAN_TAG1];
 316         vlan_tag2 = &key_fields[NPC_VLAN_TAG2];
 317
 318         /* if key profile programmed does not extract Ethertype at all */
 319         if (!etype_ether->nr_kws && !etype_tag1->nr_kws && !etype_tag2->nr_kws)
 320                 goto vlan_tci;
 321
 322         /* if key profile programmed extracts Ethertype from one layer */
 323         if (etype_ether->nr_kws && !etype_tag1->nr_kws && !etype_tag2->nr_kws)
 324                 key_fields[NPC_ETYPE] = *etype_ether;
 325         if (!etype_ether->nr_kws && etype_tag1->nr_kws && !etype_tag2->nr_kws)
 326                 key_fields[NPC_ETYPE] = *etype_tag1;
 327         if (!etype_ether->nr_kws && !etype_tag1->nr_kws && etype_tag2->nr_kws)
 328                 key_fields[NPC_ETYPE] = *etype_tag2;
 329
 330         /* if key profile programmed extracts Ethertype from multiple layers */
 331         if (etype_ether->nr_kws && etype_tag1->nr_kws) {
 332                 for (i = 0; i < NPC_MAX_KWS_IN_KEY; i++) {
 333                         if (etype_ether->kw_mask[i] != etype_tag1->kw_mask[i])
 334                                 goto vlan_tci;
 335                 }
 336                 key_fields[NPC_ETYPE] = *etype_tag1;
 337         }
 338         if (etype_ether->nr_kws && etype_tag2->nr_kws) {
 339                 for (i = 0; i < NPC_MAX_KWS_IN_KEY; i++) {
 340                         if (etype_ether->kw_mask[i] != etype_tag2->kw_mask[i])
 341                                 goto vlan_tci;
 342                 }
 343                 key_fields[NPC_ETYPE] = *etype_tag2;
 344         }
 345         if (etype_tag1->nr_kws && etype_tag2->nr_kws) {
 346                 for (i = 0; i < NPC_MAX_KWS_IN_KEY; i++) {
 347                         if (etype_tag1->kw_mask[i] != etype_tag2->kw_mask[i])
 348                                 goto vlan_tci;
 349                 }
 350                 key_fields[NPC_ETYPE] = *etype_tag2;
 351         }
 352
 353         /* check none of higher layers overwrite Ethertype */
 354         start_lid = key_fields[NPC_ETYPE].layer_mdata.lid + 1;
 355         if (npc_check_overlap(rvu, blkaddr, NPC_ETYPE, start_lid, intf))
 356                 goto vlan_tci;
 357         *features |= BIT_ULL(NPC_ETYPE);
 358 vlan_tci:
 359         /* if key profile does not extract outer vlan tci at all */
 360         if (!vlan_tag1->nr_kws && !vlan_tag2->nr_kws)
 361                 goto done;
 362
 363         /* if key profile extracts outer vlan tci from one layer */
 364         if (vlan_tag1->nr_kws && !vlan_tag2->nr_kws)
 365                 key_fields[NPC_OUTER_VID] = *vlan_tag1;
 366         if (!vlan_tag1->nr_kws && vlan_tag2->nr_kws)
 367                 key_fields[NPC_OUTER_VID] = *vlan_tag2;
 368
 369         /* if key profile extracts outer vlan tci from multiple layers */
 370         if (vlan_tag1->nr_kws && vlan_tag2->nr_kws) {
 371                 for (i = 0; i < NPC_MAX_KWS_IN_KEY; i++) {
 372                         if (vlan_tag1->kw_mask[i] != vlan_tag2->kw_mask[i])
 373                                 goto done;
 374                 }
 375                 key_fields[NPC_OUTER_VID] = *vlan_tag2;
 376         }
 377         /* check none of higher layers overwrite outer vlan tci */
 378         start_lid = key_fields[NPC_OUTER_VID].layer_mdata.lid + 1;
 379         if (npc_check_overlap(rvu, blkaddr, NPC_OUTER_VID, start_lid, intf))
 380                 goto done;
 381         *features |= BIT_ULL(NPC_OUTER_VID);
 382 done:
 383         return;
 384 }
 385
 386 static void npc_scan_ldata(struct rvu *rvu, int blkaddr, u8 lid,
 387                            u8 lt, u64 cfg, u8 intf)
 388 {
 389         struct npc_mcam *mcam = &rvu->hw->mcam;
 390         u8 hdr, key, nr_bytes, bit_offset;
 391         u8 la_ltype, la_start;
 392         /* starting KW index and starting bit position */
 393         int start_kwi, offset;
 394
 395         nr_bytes = FIELD_GET(NPC_BYTESM, cfg) + 1;
 396         hdr = FIELD_GET(NPC_HDR_OFFSET, cfg);
 397         key = FIELD_GET(NPC_KEY_OFFSET, cfg);
 398         start_kwi = key / 8;
 399         offset = (key * 8) % 64;
 400
 401         /* For Tx, Layer A has NIX_INST_HDR_S(64 bytes) preceding
 402          * ethernet header.
 403          */
 404         if (is_npc_intf_tx(intf)) {
 405                 la_ltype = NPC_LT_LA_IH_NIX_ETHER;
 406                 la_start = 8;
 407         } else {
 408                 la_ltype = NPC_LT_LA_ETHER;
 409                 la_start = 0;
 410         }
 411
 412 #define NPC_SCAN_HDR(name, hlid, hlt, hstart, hlen)                            \
 413 do {                                                                           \
 414         if (lid == (hlid) && lt == (hlt)) {                                    \
 415                 if ((hstart) >= hdr &&                                         \
 416                     ((hstart) + (hlen)) <= (hdr + nr_bytes)) {                 \
 417                         bit_offset = (hdr + nr_bytes - (hstart) - (hlen)) * 8; \
 418                         npc_set_layer_mdata(mcam, (name), cfg, lid, lt, intf); \
 419                         npc_set_kw_masks(mcam, (name), (hlen) * 8,             \
 420                                          start_kwi, offset + bit_offset, intf);\
 421                 }                                                              \
 422         }                                                                      \
 423 } while (0)
 424
 425         /* List LID, LTYPE, start offset from layer and length(in bytes) of
 426          * packet header fields below.
 427          * Example: Source IP is 4 bytes and starts at 12th byte of IP header
 428          */
 429         NPC_SCAN_HDR(NPC_TOS, NPC_LID_LC, NPC_LT_LC_IP, 1, 1);
 430         NPC_SCAN_HDR(NPC_SIP_IPV4, NPC_LID_LC, NPC_LT_LC_IP, 12, 4);
 431         NPC_SCAN_HDR(NPC_DIP_IPV4, NPC_LID_LC, NPC_LT_LC_IP, 16, 4);
 432         NPC_SCAN_HDR(NPC_SIP_IPV6, NPC_LID_LC, NPC_LT_LC_IP6, 8, 16);
 433         NPC_SCAN_HDR(NPC_DIP_IPV6, NPC_LID_LC, NPC_LT_LC_IP6, 24, 16);
 434         NPC_SCAN_HDR(NPC_SPORT_UDP, NPC_LID_LD, NPC_LT_LD_UDP, 0, 2);
 435         NPC_SCAN_HDR(NPC_DPORT_UDP, NPC_LID_LD, NPC_LT_LD_UDP, 2, 2);
 436         NPC_SCAN_HDR(NPC_SPORT_TCP, NPC_LID_LD, NPC_LT_LD_TCP, 0, 2);
 437         NPC_SCAN_HDR(NPC_DPORT_TCP, NPC_LID_LD, NPC_LT_LD_TCP, 2, 2);
 438         NPC_SCAN_HDR(NPC_SPORT_SCTP, NPC_LID_LD, NPC_LT_LD_SCTP, 0, 2);
 439         NPC_SCAN_HDR(NPC_DPORT_SCTP, NPC_LID_LD, NPC_LT_LD_SCTP, 2, 2);
 440         NPC_SCAN_HDR(NPC_ETYPE_ETHER, NPC_LID_LA, NPC_LT_LA_ETHER, 12, 2);
 441         NPC_SCAN_HDR(NPC_ETYPE_TAG1, NPC_LID_LB, NPC_LT_LB_CTAG, 4, 2);
 442         NPC_SCAN_HDR(NPC_ETYPE_TAG2, NPC_LID_LB, NPC_LT_LB_STAG_QINQ, 8, 2);
 443         NPC_SCAN_HDR(NPC_VLAN_TAG1, NPC_LID_LB, NPC_LT_LB_CTAG, 2, 2);
 444         NPC_SCAN_HDR(NPC_VLAN_TAG2, NPC_LID_LB, NPC_LT_LB_STAG_QINQ, 2, 2);
 445         NPC_SCAN_HDR(NPC_DMAC, NPC_LID_LA, la_ltype, la_start, 6);
 446         NPC_SCAN_HDR(NPC_SMAC, NPC_LID_LA, la_ltype, la_start, 6);
 447         /* PF_FUNC is 2 bytes at 0th byte of NPC_LT_LA_IH_NIX_ETHER */
 448         NPC_SCAN_HDR(NPC_PF_FUNC, NPC_LID_LA, NPC_LT_LA_IH_NIX_ETHER, 0, 2);
 449 }
 450
 451 static void npc_set_features(struct rvu *rvu, int blkaddr, u8 intf)
 452 {
 453         struct npc_mcam *mcam = &rvu->hw->mcam;
 454         u64 *features = &mcam->rx_features;
 455         u64 tcp_udp_sctp;
 456         int hdr;
 457
 458         if (is_npc_intf_tx(intf))
 459                 features = &mcam->tx_features;
 460
 461         for (hdr = NPC_DMAC; hdr < NPC_HEADER_FIELDS_MAX; hdr++) {
 462                 if (npc_check_field(rvu, blkaddr, hdr, intf))
 463                         *features |= BIT_ULL(hdr);
 464         }
 465
 466         tcp_udp_sctp = BIT_ULL(NPC_SPORT_TCP) | BIT_ULL(NPC_SPORT_UDP) |
 467                        BIT_ULL(NPC_DPORT_TCP) | BIT_ULL(NPC_DPORT_UDP) |
 468                        BIT_ULL(NPC_SPORT_SCTP) | BIT_ULL(NPC_DPORT_SCTP);
 469
 470         /* for tcp/udp/sctp corresponding layer type should be in the key */
 471         if (*features & tcp_udp_sctp) {
 472                 if (!npc_check_field(rvu, blkaddr, NPC_LD, intf))
 473                         *features &= ~tcp_udp_sctp;
 474                 else
 475                         *features |= BIT_ULL(NPC_IPPROTO_TCP) |
 476                                      BIT_ULL(NPC_IPPROTO_UDP) |
 477                                      BIT_ULL(NPC_IPPROTO_SCTP);
 478         }
 479
 480         /* for AH/ICMP/ICMPv6/, check if corresponding layer type is present in the key */
 481         if (npc_check_field(rvu, blkaddr, NPC_LD, intf)) {
 482                 *features |= BIT_ULL(NPC_IPPROTO_AH);
 483                 *features |= BIT_ULL(NPC_IPPROTO_ICMP);
 484                 *features |= BIT_ULL(NPC_IPPROTO_ICMP6);
 485         }
 486
 487         /* for ESP, check if corresponding layer type is present in the key */
 488         if (npc_check_field(rvu, blkaddr, NPC_LE, intf))
 489                 *features |= BIT_ULL(NPC_IPPROTO_ESP);
 490
 491         /* for vlan corresponding layer type should be in the key */
 492         if (*features & BIT_ULL(NPC_OUTER_VID))
 493                 if (!npc_check_field(rvu, blkaddr, NPC_LB, intf))
 494                         *features &= ~BIT_ULL(NPC_OUTER_VID);
 495 }
 496
 497 /* Scan key extraction profile and record how fields of our interest
 498  * fill the key structure. Also verify Channel and DMAC exists in
 499  * key and not overwritten by other header fields.
 500  */
 501 static int npc_scan_kex(struct rvu *rvu, int blkaddr, u8 intf)
 502 {
 503         struct npc_mcam *mcam = &rvu->hw->mcam;
 504         u8 lid, lt, ld, bitnr;
 505         u8 key_nibble = 0;
 506         u64 cfg;
 507
 508         /* Scan and note how parse result is going to be in key.
 509          * A bit set in PARSE_NIBBLE_ENA corresponds to a nibble from
 510          * parse result in the key. The enabled nibbles from parse result
 511          * will be concatenated in key.
 512          */
 513         cfg = rvu_read64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf));
 514         cfg &= NPC_PARSE_NIBBLE;
 515         for_each_set_bit(bitnr, (unsigned long *)&cfg, 31) {
 516                 npc_scan_parse_result(mcam, bitnr, key_nibble, intf);
 517                 key_nibble++;
 518         }
 519
 520         /* Scan and note how layer data is going to be in key */
 521         for (lid = 0; lid < NPC_MAX_LID; lid++) {
 522                 for (lt = 0; lt < NPC_MAX_LT; lt++) {
 523                         for (ld = 0; ld < NPC_MAX_LD; ld++) {
 524                                 cfg = rvu_read64(rvu, blkaddr,
 525                                                  NPC_AF_INTFX_LIDX_LTX_LDX_CFG
 526                                                  (intf, lid, lt, ld));
 527                                 if (!FIELD_GET(NPC_LDATA_EN, cfg))
 528                                         continue;
 529                                 npc_scan_ldata(rvu, blkaddr, lid, lt, cfg,
 530                                                intf);
 531                         }
 532                 }
 533         }
 534
 535         return 0;
 536 }
 537
 538 static int npc_scan_verify_kex(struct rvu *rvu, int blkaddr)
 539 {
 540         int err;
 541
 542         err = npc_scan_kex(rvu, blkaddr, NIX_INTF_RX);
 543         if (err)
 544                 return err;
 545
 546         err = npc_scan_kex(rvu, blkaddr, NIX_INTF_TX);
 547         if (err)
 548                 return err;
 549
 550         /* Channel is mandatory */
 551         if (!npc_is_field_present(rvu, NPC_CHAN, NIX_INTF_RX)) {
 552                 dev_err(rvu->dev, "Channel not present in Key\n");
 553                 return -EINVAL;
 554         }
 555         /* check that none of the fields overwrite channel */
 556         if (npc_check_overlap(rvu, blkaddr, NPC_CHAN, 0, NIX_INTF_RX)) {
 557                 dev_err(rvu->dev, "Channel cannot be overwritten\n");
 558                 return -EINVAL;
 559         }
 560         /* DMAC should be present in key for unicast filter to work */
 561         if (!npc_is_field_present(rvu, NPC_DMAC, NIX_INTF_RX)) {
 562                 dev_err(rvu->dev, "DMAC not present in Key\n");
 563                 return -EINVAL;
 564         }
 565         /* check that none of the fields overwrite DMAC */
 566         if (npc_check_overlap(rvu, blkaddr, NPC_DMAC, 0, NIX_INTF_RX)) {
 567                 dev_err(rvu->dev, "DMAC cannot be overwritten\n");
 568                 return -EINVAL;
 569         }
 570
 571         npc_set_features(rvu, blkaddr, NIX_INTF_TX);
 572         npc_set_features(rvu, blkaddr, NIX_INTF_RX);
 573         npc_handle_multi_layer_fields(rvu, blkaddr, NIX_INTF_TX);
 574         npc_handle_multi_layer_fields(rvu, blkaddr, NIX_INTF_RX);
 575
 576         return 0;
 577 }
 578
 579 int npc_flow_steering_init(struct rvu *rvu, int blkaddr)
 580 {
 581         struct npc_mcam *mcam = &rvu->hw->mcam;
 582
 583         INIT_LIST_HEAD(&mcam->mcam_rules);
 584
 585         return npc_scan_verify_kex(rvu, blkaddr);
 586 }
 587
 588 static int npc_check_unsupported_flows(struct rvu *rvu, u64 features, u8 intf)
 589 {
 590         struct npc_mcam *mcam = &rvu->hw->mcam;
 591         u64 *mcam_features = &mcam->rx_features;
 592         u64 unsupported;
 593         u8 bit;
 594
 595         if (is_npc_intf_tx(intf))
 596                 mcam_features = &mcam->tx_features;
 597
 598         unsupported = (*mcam_features ^ features) & ~(*mcam_features);
 599         if (unsupported) {
 600                 dev_info(rvu->dev, "Unsupported flow(s):\n");
 601                 for_each_set_bit(bit, (unsigned long *)&unsupported, 64)
 602                         dev_info(rvu->dev, "%s ", npc_get_field_name(bit));
 603                 return NIX_AF_ERR_NPC_KEY_NOT_SUPP;
 604         }
 605
 606         return 0;
 607 }
 608
 609 /* npc_update_entry - Based on the masks generated during
 610  * the key scanning, updates the given entry with value and
 611  * masks for the field of interest. Maximum 16 bytes of a packet
 612  * header can be extracted by HW hence lo and hi are sufficient.
 613  * When field bytes are less than or equal to 8 then hi should be
 614  * 0 for value and mask.
 615  *
 616  * If exact match of value is required then mask should be all 1's.
 617  * If any bits in mask are 0 then corresponding bits in value are
 618  * dont care.
 619  */
 620 static void npc_update_entry(struct rvu *rvu, enum key_fields type,
 621                              struct mcam_entry *entry, u64 val_lo,
 622                              u64 val_hi, u64 mask_lo, u64 mask_hi, u8 intf)
 623 {
 624         struct npc_mcam *mcam = &rvu->hw->mcam;
 625         struct mcam_entry dummy = { {0} };
 626         struct npc_key_field *field;
 627         u64 kw1, kw2, kw3;
 628         u8 shift;
 629         int i;
 630
 631         field = &mcam->rx_key_fields[type];
 632         if (is_npc_intf_tx(intf))
 633                 field = &mcam->tx_key_fields[type];
 634
 635         if (!field->nr_kws)
 636                 return;
 637
 638         for (i = 0; i < NPC_MAX_KWS_IN_KEY; i++) {
 639                 if (!field->kw_mask[i])
 640                         continue;
 641                 /* place key value in kw[x] */
 642                 shift = __ffs64(field->kw_mask[i]);
 643                 /* update entry value */
 644                 kw1 = (val_lo << shift) & field->kw_mask[i];
 645                 dummy.kw[i] = kw1;
 646                 /* update entry mask */
 647                 kw1 = (mask_lo << shift) & field->kw_mask[i];
 648                 dummy.kw_mask[i] = kw1;
 649
 650                 if (field->nr_kws == 1)
 651                         break;
 652                 /* place remaining bits of key value in kw[x + 1] */
 653                 if (field->nr_kws == 2) {
 654                         /* update entry value */
 655                         kw2 = shift ? val_lo >> (64 - shift) : 0;
 656                         kw2 |= (val_hi << shift);
 657                         kw2 &= field->kw_mask[i + 1];
 658                         dummy.kw[i + 1] = kw2;
 659                         /* update entry mask */
 660                         kw2 = shift ? mask_lo >> (64 - shift) : 0;
 661                         kw2 |= (mask_hi << shift);
 662                         kw2 &= field->kw_mask[i + 1];
 663                         dummy.kw_mask[i + 1] = kw2;
 664                         break;
 665                 }
 666                 /* place remaining bits of key value in kw[x + 1], kw[x + 2] */
 667                 if (field->nr_kws == 3) {
 668                         /* update entry value */
 669                         kw2 = shift ? val_lo >> (64 - shift) : 0;
 670                         kw2 |= (val_hi << shift);
 671                         kw2 &= field->kw_mask[i + 1];
 672                         kw3 = shift ? val_hi >> (64 - shift) : 0;
 673                         kw3 &= field->kw_mask[i + 2];
 674                         dummy.kw[i + 1] = kw2;
 675                         dummy.kw[i + 2] = kw3;
 676                         /* update entry mask */
 677                         kw2 = shift ? mask_lo >> (64 - shift) : 0;
 678                         kw2 |= (mask_hi << shift);
 679                         kw2 &= field->kw_mask[i + 1];
 680                         kw3 = shift ? mask_hi >> (64 - shift) : 0;
 681                         kw3 &= field->kw_mask[i + 2];
 682                         dummy.kw_mask[i + 1] = kw2;
 683                         dummy.kw_mask[i + 2] = kw3;
 684                         break;
 685                 }
 686         }
 687         /* dummy is ready with values and masks for given key
 688          * field now clear and update input entry with those
 689          */
 690         for (i = 0; i < NPC_MAX_KWS_IN_KEY; i++) {
 691                 if (!field->kw_mask[i])
 692                         continue;
 693                 entry->kw[i] &= ~field->kw_mask[i];
 694                 entry->kw_mask[i] &= ~field->kw_mask[i];
 695
 696                 entry->kw[i] |= dummy.kw[i];
 697                 entry->kw_mask[i] |= dummy.kw_mask[i];
 698         }
 699 }
 700
 701 #define IPV6_WORDS     4
 702
 703 static void npc_update_ipv6_flow(struct rvu *rvu, struct mcam_entry *entry,
 704                                  u64 features, struct flow_msg *pkt,
 705                                  struct flow_msg *mask,
 706                                  struct rvu_npc_mcam_rule *output, u8 intf)
 707 {
 708         u32 src_ip[IPV6_WORDS], src_ip_mask[IPV6_WORDS];
 709         u32 dst_ip[IPV6_WORDS], dst_ip_mask[IPV6_WORDS];
 710         struct flow_msg *opkt = &output->packet;
 711         struct flow_msg *omask = &output->mask;
 712         u64 mask_lo, mask_hi;
 713         u64 val_lo, val_hi;
 714
 715         /* For an ipv6 address fe80::2c68:63ff:fe5e:2d0a the packet
 716          * values to be programmed in MCAM should as below:
 717          * val_high: 0xfe80000000000000
 718          * val_low: 0x2c6863fffe5e2d0a
 719          */
 720         if (features & BIT_ULL(NPC_SIP_IPV6)) {
 721                 be32_to_cpu_array(src_ip_mask, mask->ip6src, IPV6_WORDS);
 722                 be32_to_cpu_array(src_ip, pkt->ip6src, IPV6_WORDS);
 723
 724                 mask_hi = (u64)src_ip_mask[0] << 32 | src_ip_mask[1];
 725                 mask_lo = (u64)src_ip_mask[2] << 32 | src_ip_mask[3];
 726                 val_hi = (u64)src_ip[0] << 32 | src_ip[1];
 727                 val_lo = (u64)src_ip[2] << 32 | src_ip[3];
 728
 729                 npc_update_entry(rvu, NPC_SIP_IPV6, entry, val_lo, val_hi,
 730                                  mask_lo, mask_hi, intf);
 731                 memcpy(opkt->ip6src, pkt->ip6src, sizeof(opkt->ip6src));
 732                 memcpy(omask->ip6src, mask->ip6src, sizeof(omask->ip6src));
 733         }
 734         if (features & BIT_ULL(NPC_DIP_IPV6)) {
 735                 be32_to_cpu_array(dst_ip_mask, mask->ip6dst, IPV6_WORDS);
 736                 be32_to_cpu_array(dst_ip, pkt->ip6dst, IPV6_WORDS);
 737
 738                 mask_hi = (u64)dst_ip_mask[0] << 32 | dst_ip_mask[1];
 739                 mask_lo = (u64)dst_ip_mask[2] << 32 | dst_ip_mask[3];
 740                 val_hi = (u64)dst_ip[0] << 32 | dst_ip[1];
 741                 val_lo = (u64)dst_ip[2] << 32 | dst_ip[3];
 742
 743                 npc_update_entry(rvu, NPC_DIP_IPV6, entry, val_lo, val_hi,
 744                                  mask_lo, mask_hi, intf);
 745                 memcpy(opkt->ip6dst, pkt->ip6dst, sizeof(opkt->ip6dst));
 746                 memcpy(omask->ip6dst, mask->ip6dst, sizeof(omask->ip6dst));
 747         }
 748 }
 749
 750 static void npc_update_flow(struct rvu *rvu, struct mcam_entry *entry,
 751                             u64 features, struct flow_msg *pkt,
 752                             struct flow_msg *mask,
 753                             struct rvu_npc_mcam_rule *output, u8 intf)
 754 {
 755         u64 dmac_mask = ether_addr_to_u64(mask->dmac);
 756         u64 smac_mask = ether_addr_to_u64(mask->smac);
 757         u64 dmac_val = ether_addr_to_u64(pkt->dmac);
 758         u64 smac_val = ether_addr_to_u64(pkt->smac);
 759         struct flow_msg *opkt = &output->packet;
 760         struct flow_msg *omask = &output->mask;
 761
 762         if (!features)
 763                 return;
 764
 765         /* For tcp/udp/sctp LTYPE should be present in entry */
 766         if (features & BIT_ULL(NPC_IPPROTO_TCP))
 767                 npc_update_entry(rvu, NPC_LD, entry, NPC_LT_LD_TCP,
 768                                  0, ~0ULL, 0, intf);
 769         if (features & BIT_ULL(NPC_IPPROTO_UDP))
 770                 npc_update_entry(rvu, NPC_LD, entry, NPC_LT_LD_UDP,
 771                                  0, ~0ULL, 0, intf);
 772         if (features & BIT_ULL(NPC_IPPROTO_SCTP))
 773                 npc_update_entry(rvu, NPC_LD, entry, NPC_LT_LD_SCTP,
 774                                  0, ~0ULL, 0, intf);
 775         if (features & BIT_ULL(NPC_IPPROTO_ICMP))
 776                 npc_update_entry(rvu, NPC_LD, entry, NPC_LT_LD_ICMP,
 777                                  0, ~0ULL, 0, intf);
 778         if (features & BIT_ULL(NPC_IPPROTO_ICMP6))
 779                 npc_update_entry(rvu, NPC_LD, entry, NPC_LT_LD_ICMP6,
 780                                  0, ~0ULL, 0, intf);
 781
 782         if (features & BIT_ULL(NPC_OUTER_VID))
 783                 npc_update_entry(rvu, NPC_LB, entry,
 784                                  NPC_LT_LB_STAG_QINQ | NPC_LT_LB_CTAG, 0,
 785                                  NPC_LT_LB_STAG_QINQ & NPC_LT_LB_CTAG, 0, intf);
 786
 787         /* For AH, LTYPE should be present in entry */
 788         if (features & BIT_ULL(NPC_IPPROTO_AH))
 789                 npc_update_entry(rvu, NPC_LD, entry, NPC_LT_LD_AH,
 790                                  0, ~0ULL, 0, intf);
 791         /* For ESP, LTYPE should be present in entry */
 792         if (features & BIT_ULL(NPC_IPPROTO_ESP))
 793                 npc_update_entry(rvu, NPC_LE, entry, NPC_LT_LE_ESP,
 794                                  0, ~0ULL, 0, intf);
 795
 796 #define NPC_WRITE_FLOW(field, member, val_lo, val_hi, mask_lo, mask_hi)       \
 797 do {                                                                          \
 798         if (features & BIT_ULL((field))) {                                    \
 799                 npc_update_entry(rvu, (field), entry, (val_lo), (val_hi),     \
 800                                  (mask_lo), (mask_hi), intf);                 \
 801                 memcpy(&opkt->member, &pkt->member, sizeof(pkt->member));     \
 802                 memcpy(&omask->member, &mask->member, sizeof(mask->member));  \
 803         }                                                                     \
 804 } while (0)
 805
 806         NPC_WRITE_FLOW(NPC_DMAC, dmac, dmac_val, 0, dmac_mask, 0);
 807         NPC_WRITE_FLOW(NPC_SMAC, smac, smac_val, 0, smac_mask, 0);
 808         NPC_WRITE_FLOW(NPC_ETYPE, etype, ntohs(pkt->etype), 0,
 809                        ntohs(mask->etype), 0);
 810         NPC_WRITE_FLOW(NPC_TOS, tos, pkt->tos, 0, mask->tos, 0);
 811         NPC_WRITE_FLOW(NPC_SIP_IPV4, ip4src, ntohl(pkt->ip4src), 0,
 812                        ntohl(mask->ip4src), 0);
 813         NPC_WRITE_FLOW(NPC_DIP_IPV4, ip4dst, ntohl(pkt->ip4dst), 0,
 814                        ntohl(mask->ip4dst), 0);
 815         NPC_WRITE_FLOW(NPC_SPORT_TCP, sport, ntohs(pkt->sport), 0,
 816                        ntohs(mask->sport), 0);
 817         NPC_WRITE_FLOW(NPC_SPORT_UDP, sport, ntohs(pkt->sport), 0,
 818                        ntohs(mask->sport), 0);
 819         NPC_WRITE_FLOW(NPC_DPORT_TCP, dport, ntohs(pkt->dport), 0,
 820                        ntohs(mask->dport), 0);
 821         NPC_WRITE_FLOW(NPC_DPORT_UDP, dport, ntohs(pkt->dport), 0,
 822                        ntohs(mask->dport), 0);
 823         NPC_WRITE_FLOW(NPC_SPORT_SCTP, sport, ntohs(pkt->sport), 0,
 824                        ntohs(mask->sport), 0);
 825         NPC_WRITE_FLOW(NPC_DPORT_SCTP, dport, ntohs(pkt->dport), 0,
 826                        ntohs(mask->dport), 0);
 827
 828         NPC_WRITE_FLOW(NPC_OUTER_VID, vlan_tci, ntohs(pkt->vlan_tci), 0,
 829                        ntohs(mask->vlan_tci), 0);
 830
 831         npc_update_ipv6_flow(rvu, entry, features, pkt, mask, output, intf);
 832 }
 833
 834 static struct rvu_npc_mcam_rule *rvu_mcam_find_rule(struct npc_mcam *mcam,
 835                                                     u16 entry)
 836 {
 837         struct rvu_npc_mcam_rule *iter;
 838
 839         mutex_lock(&mcam->lock);
 840         list_for_each_entry(iter, &mcam->mcam_rules, list) {
 841                 if (iter->entry == entry) {
 842                         mutex_unlock(&mcam->lock);
 843                         return iter;
 844                 }
 845         }
 846         mutex_unlock(&mcam->lock);
 847
 848         return NULL;
 849 }
 850
 851 static void rvu_mcam_add_rule(struct npc_mcam *mcam,
 852                               struct rvu_npc_mcam_rule *rule)
 853 {
 854         struct list_head *head = &mcam->mcam_rules;
 855         struct rvu_npc_mcam_rule *iter;
 856
 857         mutex_lock(&mcam->lock);
 858         list_for_each_entry(iter, &mcam->mcam_rules, list) {
 859                 if (iter->entry > rule->entry)
 860                         break;
 861                 head = &iter->list;
 862         }
 863
 864         list_add(&rule->list, head);
 865         mutex_unlock(&mcam->lock);
 866 }
 867
 868 static void rvu_mcam_remove_counter_from_rule(struct rvu *rvu, u16 pcifunc,
 869                                               struct rvu_npc_mcam_rule *rule)
 870 {
 871         struct npc_mcam_oper_counter_req free_req = { 0 };
 872         struct msg_rsp free_rsp;
 873
 874         if (!rule->has_cntr)
 875                 return;
 876
 877         free_req.hdr.pcifunc = pcifunc;
 878         free_req.cntr = rule->cntr;
 879
 880         rvu_mbox_handler_npc_mcam_free_counter(rvu, &free_req, &free_rsp);
 881         rule->has_cntr = false;
 882 }
 883
 884 static void rvu_mcam_add_counter_to_rule(struct rvu *rvu, u16 pcifunc,
 885                                          struct rvu_npc_mcam_rule *rule,
 886                                          struct npc_install_flow_rsp *rsp)
 887 {
 888         struct npc_mcam_alloc_counter_req cntr_req = { 0 };
 889         struct npc_mcam_alloc_counter_rsp cntr_rsp = { 0 };
 890         int err;
 891
 892         cntr_req.hdr.pcifunc = pcifunc;
 893         cntr_req.contig = true;
 894         cntr_req.count = 1;
 895
 896         /* we try to allocate a counter to track the stats of this
 897          * rule. If counter could not be allocated then proceed
 898          * without counter because counters are limited than entries.
 899          */
 900         err = rvu_mbox_handler_npc_mcam_alloc_counter(rvu, &cntr_req,
 901                                                       &cntr_rsp);
 902         if (!err && cntr_rsp.count) {
 903                 rule->cntr = cntr_rsp.cntr;
 904                 rule->has_cntr = true;
 905                 rsp->counter = rule->cntr;
 906         } else {
 907                 rsp->counter = err;
 908         }
 909 }
 910
 911 static void npc_update_rx_entry(struct rvu *rvu, struct rvu_pfvf *pfvf,
 912                                 struct mcam_entry *entry,
 913                                 struct npc_install_flow_req *req, u16 target)
 914 {
 915         struct nix_rx_action action;
 916         u64 chan_mask;
 917
 918         chan_mask = req->chan_mask ? req->chan_mask : ~0ULL;
 919         npc_update_entry(rvu, NPC_CHAN, entry, req->channel, 0, chan_mask, 0,
 920                          NIX_INTF_RX);
 921
 922         *(u64 *)&action = 0x00;
 923         action.pf_func = target;
 924         action.op = req->op;
 925         action.index = req->index;
 926         action.match_id = req->match_id;
 927         action.flow_key_alg = req->flow_key_alg;
 928
 929         if (req->op == NIX_RX_ACTION_DEFAULT && pfvf->def_ucast_rule)
 930                 action = pfvf->def_ucast_rule->rx_action;
 931
 932         entry->action = *(u64 *)&action;
 933
 934         /* VTAG0 starts at 0th byte of LID_B.
 935          * VTAG1 starts at 4th byte of LID_B.
 936          */
 937         entry->vtag_action = FIELD_PREP(RX_VTAG0_VALID_BIT, req->vtag0_valid) |
 938                              FIELD_PREP(RX_VTAG0_TYPE_MASK, req->vtag0_type) |
 939                              FIELD_PREP(RX_VTAG0_LID_MASK, NPC_LID_LB) |
 940                              FIELD_PREP(RX_VTAG0_RELPTR_MASK, 0) |
 941                              FIELD_PREP(RX_VTAG1_VALID_BIT, req->vtag1_valid) |
 942                              FIELD_PREP(RX_VTAG1_TYPE_MASK, req->vtag1_type) |
 943                              FIELD_PREP(RX_VTAG1_LID_MASK, NPC_LID_LB) |
 944                              FIELD_PREP(RX_VTAG1_RELPTR_MASK, 4);
 945 }
 946
 947 static void npc_update_tx_entry(struct rvu *rvu, struct rvu_pfvf *pfvf,
 948                                 struct mcam_entry *entry,
 949                                 struct npc_install_flow_req *req, u16 target)
 950 {
 951         struct nix_tx_action action;
 952
 953         npc_update_entry(rvu, NPC_PF_FUNC, entry, (__force u16)htons(target),
 954                          0, ~0ULL, 0, NIX_INTF_TX);
 955
 956         *(u64 *)&action = 0x00;
 957         action.op = req->op;
 958         action.index = req->index;
 959         action.match_id = req->match_id;
 960
 961         entry->action = *(u64 *)&action;
 962
 963         /* VTAG0 starts at 0th byte of LID_B.
 964          * VTAG1 starts at 4th byte of LID_B.
 965          */
 966         entry->vtag_action = FIELD_PREP(TX_VTAG0_DEF_MASK, req->vtag0_def) |
 967                              FIELD_PREP(TX_VTAG0_OP_MASK, req->vtag0_op) |
 968                              FIELD_PREP(TX_VTAG0_LID_MASK, NPC_LID_LA) |
 969                              FIELD_PREP(TX_VTAG0_RELPTR_MASK, 20) |
 970                              FIELD_PREP(TX_VTAG1_DEF_MASK, req->vtag1_def) |
 971                              FIELD_PREP(TX_VTAG1_OP_MASK, req->vtag1_op) |
 972                              FIELD_PREP(TX_VTAG1_LID_MASK, NPC_LID_LA) |
 973                              FIELD_PREP(TX_VTAG1_RELPTR_MASK, 24);
 974 }
 975
 976 static int npc_install_flow(struct rvu *rvu, int blkaddr, u16 target,
 977                             int nixlf, struct rvu_pfvf *pfvf,
 978                             struct npc_install_flow_req *req,
 979                             struct npc_install_flow_rsp *rsp, bool enable,
 980                             bool pf_set_vfs_mac)
 981 {
 982         struct rvu_npc_mcam_rule *def_ucast_rule = pfvf->def_ucast_rule;
 983         u64 features, installed_features, missing_features = 0;
 984         struct npc_mcam_write_entry_req write_req = { 0 };
 985         struct npc_mcam *mcam = &rvu->hw->mcam;
 986         struct rvu_npc_mcam_rule dummy = { 0 };
 987         struct rvu_npc_mcam_rule *rule;
 988         bool new = false, msg_from_vf;
 989         u16 owner = req->hdr.pcifunc;
 990         struct msg_rsp write_rsp;
 991         struct mcam_entry *entry;
 992         int entry_index, err;
 993
 994         msg_from_vf = !!(owner & RVU_PFVF_FUNC_MASK);
 995
 996         installed_features = req->features;
 997         features = req->features;
 998         entry = &write_req.entry_data;
 999         entry_index = req->entry;
1000
1001         npc_update_flow(rvu, entry, features, &req->packet, &req->mask, &dummy,
1002                         req->intf);
1003
1004         if (is_npc_intf_rx(req->intf))
1005                 npc_update_rx_entry(rvu, pfvf, entry, req, target);
1006         else
1007                 npc_update_tx_entry(rvu, pfvf, entry, req, target);
1008
1009         /* Default unicast rules do not exist for TX */
1010         if (is_npc_intf_tx(req->intf))
1011                 goto find_rule;
1012
1013         if (req->default_rule) {
1014                 entry_index = npc_get_nixlf_mcam_index(mcam, target, nixlf,
1015                                                        NIXLF_UCAST_ENTRY);
1016                 enable = is_mcam_entry_enabled(rvu, mcam, blkaddr, entry_index);
1017         }
1018
1019         /* update mcam entry with default unicast rule attributes */
1020         if (def_ucast_rule && (msg_from_vf || (req->default_rule && req->append))) {
1021                 missing_features = (def_ucast_rule->features ^ features) &
1022                                         def_ucast_rule->features;
1023                 if (missing_features)
1024                         npc_update_flow(rvu, entry, missing_features,
1025                                         &def_ucast_rule->packet,
1026                                         &def_ucast_rule->mask,
1027                                         &dummy, req->intf);
1028                 installed_features = req->features | missing_features;
1029         }
1030
1031 find_rule:
1032         rule = rvu_mcam_find_rule(mcam, entry_index);
1033         if (!rule) {
1034                 rule = kzalloc(sizeof(*rule), GFP_KERNEL);
1035                 if (!rule)
1036                         return -ENOMEM;
1037                 new = true;
1038         }
1039
1040         /* allocate new counter if rule has no counter */
1041         if (!req->default_rule && req->set_cntr && !rule->has_cntr)
1042                 rvu_mcam_add_counter_to_rule(rvu, owner, rule, rsp);
1043
1044         /* if user wants to delete an existing counter for a rule then
1045          * free the counter
1046          */
1047         if (!req->set_cntr && rule->has_cntr)
1048                 rvu_mcam_remove_counter_from_rule(rvu, owner, rule);
1049
1050         write_req.hdr.pcifunc = owner;
1051
1052         /* AF owns the default rules so change the owner just to relax
1053          * the checks in rvu_mbox_handler_npc_mcam_write_entry
1054          */
1055         if (req->default_rule)
1056                 write_req.hdr.pcifunc = 0;
1057
1058         write_req.entry = entry_index;
1059         write_req.intf = req->intf;
1060         write_req.enable_entry = (u8)enable;
1061         /* if counter is available then clear and use it */
1062         if (req->set_cntr && rule->has_cntr) {
1063                 rvu_write64(rvu, blkaddr, NPC_AF_MATCH_STATX(rule->cntr), 0x00);
1064                 write_req.set_cntr = 1;
1065                 write_req.cntr = rule->cntr;
1066         }
1067
1068         err = rvu_mbox_handler_npc_mcam_write_entry(rvu, &write_req,
1069                                                     &write_rsp);
1070         if (err) {
1071                 rvu_mcam_remove_counter_from_rule(rvu, owner, rule);
1072                 if (new)
1073                         kfree(rule);
1074                 return err;
1075         }
1076         /* update rule */
1077         memcpy(&rule->packet, &dummy.packet, sizeof(rule->packet));
1078         memcpy(&rule->mask, &dummy.mask, sizeof(rule->mask));
1079         rule->entry = entry_index;
1080         memcpy(&rule->rx_action, &entry->action, sizeof(struct nix_rx_action));
1081         if (is_npc_intf_tx(req->intf))
1082                 memcpy(&rule->tx_action, &entry->action,
1083                        sizeof(struct nix_tx_action));
1084         rule->vtag_action = entry->vtag_action;
1085         rule->features = installed_features;
1086         rule->default_rule = req->default_rule;
1087         rule->owner = owner;
1088         rule->enable = enable;
1089         if (is_npc_intf_tx(req->intf))
1090                 rule->intf = pfvf->nix_tx_intf;
1091         else
1092                 rule->intf = pfvf->nix_rx_intf;
1093
1094         if (new)
1095                 rvu_mcam_add_rule(mcam, rule);
1096         if (req->default_rule)
1097                 pfvf->def_ucast_rule = rule;
1098
1099         /* VF's MAC address is being changed via PF  */
1100         if (pf_set_vfs_mac) {
1101                 ether_addr_copy(pfvf->default_mac, req->packet.dmac);
1102                 ether_addr_copy(pfvf->mac_addr, req->packet.dmac);
1103                 set_bit(PF_SET_VF_MAC, &pfvf->flags);
1104         }
1105
1106         if (test_bit(PF_SET_VF_CFG, &pfvf->flags) &&
1107             req->vtag0_type == NIX_AF_LFX_RX_VTAG_TYPE7)
1108                 rule->vfvlan_cfg = true;
1109
1110         if (is_npc_intf_rx(req->intf) && req->match_id &&
1111             (req->op == NIX_RX_ACTIONOP_UCAST || req->op == NIX_RX_ACTIONOP_RSS))
1112                 return rvu_nix_setup_ratelimit_aggr(rvu, req->hdr.pcifunc,
1113                                              req->index, req->match_id);
1114
1115         return 0;
1116 }
1117
1118 int rvu_mbox_handler_npc_install_flow(struct rvu *rvu,
1119                                       struct npc_install_flow_req *req,
1120                                       struct npc_install_flow_rsp *rsp)
1121 {
1122         bool from_vf = !!(req->hdr.pcifunc & RVU_PFVF_FUNC_MASK);
1123         int blkaddr, nixlf, err;
1124         struct rvu_pfvf *pfvf;
1125         bool pf_set_vfs_mac = false;
1126         bool enable = true;
1127         u16 target;
1128
1129         blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
1130         if (blkaddr < 0) {
1131                 dev_err(rvu->dev, "%s: NPC block not implemented\n", __func__);
1132                 return -ENODEV;
1133         }
1134
1135         if (!is_npc_interface_valid(rvu, req->intf))
1136                 return -EINVAL;
1137
1138         if (from_vf && req->default_rule)
1139                 return NPC_MCAM_PERM_DENIED;
1140
1141         /* Each PF/VF info is maintained in struct rvu_pfvf.
1142          * rvu_pfvf for the target PF/VF needs to be retrieved
1143          * hence modify pcifunc accordingly.
1144          */
1145
1146         /* AF installing for a PF/VF */
1147         if (!req->hdr.pcifunc)
1148                 target = req->vf;
1149         /* PF installing for its VF */
1150         else if (!from_vf && req->vf) {
1151                 target = (req->hdr.pcifunc & ~RVU_PFVF_FUNC_MASK) | req->vf;
1152                 pf_set_vfs_mac = req->default_rule &&
1153                                 (req->features & BIT_ULL(NPC_DMAC));
1154         }
1155         /* msg received from PF/VF */
1156         else
1157                 target = req->hdr.pcifunc;
1158
1159         /* ignore chan_mask in case pf func is not AF, revisit later */
1160         if (!is_pffunc_af(req->hdr.pcifunc))
1161                 req->chan_mask = 0xFFF;
1162
1163         err = npc_check_unsupported_flows(rvu, req->features, req->intf);
1164         if (err)
1165                 return err;
1166
1167         if (npc_mcam_verify_channel(rvu, target, req->intf, req->channel))
1168                 return -EINVAL;
1169
1170         pfvf = rvu_get_pfvf(rvu, target);
1171
1172         /* PF installing for its VF */
1173         if (req->hdr.pcifunc && !from_vf && req->vf)
1174                 set_bit(PF_SET_VF_CFG, &pfvf->flags);
1175
1176         /* update req destination mac addr */
1177         if ((req->features & BIT_ULL(NPC_DMAC)) && is_npc_intf_rx(req->intf) &&
1178             is_zero_ether_addr(req->packet.dmac)) {
1179                 ether_addr_copy(req->packet.dmac, pfvf->mac_addr);
1180                 eth_broadcast_addr((u8 *)&req->mask.dmac);
1181         }
1182
1183         err = nix_get_nixlf(rvu, target, &nixlf, NULL);
1184         if (err && is_npc_intf_rx(req->intf) && !pf_set_vfs_mac)
1185                 return -EINVAL;
1186
1187         /* don't enable rule when nixlf not attached or initialized */
1188         if (!(is_nixlf_attached(rvu, target) &&
1189               test_bit(NIXLF_INITIALIZED, &pfvf->flags)))
1190                 enable = false;
1191
1192         /* Packets reaching NPC in Tx path implies that a
1193          * NIXLF is properly setup and transmitting.
1194          * Hence rules can be enabled for Tx.
1195          */
1196         if (is_npc_intf_tx(req->intf))
1197                 enable = true;
1198
1199         /* Do not allow requests from uninitialized VFs */
1200         if (from_vf && !enable)
1201                 return -EINVAL;
1202
1203         /* PF sets VF mac & VF NIXLF is not attached, update the mac addr */
1204         if (pf_set_vfs_mac && !enable) {
1205                 ether_addr_copy(pfvf->default_mac, req->packet.dmac);
1206                 ether_addr_copy(pfvf->mac_addr, req->packet.dmac);
1207                 set_bit(PF_SET_VF_MAC, &pfvf->flags);
1208                 return 0;
1209         }
1210
1211         /* If message is from VF then its flow should not overlap with
1212          * reserved unicast flow.
1213          */
1214         if (from_vf && pfvf->def_ucast_rule && is_npc_intf_rx(req->intf) &&
1215             pfvf->def_ucast_rule->features & req->features)
1216                 return -EINVAL;
1217
1218         return npc_install_flow(rvu, blkaddr, target, nixlf, pfvf, req, rsp,
1219                                 enable, pf_set_vfs_mac);
1220 }
1221
1222 static int npc_delete_flow(struct rvu *rvu, struct rvu_npc_mcam_rule *rule,
1223                            u16 pcifunc)
1224 {
1225         struct npc_mcam_ena_dis_entry_req dis_req = { 0 };
1226         struct msg_rsp dis_rsp;
1227
1228         if (rule->default_rule)
1229                 return 0;
1230
1231         if (rule->has_cntr)
1232                 rvu_mcam_remove_counter_from_rule(rvu, pcifunc, rule);
1233
1234         dis_req.hdr.pcifunc = pcifunc;
1235         dis_req.entry = rule->entry;
1236
1237         list_del(&rule->list);
1238         kfree(rule);
1239
1240         return rvu_mbox_handler_npc_mcam_dis_entry(rvu, &dis_req, &dis_rsp);
1241 }
1242
1243 int rvu_mbox_handler_npc_delete_flow(struct rvu *rvu,
1244                                      struct npc_delete_flow_req *req,
1245                                      struct msg_rsp *rsp)
1246 {
1247         struct npc_mcam *mcam = &rvu->hw->mcam;
1248         struct rvu_npc_mcam_rule *iter, *tmp;
1249         u16 pcifunc = req->hdr.pcifunc;
1250         struct list_head del_list;
1251
1252         INIT_LIST_HEAD(&del_list);
1253
1254         mutex_lock(&mcam->lock);
1255         list_for_each_entry_safe(iter, tmp, &mcam->mcam_rules, list) {
1256                 if (iter->owner == pcifunc) {
1257                         /* All rules */
1258                         if (req->all) {
1259                                 list_move_tail(&iter->list, &del_list);
1260                         /* Range of rules */
1261                         } else if (req->end && iter->entry >= req->start &&
1262                                    iter->entry <= req->end) {
1263                                 list_move_tail(&iter->list, &del_list);
1264                         /* single rule */
1265                         } else if (req->entry == iter->entry) {
1266                                 list_move_tail(&iter->list, &del_list);
1267                                 break;
1268                         }
1269                 }
1270         }
1271         mutex_unlock(&mcam->lock);
1272
1273         list_for_each_entry_safe(iter, tmp, &del_list, list) {
1274                 u16 entry = iter->entry;
1275
1276                 /* clear the mcam entry target pcifunc */
1277                 mcam->entry2target_pffunc[entry] = 0x0;
1278                 if (npc_delete_flow(rvu, iter, pcifunc))
1279                         dev_err(rvu->dev, "rule deletion failed for entry:%u",
1280                                 entry);
1281         }
1282
1283         return 0;
1284 }
1285
1286 static int npc_update_dmac_value(struct rvu *rvu, int npcblkaddr,
1287                                  struct rvu_npc_mcam_rule *rule,
1288                                  struct rvu_pfvf *pfvf)
1289 {
1290         struct npc_mcam_write_entry_req write_req = { 0 };
1291         struct mcam_entry *entry = &write_req.entry_data;
1292         struct npc_mcam *mcam = &rvu->hw->mcam;
1293         struct msg_rsp rsp;
1294         u8 intf, enable;
1295         int err;
1296
1297         ether_addr_copy(rule->packet.dmac, pfvf->mac_addr);
1298
1299         npc_read_mcam_entry(rvu, mcam, npcblkaddr, rule->entry,
1300                             entry, &intf,  &enable);
1301
1302         npc_update_entry(rvu, NPC_DMAC, entry,
1303                          ether_addr_to_u64(pfvf->mac_addr), 0,
1304                          0xffffffffffffull, 0, intf);
1305
1306         write_req.hdr.pcifunc = rule->owner;
1307         write_req.entry = rule->entry;
1308         write_req.intf = pfvf->nix_rx_intf;
1309
1310         mutex_unlock(&mcam->lock);
1311         err = rvu_mbox_handler_npc_mcam_write_entry(rvu, &write_req, &rsp);
1312         mutex_lock(&mcam->lock);
1313
1314         return err;
1315 }
1316
1317 void npc_mcam_enable_flows(struct rvu *rvu, u16 target)
1318 {
1319         struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, target);
1320         struct rvu_npc_mcam_rule *def_ucast_rule;
1321         struct npc_mcam *mcam = &rvu->hw->mcam;
1322         struct rvu_npc_mcam_rule *rule;
1323         int blkaddr, bank, index;
1324         u64 def_action;
1325
1326         blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
1327         if (blkaddr < 0)
1328                 return;
1329
1330         def_ucast_rule = pfvf->def_ucast_rule;
1331
1332         mutex_lock(&mcam->lock);
1333         list_for_each_entry(rule, &mcam->mcam_rules, list) {
1334                 if (is_npc_intf_rx(rule->intf) &&
1335                     rule->rx_action.pf_func == target && !rule->enable) {
1336                         if (rule->default_rule) {
1337                                 npc_enable_mcam_entry(rvu, mcam, blkaddr,
1338                                                       rule->entry, true);
1339                                 rule->enable = true;
1340                                 continue;
1341                         }
1342
1343                         if (rule->vfvlan_cfg)
1344                                 npc_update_dmac_value(rvu, blkaddr, rule, pfvf);
1345
1346                         if (rule->rx_action.op == NIX_RX_ACTION_DEFAULT) {
1347                                 if (!def_ucast_rule)
1348                                         continue;
1349                                 /* Use default unicast entry action */
1350                                 rule->rx_action = def_ucast_rule->rx_action;
1351                                 def_action = *(u64 *)&def_ucast_rule->rx_action;
1352                                 bank = npc_get_bank(mcam, rule->entry);
1353                                 rvu_write64(rvu, blkaddr,
1354                                             NPC_AF_MCAMEX_BANKX_ACTION
1355                                             (rule->entry, bank), def_action);
1356                         }
1357
1358                         npc_enable_mcam_entry(rvu, mcam, blkaddr,
1359                                               rule->entry, true);
1360                         rule->enable = true;
1361                 }
1362         }
1363
1364         /* Enable MCAM entries installed by PF with target as VF pcifunc */
1365         for (index = 0; index < mcam->bmap_entries; index++) {
1366                 if (mcam->entry2target_pffunc[index] == target)
1367                         npc_enable_mcam_entry(rvu, mcam, blkaddr,
1368                                               index, true);
1369         }
1370         mutex_unlock(&mcam->lock);
1371 }
1372
1373 void npc_mcam_disable_flows(struct rvu *rvu, u16 target)
1374 {
1375         struct npc_mcam *mcam = &rvu->hw->mcam;
1376         int blkaddr, index;
1377
1378         blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
1379         if (blkaddr < 0)
1380                 return;
1381
1382         mutex_lock(&mcam->lock);
1383         /* Disable MCAM entries installed by PF with target as VF pcifunc */
1384         for (index = 0; index < mcam->bmap_entries; index++) {
1385                 if (mcam->entry2target_pffunc[index] == target)
1386                         npc_enable_mcam_entry(rvu, mcam, blkaddr,
1387                                               index, false);
1388         }
1389         mutex_unlock(&mcam->lock);
1390 }