drivers/net/ethernet/sfc/mcdi_filters.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /****************************************************************************
   3  * Driver for Solarflare network controllers and boards
   4  * Copyright 2005-2018 Solarflare Communications Inc.
   5  * Copyright 2019-2020 Xilinx Inc.
   6  *
   7  * This program is free software; you can redistribute it and/or modify it
   8  * under the terms of the GNU General Public License version 2 as published
   9  * by the Free Software Foundation, incorporated herein by reference.
  10  */
  11
  12 #include "mcdi_filters.h"
  13 #include "mcdi.h"
  14 #include "nic.h"
  15 #include "rx_common.h"
  16
  17 /* The maximum size of a shared RSS context */
  18 /* TODO: this should really be from the mcdi protocol export */
  19 #define EFX_EF10_MAX_SHARED_RSS_CONTEXT_SIZE 64UL
  20
  21 #define EFX_EF10_FILTER_ID_INVALID 0xffff
  22
  23 /* An arbitrary search limit for the software hash table */
  24 #define EFX_EF10_FILTER_SEARCH_LIMIT 200
  25
  26 static struct efx_filter_spec *
  27 efx_mcdi_filter_entry_spec(const struct efx_mcdi_filter_table *table,
  28                            unsigned int filter_idx)
  29 {
  30         return (struct efx_filter_spec *)(table->entry[filter_idx].spec &
  31                                           ~EFX_EF10_FILTER_FLAGS);
  32 }
  33
  34 static unsigned int
  35 efx_mcdi_filter_entry_flags(const struct efx_mcdi_filter_table *table,
  36                            unsigned int filter_idx)
  37 {
  38         return table->entry[filter_idx].spec & EFX_EF10_FILTER_FLAGS;
  39 }
  40
  41 static u32 efx_mcdi_filter_get_unsafe_id(u32 filter_id)
  42 {
  43         WARN_ON_ONCE(filter_id == EFX_EF10_FILTER_ID_INVALID);
  44         return filter_id & (EFX_MCDI_FILTER_TBL_ROWS - 1);
  45 }
  46
  47 static unsigned int efx_mcdi_filter_get_unsafe_pri(u32 filter_id)
  48 {
  49         return filter_id / (EFX_MCDI_FILTER_TBL_ROWS * 2);
  50 }
  51
  52 static u32 efx_mcdi_filter_make_filter_id(unsigned int pri, u16 idx)
  53 {
  54         return pri * EFX_MCDI_FILTER_TBL_ROWS * 2 + idx;
  55 }
  56
  57 /*
  58  * Decide whether a filter should be exclusive or else should allow
  59  * delivery to additional recipients.  Currently we decide that
  60  * filters for specific local unicast MAC and IP addresses are
  61  * exclusive.
  62  */
  63 static bool efx_mcdi_filter_is_exclusive(const struct efx_filter_spec *spec)
  64 {
  65         if (spec->match_flags & EFX_FILTER_MATCH_LOC_MAC &&
  66             !is_multicast_ether_addr(spec->loc_mac))
  67                 return true;
  68
  69         if ((spec->match_flags &
  70              (EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_LOC_HOST)) ==
  71             (EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_LOC_HOST)) {
  72                 if (spec->ether_type == htons(ETH_P_IP) &&
  73                     !ipv4_is_multicast(spec->loc_host[0]))
  74                         return true;
  75                 if (spec->ether_type == htons(ETH_P_IPV6) &&
  76                     ((const u8 *)spec->loc_host)[0] != 0xff)
  77                         return true;
  78         }
  79
  80         return false;
  81 }
  82
  83 static void
  84 efx_mcdi_filter_set_entry(struct efx_mcdi_filter_table *table,
  85                           unsigned int filter_idx,
  86                           const struct efx_filter_spec *spec,
  87                           unsigned int flags)
  88 {
  89         table->entry[filter_idx].spec = (unsigned long)spec | flags;
  90 }
  91
  92 static void
  93 efx_mcdi_filter_push_prep_set_match_fields(struct efx_nic *efx,
  94                                            const struct efx_filter_spec *spec,
  95                                            efx_dword_t *inbuf)
  96 {
  97         enum efx_encap_type encap_type = efx_filter_get_encap_type(spec);
  98         u32 match_fields = 0, uc_match, mc_match;
  99
 100         MCDI_SET_DWORD(inbuf, FILTER_OP_IN_OP,
 101                        efx_mcdi_filter_is_exclusive(spec) ?
 102                        MC_CMD_FILTER_OP_IN_OP_INSERT :
 103                        MC_CMD_FILTER_OP_IN_OP_SUBSCRIBE);
 104
 105         /*
 106          * Convert match flags and values.  Unlike almost
 107          * everything else in MCDI, these fields are in
 108          * network byte order.
 109          */
 110 #define COPY_VALUE(value, mcdi_field)                                        \
 111         do {                                                         \
 112                 match_fields |=                                      \
 113                         1 << MC_CMD_FILTER_OP_IN_MATCH_ ##           \
 114                         mcdi_field ## _LBN;                          \
 115                 BUILD_BUG_ON(                                        \
 116                         MC_CMD_FILTER_OP_IN_ ## mcdi_field ## _LEN < \
 117                         sizeof(value));                              \
 118                 memcpy(MCDI_PTR(inbuf, FILTER_OP_IN_ ## mcdi_field), \
 119                        &value, sizeof(value));                       \
 120         } while (0)
 121 #define COPY_FIELD(gen_flag, gen_field, mcdi_field)                          \
 122         if (spec->match_flags & EFX_FILTER_MATCH_ ## gen_flag) {     \
 123                 COPY_VALUE(spec->gen_field, mcdi_field);             \
 124         }
 125         /*
 126          * Handle encap filters first.  They will always be mismatch
 127          * (unknown UC or MC) filters
 128          */
 129         if (encap_type) {
 130                 /*
 131                  * ether_type and outer_ip_proto need to be variables
 132                  * because COPY_VALUE wants to memcpy them
 133                  */
 134                 __be16 ether_type =
 135                         htons(encap_type & EFX_ENCAP_FLAG_IPV6 ?
 136                               ETH_P_IPV6 : ETH_P_IP);
 137                 u8 vni_type = MC_CMD_FILTER_OP_EXT_IN_VNI_TYPE_GENEVE;
 138                 u8 outer_ip_proto;
 139
 140                 switch (encap_type & EFX_ENCAP_TYPES_MASK) {
 141                 case EFX_ENCAP_TYPE_VXLAN:
 142                         vni_type = MC_CMD_FILTER_OP_EXT_IN_VNI_TYPE_VXLAN;
 143                         /* fallthrough */
 144                 case EFX_ENCAP_TYPE_GENEVE:
 145                         COPY_VALUE(ether_type, ETHER_TYPE);
 146                         outer_ip_proto = IPPROTO_UDP;
 147                         COPY_VALUE(outer_ip_proto, IP_PROTO);
 148                         /*
 149                          * We always need to set the type field, even
 150                          * though we're not matching on the TNI.
 151                          */
 152                         MCDI_POPULATE_DWORD_1(inbuf,
 153                                 FILTER_OP_EXT_IN_VNI_OR_VSID,
 154                                 FILTER_OP_EXT_IN_VNI_TYPE,
 155                                 vni_type);
 156                         break;
 157                 case EFX_ENCAP_TYPE_NVGRE:
 158                         COPY_VALUE(ether_type, ETHER_TYPE);
 159                         outer_ip_proto = IPPROTO_GRE;
 160                         COPY_VALUE(outer_ip_proto, IP_PROTO);
 161                         break;
 162                 default:
 163                         WARN_ON(1);
 164                 }
 165
 166                 uc_match = MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_UNKNOWN_UCAST_DST_LBN;
 167                 mc_match = MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_UNKNOWN_MCAST_DST_LBN;
 168         } else {
 169                 uc_match = MC_CMD_FILTER_OP_EXT_IN_MATCH_UNKNOWN_UCAST_DST_LBN;
 170                 mc_match = MC_CMD_FILTER_OP_EXT_IN_MATCH_UNKNOWN_MCAST_DST_LBN;
 171         }
 172
 173         if (spec->match_flags & EFX_FILTER_MATCH_LOC_MAC_IG)
 174                 match_fields |=
 175                         is_multicast_ether_addr(spec->loc_mac) ?
 176                         1 << mc_match :
 177                         1 << uc_match;
 178         COPY_FIELD(REM_HOST, rem_host, SRC_IP);
 179         COPY_FIELD(LOC_HOST, loc_host, DST_IP);
 180         COPY_FIELD(REM_MAC, rem_mac, SRC_MAC);
 181         COPY_FIELD(REM_PORT, rem_port, SRC_PORT);
 182         COPY_FIELD(LOC_MAC, loc_mac, DST_MAC);
 183         COPY_FIELD(LOC_PORT, loc_port, DST_PORT);
 184         COPY_FIELD(ETHER_TYPE, ether_type, ETHER_TYPE);
 185         COPY_FIELD(INNER_VID, inner_vid, INNER_VLAN);
 186         COPY_FIELD(OUTER_VID, outer_vid, OUTER_VLAN);
 187         COPY_FIELD(IP_PROTO, ip_proto, IP_PROTO);
 188 #undef COPY_FIELD
 189 #undef COPY_VALUE
 190         MCDI_SET_DWORD(inbuf, FILTER_OP_IN_MATCH_FIELDS,
 191                        match_fields);
 192 }
 193
 194 static void efx_mcdi_filter_push_prep(struct efx_nic *efx,
 195                                       const struct efx_filter_spec *spec,
 196                                       efx_dword_t *inbuf, u64 handle,
 197                                       struct efx_rss_context *ctx,
 198                                       bool replacing)
 199 {
 200         u32 flags = spec->flags;
 201
 202         memset(inbuf, 0, MC_CMD_FILTER_OP_EXT_IN_LEN);
 203
 204         /* If RSS filter, caller better have given us an RSS context */
 205         if (flags & EFX_FILTER_FLAG_RX_RSS) {
 206                 /*
 207                  * We don't have the ability to return an error, so we'll just
 208                  * log a warning and disable RSS for the filter.
 209                  */
 210                 if (WARN_ON_ONCE(!ctx))
 211                         flags &= ~EFX_FILTER_FLAG_RX_RSS;
 212                 else if (WARN_ON_ONCE(ctx->context_id == EFX_MCDI_RSS_CONTEXT_INVALID))
 213                         flags &= ~EFX_FILTER_FLAG_RX_RSS;
 214         }
 215
 216         if (replacing) {
 217                 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_OP,
 218                                MC_CMD_FILTER_OP_IN_OP_REPLACE);
 219                 MCDI_SET_QWORD(inbuf, FILTER_OP_IN_HANDLE, handle);
 220         } else {
 221                 efx_mcdi_filter_push_prep_set_match_fields(efx, spec, inbuf);
 222         }
 223
 224         MCDI_SET_DWORD(inbuf, FILTER_OP_IN_PORT_ID, efx->vport_id);
 225         MCDI_SET_DWORD(inbuf, FILTER_OP_IN_RX_DEST,
 226                        spec->dmaq_id == EFX_FILTER_RX_DMAQ_ID_DROP ?
 227                        MC_CMD_FILTER_OP_IN_RX_DEST_DROP :
 228                        MC_CMD_FILTER_OP_IN_RX_DEST_HOST);
 229         MCDI_SET_DWORD(inbuf, FILTER_OP_IN_TX_DOMAIN, 0);
 230         MCDI_SET_DWORD(inbuf, FILTER_OP_IN_TX_DEST,
 231                        MC_CMD_FILTER_OP_IN_TX_DEST_DEFAULT);
 232         MCDI_SET_DWORD(inbuf, FILTER_OP_IN_RX_QUEUE,
 233                        spec->dmaq_id == EFX_FILTER_RX_DMAQ_ID_DROP ?
 234                        0 : spec->dmaq_id);
 235         MCDI_SET_DWORD(inbuf, FILTER_OP_IN_RX_MODE,
 236                        (flags & EFX_FILTER_FLAG_RX_RSS) ?
 237                        MC_CMD_FILTER_OP_IN_RX_MODE_RSS :
 238                        MC_CMD_FILTER_OP_IN_RX_MODE_SIMPLE);
 239         if (flags & EFX_FILTER_FLAG_RX_RSS)
 240                 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_RX_CONTEXT, ctx->context_id);
 241 }
 242
 243 static int efx_mcdi_filter_push(struct efx_nic *efx,
 244                                 const struct efx_filter_spec *spec, u64 *handle,
 245                                 struct efx_rss_context *ctx, bool replacing)
 246 {
 247         MCDI_DECLARE_BUF(inbuf, MC_CMD_FILTER_OP_EXT_IN_LEN);
 248         MCDI_DECLARE_BUF(outbuf, MC_CMD_FILTER_OP_EXT_OUT_LEN);
 249         size_t outlen;
 250         int rc;
 251
 252         efx_mcdi_filter_push_prep(efx, spec, inbuf, *handle, ctx, replacing);
 253         rc = efx_mcdi_rpc_quiet(efx, MC_CMD_FILTER_OP, inbuf, sizeof(inbuf),
 254                                 outbuf, sizeof(outbuf), &outlen);
 255         if (rc && spec->priority != EFX_FILTER_PRI_HINT)
 256                 efx_mcdi_display_error(efx, MC_CMD_FILTER_OP, sizeof(inbuf),
 257                                        outbuf, outlen, rc);
 258         if (rc == 0)
 259                 *handle = MCDI_QWORD(outbuf, FILTER_OP_OUT_HANDLE);
 260         if (rc == -ENOSPC)
 261                 rc = -EBUSY; /* to match efx_farch_filter_insert() */
 262         return rc;
 263 }
 264
 265 static u32 efx_mcdi_filter_mcdi_flags_from_spec(const struct efx_filter_spec *spec)
 266 {
 267         enum efx_encap_type encap_type = efx_filter_get_encap_type(spec);
 268         unsigned int match_flags = spec->match_flags;
 269         unsigned int uc_match, mc_match;
 270         u32 mcdi_flags = 0;
 271
 272 #define MAP_FILTER_TO_MCDI_FLAG(gen_flag, mcdi_field, encap) {          \
 273                 unsigned int  old_match_flags = match_flags;            \
 274                 match_flags &= ~EFX_FILTER_MATCH_ ## gen_flag;          \
 275                 if (match_flags != old_match_flags)                     \
 276                         mcdi_flags |=                                   \
 277                                 (1 << ((encap) ?                        \
 278                                        MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_ ## \
 279                                        mcdi_field ## _LBN :             \
 280                                        MC_CMD_FILTER_OP_EXT_IN_MATCH_ ##\
 281                                        mcdi_field ## _LBN));            \
 282         }
 283         /* inner or outer based on encap type */
 284         MAP_FILTER_TO_MCDI_FLAG(REM_HOST, SRC_IP, encap_type);
 285         MAP_FILTER_TO_MCDI_FLAG(LOC_HOST, DST_IP, encap_type);
 286         MAP_FILTER_TO_MCDI_FLAG(REM_MAC, SRC_MAC, encap_type);
 287         MAP_FILTER_TO_MCDI_FLAG(REM_PORT, SRC_PORT, encap_type);
 288         MAP_FILTER_TO_MCDI_FLAG(LOC_MAC, DST_MAC, encap_type);
 289         MAP_FILTER_TO_MCDI_FLAG(LOC_PORT, DST_PORT, encap_type);
 290         MAP_FILTER_TO_MCDI_FLAG(ETHER_TYPE, ETHER_TYPE, encap_type);
 291         MAP_FILTER_TO_MCDI_FLAG(IP_PROTO, IP_PROTO, encap_type);
 292         /* always outer */
 293         MAP_FILTER_TO_MCDI_FLAG(INNER_VID, INNER_VLAN, false);
 294         MAP_FILTER_TO_MCDI_FLAG(OUTER_VID, OUTER_VLAN, false);
 295 #undef MAP_FILTER_TO_MCDI_FLAG
 296
 297         /* special handling for encap type, and mismatch */
 298         if (encap_type) {
 299                 match_flags &= ~EFX_FILTER_MATCH_ENCAP_TYPE;
 300                 mcdi_flags |=
 301                         (1 << MC_CMD_FILTER_OP_EXT_IN_MATCH_ETHER_TYPE_LBN);
 302                 mcdi_flags |= (1 << MC_CMD_FILTER_OP_EXT_IN_MATCH_IP_PROTO_LBN);
 303
 304                 uc_match = MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_UNKNOWN_UCAST_DST_LBN;
 305                 mc_match = MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_UNKNOWN_MCAST_DST_LBN;
 306         } else {
 307                 uc_match = MC_CMD_FILTER_OP_EXT_IN_MATCH_UNKNOWN_UCAST_DST_LBN;
 308                 mc_match = MC_CMD_FILTER_OP_EXT_IN_MATCH_UNKNOWN_MCAST_DST_LBN;
 309         }
 310
 311         if (match_flags & EFX_FILTER_MATCH_LOC_MAC_IG) {
 312                 match_flags &= ~EFX_FILTER_MATCH_LOC_MAC_IG;
 313                 mcdi_flags |=
 314                         is_multicast_ether_addr(spec->loc_mac) ?
 315                         1 << mc_match :
 316                         1 << uc_match;
 317         }
 318
 319         /* Did we map them all? */
 320         WARN_ON_ONCE(match_flags);
 321
 322         return mcdi_flags;
 323 }
 324
 325 static int efx_mcdi_filter_pri(struct efx_mcdi_filter_table *table,
 326                                const struct efx_filter_spec *spec)
 327 {
 328         u32 mcdi_flags = efx_mcdi_filter_mcdi_flags_from_spec(spec);
 329         unsigned int match_pri;
 330
 331         for (match_pri = 0;
 332              match_pri < table->rx_match_count;
 333              match_pri++)
 334                 if (table->rx_match_mcdi_flags[match_pri] == mcdi_flags)
 335                         return match_pri;
 336
 337         return -EPROTONOSUPPORT;
 338 }
 339
 340 static s32 efx_mcdi_filter_insert_locked(struct efx_nic *efx,
 341                                          struct efx_filter_spec *spec,
 342                                          bool replace_equal)
 343 {
 344         DECLARE_BITMAP(mc_rem_map, EFX_EF10_FILTER_SEARCH_LIMIT);
 345         struct efx_mcdi_filter_table *table;
 346         struct efx_filter_spec *saved_spec;
 347         struct efx_rss_context *ctx = NULL;
 348         unsigned int match_pri, hash;
 349         unsigned int priv_flags;
 350         bool rss_locked = false;
 351         bool replacing = false;
 352         unsigned int depth, i;
 353         int ins_index = -1;
 354         DEFINE_WAIT(wait);
 355         bool is_mc_recip;
 356         s32 rc;
 357
 358         WARN_ON(!rwsem_is_locked(&efx->filter_sem));
 359         table = efx->filter_state;
 360         down_write(&table->lock);
 361
 362         /* For now, only support RX filters */
 363         if ((spec->flags & (EFX_FILTER_FLAG_RX | EFX_FILTER_FLAG_TX)) !=
 364             EFX_FILTER_FLAG_RX) {
 365                 rc = -EINVAL;
 366                 goto out_unlock;
 367         }
 368
 369         rc = efx_mcdi_filter_pri(table, spec);
 370         if (rc < 0)
 371                 goto out_unlock;
 372         match_pri = rc;
 373
 374         hash = efx_filter_spec_hash(spec);
 375         is_mc_recip = efx_filter_is_mc_recipient(spec);
 376         if (is_mc_recip)
 377                 bitmap_zero(mc_rem_map, EFX_EF10_FILTER_SEARCH_LIMIT);
 378
 379         if (spec->flags & EFX_FILTER_FLAG_RX_RSS) {
 380                 mutex_lock(&efx->rss_lock);
 381                 rss_locked = true;
 382                 if (spec->rss_context)
 383                         ctx = efx_find_rss_context_entry(efx, spec->rss_context);
 384                 else
 385                         ctx = &efx->rss_context;
 386                 if (!ctx) {
 387                         rc = -ENOENT;
 388                         goto out_unlock;
 389                 }
 390                 if (ctx->context_id == EFX_MCDI_RSS_CONTEXT_INVALID) {
 391                         rc = -EOPNOTSUPP;
 392                         goto out_unlock;
 393                 }
 394         }
 395
 396         /* Find any existing filters with the same match tuple or
 397          * else a free slot to insert at.
 398          */
 399         for (depth = 1; depth < EFX_EF10_FILTER_SEARCH_LIMIT; depth++) {
 400                 i = (hash + depth) & (EFX_MCDI_FILTER_TBL_ROWS - 1);
 401                 saved_spec = efx_mcdi_filter_entry_spec(table, i);
 402
 403                 if (!saved_spec) {
 404                         if (ins_index < 0)
 405                                 ins_index = i;
 406                 } else if (efx_filter_spec_equal(spec, saved_spec)) {
 407                         if (spec->priority < saved_spec->priority &&
 408                             spec->priority != EFX_FILTER_PRI_AUTO) {
 409                                 rc = -EPERM;
 410                                 goto out_unlock;
 411                         }
 412                         if (!is_mc_recip) {
 413                                 /* This is the only one */
 414                                 if (spec->priority ==
 415                                     saved_spec->priority &&
 416                                     !replace_equal) {
 417                                         rc = -EEXIST;
 418                                         goto out_unlock;
 419                                 }
 420                                 ins_index = i;
 421                                 break;
 422                         } else if (spec->priority >
 423                                    saved_spec->priority ||
 424                                    (spec->priority ==
 425                                     saved_spec->priority &&
 426                                     replace_equal)) {
 427                                 if (ins_index < 0)
 428                                         ins_index = i;
 429                                 else
 430                                         __set_bit(depth, mc_rem_map);
 431                         }
 432                 }
 433         }
 434
 435         /* Once we reach the maximum search depth, use the first suitable
 436          * slot, or return -EBUSY if there was none
 437          */
 438         if (ins_index < 0) {
 439                 rc = -EBUSY;
 440                 goto out_unlock;
 441         }
 442
 443         /* Create a software table entry if necessary. */
 444         saved_spec = efx_mcdi_filter_entry_spec(table, ins_index);
 445         if (saved_spec) {
 446                 if (spec->priority == EFX_FILTER_PRI_AUTO &&
 447                     saved_spec->priority >= EFX_FILTER_PRI_AUTO) {
 448                         /* Just make sure it won't be removed */
 449                         if (saved_spec->priority > EFX_FILTER_PRI_AUTO)
 450                                 saved_spec->flags |= EFX_FILTER_FLAG_RX_OVER_AUTO;
 451                         table->entry[ins_index].spec &=
 452                                 ~EFX_EF10_FILTER_FLAG_AUTO_OLD;
 453                         rc = ins_index;
 454                         goto out_unlock;
 455                 }
 456                 replacing = true;
 457                 priv_flags = efx_mcdi_filter_entry_flags(table, ins_index);
 458         } else {
 459                 saved_spec = kmalloc(sizeof(*spec), GFP_ATOMIC);
 460                 if (!saved_spec) {
 461                         rc = -ENOMEM;
 462                         goto out_unlock;
 463                 }
 464                 *saved_spec = *spec;
 465                 priv_flags = 0;
 466         }
 467         efx_mcdi_filter_set_entry(table, ins_index, saved_spec, priv_flags);
 468
 469         /* Actually insert the filter on the HW */
 470         rc = efx_mcdi_filter_push(efx, spec, &table->entry[ins_index].handle,
 471                                   ctx, replacing);
 472
 473         if (rc == -EINVAL && efx->must_realloc_vis)
 474                 /* The MC rebooted under us, causing it to reject our filter
 475                  * insertion as pointing to an invalid VI (spec->dmaq_id).
 476                  */
 477                 rc = -EAGAIN;
 478
 479         /* Finalise the software table entry */
 480         if (rc == 0) {
 481                 if (replacing) {
 482                         /* Update the fields that may differ */
 483                         if (saved_spec->priority == EFX_FILTER_PRI_AUTO)
 484                                 saved_spec->flags |=
 485                                         EFX_FILTER_FLAG_RX_OVER_AUTO;
 486                         saved_spec->priority = spec->priority;
 487                         saved_spec->flags &= EFX_FILTER_FLAG_RX_OVER_AUTO;
 488                         saved_spec->flags |= spec->flags;
 489                         saved_spec->rss_context = spec->rss_context;
 490                         saved_spec->dmaq_id = spec->dmaq_id;
 491                 }
 492         } else if (!replacing) {
 493                 kfree(saved_spec);
 494                 saved_spec = NULL;
 495         } else {
 496                 /* We failed to replace, so the old filter is still present.
 497                  * Roll back the software table to reflect this.  In fact the
 498                  * efx_mcdi_filter_set_entry() call below will do the right
 499                  * thing, so nothing extra is needed here.
 500                  */
 501         }
 502         efx_mcdi_filter_set_entry(table, ins_index, saved_spec, priv_flags);
 503
 504         /* Remove and finalise entries for lower-priority multicast
 505          * recipients
 506          */
 507         if (is_mc_recip) {
 508                 MCDI_DECLARE_BUF(inbuf, MC_CMD_FILTER_OP_EXT_IN_LEN);
 509                 unsigned int depth, i;
 510
 511                 memset(inbuf, 0, sizeof(inbuf));
 512
 513                 for (depth = 0; depth < EFX_EF10_FILTER_SEARCH_LIMIT; depth++) {
 514                         if (!test_bit(depth, mc_rem_map))
 515                                 continue;
 516
 517                         i = (hash + depth) & (EFX_MCDI_FILTER_TBL_ROWS - 1);
 518                         saved_spec = efx_mcdi_filter_entry_spec(table, i);
 519                         priv_flags = efx_mcdi_filter_entry_flags(table, i);
 520
 521                         if (rc == 0) {
 522                                 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_OP,
 523                                                MC_CMD_FILTER_OP_IN_OP_UNSUBSCRIBE);
 524                                 MCDI_SET_QWORD(inbuf, FILTER_OP_IN_HANDLE,
 525                                                table->entry[i].handle);
 526                                 rc = efx_mcdi_rpc(efx, MC_CMD_FILTER_OP,
 527                                                   inbuf, sizeof(inbuf),
 528                                                   NULL, 0, NULL);
 529                         }
 530
 531                         if (rc == 0) {
 532                                 kfree(saved_spec);
 533                                 saved_spec = NULL;
 534                                 priv_flags = 0;
 535                         }
 536                         efx_mcdi_filter_set_entry(table, i, saved_spec,
 537                                                   priv_flags);
 538                 }
 539         }
 540
 541         /* If successful, return the inserted filter ID */
 542         if (rc == 0)
 543                 rc = efx_mcdi_filter_make_filter_id(match_pri, ins_index);
 544
 545 out_unlock:
 546         if (rss_locked)
 547                 mutex_unlock(&efx->rss_lock);
 548         up_write(&table->lock);
 549         return rc;
 550 }
 551
 552 s32 efx_mcdi_filter_insert(struct efx_nic *efx, struct efx_filter_spec *spec,
 553                            bool replace_equal)
 554 {
 555         s32 ret;
 556
 557         down_read(&efx->filter_sem);
 558         ret = efx_mcdi_filter_insert_locked(efx, spec, replace_equal);
 559         up_read(&efx->filter_sem);
 560
 561         return ret;
 562 }
 563
 564 /*
 565  * Remove a filter.
 566  * If !by_index, remove by ID
 567  * If by_index, remove by index
 568  * Filter ID may come from userland and must be range-checked.
 569  * Caller must hold efx->filter_sem for read, and efx->filter_state->lock
 570  * for write.
 571  */
 572 static int efx_mcdi_filter_remove_internal(struct efx_nic *efx,
 573                                            unsigned int priority_mask,
 574                                            u32 filter_id, bool by_index)
 575 {
 576         unsigned int filter_idx = efx_mcdi_filter_get_unsafe_id(filter_id);
 577         struct efx_mcdi_filter_table *table = efx->filter_state;
 578         MCDI_DECLARE_BUF(inbuf,
 579                          MC_CMD_FILTER_OP_IN_HANDLE_OFST +
 580                          MC_CMD_FILTER_OP_IN_HANDLE_LEN);
 581         struct efx_filter_spec *spec;
 582         DEFINE_WAIT(wait);
 583         int rc;
 584
 585         spec = efx_mcdi_filter_entry_spec(table, filter_idx);
 586         if (!spec ||
 587             (!by_index &&
 588              efx_mcdi_filter_pri(table, spec) !=
 589              efx_mcdi_filter_get_unsafe_pri(filter_id)))
 590                 return -ENOENT;
 591
 592         if (spec->flags & EFX_FILTER_FLAG_RX_OVER_AUTO &&
 593             priority_mask == (1U << EFX_FILTER_PRI_AUTO)) {
 594                 /* Just remove flags */
 595                 spec->flags &= ~EFX_FILTER_FLAG_RX_OVER_AUTO;
 596                 table->entry[filter_idx].spec &= ~EFX_EF10_FILTER_FLAG_AUTO_OLD;
 597                 return 0;
 598         }
 599
 600         if (!(priority_mask & (1U << spec->priority)))
 601                 return -ENOENT;
 602
 603         if (spec->flags & EFX_FILTER_FLAG_RX_OVER_AUTO) {
 604                 /* Reset to an automatic filter */
 605
 606                 struct efx_filter_spec new_spec = *spec;
 607
 608                 new_spec.priority = EFX_FILTER_PRI_AUTO;
 609                 new_spec.flags = (EFX_FILTER_FLAG_RX |
 610                                   (efx_rss_active(&efx->rss_context) ?
 611                                    EFX_FILTER_FLAG_RX_RSS : 0));
 612                 new_spec.dmaq_id = 0;
 613                 new_spec.rss_context = 0;
 614                 rc = efx_mcdi_filter_push(efx, &new_spec,
 615                                           &table->entry[filter_idx].handle,
 616                                           &efx->rss_context,
 617                                           true);
 618
 619                 if (rc == 0)
 620                         *spec = new_spec;
 621         } else {
 622                 /* Really remove the filter */
 623
 624                 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_OP,
 625                                efx_mcdi_filter_is_exclusive(spec) ?
 626                                MC_CMD_FILTER_OP_IN_OP_REMOVE :
 627                                MC_CMD_FILTER_OP_IN_OP_UNSUBSCRIBE);
 628                 MCDI_SET_QWORD(inbuf, FILTER_OP_IN_HANDLE,
 629                                table->entry[filter_idx].handle);
 630                 rc = efx_mcdi_rpc_quiet(efx, MC_CMD_FILTER_OP,
 631                                         inbuf, sizeof(inbuf), NULL, 0, NULL);
 632
 633                 if ((rc == 0) || (rc == -ENOENT)) {
 634                         /* Filter removed OK or didn't actually exist */
 635                         kfree(spec);
 636                         efx_mcdi_filter_set_entry(table, filter_idx, NULL, 0);
 637                 } else {
 638                         efx_mcdi_display_error(efx, MC_CMD_FILTER_OP,
 639                                                MC_CMD_FILTER_OP_EXT_IN_LEN,
 640                                                NULL, 0, rc);
 641                 }
 642         }
 643
 644         return rc;
 645 }
 646
 647 /* Remove filters that weren't renewed. */
 648 static void efx_mcdi_filter_remove_old(struct efx_nic *efx)
 649 {
 650         struct efx_mcdi_filter_table *table = efx->filter_state;
 651         int remove_failed = 0;
 652         int remove_noent = 0;
 653         int rc;
 654         int i;
 655
 656         down_write(&table->lock);
 657         for (i = 0; i < EFX_MCDI_FILTER_TBL_ROWS; i++) {
 658                 if (READ_ONCE(table->entry[i].spec) &
 659                     EFX_EF10_FILTER_FLAG_AUTO_OLD) {
 660                         rc = efx_mcdi_filter_remove_internal(efx,
 661                                         1U << EFX_FILTER_PRI_AUTO, i, true);
 662                         if (rc == -ENOENT)
 663                                 remove_noent++;
 664                         else if (rc)
 665                                 remove_failed++;
 666                 }
 667         }
 668         up_write(&table->lock);
 669
 670         if (remove_failed)
 671                 netif_info(efx, drv, efx->net_dev,
 672                            "%s: failed to remove %d filters\n",
 673                            __func__, remove_failed);
 674         if (remove_noent)
 675                 netif_info(efx, drv, efx->net_dev,
 676                            "%s: failed to remove %d non-existent filters\n",
 677                            __func__, remove_noent);
 678 }
 679
 680 int efx_mcdi_filter_remove_safe(struct efx_nic *efx,
 681                                 enum efx_filter_priority priority,
 682                                 u32 filter_id)
 683 {
 684         struct efx_mcdi_filter_table *table;
 685         int rc;
 686
 687         down_read(&efx->filter_sem);
 688         table = efx->filter_state;
 689         down_write(&table->lock);
 690         rc = efx_mcdi_filter_remove_internal(efx, 1U << priority, filter_id,
 691                                              false);
 692         up_write(&table->lock);
 693         up_read(&efx->filter_sem);
 694         return rc;
 695 }
 696
 697 /* Caller must hold efx->filter_sem for read */
 698 static void efx_mcdi_filter_remove_unsafe(struct efx_nic *efx,
 699                                           enum efx_filter_priority priority,
 700                                           u32 filter_id)
 701 {
 702         struct efx_mcdi_filter_table *table = efx->filter_state;
 703
 704         if (filter_id == EFX_EF10_FILTER_ID_INVALID)
 705                 return;
 706
 707         down_write(&table->lock);
 708         efx_mcdi_filter_remove_internal(efx, 1U << priority, filter_id,
 709                                         true);
 710         up_write(&table->lock);
 711 }
 712
 713 int efx_mcdi_filter_get_safe(struct efx_nic *efx,
 714                              enum efx_filter_priority priority,
 715                              u32 filter_id, struct efx_filter_spec *spec)
 716 {
 717         unsigned int filter_idx = efx_mcdi_filter_get_unsafe_id(filter_id);
 718         const struct efx_filter_spec *saved_spec;
 719         struct efx_mcdi_filter_table *table;
 720         int rc;
 721
 722         down_read(&efx->filter_sem);
 723         table = efx->filter_state;
 724         down_read(&table->lock);
 725         saved_spec = efx_mcdi_filter_entry_spec(table, filter_idx);
 726         if (saved_spec && saved_spec->priority == priority &&
 727             efx_mcdi_filter_pri(table, saved_spec) ==
 728             efx_mcdi_filter_get_unsafe_pri(filter_id)) {
 729                 *spec = *saved_spec;
 730                 rc = 0;
 731         } else {
 732                 rc = -ENOENT;
 733         }
 734         up_read(&table->lock);
 735         up_read(&efx->filter_sem);
 736         return rc;
 737 }
 738
 739 static int efx_mcdi_filter_insert_addr_list(struct efx_nic *efx,
 740                                             struct efx_mcdi_filter_vlan *vlan,
 741                                             bool multicast, bool rollback)
 742 {
 743         struct efx_mcdi_filter_table *table = efx->filter_state;
 744         struct efx_mcdi_dev_addr *addr_list;
 745         enum efx_filter_flags filter_flags;
 746         struct efx_filter_spec spec;
 747         u8 baddr[ETH_ALEN];
 748         unsigned int i, j;
 749         int addr_count;
 750         u16 *ids;
 751         int rc;
 752
 753         if (multicast) {
 754                 addr_list = table->dev_mc_list;
 755                 addr_count = table->dev_mc_count;
 756                 ids = vlan->mc;
 757         } else {
 758                 addr_list = table->dev_uc_list;
 759                 addr_count = table->dev_uc_count;
 760                 ids = vlan->uc;
 761         }
 762
 763         filter_flags = efx_rss_active(&efx->rss_context) ? EFX_FILTER_FLAG_RX_RSS : 0;
 764
 765         /* Insert/renew filters */
 766         for (i = 0; i < addr_count; i++) {
 767                 EFX_WARN_ON_PARANOID(ids[i] != EFX_EF10_FILTER_ID_INVALID);
 768                 efx_filter_init_rx(&spec, EFX_FILTER_PRI_AUTO, filter_flags, 0);
 769                 efx_filter_set_eth_local(&spec, vlan->vid, addr_list[i].addr);
 770                 rc = efx_mcdi_filter_insert_locked(efx, &spec, true);
 771                 if (rc < 0) {
 772                         if (rollback) {
 773                                 netif_info(efx, drv, efx->net_dev,
 774                                            "efx_mcdi_filter_insert failed rc=%d\n",
 775                                            rc);
 776                                 /* Fall back to promiscuous */
 777                                 for (j = 0; j < i; j++) {
 778                                         efx_mcdi_filter_remove_unsafe(
 779                                                 efx, EFX_FILTER_PRI_AUTO,
 780                                                 ids[j]);
 781                                         ids[j] = EFX_EF10_FILTER_ID_INVALID;
 782                                 }
 783                                 return rc;
 784                         } else {
 785                                 /* keep invalid ID, and carry on */
 786                         }
 787                 } else {
 788                         ids[i] = efx_mcdi_filter_get_unsafe_id(rc);
 789                 }
 790         }
 791
 792         if (multicast && rollback) {
 793                 /* Also need an Ethernet broadcast filter */
 794                 EFX_WARN_ON_PARANOID(vlan->default_filters[EFX_EF10_BCAST] !=
 795                                      EFX_EF10_FILTER_ID_INVALID);
 796                 efx_filter_init_rx(&spec, EFX_FILTER_PRI_AUTO, filter_flags, 0);
 797                 eth_broadcast_addr(baddr);
 798                 efx_filter_set_eth_local(&spec, vlan->vid, baddr);
 799                 rc = efx_mcdi_filter_insert_locked(efx, &spec, true);
 800                 if (rc < 0) {
 801                         netif_warn(efx, drv, efx->net_dev,
 802                                    "Broadcast filter insert failed rc=%d\n", rc);
 803                         /* Fall back to promiscuous */
 804                         for (j = 0; j < i; j++) {
 805                                 efx_mcdi_filter_remove_unsafe(
 806                                         efx, EFX_FILTER_PRI_AUTO,
 807                                         ids[j]);
 808                                 ids[j] = EFX_EF10_FILTER_ID_INVALID;
 809                         }
 810                         return rc;
 811                 } else {
 812                         vlan->default_filters[EFX_EF10_BCAST] =
 813                                 efx_mcdi_filter_get_unsafe_id(rc);
 814                 }
 815         }
 816
 817         return 0;
 818 }
 819
 820 static int efx_mcdi_filter_insert_def(struct efx_nic *efx,
 821                                       struct efx_mcdi_filter_vlan *vlan,
 822                                       enum efx_encap_type encap_type,
 823                                       bool multicast, bool rollback)
 824 {
 825         struct efx_mcdi_filter_table *table = efx->filter_state;
 826         enum efx_filter_flags filter_flags;
 827         struct efx_filter_spec spec;
 828         u8 baddr[ETH_ALEN];
 829         int rc;
 830         u16 *id;
 831
 832         filter_flags = efx_rss_active(&efx->rss_context) ? EFX_FILTER_FLAG_RX_RSS : 0;
 833
 834         efx_filter_init_rx(&spec, EFX_FILTER_PRI_AUTO, filter_flags, 0);
 835
 836         if (multicast)
 837                 efx_filter_set_mc_def(&spec);
 838         else
 839                 efx_filter_set_uc_def(&spec);
 840
 841         if (encap_type) {
 842                 if (efx_has_cap(efx, VXLAN_NVGRE))
 843                         efx_filter_set_encap_type(&spec, encap_type);
 844                 else
 845                         /*
 846                          * don't insert encap filters on non-supporting
 847                          * platforms. ID will be left as INVALID.
 848                          */
 849                         return 0;
 850         }
 851
 852         if (vlan->vid != EFX_FILTER_VID_UNSPEC)
 853                 efx_filter_set_eth_local(&spec, vlan->vid, NULL);
 854
 855         rc = efx_mcdi_filter_insert_locked(efx, &spec, true);
 856         if (rc < 0) {
 857                 const char *um = multicast ? "Multicast" : "Unicast";
 858                 const char *encap_name = "";
 859                 const char *encap_ipv = "";
 860
 861                 if ((encap_type & EFX_ENCAP_TYPES_MASK) ==
 862                     EFX_ENCAP_TYPE_VXLAN)
 863                         encap_name = "VXLAN ";
 864                 else if ((encap_type & EFX_ENCAP_TYPES_MASK) ==
 865                          EFX_ENCAP_TYPE_NVGRE)
 866                         encap_name = "NVGRE ";
 867                 else if ((encap_type & EFX_ENCAP_TYPES_MASK) ==
 868                          EFX_ENCAP_TYPE_GENEVE)
 869                         encap_name = "GENEVE ";
 870                 if (encap_type & EFX_ENCAP_FLAG_IPV6)
 871                         encap_ipv = "IPv6 ";
 872                 else if (encap_type)
 873                         encap_ipv = "IPv4 ";
 874
 875                 /*
 876                  * unprivileged functions can't insert mismatch filters
 877                  * for encapsulated or unicast traffic, so downgrade
 878                  * those warnings to debug.
 879                  */
 880                 netif_cond_dbg(efx, drv, efx->net_dev,
 881                                rc == -EPERM && (encap_type || !multicast), warn,
 882                                "%s%s%s mismatch filter insert failed rc=%d\n",
 883                                encap_name, encap_ipv, um, rc);
 884         } else if (multicast) {
 885                 /* mapping from encap types to default filter IDs (multicast) */
 886                 static enum efx_mcdi_filter_default_filters map[] = {
 887                         [EFX_ENCAP_TYPE_NONE] = EFX_EF10_MCDEF,
 888                         [EFX_ENCAP_TYPE_VXLAN] = EFX_EF10_VXLAN4_MCDEF,
 889                         [EFX_ENCAP_TYPE_NVGRE] = EFX_EF10_NVGRE4_MCDEF,
 890                         [EFX_ENCAP_TYPE_GENEVE] = EFX_EF10_GENEVE4_MCDEF,
 891                         [EFX_ENCAP_TYPE_VXLAN | EFX_ENCAP_FLAG_IPV6] =
 892                                 EFX_EF10_VXLAN6_MCDEF,
 893                         [EFX_ENCAP_TYPE_NVGRE | EFX_ENCAP_FLAG_IPV6] =
 894                                 EFX_EF10_NVGRE6_MCDEF,
 895                         [EFX_ENCAP_TYPE_GENEVE | EFX_ENCAP_FLAG_IPV6] =
 896                                 EFX_EF10_GENEVE6_MCDEF,
 897                 };
 898
 899                 /* quick bounds check (BCAST result impossible) */
 900                 BUILD_BUG_ON(EFX_EF10_BCAST != 0);
 901                 if (encap_type >= ARRAY_SIZE(map) || map[encap_type] == 0) {
 902                         WARN_ON(1);
 903                         return -EINVAL;
 904                 }
 905                 /* then follow map */
 906                 id = &vlan->default_filters[map[encap_type]];
 907
 908                 EFX_WARN_ON_PARANOID(*id != EFX_EF10_FILTER_ID_INVALID);
 909                 *id = efx_mcdi_filter_get_unsafe_id(rc);
 910                 if (!table->mc_chaining && !encap_type) {
 911                         /* Also need an Ethernet broadcast filter */
 912                         efx_filter_init_rx(&spec, EFX_FILTER_PRI_AUTO,
 913                                            filter_flags, 0);
 914                         eth_broadcast_addr(baddr);
 915                         efx_filter_set_eth_local(&spec, vlan->vid, baddr);
 916                         rc = efx_mcdi_filter_insert_locked(efx, &spec, true);
 917                         if (rc < 0) {
 918                                 netif_warn(efx, drv, efx->net_dev,
 919                                            "Broadcast filter insert failed rc=%d\n",
 920                                            rc);
 921                                 if (rollback) {
 922                                         /* Roll back the mc_def filter */
 923                                         efx_mcdi_filter_remove_unsafe(
 924                                                         efx, EFX_FILTER_PRI_AUTO,
 925                                                         *id);
 926                                         *id = EFX_EF10_FILTER_ID_INVALID;
 927                                         return rc;
 928                                 }
 929                         } else {
 930                                 EFX_WARN_ON_PARANOID(
 931                                         vlan->default_filters[EFX_EF10_BCAST] !=
 932                                         EFX_EF10_FILTER_ID_INVALID);
 933                                 vlan->default_filters[EFX_EF10_BCAST] =
 934                                         efx_mcdi_filter_get_unsafe_id(rc);
 935                         }
 936                 }
 937                 rc = 0;
 938         } else {
 939                 /* mapping from encap types to default filter IDs (unicast) */
 940                 static enum efx_mcdi_filter_default_filters map[] = {
 941                         [EFX_ENCAP_TYPE_NONE] = EFX_EF10_UCDEF,
 942                         [EFX_ENCAP_TYPE_VXLAN] = EFX_EF10_VXLAN4_UCDEF,
 943                         [EFX_ENCAP_TYPE_NVGRE] = EFX_EF10_NVGRE4_UCDEF,
 944                         [EFX_ENCAP_TYPE_GENEVE] = EFX_EF10_GENEVE4_UCDEF,
 945                         [EFX_ENCAP_TYPE_VXLAN | EFX_ENCAP_FLAG_IPV6] =
 946                                 EFX_EF10_VXLAN6_UCDEF,
 947                         [EFX_ENCAP_TYPE_NVGRE | EFX_ENCAP_FLAG_IPV6] =
 948                                 EFX_EF10_NVGRE6_UCDEF,
 949                         [EFX_ENCAP_TYPE_GENEVE | EFX_ENCAP_FLAG_IPV6] =
 950                                 EFX_EF10_GENEVE6_UCDEF,
 951                 };
 952
 953                 /* quick bounds check (BCAST result impossible) */
 954                 BUILD_BUG_ON(EFX_EF10_BCAST != 0);
 955                 if (encap_type >= ARRAY_SIZE(map) || map[encap_type] == 0) {
 956                         WARN_ON(1);
 957                         return -EINVAL;
 958                 }
 959                 /* then follow map */
 960                 id = &vlan->default_filters[map[encap_type]];
 961                 EFX_WARN_ON_PARANOID(*id != EFX_EF10_FILTER_ID_INVALID);
 962                 *id = rc;
 963                 rc = 0;
 964         }
 965         return rc;
 966 }
 967
 968 /*
 969  * Caller must hold efx->filter_sem for read if race against
 970  * efx_mcdi_filter_table_remove() is possible
 971  */
 972 static void efx_mcdi_filter_vlan_sync_rx_mode(struct efx_nic *efx,
 973                                               struct efx_mcdi_filter_vlan *vlan)
 974 {
 975         struct efx_mcdi_filter_table *table = efx->filter_state;
 976
 977         /*
 978          * Do not install unspecified VID if VLAN filtering is enabled.
 979          * Do not install all specified VIDs if VLAN filtering is disabled.
 980          */
 981         if ((vlan->vid == EFX_FILTER_VID_UNSPEC) == table->vlan_filter)
 982                 return;
 983
 984         /* Insert/renew unicast filters */
 985         if (table->uc_promisc) {
 986                 efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_NONE,
 987                                            false, false);
 988                 efx_mcdi_filter_insert_addr_list(efx, vlan, false, false);
 989         } else {
 990                 /*
 991                  * If any of the filters failed to insert, fall back to
 992                  * promiscuous mode - add in the uc_def filter.  But keep
 993                  * our individual unicast filters.
 994                  */
 995                 if (efx_mcdi_filter_insert_addr_list(efx, vlan, false, false))
 996                         efx_mcdi_filter_insert_def(efx, vlan,
 997                                                    EFX_ENCAP_TYPE_NONE,
 998                                                    false, false);
 999         }
1000         efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_VXLAN,
1001                                    false, false);
1002         efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_VXLAN |
1003                                               EFX_ENCAP_FLAG_IPV6,
1004                                    false, false);
1005         efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_NVGRE,
1006                                    false, false);
1007         efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_NVGRE |
1008                                               EFX_ENCAP_FLAG_IPV6,
1009                                    false, false);
1010         efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_GENEVE,
1011                                    false, false);
1012         efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_GENEVE |
1013                                               EFX_ENCAP_FLAG_IPV6,
1014                                    false, false);
1015
1016         /*
1017          * Insert/renew multicast filters
1018          *
1019          * If changing promiscuous state with cascaded multicast filters, remove
1020          * old filters first, so that packets are dropped rather than duplicated
1021          */
1022         if (table->mc_chaining && table->mc_promisc_last != table->mc_promisc)
1023                 efx_mcdi_filter_remove_old(efx);
1024         if (table->mc_promisc) {
1025                 if (table->mc_chaining) {
1026                         /*
1027                          * If we failed to insert promiscuous filters, rollback
1028                          * and fall back to individual multicast filters
1029                          */
1030                         if (efx_mcdi_filter_insert_def(efx, vlan,
1031                                                        EFX_ENCAP_TYPE_NONE,
1032                                                        true, true)) {
1033                                 /* Changing promisc state, so remove old filters */
1034                                 efx_mcdi_filter_remove_old(efx);
1035                                 efx_mcdi_filter_insert_addr_list(efx, vlan,
1036                                                                  true, false);
1037                         }
1038                 } else {
1039                         /*
1040                          * If we failed to insert promiscuous filters, don't
1041                          * rollback.  Regardless, also insert the mc_list,
1042                          * unless it's incomplete due to overflow
1043                          */
1044                         efx_mcdi_filter_insert_def(efx, vlan,
1045                                                    EFX_ENCAP_TYPE_NONE,
1046                                                    true, false);
1047                         if (!table->mc_overflow)
1048                                 efx_mcdi_filter_insert_addr_list(efx, vlan,
1049                                                                  true, false);
1050                 }
1051         } else {
1052                 /*
1053                  * If any filters failed to insert, rollback and fall back to
1054                  * promiscuous mode - mc_def filter and maybe broadcast.  If
1055                  * that fails, roll back again and insert as many of our
1056                  * individual multicast filters as we can.
1057                  */
1058                 if (efx_mcdi_filter_insert_addr_list(efx, vlan, true, true)) {
1059                         /* Changing promisc state, so remove old filters */
1060                         if (table->mc_chaining)
1061                                 efx_mcdi_filter_remove_old(efx);
1062                         if (efx_mcdi_filter_insert_def(efx, vlan,
1063                                                        EFX_ENCAP_TYPE_NONE,
1064                                                        true, true))
1065                                 efx_mcdi_filter_insert_addr_list(efx, vlan,
1066                                                                  true, false);
1067                 }
1068         }
1069         efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_VXLAN,
1070                                    true, false);
1071         efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_VXLAN |
1072                                               EFX_ENCAP_FLAG_IPV6,
1073                                    true, false);
1074         efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_NVGRE,
1075                                    true, false);
1076         efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_NVGRE |
1077                                               EFX_ENCAP_FLAG_IPV6,
1078                                    true, false);
1079         efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_GENEVE,
1080                                    true, false);
1081         efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_GENEVE |
1082                                               EFX_ENCAP_FLAG_IPV6,
1083                                    true, false);
1084 }
1085
1086 int efx_mcdi_filter_clear_rx(struct efx_nic *efx,
1087                              enum efx_filter_priority priority)
1088 {
1089         struct efx_mcdi_filter_table *table;
1090         unsigned int priority_mask;
1091         unsigned int i;
1092         int rc;
1093
1094         priority_mask = (((1U << (priority + 1)) - 1) &
1095                          ~(1U << EFX_FILTER_PRI_AUTO));
1096
1097         down_read(&efx->filter_sem);
1098         table = efx->filter_state;
1099         down_write(&table->lock);
1100         for (i = 0; i < EFX_MCDI_FILTER_TBL_ROWS; i++) {
1101                 rc = efx_mcdi_filter_remove_internal(efx, priority_mask,
1102                                                      i, true);
1103                 if (rc && rc != -ENOENT)
1104                         break;
1105                 rc = 0;
1106         }
1107
1108         up_write(&table->lock);
1109         up_read(&efx->filter_sem);
1110         return rc;
1111 }
1112
1113 u32 efx_mcdi_filter_count_rx_used(struct efx_nic *efx,
1114                                  enum efx_filter_priority priority)
1115 {
1116         struct efx_mcdi_filter_table *table;
1117         unsigned int filter_idx;
1118         s32 count = 0;
1119
1120         down_read(&efx->filter_sem);
1121         table = efx->filter_state;
1122         down_read(&table->lock);
1123         for (filter_idx = 0; filter_idx < EFX_MCDI_FILTER_TBL_ROWS; filter_idx++) {
1124                 if (table->entry[filter_idx].spec &&
1125                     efx_mcdi_filter_entry_spec(table, filter_idx)->priority ==
1126                     priority)
1127                         ++count;
1128         }
1129         up_read(&table->lock);
1130         up_read(&efx->filter_sem);
1131         return count;
1132 }
1133
1134 u32 efx_mcdi_filter_get_rx_id_limit(struct efx_nic *efx)
1135 {
1136         struct efx_mcdi_filter_table *table = efx->filter_state;
1137
1138         return table->rx_match_count * EFX_MCDI_FILTER_TBL_ROWS * 2;
1139 }
1140
1141 s32 efx_mcdi_filter_get_rx_ids(struct efx_nic *efx,
1142                                enum efx_filter_priority priority,
1143                                u32 *buf, u32 size)
1144 {
1145         struct efx_mcdi_filter_table *table;
1146         struct efx_filter_spec *spec;
1147         unsigned int filter_idx;
1148         s32 count = 0;
1149
1150         down_read(&efx->filter_sem);
1151         table = efx->filter_state;
1152         down_read(&table->lock);
1153
1154         for (filter_idx = 0; filter_idx < EFX_MCDI_FILTER_TBL_ROWS; filter_idx++) {
1155                 spec = efx_mcdi_filter_entry_spec(table, filter_idx);
1156                 if (spec && spec->priority == priority) {
1157                         if (count == size) {
1158                                 count = -EMSGSIZE;
1159                                 break;
1160                         }
1161                         buf[count++] =
1162                                 efx_mcdi_filter_make_filter_id(
1163                                         efx_mcdi_filter_pri(table, spec),
1164                                         filter_idx);
1165                 }
1166         }
1167         up_read(&table->lock);
1168         up_read(&efx->filter_sem);
1169         return count;
1170 }
1171
1172 static int efx_mcdi_filter_match_flags_from_mcdi(bool encap, u32 mcdi_flags)
1173 {
1174         int match_flags = 0;
1175
1176 #define MAP_FLAG(gen_flag, mcdi_field) do {                             \
1177                 u32 old_mcdi_flags = mcdi_flags;                        \
1178                 mcdi_flags &= ~(1 << MC_CMD_FILTER_OP_EXT_IN_MATCH_ ##  \
1179                                      mcdi_field ## _LBN);               \
1180                 if (mcdi_flags != old_mcdi_flags)                       \
1181                         match_flags |= EFX_FILTER_MATCH_ ## gen_flag;   \
1182         } while (0)
1183
1184         if (encap) {
1185                 /* encap filters must specify encap type */
1186                 match_flags |= EFX_FILTER_MATCH_ENCAP_TYPE;
1187                 /* and imply ethertype and ip proto */
1188                 mcdi_flags &=
1189                         ~(1 << MC_CMD_FILTER_OP_EXT_IN_MATCH_IP_PROTO_LBN);
1190                 mcdi_flags &=
1191                         ~(1 << MC_CMD_FILTER_OP_EXT_IN_MATCH_ETHER_TYPE_LBN);
1192                 /* VLAN tags refer to the outer packet */
1193                 MAP_FLAG(INNER_VID, INNER_VLAN);
1194                 MAP_FLAG(OUTER_VID, OUTER_VLAN);
1195                 /* everything else refers to the inner packet */
1196                 MAP_FLAG(LOC_MAC_IG, IFRM_UNKNOWN_UCAST_DST);
1197                 MAP_FLAG(LOC_MAC_IG, IFRM_UNKNOWN_MCAST_DST);
1198                 MAP_FLAG(REM_HOST, IFRM_SRC_IP);
1199                 MAP_FLAG(LOC_HOST, IFRM_DST_IP);
1200                 MAP_FLAG(REM_MAC, IFRM_SRC_MAC);
1201                 MAP_FLAG(REM_PORT, IFRM_SRC_PORT);
1202                 MAP_FLAG(LOC_MAC, IFRM_DST_MAC);
1203                 MAP_FLAG(LOC_PORT, IFRM_DST_PORT);
1204                 MAP_FLAG(ETHER_TYPE, IFRM_ETHER_TYPE);
1205                 MAP_FLAG(IP_PROTO, IFRM_IP_PROTO);
1206         } else {
1207                 MAP_FLAG(LOC_MAC_IG, UNKNOWN_UCAST_DST);
1208                 MAP_FLAG(LOC_MAC_IG, UNKNOWN_MCAST_DST);
1209                 MAP_FLAG(REM_HOST, SRC_IP);
1210                 MAP_FLAG(LOC_HOST, DST_IP);
1211                 MAP_FLAG(REM_MAC, SRC_MAC);
1212                 MAP_FLAG(REM_PORT, SRC_PORT);
1213                 MAP_FLAG(LOC_MAC, DST_MAC);
1214                 MAP_FLAG(LOC_PORT, DST_PORT);
1215                 MAP_FLAG(ETHER_TYPE, ETHER_TYPE);
1216                 MAP_FLAG(INNER_VID, INNER_VLAN);
1217                 MAP_FLAG(OUTER_VID, OUTER_VLAN);
1218                 MAP_FLAG(IP_PROTO, IP_PROTO);
1219         }
1220 #undef MAP_FLAG
1221
1222         /* Did we map them all? */
1223         if (mcdi_flags)
1224                 return -EINVAL;
1225
1226         return match_flags;
1227 }
1228
1229 bool efx_mcdi_filter_match_supported(struct efx_mcdi_filter_table *table,
1230                                      bool encap,
1231                                      enum efx_filter_match_flags match_flags)
1232 {
1233         unsigned int match_pri;
1234         int mf;
1235
1236         for (match_pri = 0;
1237              match_pri < table->rx_match_count;
1238              match_pri++) {
1239                 mf = efx_mcdi_filter_match_flags_from_mcdi(encap,
1240                                 table->rx_match_mcdi_flags[match_pri]);
1241                 if (mf == match_flags)
1242                         return true;
1243         }
1244
1245         return false;
1246 }
1247
1248 static int
1249 efx_mcdi_filter_table_probe_matches(struct efx_nic *efx,
1250                                     struct efx_mcdi_filter_table *table,
1251                                     bool encap)
1252 {
1253         MCDI_DECLARE_BUF(inbuf, MC_CMD_GET_PARSER_DISP_INFO_IN_LEN);
1254         MCDI_DECLARE_BUF(outbuf, MC_CMD_GET_PARSER_DISP_INFO_OUT_LENMAX);
1255         unsigned int pd_match_pri, pd_match_count;
1256         size_t outlen;
1257         int rc;
1258
1259         /* Find out which RX filter types are supported, and their priorities */
1260         MCDI_SET_DWORD(inbuf, GET_PARSER_DISP_INFO_IN_OP,
1261                        encap ?
1262                        MC_CMD_GET_PARSER_DISP_INFO_IN_OP_GET_SUPPORTED_ENCAP_RX_MATCHES :
1263                        MC_CMD_GET_PARSER_DISP_INFO_IN_OP_GET_SUPPORTED_RX_MATCHES);
1264         rc = efx_mcdi_rpc(efx, MC_CMD_GET_PARSER_DISP_INFO,
1265                           inbuf, sizeof(inbuf), outbuf, sizeof(outbuf),
1266                           &outlen);
1267         if (rc)
1268                 return rc;
1269
1270         pd_match_count = MCDI_VAR_ARRAY_LEN(
1271                 outlen, GET_PARSER_DISP_INFO_OUT_SUPPORTED_MATCHES);
1272
1273         for (pd_match_pri = 0; pd_match_pri < pd_match_count; pd_match_pri++) {
1274                 u32 mcdi_flags =
1275                         MCDI_ARRAY_DWORD(
1276                                 outbuf,
1277                                 GET_PARSER_DISP_INFO_OUT_SUPPORTED_MATCHES,
1278                                 pd_match_pri);
1279                 rc = efx_mcdi_filter_match_flags_from_mcdi(encap, mcdi_flags);
1280                 if (rc < 0) {
1281                         netif_dbg(efx, probe, efx->net_dev,
1282                                   "%s: fw flags %#x pri %u not supported in driver\n",
1283                                   __func__, mcdi_flags, pd_match_pri);
1284                 } else {
1285                         netif_dbg(efx, probe, efx->net_dev,
1286                                   "%s: fw flags %#x pri %u supported as driver flags %#x pri %u\n",
1287                                   __func__, mcdi_flags, pd_match_pri,
1288                                   rc, table->rx_match_count);
1289                         table->rx_match_mcdi_flags[table->rx_match_count] = mcdi_flags;
1290                         table->rx_match_count++;
1291                 }
1292         }
1293
1294         return 0;
1295 }
1296
1297 int efx_mcdi_filter_table_probe(struct efx_nic *efx, bool multicast_chaining)
1298 {
1299         struct net_device *net_dev = efx->net_dev;
1300         struct efx_mcdi_filter_table *table;
1301         int rc;
1302
1303         if (!efx_rwsem_assert_write_locked(&efx->filter_sem))
1304                 return -EINVAL;
1305
1306         if (efx->filter_state) /* already probed */
1307                 return 0;
1308
1309         table = kzalloc(sizeof(*table), GFP_KERNEL);
1310         if (!table)
1311                 return -ENOMEM;
1312
1313         table->mc_chaining = multicast_chaining;
1314         table->rx_match_count = 0;
1315         rc = efx_mcdi_filter_table_probe_matches(efx, table, false);
1316         if (rc)
1317                 goto fail;
1318         if (efx_has_cap(efx, VXLAN_NVGRE))
1319                 rc = efx_mcdi_filter_table_probe_matches(efx, table, true);
1320         if (rc)
1321                 goto fail;
1322         if ((efx_supported_features(efx) & NETIF_F_HW_VLAN_CTAG_FILTER) &&
1323             !(efx_mcdi_filter_match_supported(table, false,
1324                 (EFX_FILTER_MATCH_OUTER_VID | EFX_FILTER_MATCH_LOC_MAC)) &&
1325               efx_mcdi_filter_match_supported(table, false,
1326                 (EFX_FILTER_MATCH_OUTER_VID | EFX_FILTER_MATCH_LOC_MAC_IG)))) {
1327                 netif_info(efx, probe, net_dev,
1328                            "VLAN filters are not supported in this firmware variant\n");
1329                 net_dev->features &= ~NETIF_F_HW_VLAN_CTAG_FILTER;
1330                 efx->fixed_features &= ~NETIF_F_HW_VLAN_CTAG_FILTER;
1331                 net_dev->hw_features &= ~NETIF_F_HW_VLAN_CTAG_FILTER;
1332         }
1333
1334         table->entry = vzalloc(array_size(EFX_MCDI_FILTER_TBL_ROWS,
1335                                           sizeof(*table->entry)));
1336         if (!table->entry) {
1337                 rc = -ENOMEM;
1338                 goto fail;
1339         }
1340
1341         table->mc_promisc_last = false;
1342         table->vlan_filter =
1343                 !!(efx->net_dev->features & NETIF_F_HW_VLAN_CTAG_FILTER);
1344         INIT_LIST_HEAD(&table->vlan_list);
1345         init_rwsem(&table->lock);
1346
1347         efx->filter_state = table;
1348
1349         return 0;
1350 fail:
1351         kfree(table);
1352         return rc;
1353 }
1354
1355 void efx_mcdi_filter_table_reset_mc_allocations(struct efx_nic *efx)
1356 {
1357         struct efx_mcdi_filter_table *table = efx->filter_state;
1358
1359         if (table) {
1360                 table->must_restore_filters = true;
1361                 table->must_restore_rss_contexts = true;
1362         }
1363 }
1364
1365 /*
1366  * Caller must hold efx->filter_sem for read if race against
1367  * efx_mcdi_filter_table_remove() is possible
1368  */
1369 void efx_mcdi_filter_table_restore(struct efx_nic *efx)
1370 {
1371         struct efx_mcdi_filter_table *table = efx->filter_state;
1372         unsigned int invalid_filters = 0, failed = 0;
1373         struct efx_mcdi_filter_vlan *vlan;
1374         struct efx_filter_spec *spec;
1375         struct efx_rss_context *ctx;
1376         unsigned int filter_idx;
1377         u32 mcdi_flags;
1378         int match_pri;
1379         int rc, i;
1380
1381         WARN_ON(!rwsem_is_locked(&efx->filter_sem));
1382
1383         if (!table || !table->must_restore_filters)
1384                 return;
1385
1386         down_write(&table->lock);
1387         mutex_lock(&efx->rss_lock);
1388
1389         for (filter_idx = 0; filter_idx < EFX_MCDI_FILTER_TBL_ROWS; filter_idx++) {
1390                 spec = efx_mcdi_filter_entry_spec(table, filter_idx);
1391                 if (!spec)
1392                         continue;
1393
1394                 mcdi_flags = efx_mcdi_filter_mcdi_flags_from_spec(spec);
1395                 match_pri = 0;
1396                 while (match_pri < table->rx_match_count &&
1397                        table->rx_match_mcdi_flags[match_pri] != mcdi_flags)
1398                         ++match_pri;
1399                 if (match_pri >= table->rx_match_count) {
1400                         invalid_filters++;
1401                         goto not_restored;
1402                 }
1403                 if (spec->rss_context)
1404                         ctx = efx_find_rss_context_entry(efx, spec->rss_context);
1405                 else
1406                         ctx = &efx->rss_context;
1407                 if (spec->flags & EFX_FILTER_FLAG_RX_RSS) {
1408                         if (!ctx) {
1409                                 netif_warn(efx, drv, efx->net_dev,
1410                                            "Warning: unable to restore a filter with nonexistent RSS context %u.\n",
1411                                            spec->rss_context);
1412                                 invalid_filters++;
1413                                 goto not_restored;
1414                         }
1415                         if (ctx->context_id == EFX_MCDI_RSS_CONTEXT_INVALID) {
1416                                 netif_warn(efx, drv, efx->net_dev,
1417                                            "Warning: unable to restore a filter with RSS context %u as it was not created.\n",
1418                                            spec->rss_context);
1419                                 invalid_filters++;
1420                                 goto not_restored;
1421                         }
1422                 }
1423
1424                 rc = efx_mcdi_filter_push(efx, spec,
1425                                           &table->entry[filter_idx].handle,
1426                                           ctx, false);
1427                 if (rc)
1428                         failed++;
1429
1430                 if (rc) {
1431 not_restored:
1432                         list_for_each_entry(vlan, &table->vlan_list, list)
1433                                 for (i = 0; i < EFX_EF10_NUM_DEFAULT_FILTERS; ++i)
1434                                         if (vlan->default_filters[i] == filter_idx)
1435                                                 vlan->default_filters[i] =
1436                                                         EFX_EF10_FILTER_ID_INVALID;
1437
1438                         kfree(spec);
1439                         efx_mcdi_filter_set_entry(table, filter_idx, NULL, 0);
1440                 }
1441         }
1442
1443         mutex_unlock(&efx->rss_lock);
1444         up_write(&table->lock);
1445
1446         /*
1447          * This can happen validly if the MC's capabilities have changed, so
1448          * is not an error.
1449          */
1450         if (invalid_filters)
1451                 netif_dbg(efx, drv, efx->net_dev,
1452                           "Did not restore %u filters that are now unsupported.\n",
1453                           invalid_filters);
1454
1455         if (failed)
1456                 netif_err(efx, hw, efx->net_dev,
1457                           "unable to restore %u filters\n", failed);
1458         else
1459                 table->must_restore_filters = false;
1460 }
1461
1462 void efx_mcdi_filter_table_down(struct efx_nic *efx)
1463 {
1464         struct efx_mcdi_filter_table *table = efx->filter_state;
1465         MCDI_DECLARE_BUF(inbuf, MC_CMD_FILTER_OP_EXT_IN_LEN);
1466         struct efx_filter_spec *spec;
1467         unsigned int filter_idx;
1468         int rc;
1469
1470         if (!table)
1471                 return;
1472
1473         efx_mcdi_filter_cleanup_vlans(efx);
1474
1475         for (filter_idx = 0; filter_idx < EFX_MCDI_FILTER_TBL_ROWS; filter_idx++) {
1476                 spec = efx_mcdi_filter_entry_spec(table, filter_idx);
1477                 if (!spec)
1478                         continue;
1479
1480                 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_OP,
1481                                efx_mcdi_filter_is_exclusive(spec) ?
1482                                MC_CMD_FILTER_OP_IN_OP_REMOVE :
1483                                MC_CMD_FILTER_OP_IN_OP_UNSUBSCRIBE);
1484                 MCDI_SET_QWORD(inbuf, FILTER_OP_IN_HANDLE,
1485                                table->entry[filter_idx].handle);
1486                 rc = efx_mcdi_rpc_quiet(efx, MC_CMD_FILTER_OP, inbuf,
1487                                         sizeof(inbuf), NULL, 0, NULL);
1488                 if (rc)
1489                         netif_info(efx, drv, efx->net_dev,
1490                                    "%s: filter %04x remove failed\n",
1491                                    __func__, filter_idx);
1492                 kfree(spec);
1493         }
1494 }
1495
1496 void efx_mcdi_filter_table_remove(struct efx_nic *efx)
1497 {
1498         struct efx_mcdi_filter_table *table = efx->filter_state;
1499
1500         efx_mcdi_filter_table_down(efx);
1501
1502         efx->filter_state = NULL;
1503         /*
1504          * If we were called without locking, then it's not safe to free
1505          * the table as others might be using it.  So we just WARN, leak
1506          * the memory, and potentially get an inconsistent filter table
1507          * state.
1508          * This should never actually happen.
1509          */
1510         if (!efx_rwsem_assert_write_locked(&efx->filter_sem))
1511                 return;
1512
1513         if (!table)
1514                 return;
1515
1516         vfree(table->entry);
1517         kfree(table);
1518 }
1519
1520 static void efx_mcdi_filter_mark_one_old(struct efx_nic *efx, uint16_t *id)
1521 {
1522         struct efx_mcdi_filter_table *table = efx->filter_state;
1523         unsigned int filter_idx;
1524
1525         efx_rwsem_assert_write_locked(&table->lock);
1526
1527         if (*id != EFX_EF10_FILTER_ID_INVALID) {
1528                 filter_idx = efx_mcdi_filter_get_unsafe_id(*id);
1529                 if (!table->entry[filter_idx].spec)
1530                         netif_dbg(efx, drv, efx->net_dev,
1531                                   "marked null spec old %04x:%04x\n", *id,
1532                                   filter_idx);
1533                 table->entry[filter_idx].spec |= EFX_EF10_FILTER_FLAG_AUTO_OLD;
1534                 *id = EFX_EF10_FILTER_ID_INVALID;
1535         }
1536 }
1537
1538 /* Mark old per-VLAN filters that may need to be removed */
1539 static void _efx_mcdi_filter_vlan_mark_old(struct efx_nic *efx,
1540                                            struct efx_mcdi_filter_vlan *vlan)
1541 {
1542         struct efx_mcdi_filter_table *table = efx->filter_state;
1543         unsigned int i;
1544
1545         for (i = 0; i < table->dev_uc_count; i++)
1546                 efx_mcdi_filter_mark_one_old(efx, &vlan->uc[i]);
1547         for (i = 0; i < table->dev_mc_count; i++)
1548                 efx_mcdi_filter_mark_one_old(efx, &vlan->mc[i]);
1549         for (i = 0; i < EFX_EF10_NUM_DEFAULT_FILTERS; i++)
1550                 efx_mcdi_filter_mark_one_old(efx, &vlan->default_filters[i]);
1551 }
1552
1553 /*
1554  * Mark old filters that may need to be removed.
1555  * Caller must hold efx->filter_sem for read if race against
1556  * efx_mcdi_filter_table_remove() is possible
1557  */
1558 static void efx_mcdi_filter_mark_old(struct efx_nic *efx)
1559 {
1560         struct efx_mcdi_filter_table *table = efx->filter_state;
1561         struct efx_mcdi_filter_vlan *vlan;
1562
1563         down_write(&table->lock);
1564         list_for_each_entry(vlan, &table->vlan_list, list)
1565                 _efx_mcdi_filter_vlan_mark_old(efx, vlan);
1566         up_write(&table->lock);
1567 }
1568
1569 int efx_mcdi_filter_add_vlan(struct efx_nic *efx, u16 vid)
1570 {
1571         struct efx_mcdi_filter_table *table = efx->filter_state;
1572         struct efx_mcdi_filter_vlan *vlan;
1573         unsigned int i;
1574
1575         if (!efx_rwsem_assert_write_locked(&efx->filter_sem))
1576                 return -EINVAL;
1577
1578         vlan = efx_mcdi_filter_find_vlan(efx, vid);
1579         if (WARN_ON(vlan)) {
1580                 netif_err(efx, drv, efx->net_dev,
1581                           "VLAN %u already added\n", vid);
1582                 return -EALREADY;
1583         }
1584
1585         vlan = kzalloc(sizeof(*vlan), GFP_KERNEL);
1586         if (!vlan)
1587                 return -ENOMEM;
1588
1589         vlan->vid = vid;
1590
1591         for (i = 0; i < ARRAY_SIZE(vlan->uc); i++)
1592                 vlan->uc[i] = EFX_EF10_FILTER_ID_INVALID;
1593         for (i = 0; i < ARRAY_SIZE(vlan->mc); i++)
1594                 vlan->mc[i] = EFX_EF10_FILTER_ID_INVALID;
1595         for (i = 0; i < EFX_EF10_NUM_DEFAULT_FILTERS; i++)
1596                 vlan->default_filters[i] = EFX_EF10_FILTER_ID_INVALID;
1597
1598         list_add_tail(&vlan->list, &table->vlan_list);
1599
1600         if (efx_dev_registered(efx))
1601                 efx_mcdi_filter_vlan_sync_rx_mode(efx, vlan);
1602
1603         return 0;
1604 }
1605
1606 static void efx_mcdi_filter_del_vlan_internal(struct efx_nic *efx,
1607                                               struct efx_mcdi_filter_vlan *vlan)
1608 {
1609         unsigned int i;
1610
1611         /* See comment in efx_mcdi_filter_table_remove() */
1612         if (!efx_rwsem_assert_write_locked(&efx->filter_sem))
1613                 return;
1614
1615         list_del(&vlan->list);
1616
1617         for (i = 0; i < ARRAY_SIZE(vlan->uc); i++)
1618                 efx_mcdi_filter_remove_unsafe(efx, EFX_FILTER_PRI_AUTO,
1619                                               vlan->uc[i]);
1620         for (i = 0; i < ARRAY_SIZE(vlan->mc); i++)
1621                 efx_mcdi_filter_remove_unsafe(efx, EFX_FILTER_PRI_AUTO,
1622                                               vlan->mc[i]);
1623         for (i = 0; i < EFX_EF10_NUM_DEFAULT_FILTERS; i++)
1624                 if (vlan->default_filters[i] != EFX_EF10_FILTER_ID_INVALID)
1625                         efx_mcdi_filter_remove_unsafe(efx, EFX_FILTER_PRI_AUTO,
1626                                                       vlan->default_filters[i]);
1627
1628         kfree(vlan);
1629 }
1630
1631 void efx_mcdi_filter_del_vlan(struct efx_nic *efx, u16 vid)
1632 {
1633         struct efx_mcdi_filter_vlan *vlan;
1634
1635         /* See comment in efx_mcdi_filter_table_remove() */
1636         if (!efx_rwsem_assert_write_locked(&efx->filter_sem))
1637                 return;
1638
1639         vlan = efx_mcdi_filter_find_vlan(efx, vid);
1640         if (!vlan) {
1641                 netif_err(efx, drv, efx->net_dev,
1642                           "VLAN %u not found in filter state\n", vid);
1643                 return;
1644         }
1645
1646         efx_mcdi_filter_del_vlan_internal(efx, vlan);
1647 }
1648
1649 struct efx_mcdi_filter_vlan *efx_mcdi_filter_find_vlan(struct efx_nic *efx,
1650                                                        u16 vid)
1651 {
1652         struct efx_mcdi_filter_table *table = efx->filter_state;
1653         struct efx_mcdi_filter_vlan *vlan;
1654
1655         WARN_ON(!rwsem_is_locked(&efx->filter_sem));
1656
1657         list_for_each_entry(vlan, &table->vlan_list, list) {
1658                 if (vlan->vid == vid)
1659                         return vlan;
1660         }
1661
1662         return NULL;
1663 }
1664
1665 void efx_mcdi_filter_cleanup_vlans(struct efx_nic *efx)
1666 {
1667         struct efx_mcdi_filter_table *table = efx->filter_state;
1668         struct efx_mcdi_filter_vlan *vlan, *next_vlan;
1669
1670         /* See comment in efx_mcdi_filter_table_remove() */
1671         if (!efx_rwsem_assert_write_locked(&efx->filter_sem))
1672                 return;
1673
1674         if (!table)
1675                 return;
1676
1677         list_for_each_entry_safe(vlan, next_vlan, &table->vlan_list, list)
1678                 efx_mcdi_filter_del_vlan_internal(efx, vlan);
1679 }
1680
1681 static void efx_mcdi_filter_uc_addr_list(struct efx_nic *efx)
1682 {
1683         struct efx_mcdi_filter_table *table = efx->filter_state;
1684         struct net_device *net_dev = efx->net_dev;
1685         struct netdev_hw_addr *uc;
1686         unsigned int i;
1687
1688         table->uc_promisc = !!(net_dev->flags & IFF_PROMISC);
1689         ether_addr_copy(table->dev_uc_list[0].addr, net_dev->dev_addr);
1690         i = 1;
1691         netdev_for_each_uc_addr(uc, net_dev) {
1692                 if (i >= EFX_EF10_FILTER_DEV_UC_MAX) {
1693                         table->uc_promisc = true;
1694                         break;
1695                 }
1696                 ether_addr_copy(table->dev_uc_list[i].addr, uc->addr);
1697                 i++;
1698         }
1699
1700         table->dev_uc_count = i;
1701 }
1702
1703 static void efx_mcdi_filter_mc_addr_list(struct efx_nic *efx)
1704 {
1705         struct efx_mcdi_filter_table *table = efx->filter_state;
1706         struct net_device *net_dev = efx->net_dev;
1707         struct netdev_hw_addr *mc;
1708         unsigned int i;
1709
1710         table->mc_overflow = false;
1711         table->mc_promisc = !!(net_dev->flags & (IFF_PROMISC | IFF_ALLMULTI));
1712
1713         i = 0;
1714         netdev_for_each_mc_addr(mc, net_dev) {
1715                 if (i >= EFX_EF10_FILTER_DEV_MC_MAX) {
1716                         table->mc_promisc = true;
1717                         table->mc_overflow = true;
1718                         break;
1719                 }
1720                 ether_addr_copy(table->dev_mc_list[i].addr, mc->addr);
1721                 i++;
1722         }
1723
1724         table->dev_mc_count = i;
1725 }
1726
1727 /*
1728  * Caller must hold efx->filter_sem for read if race against
1729  * efx_mcdi_filter_table_remove() is possible
1730  */
1731 void efx_mcdi_filter_sync_rx_mode(struct efx_nic *efx)
1732 {
1733         struct efx_mcdi_filter_table *table = efx->filter_state;
1734         struct net_device *net_dev = efx->net_dev;
1735         struct efx_mcdi_filter_vlan *vlan;
1736         bool vlan_filter;
1737
1738         if (!efx_dev_registered(efx))
1739                 return;
1740
1741         if (!table)
1742                 return;
1743
1744         efx_mcdi_filter_mark_old(efx);
1745
1746         /*
1747          * Copy/convert the address lists; add the primary station
1748          * address and broadcast address
1749          */
1750         netif_addr_lock_bh(net_dev);
1751         efx_mcdi_filter_uc_addr_list(efx);
1752         efx_mcdi_filter_mc_addr_list(efx);
1753         netif_addr_unlock_bh(net_dev);
1754
1755         /*
1756          * If VLAN filtering changes, all old filters are finally removed.
1757          * Do it in advance to avoid conflicts for unicast untagged and
1758          * VLAN 0 tagged filters.
1759          */
1760         vlan_filter = !!(net_dev->features & NETIF_F_HW_VLAN_CTAG_FILTER);
1761         if (table->vlan_filter != vlan_filter) {
1762                 table->vlan_filter = vlan_filter;
1763                 efx_mcdi_filter_remove_old(efx);
1764         }
1765
1766         list_for_each_entry(vlan, &table->vlan_list, list)
1767                 efx_mcdi_filter_vlan_sync_rx_mode(efx, vlan);
1768
1769         efx_mcdi_filter_remove_old(efx);
1770         table->mc_promisc_last = table->mc_promisc;
1771 }
1772
1773 #ifdef CONFIG_RFS_ACCEL
1774
1775 bool efx_mcdi_filter_rfs_expire_one(struct efx_nic *efx, u32 flow_id,
1776                                     unsigned int filter_idx)
1777 {
1778         struct efx_filter_spec *spec, saved_spec;
1779         struct efx_mcdi_filter_table *table;
1780         struct efx_arfs_rule *rule = NULL;
1781         bool ret = true, force = false;
1782         u16 arfs_id;
1783
1784         down_read(&efx->filter_sem);
1785         table = efx->filter_state;
1786         down_write(&table->lock);
1787         spec = efx_mcdi_filter_entry_spec(table, filter_idx);
1788
1789         if (!spec || spec->priority != EFX_FILTER_PRI_HINT)
1790                 goto out_unlock;
1791
1792         spin_lock_bh(&efx->rps_hash_lock);
1793         if (!efx->rps_hash_table) {
1794                 /* In the absence of the table, we always return 0 to ARFS. */
1795                 arfs_id = 0;
1796         } else {
1797                 rule = efx_rps_hash_find(efx, spec);
1798                 if (!rule)
1799                         /* ARFS table doesn't know of this filter, so remove it */
1800                         goto expire;
1801                 arfs_id = rule->arfs_id;
1802                 ret = efx_rps_check_rule(rule, filter_idx, &force);
1803                 if (force)
1804                         goto expire;
1805                 if (!ret) {
1806                         spin_unlock_bh(&efx->rps_hash_lock);
1807                         goto out_unlock;
1808                 }
1809         }
1810         if (!rps_may_expire_flow(efx->net_dev, spec->dmaq_id, flow_id, arfs_id))
1811                 ret = false;
1812         else if (rule)
1813                 rule->filter_id = EFX_ARFS_FILTER_ID_REMOVING;
1814 expire:
1815         saved_spec = *spec; /* remove operation will kfree spec */
1816         spin_unlock_bh(&efx->rps_hash_lock);
1817         /*
1818          * At this point (since we dropped the lock), another thread might queue
1819          * up a fresh insertion request (but the actual insertion will be held
1820          * up by our possession of the filter table lock).  In that case, it
1821          * will set rule->filter_id to EFX_ARFS_FILTER_ID_PENDING, meaning that
1822          * the rule is not removed by efx_rps_hash_del() below.
1823          */
1824         if (ret)
1825                 ret = efx_mcdi_filter_remove_internal(efx, 1U << spec->priority,
1826                                                       filter_idx, true) == 0;
1827         /*
1828          * While we can't safely dereference rule (we dropped the lock), we can
1829          * still test it for NULL.
1830          */
1831         if (ret && rule) {
1832                 /* Expiring, so remove entry from ARFS table */
1833                 spin_lock_bh(&efx->rps_hash_lock);
1834                 efx_rps_hash_del(efx, &saved_spec);
1835                 spin_unlock_bh(&efx->rps_hash_lock);
1836         }
1837 out_unlock:
1838         up_write(&table->lock);
1839         up_read(&efx->filter_sem);
1840         return ret;
1841 }
1842
1843 #endif /* CONFIG_RFS_ACCEL */
1844
1845 #define RSS_MODE_HASH_ADDRS     (1 << RSS_MODE_HASH_SRC_ADDR_LBN |\
1846                                  1 << RSS_MODE_HASH_DST_ADDR_LBN)
1847 #define RSS_MODE_HASH_PORTS     (1 << RSS_MODE_HASH_SRC_PORT_LBN |\
1848                                  1 << RSS_MODE_HASH_DST_PORT_LBN)
1849 #define RSS_CONTEXT_FLAGS_DEFAULT       (1 << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_TOEPLITZ_IPV4_EN_LBN |\
1850                                          1 << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_TOEPLITZ_TCPV4_EN_LBN |\
1851                                          1 << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_TOEPLITZ_IPV6_EN_LBN |\
1852                                          1 << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_TOEPLITZ_TCPV6_EN_LBN |\
1853                                          (RSS_MODE_HASH_ADDRS | RSS_MODE_HASH_PORTS) << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_TCP_IPV4_RSS_MODE_LBN |\
1854                                          RSS_MODE_HASH_ADDRS << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_UDP_IPV4_RSS_MODE_LBN |\
1855                                          RSS_MODE_HASH_ADDRS << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_OTHER_IPV4_RSS_MODE_LBN |\
1856                                          (RSS_MODE_HASH_ADDRS | RSS_MODE_HASH_PORTS) << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_TCP_IPV6_RSS_MODE_LBN |\
1857                                          RSS_MODE_HASH_ADDRS << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_UDP_IPV6_RSS_MODE_LBN |\
1858                                          RSS_MODE_HASH_ADDRS << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_OTHER_IPV6_RSS_MODE_LBN)
1859
1860 int efx_mcdi_get_rss_context_flags(struct efx_nic *efx, u32 context, u32 *flags)
1861 {
1862         /*
1863          * Firmware had a bug (sfc bug 61952) where it would not actually
1864          * fill in the flags field in the response to MC_CMD_RSS_CONTEXT_GET_FLAGS.
1865          * This meant that it would always contain whatever was previously
1866          * in the MCDI buffer.  Fortunately, all firmware versions with
1867          * this bug have the same default flags value for a newly-allocated
1868          * RSS context, and the only time we want to get the flags is just
1869          * after allocating.  Moreover, the response has a 32-bit hole
1870          * where the context ID would be in the request, so we can use an
1871          * overlength buffer in the request and pre-fill the flags field
1872          * with what we believe the default to be.  Thus if the firmware
1873          * has the bug, it will leave our pre-filled value in the flags
1874          * field of the response, and we will get the right answer.
1875          *
1876          * However, this does mean that this function should NOT be used if
1877          * the RSS context flags might not be their defaults - it is ONLY
1878          * reliably correct for a newly-allocated RSS context.
1879          */
1880         MCDI_DECLARE_BUF(inbuf, MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_LEN);
1881         MCDI_DECLARE_BUF(outbuf, MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_LEN);
1882         size_t outlen;
1883         int rc;
1884
1885         /* Check we have a hole for the context ID */
1886         BUILD_BUG_ON(MC_CMD_RSS_CONTEXT_GET_FLAGS_IN_LEN != MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_FLAGS_OFST);
1887         MCDI_SET_DWORD(inbuf, RSS_CONTEXT_GET_FLAGS_IN_RSS_CONTEXT_ID, context);
1888         MCDI_SET_DWORD(inbuf, RSS_CONTEXT_GET_FLAGS_OUT_FLAGS,
1889                        RSS_CONTEXT_FLAGS_DEFAULT);
1890         rc = efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_GET_FLAGS, inbuf,
1891                           sizeof(inbuf), outbuf, sizeof(outbuf), &outlen);
1892         if (rc == 0) {
1893                 if (outlen < MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_LEN)
1894                         rc = -EIO;
1895                 else
1896                         *flags = MCDI_DWORD(outbuf, RSS_CONTEXT_GET_FLAGS_OUT_FLAGS);
1897         }
1898         return rc;
1899 }
1900
1901 /*
1902  * Attempt to enable 4-tuple UDP hashing on the specified RSS context.
1903  * If we fail, we just leave the RSS context at its default hash settings,
1904  * which is safe but may slightly reduce performance.
1905  * Defaults are 4-tuple for TCP and 2-tuple for UDP and other-IP, so we
1906  * just need to set the UDP ports flags (for both IP versions).
1907  */
1908 void efx_mcdi_set_rss_context_flags(struct efx_nic *efx,
1909                                     struct efx_rss_context *ctx)
1910 {
1911         MCDI_DECLARE_BUF(inbuf, MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN);
1912         u32 flags;
1913
1914         BUILD_BUG_ON(MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN != 0);
1915
1916         if (efx_mcdi_get_rss_context_flags(efx, ctx->context_id, &flags) != 0)
1917                 return;
1918         MCDI_SET_DWORD(inbuf, RSS_CONTEXT_SET_FLAGS_IN_RSS_CONTEXT_ID,
1919                        ctx->context_id);
1920         flags |= RSS_MODE_HASH_PORTS << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_UDP_IPV4_RSS_MODE_LBN;
1921         flags |= RSS_MODE_HASH_PORTS << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_UDP_IPV6_RSS_MODE_LBN;
1922         MCDI_SET_DWORD(inbuf, RSS_CONTEXT_SET_FLAGS_IN_FLAGS, flags);
1923         if (!efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_SET_FLAGS, inbuf, sizeof(inbuf),
1924                           NULL, 0, NULL))
1925                 /* Succeeded, so UDP 4-tuple is now enabled */
1926                 ctx->rx_hash_udp_4tuple = true;
1927 }
1928
1929 static int efx_mcdi_filter_alloc_rss_context(struct efx_nic *efx, bool exclusive,
1930                                              struct efx_rss_context *ctx,
1931                                              unsigned *context_size)
1932 {
1933         MCDI_DECLARE_BUF(inbuf, MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN);
1934         MCDI_DECLARE_BUF(outbuf, MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN);
1935         size_t outlen;
1936         int rc;
1937         u32 alloc_type = exclusive ?
1938                                 MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_EXCLUSIVE :
1939                                 MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_SHARED;
1940         unsigned rss_spread = exclusive ?
1941                                 efx->rss_spread :
1942                                 min(rounddown_pow_of_two(efx->rss_spread),
1943                                     EFX_EF10_MAX_SHARED_RSS_CONTEXT_SIZE);
1944
1945         if (!exclusive && rss_spread == 1) {
1946                 ctx->context_id = EFX_MCDI_RSS_CONTEXT_INVALID;
1947                 if (context_size)
1948                         *context_size = 1;
1949                 return 0;
1950         }
1951
1952         if (efx_has_cap(efx, RX_RSS_LIMITED))
1953                 return -EOPNOTSUPP;
1954
1955         MCDI_SET_DWORD(inbuf, RSS_CONTEXT_ALLOC_IN_UPSTREAM_PORT_ID,
1956                        efx->vport_id);
1957         MCDI_SET_DWORD(inbuf, RSS_CONTEXT_ALLOC_IN_TYPE, alloc_type);
1958         MCDI_SET_DWORD(inbuf, RSS_CONTEXT_ALLOC_IN_NUM_QUEUES, rss_spread);
1959
1960         rc = efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_ALLOC, inbuf, sizeof(inbuf),
1961                 outbuf, sizeof(outbuf), &outlen);
1962         if (rc != 0)
1963                 return rc;
1964
1965         if (outlen < MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN)
1966                 return -EIO;
1967
1968         ctx->context_id = MCDI_DWORD(outbuf, RSS_CONTEXT_ALLOC_OUT_RSS_CONTEXT_ID);
1969
1970         if (context_size)
1971                 *context_size = rss_spread;
1972
1973         if (efx_has_cap(efx, ADDITIONAL_RSS_MODES))
1974                 efx_mcdi_set_rss_context_flags(efx, ctx);
1975
1976         return 0;
1977 }
1978
1979 static int efx_mcdi_filter_free_rss_context(struct efx_nic *efx, u32 context)
1980 {
1981         MCDI_DECLARE_BUF(inbuf, MC_CMD_RSS_CONTEXT_FREE_IN_LEN);
1982
1983         MCDI_SET_DWORD(inbuf, RSS_CONTEXT_FREE_IN_RSS_CONTEXT_ID,
1984                        context);
1985         return efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_FREE, inbuf, sizeof(inbuf),
1986                             NULL, 0, NULL);
1987 }
1988
1989 static int efx_mcdi_filter_populate_rss_table(struct efx_nic *efx, u32 context,
1990                                        const u32 *rx_indir_table, const u8 *key)
1991 {
1992         MCDI_DECLARE_BUF(tablebuf, MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN);
1993         MCDI_DECLARE_BUF(keybuf, MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN);
1994         int i, rc;
1995
1996         MCDI_SET_DWORD(tablebuf, RSS_CONTEXT_SET_TABLE_IN_RSS_CONTEXT_ID,
1997                        context);
1998         BUILD_BUG_ON(ARRAY_SIZE(efx->rss_context.rx_indir_table) !=
1999                      MC_CMD_RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE_LEN);
2000
2001         /* This iterates over the length of efx->rss_context.rx_indir_table, but
2002          * copies bytes from rx_indir_table.  That's because the latter is a
2003          * pointer rather than an array, but should have the same length.
2004          * The efx->rss_context.rx_hash_key loop below is similar.
2005          */
2006         for (i = 0; i < ARRAY_SIZE(efx->rss_context.rx_indir_table); ++i)
2007                 MCDI_PTR(tablebuf,
2008                          RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE)[i] =
2009                                 (u8) rx_indir_table[i];
2010
2011         rc = efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_SET_TABLE, tablebuf,
2012                           sizeof(tablebuf), NULL, 0, NULL);
2013         if (rc != 0)
2014                 return rc;
2015
2016         MCDI_SET_DWORD(keybuf, RSS_CONTEXT_SET_KEY_IN_RSS_CONTEXT_ID,
2017                        context);
2018         BUILD_BUG_ON(ARRAY_SIZE(efx->rss_context.rx_hash_key) !=
2019                      MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN);
2020         for (i = 0; i < ARRAY_SIZE(efx->rss_context.rx_hash_key); ++i)
2021                 MCDI_PTR(keybuf, RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY)[i] = key[i];
2022
2023         return efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_SET_KEY, keybuf,
2024                             sizeof(keybuf), NULL, 0, NULL);
2025 }
2026
2027 void efx_mcdi_rx_free_indir_table(struct efx_nic *efx)
2028 {
2029         int rc;
2030
2031         if (efx->rss_context.context_id != EFX_MCDI_RSS_CONTEXT_INVALID) {
2032                 rc = efx_mcdi_filter_free_rss_context(efx, efx->rss_context.context_id);
2033                 WARN_ON(rc != 0);
2034         }
2035         efx->rss_context.context_id = EFX_MCDI_RSS_CONTEXT_INVALID;
2036 }
2037
2038 static int efx_mcdi_filter_rx_push_shared_rss_config(struct efx_nic *efx,
2039                                               unsigned *context_size)
2040 {
2041         struct efx_mcdi_filter_table *table = efx->filter_state;
2042         int rc = efx_mcdi_filter_alloc_rss_context(efx, false, &efx->rss_context,
2043                                             context_size);
2044
2045         if (rc != 0)
2046                 return rc;
2047
2048         table->rx_rss_context_exclusive = false;
2049         efx_set_default_rx_indir_table(efx, &efx->rss_context);
2050         return 0;
2051 }
2052
2053 static int efx_mcdi_filter_rx_push_exclusive_rss_config(struct efx_nic *efx,
2054                                                  const u32 *rx_indir_table,
2055                                                  const u8 *key)
2056 {
2057         struct efx_mcdi_filter_table *table = efx->filter_state;
2058         u32 old_rx_rss_context = efx->rss_context.context_id;
2059         int rc;
2060
2061         if (efx->rss_context.context_id == EFX_MCDI_RSS_CONTEXT_INVALID ||
2062             !table->rx_rss_context_exclusive) {
2063                 rc = efx_mcdi_filter_alloc_rss_context(efx, true, &efx->rss_context,
2064                                                 NULL);
2065                 if (rc == -EOPNOTSUPP)
2066                         return rc;
2067                 else if (rc != 0)
2068                         goto fail1;
2069         }
2070
2071         rc = efx_mcdi_filter_populate_rss_table(efx, efx->rss_context.context_id,
2072                                          rx_indir_table, key);
2073         if (rc != 0)
2074                 goto fail2;
2075
2076         if (efx->rss_context.context_id != old_rx_rss_context &&
2077             old_rx_rss_context != EFX_MCDI_RSS_CONTEXT_INVALID)
2078                 WARN_ON(efx_mcdi_filter_free_rss_context(efx, old_rx_rss_context) != 0);
2079         table->rx_rss_context_exclusive = true;
2080         if (rx_indir_table != efx->rss_context.rx_indir_table)
2081                 memcpy(efx->rss_context.rx_indir_table, rx_indir_table,
2082                        sizeof(efx->rss_context.rx_indir_table));
2083         if (key != efx->rss_context.rx_hash_key)
2084                 memcpy(efx->rss_context.rx_hash_key, key,
2085                        efx->type->rx_hash_key_size);
2086
2087         return 0;
2088
2089 fail2:
2090         if (old_rx_rss_context != efx->rss_context.context_id) {
2091                 WARN_ON(efx_mcdi_filter_free_rss_context(efx, efx->rss_context.context_id) != 0);
2092                 efx->rss_context.context_id = old_rx_rss_context;
2093         }
2094 fail1:
2095         netif_err(efx, hw, efx->net_dev, "%s: failed rc=%d\n", __func__, rc);
2096         return rc;
2097 }
2098
2099 int efx_mcdi_rx_push_rss_context_config(struct efx_nic *efx,
2100                                         struct efx_rss_context *ctx,
2101                                         const u32 *rx_indir_table,
2102                                         const u8 *key)
2103 {
2104         int rc;
2105
2106         WARN_ON(!mutex_is_locked(&efx->rss_lock));
2107
2108         if (ctx->context_id == EFX_MCDI_RSS_CONTEXT_INVALID) {
2109                 rc = efx_mcdi_filter_alloc_rss_context(efx, true, ctx, NULL);
2110                 if (rc)
2111                         return rc;
2112         }
2113
2114         if (!rx_indir_table) /* Delete this context */
2115                 return efx_mcdi_filter_free_rss_context(efx, ctx->context_id);
2116
2117         rc = efx_mcdi_filter_populate_rss_table(efx, ctx->context_id,
2118                                          rx_indir_table, key);
2119         if (rc)
2120                 return rc;
2121
2122         memcpy(ctx->rx_indir_table, rx_indir_table,
2123                sizeof(efx->rss_context.rx_indir_table));
2124         memcpy(ctx->rx_hash_key, key, efx->type->rx_hash_key_size);
2125
2126         return 0;
2127 }
2128
2129 int efx_mcdi_rx_pull_rss_context_config(struct efx_nic *efx,
2130                                         struct efx_rss_context *ctx)
2131 {
2132         MCDI_DECLARE_BUF(inbuf, MC_CMD_RSS_CONTEXT_GET_TABLE_IN_LEN);
2133         MCDI_DECLARE_BUF(tablebuf, MC_CMD_RSS_CONTEXT_GET_TABLE_OUT_LEN);
2134         MCDI_DECLARE_BUF(keybuf, MC_CMD_RSS_CONTEXT_GET_KEY_OUT_LEN);
2135         size_t outlen;
2136         int rc, i;
2137
2138         WARN_ON(!mutex_is_locked(&efx->rss_lock));
2139
2140         BUILD_BUG_ON(MC_CMD_RSS_CONTEXT_GET_TABLE_IN_LEN !=
2141                      MC_CMD_RSS_CONTEXT_GET_KEY_IN_LEN);
2142
2143         if (ctx->context_id == EFX_MCDI_RSS_CONTEXT_INVALID)
2144                 return -ENOENT;
2145
2146         MCDI_SET_DWORD(inbuf, RSS_CONTEXT_GET_TABLE_IN_RSS_CONTEXT_ID,
2147                        ctx->context_id);
2148         BUILD_BUG_ON(ARRAY_SIZE(ctx->rx_indir_table) !=
2149                      MC_CMD_RSS_CONTEXT_GET_TABLE_OUT_INDIRECTION_TABLE_LEN);
2150         rc = efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_GET_TABLE, inbuf, sizeof(inbuf),
2151                           tablebuf, sizeof(tablebuf), &outlen);
2152         if (rc != 0)
2153                 return rc;
2154
2155         if (WARN_ON(outlen != MC_CMD_RSS_CONTEXT_GET_TABLE_OUT_LEN))
2156                 return -EIO;
2157
2158         for (i = 0; i < ARRAY_SIZE(ctx->rx_indir_table); i++)
2159                 ctx->rx_indir_table[i] = MCDI_PTR(tablebuf,
2160                                 RSS_CONTEXT_GET_TABLE_OUT_INDIRECTION_TABLE)[i];
2161
2162         MCDI_SET_DWORD(inbuf, RSS_CONTEXT_GET_KEY_IN_RSS_CONTEXT_ID,
2163                        ctx->context_id);
2164         BUILD_BUG_ON(ARRAY_SIZE(ctx->rx_hash_key) !=
2165                      MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN);
2166         rc = efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_GET_KEY, inbuf, sizeof(inbuf),
2167                           keybuf, sizeof(keybuf), &outlen);
2168         if (rc != 0)
2169                 return rc;
2170
2171         if (WARN_ON(outlen != MC_CMD_RSS_CONTEXT_GET_KEY_OUT_LEN))
2172                 return -EIO;
2173
2174         for (i = 0; i < ARRAY_SIZE(ctx->rx_hash_key); ++i)
2175                 ctx->rx_hash_key[i] = MCDI_PTR(
2176                                 keybuf, RSS_CONTEXT_GET_KEY_OUT_TOEPLITZ_KEY)[i];
2177
2178         return 0;
2179 }
2180
2181 int efx_mcdi_rx_pull_rss_config(struct efx_nic *efx)
2182 {
2183         int rc;
2184
2185         mutex_lock(&efx->rss_lock);
2186         rc = efx_mcdi_rx_pull_rss_context_config(efx, &efx->rss_context);
2187         mutex_unlock(&efx->rss_lock);
2188         return rc;
2189 }
2190
2191 void efx_mcdi_rx_restore_rss_contexts(struct efx_nic *efx)
2192 {
2193         struct efx_mcdi_filter_table *table = efx->filter_state;
2194         struct efx_rss_context *ctx;
2195         int rc;
2196
2197         WARN_ON(!mutex_is_locked(&efx->rss_lock));
2198
2199         if (!table->must_restore_rss_contexts)
2200                 return;
2201
2202         list_for_each_entry(ctx, &efx->rss_context.list, list) {
2203                 /* previous NIC RSS context is gone */
2204                 ctx->context_id = EFX_MCDI_RSS_CONTEXT_INVALID;
2205                 /* so try to allocate a new one */
2206                 rc = efx_mcdi_rx_push_rss_context_config(efx, ctx,
2207                                                          ctx->rx_indir_table,
2208                                                          ctx->rx_hash_key);
2209                 if (rc)
2210                         netif_warn(efx, probe, efx->net_dev,
2211                                    "failed to restore RSS context %u, rc=%d"
2212                                    "; RSS filters may fail to be applied\n",
2213                                    ctx->user_id, rc);
2214         }
2215         table->must_restore_rss_contexts = false;
2216 }
2217
2218 int efx_mcdi_pf_rx_push_rss_config(struct efx_nic *efx, bool user,
2219                                    const u32 *rx_indir_table,
2220                                    const u8 *key)
2221 {
2222         int rc;
2223
2224         if (efx->rss_spread == 1)
2225                 return 0;
2226
2227         if (!key)
2228                 key = efx->rss_context.rx_hash_key;
2229
2230         rc = efx_mcdi_filter_rx_push_exclusive_rss_config(efx, rx_indir_table, key);
2231
2232         if (rc == -ENOBUFS && !user) {
2233                 unsigned context_size;
2234                 bool mismatch = false;
2235                 size_t i;
2236
2237                 for (i = 0;
2238                      i < ARRAY_SIZE(efx->rss_context.rx_indir_table) && !mismatch;
2239                      i++)
2240                         mismatch = rx_indir_table[i] !=
2241                                 ethtool_rxfh_indir_default(i, efx->rss_spread);
2242
2243                 rc = efx_mcdi_filter_rx_push_shared_rss_config(efx, &context_size);
2244                 if (rc == 0) {
2245                         if (context_size != efx->rss_spread)
2246                                 netif_warn(efx, probe, efx->net_dev,
2247                                            "Could not allocate an exclusive RSS"
2248                                            " context; allocated a shared one of"
2249                                            " different size."
2250                                            " Wanted %u, got %u.\n",
2251                                            efx->rss_spread, context_size);
2252                         else if (mismatch)
2253                                 netif_warn(efx, probe, efx->net_dev,
2254                                            "Could not allocate an exclusive RSS"
2255                                            " context; allocated a shared one but"
2256                                            " could not apply custom"
2257                                            " indirection.\n");
2258                         else
2259                                 netif_info(efx, probe, efx->net_dev,
2260                                            "Could not allocate an exclusive RSS"
2261                                            " context; allocated a shared one.\n");
2262                 }
2263         }
2264         return rc;
2265 }
2266
2267 int efx_mcdi_vf_rx_push_rss_config(struct efx_nic *efx, bool user,
2268                                    const u32 *rx_indir_table
2269                                    __attribute__ ((unused)),
2270                                    const u8 *key
2271                                    __attribute__ ((unused)))
2272 {
2273         if (user)
2274                 return -EOPNOTSUPP;
2275         if (efx->rss_context.context_id != EFX_MCDI_RSS_CONTEXT_INVALID)
2276                 return 0;
2277         return efx_mcdi_filter_rx_push_shared_rss_config(efx, NULL);
2278 }
2279
2280 int efx_mcdi_push_default_indir_table(struct efx_nic *efx,
2281                                       unsigned int rss_spread)
2282 {
2283         int rc = 0;
2284
2285         if (efx->rss_spread == rss_spread)
2286                 return 0;
2287
2288         efx->rss_spread = rss_spread;
2289         if (!efx->filter_state)
2290                 return 0;
2291
2292         efx_mcdi_rx_free_indir_table(efx);
2293         if (rss_spread > 1) {
2294                 efx_set_default_rx_indir_table(efx, &efx->rss_context);
2295                 rc = efx->type->rx_push_rss_config(efx, false,
2296                                    efx->rss_context.rx_indir_table, NULL);
2297         }
2298         return rc;
2299 }