drivers/net/ethernet/mellanox/mlxsw/spectrum_nve.c

   1 // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0
   2 /* Copyright (c) 2018 Mellanox Technologies. All rights reserved */
   3
   4 #include <linux/err.h>
   5 #include <linux/gfp.h>
   6 #include <linux/kernel.h>
   7 #include <linux/list.h>
   8 #include <linux/netlink.h>
   9 #include <linux/rtnetlink.h>
  10 #include <linux/slab.h>
  11 #include <net/inet_ecn.h>
  12 #include <net/ipv6.h>
  13
  14 #include "reg.h"
  15 #include "spectrum.h"
  16 #include "spectrum_nve.h"
  17
  18 const struct mlxsw_sp_nve_ops *mlxsw_sp1_nve_ops_arr[] = {
  19         [MLXSW_SP_NVE_TYPE_VXLAN]       = &mlxsw_sp1_nve_vxlan_ops,
  20 };
  21
  22 const struct mlxsw_sp_nve_ops *mlxsw_sp2_nve_ops_arr[] = {
  23         [MLXSW_SP_NVE_TYPE_VXLAN]       = &mlxsw_sp2_nve_vxlan_ops,
  24 };
  25
  26 struct mlxsw_sp_nve_mc_entry;
  27 struct mlxsw_sp_nve_mc_record;
  28 struct mlxsw_sp_nve_mc_list;
  29
  30 struct mlxsw_sp_nve_mc_record_ops {
  31         enum mlxsw_reg_tnumt_record_type type;
  32         int (*entry_add)(struct mlxsw_sp_nve_mc_record *mc_record,
  33                          struct mlxsw_sp_nve_mc_entry *mc_entry,
  34                          const union mlxsw_sp_l3addr *addr);
  35         void (*entry_del)(const struct mlxsw_sp_nve_mc_record *mc_record,
  36                           const struct mlxsw_sp_nve_mc_entry *mc_entry);
  37         void (*entry_set)(const struct mlxsw_sp_nve_mc_record *mc_record,
  38                           const struct mlxsw_sp_nve_mc_entry *mc_entry,
  39                           char *tnumt_pl, unsigned int entry_index);
  40         bool (*entry_compare)(const struct mlxsw_sp_nve_mc_record *mc_record,
  41                               const struct mlxsw_sp_nve_mc_entry *mc_entry,
  42                               const union mlxsw_sp_l3addr *addr);
  43 };
  44
  45 struct mlxsw_sp_nve_mc_list_key {
  46         u16 fid_index;
  47 };
  48
  49 struct mlxsw_sp_nve_mc_ipv6_entry {
  50         struct in6_addr addr6;
  51         u32 addr6_kvdl_index;
  52 };
  53
  54 struct mlxsw_sp_nve_mc_entry {
  55         union {
  56                 __be32 addr4;
  57                 struct mlxsw_sp_nve_mc_ipv6_entry ipv6_entry;
  58         };
  59         u8 valid:1;
  60 };
  61
  62 struct mlxsw_sp_nve_mc_record {
  63         struct list_head list;
  64         enum mlxsw_sp_l3proto proto;
  65         unsigned int num_entries;
  66         struct mlxsw_sp *mlxsw_sp;
  67         struct mlxsw_sp_nve_mc_list *mc_list;
  68         const struct mlxsw_sp_nve_mc_record_ops *ops;
  69         u32 kvdl_index;
  70         struct mlxsw_sp_nve_mc_entry entries[0];
  71 };
  72
  73 struct mlxsw_sp_nve_mc_list {
  74         struct list_head records_list;
  75         struct rhash_head ht_node;
  76         struct mlxsw_sp_nve_mc_list_key key;
  77 };
  78
  79 static const struct rhashtable_params mlxsw_sp_nve_mc_list_ht_params = {
  80         .key_len = sizeof(struct mlxsw_sp_nve_mc_list_key),
  81         .key_offset = offsetof(struct mlxsw_sp_nve_mc_list, key),
  82         .head_offset = offsetof(struct mlxsw_sp_nve_mc_list, ht_node),
  83 };
  84
  85 static int
  86 mlxsw_sp_nve_mc_record_ipv4_entry_add(struct mlxsw_sp_nve_mc_record *mc_record,
  87                                       struct mlxsw_sp_nve_mc_entry *mc_entry,
  88                                       const union mlxsw_sp_l3addr *addr)
  89 {
  90         mc_entry->addr4 = addr->addr4;
  91
  92         return 0;
  93 }
  94
  95 static void
  96 mlxsw_sp_nve_mc_record_ipv4_entry_del(const struct mlxsw_sp_nve_mc_record *mc_record,
  97                                       const struct mlxsw_sp_nve_mc_entry *mc_entry)
  98 {
  99 }
 100
 101 static void
 102 mlxsw_sp_nve_mc_record_ipv4_entry_set(const struct mlxsw_sp_nve_mc_record *mc_record,
 103                                       const struct mlxsw_sp_nve_mc_entry *mc_entry,
 104                                       char *tnumt_pl, unsigned int entry_index)
 105 {
 106         u32 udip = be32_to_cpu(mc_entry->addr4);
 107
 108         mlxsw_reg_tnumt_udip_set(tnumt_pl, entry_index, udip);
 109 }
 110
 111 static bool
 112 mlxsw_sp_nve_mc_record_ipv4_entry_compare(const struct mlxsw_sp_nve_mc_record *mc_record,
 113                                           const struct mlxsw_sp_nve_mc_entry *mc_entry,
 114                                           const union mlxsw_sp_l3addr *addr)
 115 {
 116         return mc_entry->addr4 == addr->addr4;
 117 }
 118
 119 static const struct mlxsw_sp_nve_mc_record_ops
 120 mlxsw_sp_nve_mc_record_ipv4_ops = {
 121         .type           = MLXSW_REG_TNUMT_RECORD_TYPE_IPV4,
 122         .entry_add      = &mlxsw_sp_nve_mc_record_ipv4_entry_add,
 123         .entry_del      = &mlxsw_sp_nve_mc_record_ipv4_entry_del,
 124         .entry_set      = &mlxsw_sp_nve_mc_record_ipv4_entry_set,
 125         .entry_compare  = &mlxsw_sp_nve_mc_record_ipv4_entry_compare,
 126 };
 127
 128 static int
 129 mlxsw_sp_nve_mc_record_ipv6_entry_add(struct mlxsw_sp_nve_mc_record *mc_record,
 130                                       struct mlxsw_sp_nve_mc_entry *mc_entry,
 131                                       const union mlxsw_sp_l3addr *addr)
 132 {
 133         WARN_ON(1);
 134
 135         return -EINVAL;
 136 }
 137
 138 static void
 139 mlxsw_sp_nve_mc_record_ipv6_entry_del(const struct mlxsw_sp_nve_mc_record *mc_record,
 140                                       const struct mlxsw_sp_nve_mc_entry *mc_entry)
 141 {
 142 }
 143
 144 static void
 145 mlxsw_sp_nve_mc_record_ipv6_entry_set(const struct mlxsw_sp_nve_mc_record *mc_record,
 146                                       const struct mlxsw_sp_nve_mc_entry *mc_entry,
 147                                       char *tnumt_pl, unsigned int entry_index)
 148 {
 149         u32 udip_ptr = mc_entry->ipv6_entry.addr6_kvdl_index;
 150
 151         mlxsw_reg_tnumt_udip_ptr_set(tnumt_pl, entry_index, udip_ptr);
 152 }
 153
 154 static bool
 155 mlxsw_sp_nve_mc_record_ipv6_entry_compare(const struct mlxsw_sp_nve_mc_record *mc_record,
 156                                           const struct mlxsw_sp_nve_mc_entry *mc_entry,
 157                                           const union mlxsw_sp_l3addr *addr)
 158 {
 159         return ipv6_addr_equal(&mc_entry->ipv6_entry.addr6, &addr->addr6);
 160 }
 161
 162 static const struct mlxsw_sp_nve_mc_record_ops
 163 mlxsw_sp_nve_mc_record_ipv6_ops = {
 164         .type           = MLXSW_REG_TNUMT_RECORD_TYPE_IPV6,
 165         .entry_add      = &mlxsw_sp_nve_mc_record_ipv6_entry_add,
 166         .entry_del      = &mlxsw_sp_nve_mc_record_ipv6_entry_del,
 167         .entry_set      = &mlxsw_sp_nve_mc_record_ipv6_entry_set,
 168         .entry_compare  = &mlxsw_sp_nve_mc_record_ipv6_entry_compare,
 169 };
 170
 171 static const struct mlxsw_sp_nve_mc_record_ops *
 172 mlxsw_sp_nve_mc_record_ops_arr[] = {
 173         [MLXSW_SP_L3_PROTO_IPV4] = &mlxsw_sp_nve_mc_record_ipv4_ops,
 174         [MLXSW_SP_L3_PROTO_IPV6] = &mlxsw_sp_nve_mc_record_ipv6_ops,
 175 };
 176
 177 static struct mlxsw_sp_nve_mc_list *
 178 mlxsw_sp_nve_mc_list_find(struct mlxsw_sp *mlxsw_sp,
 179                           const struct mlxsw_sp_nve_mc_list_key *key)
 180 {
 181         struct mlxsw_sp_nve *nve = mlxsw_sp->nve;
 182
 183         return rhashtable_lookup_fast(&nve->mc_list_ht, key,
 184                                       mlxsw_sp_nve_mc_list_ht_params);
 185 }
 186
 187 static struct mlxsw_sp_nve_mc_list *
 188 mlxsw_sp_nve_mc_list_create(struct mlxsw_sp *mlxsw_sp,
 189                             const struct mlxsw_sp_nve_mc_list_key *key)
 190 {
 191         struct mlxsw_sp_nve *nve = mlxsw_sp->nve;
 192         struct mlxsw_sp_nve_mc_list *mc_list;
 193         int err;
 194
 195         mc_list = kmalloc(sizeof(*mc_list), GFP_KERNEL);
 196         if (!mc_list)
 197                 return ERR_PTR(-ENOMEM);
 198
 199         INIT_LIST_HEAD(&mc_list->records_list);
 200         mc_list->key = *key;
 201
 202         err = rhashtable_insert_fast(&nve->mc_list_ht, &mc_list->ht_node,
 203                                      mlxsw_sp_nve_mc_list_ht_params);
 204         if (err)
 205                 goto err_rhashtable_insert;
 206
 207         return mc_list;
 208
 209 err_rhashtable_insert:
 210         kfree(mc_list);
 211         return ERR_PTR(err);
 212 }
 213
 214 static void mlxsw_sp_nve_mc_list_destroy(struct mlxsw_sp *mlxsw_sp,
 215                                          struct mlxsw_sp_nve_mc_list *mc_list)
 216 {
 217         struct mlxsw_sp_nve *nve = mlxsw_sp->nve;
 218
 219         rhashtable_remove_fast(&nve->mc_list_ht, &mc_list->ht_node,
 220                                mlxsw_sp_nve_mc_list_ht_params);
 221         WARN_ON(!list_empty(&mc_list->records_list));
 222         kfree(mc_list);
 223 }
 224
 225 static struct mlxsw_sp_nve_mc_list *
 226 mlxsw_sp_nve_mc_list_get(struct mlxsw_sp *mlxsw_sp,
 227                          const struct mlxsw_sp_nve_mc_list_key *key)
 228 {
 229         struct mlxsw_sp_nve_mc_list *mc_list;
 230
 231         mc_list = mlxsw_sp_nve_mc_list_find(mlxsw_sp, key);
 232         if (mc_list)
 233                 return mc_list;
 234
 235         return mlxsw_sp_nve_mc_list_create(mlxsw_sp, key);
 236 }
 237
 238 static void
 239 mlxsw_sp_nve_mc_list_put(struct mlxsw_sp *mlxsw_sp,
 240                          struct mlxsw_sp_nve_mc_list *mc_list)
 241 {
 242         if (!list_empty(&mc_list->records_list))
 243                 return;
 244         mlxsw_sp_nve_mc_list_destroy(mlxsw_sp, mc_list);
 245 }
 246
 247 static struct mlxsw_sp_nve_mc_record *
 248 mlxsw_sp_nve_mc_record_create(struct mlxsw_sp *mlxsw_sp,
 249                               struct mlxsw_sp_nve_mc_list *mc_list,
 250                               enum mlxsw_sp_l3proto proto)
 251 {
 252         unsigned int num_max_entries = mlxsw_sp->nve->num_max_mc_entries[proto];
 253         struct mlxsw_sp_nve_mc_record *mc_record;
 254         int err;
 255
 256         mc_record = kzalloc(sizeof(*mc_record) + num_max_entries *
 257                             sizeof(struct mlxsw_sp_nve_mc_entry), GFP_KERNEL);
 258         if (!mc_record)
 259                 return ERR_PTR(-ENOMEM);
 260
 261         err = mlxsw_sp_kvdl_alloc(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_TNUMT, 1,
 262                                   &mc_record->kvdl_index);
 263         if (err)
 264                 goto err_kvdl_alloc;
 265
 266         mc_record->ops = mlxsw_sp_nve_mc_record_ops_arr[proto];
 267         mc_record->mlxsw_sp = mlxsw_sp;
 268         mc_record->mc_list = mc_list;
 269         mc_record->proto = proto;
 270         list_add_tail(&mc_record->list, &mc_list->records_list);
 271
 272         return mc_record;
 273
 274 err_kvdl_alloc:
 275         kfree(mc_record);
 276         return ERR_PTR(err);
 277 }
 278
 279 static void
 280 mlxsw_sp_nve_mc_record_destroy(struct mlxsw_sp_nve_mc_record *mc_record)
 281 {
 282         struct mlxsw_sp *mlxsw_sp = mc_record->mlxsw_sp;
 283
 284         list_del(&mc_record->list);
 285         mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_TNUMT, 1,
 286                            mc_record->kvdl_index);
 287         WARN_ON(mc_record->num_entries);
 288         kfree(mc_record);
 289 }
 290
 291 static struct mlxsw_sp_nve_mc_record *
 292 mlxsw_sp_nve_mc_record_get(struct mlxsw_sp *mlxsw_sp,
 293                            struct mlxsw_sp_nve_mc_list *mc_list,
 294                            enum mlxsw_sp_l3proto proto)
 295 {
 296         struct mlxsw_sp_nve_mc_record *mc_record;
 297
 298         list_for_each_entry_reverse(mc_record, &mc_list->records_list, list) {
 299                 unsigned int num_entries = mc_record->num_entries;
 300                 struct mlxsw_sp_nve *nve = mlxsw_sp->nve;
 301
 302                 if (mc_record->proto == proto &&
 303                     num_entries < nve->num_max_mc_entries[proto])
 304                         return mc_record;
 305         }
 306
 307         return mlxsw_sp_nve_mc_record_create(mlxsw_sp, mc_list, proto);
 308 }
 309
 310 static void
 311 mlxsw_sp_nve_mc_record_put(struct mlxsw_sp_nve_mc_record *mc_record)
 312 {
 313         if (mc_record->num_entries != 0)
 314                 return;
 315
 316         mlxsw_sp_nve_mc_record_destroy(mc_record);
 317 }
 318
 319 static struct mlxsw_sp_nve_mc_entry *
 320 mlxsw_sp_nve_mc_free_entry_find(struct mlxsw_sp_nve_mc_record *mc_record)
 321 {
 322         struct mlxsw_sp_nve *nve = mc_record->mlxsw_sp->nve;
 323         unsigned int num_max_entries;
 324         int i;
 325
 326         num_max_entries = nve->num_max_mc_entries[mc_record->proto];
 327         for (i = 0; i < num_max_entries; i++) {
 328                 if (mc_record->entries[i].valid)
 329                         continue;
 330                 return &mc_record->entries[i];
 331         }
 332
 333         return NULL;
 334 }
 335
 336 static int
 337 mlxsw_sp_nve_mc_record_refresh(struct mlxsw_sp_nve_mc_record *mc_record)
 338 {
 339         enum mlxsw_reg_tnumt_record_type type = mc_record->ops->type;
 340         struct mlxsw_sp_nve_mc_list *mc_list = mc_record->mc_list;
 341         struct mlxsw_sp *mlxsw_sp = mc_record->mlxsw_sp;
 342         char tnumt_pl[MLXSW_REG_TNUMT_LEN];
 343         unsigned int num_max_entries;
 344         unsigned int num_entries = 0;
 345         u32 next_kvdl_index = 0;
 346         bool next_valid = false;
 347         int i;
 348
 349         if (!list_is_last(&mc_record->list, &mc_list->records_list)) {
 350                 struct mlxsw_sp_nve_mc_record *next_record;
 351
 352                 next_record = list_next_entry(mc_record, list);
 353                 next_kvdl_index = next_record->kvdl_index;
 354                 next_valid = true;
 355         }
 356
 357         mlxsw_reg_tnumt_pack(tnumt_pl, type, MLXSW_REG_TNUMT_TUNNEL_PORT_NVE,
 358                              mc_record->kvdl_index, next_valid,
 359                              next_kvdl_index, mc_record->num_entries);
 360
 361         num_max_entries = mlxsw_sp->nve->num_max_mc_entries[mc_record->proto];
 362         for (i = 0; i < num_max_entries; i++) {
 363                 struct mlxsw_sp_nve_mc_entry *mc_entry;
 364
 365                 mc_entry = &mc_record->entries[i];
 366                 if (!mc_entry->valid)
 367                         continue;
 368                 mc_record->ops->entry_set(mc_record, mc_entry, tnumt_pl,
 369                                           num_entries++);
 370         }
 371
 372         WARN_ON(num_entries != mc_record->num_entries);
 373
 374         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(tnumt), tnumt_pl);
 375 }
 376
 377 static bool
 378 mlxsw_sp_nve_mc_record_is_first(struct mlxsw_sp_nve_mc_record *mc_record)
 379 {
 380         struct mlxsw_sp_nve_mc_list *mc_list = mc_record->mc_list;
 381         struct mlxsw_sp_nve_mc_record *first_record;
 382
 383         first_record = list_first_entry(&mc_list->records_list,
 384                                         struct mlxsw_sp_nve_mc_record, list);
 385
 386         return mc_record == first_record;
 387 }
 388
 389 static struct mlxsw_sp_nve_mc_entry *
 390 mlxsw_sp_nve_mc_entry_find(struct mlxsw_sp_nve_mc_record *mc_record,
 391                            union mlxsw_sp_l3addr *addr)
 392 {
 393         struct mlxsw_sp_nve *nve = mc_record->mlxsw_sp->nve;
 394         unsigned int num_max_entries;
 395         int i;
 396
 397         num_max_entries = nve->num_max_mc_entries[mc_record->proto];
 398         for (i = 0; i < num_max_entries; i++) {
 399                 struct mlxsw_sp_nve_mc_entry *mc_entry;
 400
 401                 mc_entry = &mc_record->entries[i];
 402                 if (!mc_entry->valid)
 403                         continue;
 404                 if (mc_record->ops->entry_compare(mc_record, mc_entry, addr))
 405                         return mc_entry;
 406         }
 407
 408         return NULL;
 409 }
 410
 411 static int
 412 mlxsw_sp_nve_mc_record_ip_add(struct mlxsw_sp_nve_mc_record *mc_record,
 413                               union mlxsw_sp_l3addr *addr)
 414 {
 415         struct mlxsw_sp_nve_mc_entry *mc_entry = NULL;
 416         int err;
 417
 418         mc_entry = mlxsw_sp_nve_mc_free_entry_find(mc_record);
 419         if (WARN_ON(!mc_entry))
 420                 return -EINVAL;
 421
 422         err = mc_record->ops->entry_add(mc_record, mc_entry, addr);
 423         if (err)
 424                 return err;
 425         mc_record->num_entries++;
 426         mc_entry->valid = true;
 427
 428         err = mlxsw_sp_nve_mc_record_refresh(mc_record);
 429         if (err)
 430                 goto err_record_refresh;
 431
 432         /* If this is a new record and not the first one, then we need to
 433          * update the next pointer of the previous entry
 434          */
 435         if (mc_record->num_entries != 1 ||
 436             mlxsw_sp_nve_mc_record_is_first(mc_record))
 437                 return 0;
 438
 439         err = mlxsw_sp_nve_mc_record_refresh(list_prev_entry(mc_record, list));
 440         if (err)
 441                 goto err_prev_record_refresh;
 442
 443         return 0;
 444
 445 err_prev_record_refresh:
 446 err_record_refresh:
 447         mc_entry->valid = false;
 448         mc_record->num_entries--;
 449         mc_record->ops->entry_del(mc_record, mc_entry);
 450         return err;
 451 }
 452
 453 static void
 454 mlxsw_sp_nve_mc_record_entry_del(struct mlxsw_sp_nve_mc_record *mc_record,
 455                                  struct mlxsw_sp_nve_mc_entry *mc_entry)
 456 {
 457         struct mlxsw_sp_nve_mc_list *mc_list = mc_record->mc_list;
 458
 459         mc_entry->valid = false;
 460         mc_record->num_entries--;
 461
 462         /* When the record continues to exist we only need to invalidate
 463          * the requested entry
 464          */
 465         if (mc_record->num_entries != 0) {
 466                 mlxsw_sp_nve_mc_record_refresh(mc_record);
 467                 mc_record->ops->entry_del(mc_record, mc_entry);
 468                 return;
 469         }
 470
 471         /* If the record needs to be deleted, but it is not the first,
 472          * then we need to make sure that the previous record no longer
 473          * points to it. Remove deleted record from the list to reflect
 474          * that and then re-add it at the end, so that it could be
 475          * properly removed by the record destruction code
 476          */
 477         if (!mlxsw_sp_nve_mc_record_is_first(mc_record)) {
 478                 struct mlxsw_sp_nve_mc_record *prev_record;
 479
 480                 prev_record = list_prev_entry(mc_record, list);
 481                 list_del(&mc_record->list);
 482                 mlxsw_sp_nve_mc_record_refresh(prev_record);
 483                 list_add_tail(&mc_record->list, &mc_list->records_list);
 484                 mc_record->ops->entry_del(mc_record, mc_entry);
 485                 return;
 486         }
 487
 488         /* If the first record needs to be deleted, but the list is not
 489          * singular, then the second record needs to be written in the
 490          * first record's address, as this address is stored as a property
 491          * of the FID
 492          */
 493         if (mlxsw_sp_nve_mc_record_is_first(mc_record) &&
 494             !list_is_singular(&mc_list->records_list)) {
 495                 struct mlxsw_sp_nve_mc_record *next_record;
 496
 497                 next_record = list_next_entry(mc_record, list);
 498                 swap(mc_record->kvdl_index, next_record->kvdl_index);
 499                 mlxsw_sp_nve_mc_record_refresh(next_record);
 500                 mc_record->ops->entry_del(mc_record, mc_entry);
 501                 return;
 502         }
 503
 504         /* This is the last case where the last remaining record needs to
 505          * be deleted. Simply delete the entry
 506          */
 507         mc_record->ops->entry_del(mc_record, mc_entry);
 508 }
 509
 510 static struct mlxsw_sp_nve_mc_record *
 511 mlxsw_sp_nve_mc_record_find(struct mlxsw_sp_nve_mc_list *mc_list,
 512                             enum mlxsw_sp_l3proto proto,
 513                             union mlxsw_sp_l3addr *addr,
 514                             struct mlxsw_sp_nve_mc_entry **mc_entry)
 515 {
 516         struct mlxsw_sp_nve_mc_record *mc_record;
 517
 518         list_for_each_entry(mc_record, &mc_list->records_list, list) {
 519                 if (mc_record->proto != proto)
 520                         continue;
 521
 522                 *mc_entry = mlxsw_sp_nve_mc_entry_find(mc_record, addr);
 523                 if (*mc_entry)
 524                         return mc_record;
 525         }
 526
 527         return NULL;
 528 }
 529
 530 static int mlxsw_sp_nve_mc_list_ip_add(struct mlxsw_sp *mlxsw_sp,
 531                                        struct mlxsw_sp_nve_mc_list *mc_list,
 532                                        enum mlxsw_sp_l3proto proto,
 533                                        union mlxsw_sp_l3addr *addr)
 534 {
 535         struct mlxsw_sp_nve_mc_record *mc_record;
 536         int err;
 537
 538         mc_record = mlxsw_sp_nve_mc_record_get(mlxsw_sp, mc_list, proto);
 539         if (IS_ERR(mc_record))
 540                 return PTR_ERR(mc_record);
 541
 542         err = mlxsw_sp_nve_mc_record_ip_add(mc_record, addr);
 543         if (err)
 544                 goto err_ip_add;
 545
 546         return 0;
 547
 548 err_ip_add:
 549         mlxsw_sp_nve_mc_record_put(mc_record);
 550         return err;
 551 }
 552
 553 static void mlxsw_sp_nve_mc_list_ip_del(struct mlxsw_sp *mlxsw_sp,
 554                                         struct mlxsw_sp_nve_mc_list *mc_list,
 555                                         enum mlxsw_sp_l3proto proto,
 556                                         union mlxsw_sp_l3addr *addr)
 557 {
 558         struct mlxsw_sp_nve_mc_record *mc_record;
 559         struct mlxsw_sp_nve_mc_entry *mc_entry;
 560
 561         mc_record = mlxsw_sp_nve_mc_record_find(mc_list, proto, addr,
 562                                                 &mc_entry);
 563         if (WARN_ON(!mc_record))
 564                 return;
 565
 566         mlxsw_sp_nve_mc_record_entry_del(mc_record, mc_entry);
 567         mlxsw_sp_nve_mc_record_put(mc_record);
 568 }
 569
 570 static int
 571 mlxsw_sp_nve_fid_flood_index_set(struct mlxsw_sp_fid *fid,
 572                                  struct mlxsw_sp_nve_mc_list *mc_list)
 573 {
 574         struct mlxsw_sp_nve_mc_record *mc_record;
 575
 576         /* The address of the first record in the list is a property of
 577          * the FID and we never change it. It only needs to be set when
 578          * a new list is created
 579          */
 580         if (mlxsw_sp_fid_nve_flood_index_is_set(fid))
 581                 return 0;
 582
 583         mc_record = list_first_entry(&mc_list->records_list,
 584                                      struct mlxsw_sp_nve_mc_record, list);
 585
 586         return mlxsw_sp_fid_nve_flood_index_set(fid, mc_record->kvdl_index);
 587 }
 588
 589 static void
 590 mlxsw_sp_nve_fid_flood_index_clear(struct mlxsw_sp_fid *fid,
 591                                    struct mlxsw_sp_nve_mc_list *mc_list)
 592 {
 593         struct mlxsw_sp_nve_mc_record *mc_record;
 594
 595         /* The address of the first record needs to be invalidated only when
 596          * the last record is about to be removed
 597          */
 598         if (!list_is_singular(&mc_list->records_list))
 599                 return;
 600
 601         mc_record = list_first_entry(&mc_list->records_list,
 602                                      struct mlxsw_sp_nve_mc_record, list);
 603         if (mc_record->num_entries != 1)
 604                 return;
 605
 606         return mlxsw_sp_fid_nve_flood_index_clear(fid);
 607 }
 608
 609 int mlxsw_sp_nve_flood_ip_add(struct mlxsw_sp *mlxsw_sp,
 610                               struct mlxsw_sp_fid *fid,
 611                               enum mlxsw_sp_l3proto proto,
 612                               union mlxsw_sp_l3addr *addr)
 613 {
 614         struct mlxsw_sp_nve_mc_list_key key = { 0 };
 615         struct mlxsw_sp_nve_mc_list *mc_list;
 616         int err;
 617
 618         key.fid_index = mlxsw_sp_fid_index(fid);
 619         mc_list = mlxsw_sp_nve_mc_list_get(mlxsw_sp, &key);
 620         if (IS_ERR(mc_list))
 621                 return PTR_ERR(mc_list);
 622
 623         err = mlxsw_sp_nve_mc_list_ip_add(mlxsw_sp, mc_list, proto, addr);
 624         if (err)
 625                 goto err_add_ip;
 626
 627         err = mlxsw_sp_nve_fid_flood_index_set(fid, mc_list);
 628         if (err)
 629                 goto err_fid_flood_index_set;
 630
 631         return 0;
 632
 633 err_fid_flood_index_set:
 634         mlxsw_sp_nve_mc_list_ip_del(mlxsw_sp, mc_list, proto, addr);
 635 err_add_ip:
 636         mlxsw_sp_nve_mc_list_put(mlxsw_sp, mc_list);
 637         return err;
 638 }
 639
 640 void mlxsw_sp_nve_flood_ip_del(struct mlxsw_sp *mlxsw_sp,
 641                                struct mlxsw_sp_fid *fid,
 642                                enum mlxsw_sp_l3proto proto,
 643                                union mlxsw_sp_l3addr *addr)
 644 {
 645         struct mlxsw_sp_nve_mc_list_key key = { 0 };
 646         struct mlxsw_sp_nve_mc_list *mc_list;
 647
 648         key.fid_index = mlxsw_sp_fid_index(fid);
 649         mc_list = mlxsw_sp_nve_mc_list_find(mlxsw_sp, &key);
 650         if (WARN_ON(!mc_list))
 651                 return;
 652
 653         mlxsw_sp_nve_fid_flood_index_clear(fid, mc_list);
 654         mlxsw_sp_nve_mc_list_ip_del(mlxsw_sp, mc_list, proto, addr);
 655         mlxsw_sp_nve_mc_list_put(mlxsw_sp, mc_list);
 656 }
 657
 658 static void
 659 mlxsw_sp_nve_mc_record_delete(struct mlxsw_sp_nve_mc_record *mc_record)
 660 {
 661         struct mlxsw_sp_nve *nve = mc_record->mlxsw_sp->nve;
 662         unsigned int num_max_entries;
 663         int i;
 664
 665         num_max_entries = nve->num_max_mc_entries[mc_record->proto];
 666         for (i = 0; i < num_max_entries; i++) {
 667                 struct mlxsw_sp_nve_mc_entry *mc_entry = &mc_record->entries[i];
 668
 669                 if (!mc_entry->valid)
 670                         continue;
 671                 mlxsw_sp_nve_mc_record_entry_del(mc_record, mc_entry);
 672         }
 673
 674         WARN_ON(mc_record->num_entries);
 675         mlxsw_sp_nve_mc_record_put(mc_record);
 676 }
 677
 678 static void mlxsw_sp_nve_flood_ip_flush(struct mlxsw_sp *mlxsw_sp,
 679                                         struct mlxsw_sp_fid *fid)
 680 {
 681         struct mlxsw_sp_nve_mc_record *mc_record, *tmp;
 682         struct mlxsw_sp_nve_mc_list_key key = { 0 };
 683         struct mlxsw_sp_nve_mc_list *mc_list;
 684
 685         if (!mlxsw_sp_fid_nve_flood_index_is_set(fid))
 686                 return;
 687
 688         mlxsw_sp_fid_nve_flood_index_clear(fid);
 689
 690         key.fid_index = mlxsw_sp_fid_index(fid);
 691         mc_list = mlxsw_sp_nve_mc_list_find(mlxsw_sp, &key);
 692         if (WARN_ON(!mc_list))
 693                 return;
 694
 695         list_for_each_entry_safe(mc_record, tmp, &mc_list->records_list, list)
 696                 mlxsw_sp_nve_mc_record_delete(mc_record);
 697
 698         WARN_ON(!list_empty(&mc_list->records_list));
 699         mlxsw_sp_nve_mc_list_put(mlxsw_sp, mc_list);
 700 }
 701
 702 u32 mlxsw_sp_nve_decap_tunnel_index_get(const struct mlxsw_sp *mlxsw_sp)
 703 {
 704         WARN_ON(mlxsw_sp->nve->num_nve_tunnels == 0);
 705
 706         return mlxsw_sp->nve->tunnel_index;
 707 }
 708
 709 bool mlxsw_sp_nve_ipv4_route_is_decap(const struct mlxsw_sp *mlxsw_sp,
 710                                       u32 tb_id, __be32 addr)
 711 {
 712         struct mlxsw_sp_nve *nve = mlxsw_sp->nve;
 713         struct mlxsw_sp_nve_config *config = &nve->config;
 714
 715         if (nve->num_nve_tunnels &&
 716             config->ul_proto == MLXSW_SP_L3_PROTO_IPV4 &&
 717             config->ul_sip.addr4 == addr && config->ul_tb_id == tb_id)
 718                 return true;
 719
 720         return false;
 721 }
 722
 723 static int mlxsw_sp_nve_tunnel_init(struct mlxsw_sp *mlxsw_sp,
 724                                     struct mlxsw_sp_nve_config *config)
 725 {
 726         struct mlxsw_sp_nve *nve = mlxsw_sp->nve;
 727         const struct mlxsw_sp_nve_ops *ops;
 728         int err;
 729
 730         if (nve->num_nve_tunnels++ != 0)
 731                 return 0;
 732
 733         err = mlxsw_sp_kvdl_alloc(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1,
 734                                   &nve->tunnel_index);
 735         if (err)
 736                 goto err_kvdl_alloc;
 737
 738         ops = nve->nve_ops_arr[config->type];
 739         err = ops->init(nve, config);
 740         if (err)
 741                 goto err_ops_init;
 742
 743         return 0;
 744
 745 err_ops_init:
 746         mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1,
 747                            nve->tunnel_index);
 748 err_kvdl_alloc:
 749         nve->num_nve_tunnels--;
 750         return err;
 751 }
 752
 753 static void mlxsw_sp_nve_tunnel_fini(struct mlxsw_sp *mlxsw_sp)
 754 {
 755         struct mlxsw_sp_nve *nve = mlxsw_sp->nve;
 756         const struct mlxsw_sp_nve_ops *ops;
 757
 758         ops = nve->nve_ops_arr[nve->config.type];
 759
 760         if (mlxsw_sp->nve->num_nve_tunnels == 1) {
 761                 ops->fini(nve);
 762                 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1,
 763                                    nve->tunnel_index);
 764         }
 765         nve->num_nve_tunnels--;
 766 }
 767
 768 static void mlxsw_sp_nve_fdb_flush_by_fid(struct mlxsw_sp *mlxsw_sp,
 769                                           u16 fid_index)
 770 {
 771         char sfdf_pl[MLXSW_REG_SFDF_LEN];
 772
 773         mlxsw_reg_sfdf_pack(sfdf_pl, MLXSW_REG_SFDF_FLUSH_PER_NVE_AND_FID);
 774         mlxsw_reg_sfdf_fid_set(sfdf_pl, fid_index);
 775         mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(sfdf), sfdf_pl);
 776 }
 777
 778 int mlxsw_sp_nve_fid_enable(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_fid *fid,
 779                             struct mlxsw_sp_nve_params *params,
 780                             struct netlink_ext_ack *extack)
 781 {
 782         struct mlxsw_sp_nve *nve = mlxsw_sp->nve;
 783         const struct mlxsw_sp_nve_ops *ops;
 784         struct mlxsw_sp_nve_config config;
 785         int err;
 786
 787         ops = nve->nve_ops_arr[params->type];
 788
 789         if (!ops->can_offload(nve, params->dev, extack))
 790                 return -EOPNOTSUPP;
 791
 792         memset(&config, 0, sizeof(config));
 793         ops->nve_config(nve, params->dev, &config);
 794         if (nve->num_nve_tunnels &&
 795             memcmp(&config, &nve->config, sizeof(config))) {
 796                 NL_SET_ERR_MSG_MOD(extack, "Conflicting NVE tunnels configuration");
 797                 return -EOPNOTSUPP;
 798         }
 799
 800         err = mlxsw_sp_nve_tunnel_init(mlxsw_sp, &config);
 801         if (err) {
 802                 NL_SET_ERR_MSG_MOD(extack, "Failed to initialize NVE tunnel");
 803                 return err;
 804         }
 805
 806         err = mlxsw_sp_fid_vni_set(fid, params->vni);
 807         if (err) {
 808                 NL_SET_ERR_MSG_MOD(extack, "Failed to set VNI on FID");
 809                 goto err_fid_vni_set;
 810         }
 811
 812         nve->config = config;
 813
 814         return 0;
 815
 816 err_fid_vni_set:
 817         mlxsw_sp_nve_tunnel_fini(mlxsw_sp);
 818         return err;
 819 }
 820
 821 void mlxsw_sp_nve_fid_disable(struct mlxsw_sp *mlxsw_sp,
 822                               struct mlxsw_sp_fid *fid)
 823 {
 824         u16 fid_index = mlxsw_sp_fid_index(fid);
 825
 826         mlxsw_sp_nve_flood_ip_flush(mlxsw_sp, fid);
 827         mlxsw_sp_nve_fdb_flush_by_fid(mlxsw_sp, fid_index);
 828         mlxsw_sp_fid_vni_clear(fid);
 829         mlxsw_sp_nve_tunnel_fini(mlxsw_sp);
 830 }
 831
 832 int mlxsw_sp_port_nve_init(struct mlxsw_sp_port *mlxsw_sp_port)
 833 {
 834         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
 835         char tnqdr_pl[MLXSW_REG_TNQDR_LEN];
 836
 837         mlxsw_reg_tnqdr_pack(tnqdr_pl, mlxsw_sp_port->local_port);
 838         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(tnqdr), tnqdr_pl);
 839 }
 840
 841 void mlxsw_sp_port_nve_fini(struct mlxsw_sp_port *mlxsw_sp_port)
 842 {
 843 }
 844
 845 static int mlxsw_sp_nve_qos_init(struct mlxsw_sp *mlxsw_sp)
 846 {
 847         char tnqcr_pl[MLXSW_REG_TNQCR_LEN];
 848
 849         mlxsw_reg_tnqcr_pack(tnqcr_pl);
 850         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(tnqcr), tnqcr_pl);
 851 }
 852
 853 static int mlxsw_sp_nve_ecn_encap_init(struct mlxsw_sp *mlxsw_sp)
 854 {
 855         int i;
 856
 857         /* Iterate over inner ECN values */
 858         for (i = INET_ECN_NOT_ECT; i <= INET_ECN_CE; i++) {
 859                 u8 outer_ecn = INET_ECN_encapsulate(0, i);
 860                 char tneem_pl[MLXSW_REG_TNEEM_LEN];
 861                 int err;
 862
 863                 mlxsw_reg_tneem_pack(tneem_pl, i, outer_ecn);
 864                 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(tneem),
 865                                       tneem_pl);
 866                 if (err)
 867                         return err;
 868         }
 869
 870         return 0;
 871 }
 872
 873 static int __mlxsw_sp_nve_ecn_decap_init(struct mlxsw_sp *mlxsw_sp,
 874                                          u8 inner_ecn, u8 outer_ecn)
 875 {
 876         char tndem_pl[MLXSW_REG_TNDEM_LEN];
 877         bool trap_en, set_ce = false;
 878         u8 new_inner_ecn;
 879
 880         trap_en = !!__INET_ECN_decapsulate(outer_ecn, inner_ecn, &set_ce);
 881         new_inner_ecn = set_ce ? INET_ECN_CE : inner_ecn;
 882
 883         mlxsw_reg_tndem_pack(tndem_pl, outer_ecn, inner_ecn, new_inner_ecn,
 884                              trap_en, trap_en ? MLXSW_TRAP_ID_DECAP_ECN0 : 0);
 885         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(tndem), tndem_pl);
 886 }
 887
 888 static int mlxsw_sp_nve_ecn_decap_init(struct mlxsw_sp *mlxsw_sp)
 889 {
 890         int i;
 891
 892         /* Iterate over inner ECN values */
 893         for (i = INET_ECN_NOT_ECT; i <= INET_ECN_CE; i++) {
 894                 int j;
 895
 896                 /* Iterate over outer ECN values */
 897                 for (j = INET_ECN_NOT_ECT; j <= INET_ECN_CE; j++) {
 898                         int err;
 899
 900                         err = __mlxsw_sp_nve_ecn_decap_init(mlxsw_sp, i, j);
 901                         if (err)
 902                                 return err;
 903                 }
 904         }
 905
 906         return 0;
 907 }
 908
 909 static int mlxsw_sp_nve_ecn_init(struct mlxsw_sp *mlxsw_sp)
 910 {
 911         int err;
 912
 913         err = mlxsw_sp_nve_ecn_encap_init(mlxsw_sp);
 914         if (err)
 915                 return err;
 916
 917         return mlxsw_sp_nve_ecn_decap_init(mlxsw_sp);
 918 }
 919
 920 static int mlxsw_sp_nve_resources_query(struct mlxsw_sp *mlxsw_sp)
 921 {
 922         unsigned int max;
 923
 924         if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_NVE_MC_ENTRIES_IPV4) ||
 925             !MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_NVE_MC_ENTRIES_IPV6))
 926                 return -EIO;
 927         max = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_NVE_MC_ENTRIES_IPV4);
 928         mlxsw_sp->nve->num_max_mc_entries[MLXSW_SP_L3_PROTO_IPV4] = max;
 929         max = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_NVE_MC_ENTRIES_IPV6);
 930         mlxsw_sp->nve->num_max_mc_entries[MLXSW_SP_L3_PROTO_IPV6] = max;
 931
 932         return 0;
 933 }
 934
 935 int mlxsw_sp_nve_init(struct mlxsw_sp *mlxsw_sp)
 936 {
 937         struct mlxsw_sp_nve *nve;
 938         int err;
 939
 940         nve = kzalloc(sizeof(*mlxsw_sp->nve), GFP_KERNEL);
 941         if (!nve)
 942                 return -ENOMEM;
 943         mlxsw_sp->nve = nve;
 944         nve->mlxsw_sp = mlxsw_sp;
 945         nve->nve_ops_arr = mlxsw_sp->nve_ops_arr;
 946
 947         err = rhashtable_init(&nve->mc_list_ht,
 948                               &mlxsw_sp_nve_mc_list_ht_params);
 949         if (err)
 950                 goto err_rhashtable_init;
 951
 952         err = mlxsw_sp_nve_qos_init(mlxsw_sp);
 953         if (err)
 954                 goto err_nve_qos_init;
 955
 956         err = mlxsw_sp_nve_ecn_init(mlxsw_sp);
 957         if (err)
 958                 goto err_nve_ecn_init;
 959
 960         err = mlxsw_sp_nve_resources_query(mlxsw_sp);
 961         if (err)
 962                 goto err_nve_resources_query;
 963
 964         return 0;
 965
 966 err_nve_resources_query:
 967 err_nve_ecn_init:
 968 err_nve_qos_init:
 969         rhashtable_destroy(&nve->mc_list_ht);
 970 err_rhashtable_init:
 971         mlxsw_sp->nve = NULL;
 972         kfree(nve);
 973         return err;
 974 }
 975
 976 void mlxsw_sp_nve_fini(struct mlxsw_sp *mlxsw_sp)
 977 {
 978         WARN_ON(mlxsw_sp->nve->num_nve_tunnels);
 979         rhashtable_destroy(&mlxsw_sp->nve->mc_list_ht);
 980         mlxsw_sp->nve = NULL;
 981         kfree(mlxsw_sp->nve);
 982 }