drivers/net/ethernet/mellanox/mlxsw/spectrum_acl_tcam.c

   1 // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0
   2 /* Copyright (c) 2017-2018 Mellanox Technologies. All rights reserved */
   3
   4 #include <linux/kernel.h>
   5 #include <linux/slab.h>
   6 #include <linux/errno.h>
   7 #include <linux/bitops.h>
   8 #include <linux/list.h>
   9 #include <linux/rhashtable.h>
  10 #include <linux/netdevice.h>
  11 #include <linux/mutex.h>
  12 #include <trace/events/mlxsw.h>
  13
  14 #include "reg.h"
  15 #include "core.h"
  16 #include "resources.h"
  17 #include "spectrum.h"
  18 #include "spectrum_acl_tcam.h"
  19 #include "core_acl_flex_keys.h"
  20
  21 size_t mlxsw_sp_acl_tcam_priv_size(struct mlxsw_sp *mlxsw_sp)
  22 {
  23         const struct mlxsw_sp_acl_tcam_ops *ops = mlxsw_sp->acl_tcam_ops;
  24
  25         return ops->priv_size;
  26 }
  27
  28 #define MLXSW_SP_ACL_TCAM_VREGION_REHASH_INTRVL_DFLT 5000 /* ms */
  29 #define MLXSW_SP_ACL_TCAM_VREGION_REHASH_INTRVL_MIN 3000 /* ms */
  30 #define MLXSW_SP_ACL_TCAM_VREGION_REHASH_CREDITS 100 /* number of entries */
  31
  32 int mlxsw_sp_acl_tcam_init(struct mlxsw_sp *mlxsw_sp,
  33                            struct mlxsw_sp_acl_tcam *tcam)
  34 {
  35         const struct mlxsw_sp_acl_tcam_ops *ops = mlxsw_sp->acl_tcam_ops;
  36         u64 max_tcam_regions;
  37         u64 max_regions;
  38         u64 max_groups;
  39         size_t alloc_size;
  40         int err;
  41
  42         mutex_init(&tcam->lock);
  43         tcam->vregion_rehash_intrvl =
  44                         MLXSW_SP_ACL_TCAM_VREGION_REHASH_INTRVL_DFLT;
  45         INIT_LIST_HEAD(&tcam->vregion_list);
  46
  47         max_tcam_regions = MLXSW_CORE_RES_GET(mlxsw_sp->core,
  48                                               ACL_MAX_TCAM_REGIONS);
  49         max_regions = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_MAX_REGIONS);
  50
  51         /* Use 1:1 mapping between ACL region and TCAM region */
  52         if (max_tcam_regions < max_regions)
  53                 max_regions = max_tcam_regions;
  54
  55         alloc_size = sizeof(tcam->used_regions[0]) * BITS_TO_LONGS(max_regions);
  56         tcam->used_regions = kzalloc(alloc_size, GFP_KERNEL);
  57         if (!tcam->used_regions)
  58                 return -ENOMEM;
  59         tcam->max_regions = max_regions;
  60
  61         max_groups = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_MAX_GROUPS);
  62         alloc_size = sizeof(tcam->used_groups[0]) * BITS_TO_LONGS(max_groups);
  63         tcam->used_groups = kzalloc(alloc_size, GFP_KERNEL);
  64         if (!tcam->used_groups) {
  65                 err = -ENOMEM;
  66                 goto err_alloc_used_groups;
  67         }
  68         tcam->max_groups = max_groups;
  69         tcam->max_group_size = MLXSW_CORE_RES_GET(mlxsw_sp->core,
  70                                                  ACL_MAX_GROUP_SIZE);
  71
  72         err = ops->init(mlxsw_sp, tcam->priv, tcam);
  73         if (err)
  74                 goto err_tcam_init;
  75
  76         return 0;
  77
  78 err_tcam_init:
  79         kfree(tcam->used_groups);
  80 err_alloc_used_groups:
  81         kfree(tcam->used_regions);
  82         return err;
  83 }
  84
  85 void mlxsw_sp_acl_tcam_fini(struct mlxsw_sp *mlxsw_sp,
  86                             struct mlxsw_sp_acl_tcam *tcam)
  87 {
  88         const struct mlxsw_sp_acl_tcam_ops *ops = mlxsw_sp->acl_tcam_ops;
  89
  90         mutex_destroy(&tcam->lock);
  91         ops->fini(mlxsw_sp, tcam->priv);
  92         kfree(tcam->used_groups);
  93         kfree(tcam->used_regions);
  94 }
  95
  96 int mlxsw_sp_acl_tcam_priority_get(struct mlxsw_sp *mlxsw_sp,
  97                                    struct mlxsw_sp_acl_rule_info *rulei,
  98                                    u32 *priority, bool fillup_priority)
  99 {
 100         u64 max_priority;
 101
 102         if (!fillup_priority) {
 103                 *priority = 0;
 104                 return 0;
 105         }
 106
 107         if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, KVD_SIZE))
 108                 return -EIO;
 109
 110         /* Priority range is 1..cap_kvd_size-1. */
 111         max_priority = MLXSW_CORE_RES_GET(mlxsw_sp->core, KVD_SIZE) - 1;
 112         if (rulei->priority >= max_priority)
 113                 return -EINVAL;
 114
 115         /* Unlike in TC, in HW, higher number means higher priority. */
 116         *priority = max_priority - rulei->priority;
 117         return 0;
 118 }
 119
 120 static int mlxsw_sp_acl_tcam_region_id_get(struct mlxsw_sp_acl_tcam *tcam,
 121                                            u16 *p_id)
 122 {
 123         u16 id;
 124
 125         id = find_first_zero_bit(tcam->used_regions, tcam->max_regions);
 126         if (id < tcam->max_regions) {
 127                 __set_bit(id, tcam->used_regions);
 128                 *p_id = id;
 129                 return 0;
 130         }
 131         return -ENOBUFS;
 132 }
 133
 134 static void mlxsw_sp_acl_tcam_region_id_put(struct mlxsw_sp_acl_tcam *tcam,
 135                                             u16 id)
 136 {
 137         __clear_bit(id, tcam->used_regions);
 138 }
 139
 140 static int mlxsw_sp_acl_tcam_group_id_get(struct mlxsw_sp_acl_tcam *tcam,
 141                                           u16 *p_id)
 142 {
 143         u16 id;
 144
 145         id = find_first_zero_bit(tcam->used_groups, tcam->max_groups);
 146         if (id < tcam->max_groups) {
 147                 __set_bit(id, tcam->used_groups);
 148                 *p_id = id;
 149                 return 0;
 150         }
 151         return -ENOBUFS;
 152 }
 153
 154 static void mlxsw_sp_acl_tcam_group_id_put(struct mlxsw_sp_acl_tcam *tcam,
 155                                            u16 id)
 156 {
 157         __clear_bit(id, tcam->used_groups);
 158 }
 159
 160 struct mlxsw_sp_acl_tcam_pattern {
 161         const enum mlxsw_afk_element *elements;
 162         unsigned int elements_count;
 163 };
 164
 165 struct mlxsw_sp_acl_tcam_group {
 166         struct mlxsw_sp_acl_tcam *tcam;
 167         u16 id;
 168         struct mutex lock; /* guards region list updates */
 169         struct list_head region_list;
 170         unsigned int region_count;
 171 };
 172
 173 struct mlxsw_sp_acl_tcam_vgroup {
 174         struct mlxsw_sp_acl_tcam_group group;
 175         struct list_head vregion_list;
 176         struct rhashtable vchunk_ht;
 177         const struct mlxsw_sp_acl_tcam_pattern *patterns;
 178         unsigned int patterns_count;
 179         bool tmplt_elusage_set;
 180         struct mlxsw_afk_element_usage tmplt_elusage;
 181         bool vregion_rehash_enabled;
 182         unsigned int *p_min_prio;
 183         unsigned int *p_max_prio;
 184 };
 185
 186 struct mlxsw_sp_acl_tcam_rehash_ctx {
 187         void *hints_priv;
 188         bool this_is_rollback;
 189         struct mlxsw_sp_acl_tcam_vchunk *current_vchunk; /* vchunk being
 190                                                           * currently migrated.
 191                                                           */
 192         struct mlxsw_sp_acl_tcam_ventry *start_ventry; /* ventry to start
 193                                                         * migration from in
 194                                                         * a vchunk being
 195                                                         * currently migrated.
 196                                                         */
 197         struct mlxsw_sp_acl_tcam_ventry *stop_ventry; /* ventry to stop
 198                                                        * migration at
 199                                                        * a vchunk being
 200                                                        * currently migrated.
 201                                                        */
 202 };
 203
 204 struct mlxsw_sp_acl_tcam_vregion {
 205         struct mutex lock; /* Protects consistency of region, region2 pointers
 206                             * and vchunk_list.
 207                             */
 208         struct mlxsw_sp_acl_tcam_region *region;
 209         struct mlxsw_sp_acl_tcam_region *region2; /* Used during migration */
 210         struct list_head list; /* Member of a TCAM group */
 211         struct list_head tlist; /* Member of a TCAM */
 212         struct list_head vchunk_list; /* List of vchunks under this vregion */
 213         struct mlxsw_afk_key_info *key_info;
 214         struct mlxsw_sp_acl_tcam *tcam;
 215         struct mlxsw_sp_acl_tcam_vgroup *vgroup;
 216         struct {
 217                 struct delayed_work dw;
 218                 struct mlxsw_sp_acl_tcam_rehash_ctx ctx;
 219         } rehash;
 220         struct mlxsw_sp *mlxsw_sp;
 221         unsigned int ref_count;
 222 };
 223
 224 struct mlxsw_sp_acl_tcam_vchunk;
 225
 226 struct mlxsw_sp_acl_tcam_chunk {
 227         struct mlxsw_sp_acl_tcam_vchunk *vchunk;
 228         struct mlxsw_sp_acl_tcam_region *region;
 229         unsigned long priv[];
 230         /* priv has to be always the last item */
 231 };
 232
 233 struct mlxsw_sp_acl_tcam_vchunk {
 234         struct mlxsw_sp_acl_tcam_chunk *chunk;
 235         struct mlxsw_sp_acl_tcam_chunk *chunk2; /* Used during migration */
 236         struct list_head list; /* Member of a TCAM vregion */
 237         struct rhash_head ht_node; /* Member of a chunk HT */
 238         struct list_head ventry_list;
 239         unsigned int priority; /* Priority within the vregion and group */
 240         struct mlxsw_sp_acl_tcam_vgroup *vgroup;
 241         struct mlxsw_sp_acl_tcam_vregion *vregion;
 242         unsigned int ref_count;
 243 };
 244
 245 struct mlxsw_sp_acl_tcam_entry {
 246         struct mlxsw_sp_acl_tcam_ventry *ventry;
 247         struct mlxsw_sp_acl_tcam_chunk *chunk;
 248         unsigned long priv[];
 249         /* priv has to be always the last item */
 250 };
 251
 252 struct mlxsw_sp_acl_tcam_ventry {
 253         struct mlxsw_sp_acl_tcam_entry *entry;
 254         struct list_head list; /* Member of a TCAM vchunk */
 255         struct mlxsw_sp_acl_tcam_vchunk *vchunk;
 256         struct mlxsw_sp_acl_rule_info *rulei;
 257 };
 258
 259 static const struct rhashtable_params mlxsw_sp_acl_tcam_vchunk_ht_params = {
 260         .key_len = sizeof(unsigned int),
 261         .key_offset = offsetof(struct mlxsw_sp_acl_tcam_vchunk, priority),
 262         .head_offset = offsetof(struct mlxsw_sp_acl_tcam_vchunk, ht_node),
 263         .automatic_shrinking = true,
 264 };
 265
 266 static int mlxsw_sp_acl_tcam_group_update(struct mlxsw_sp *mlxsw_sp,
 267                                           struct mlxsw_sp_acl_tcam_group *group)
 268 {
 269         struct mlxsw_sp_acl_tcam_region *region;
 270         char pagt_pl[MLXSW_REG_PAGT_LEN];
 271         int acl_index = 0;
 272
 273         mlxsw_reg_pagt_pack(pagt_pl, group->id);
 274         list_for_each_entry(region, &group->region_list, list) {
 275                 bool multi = false;
 276
 277                 /* Check if the next entry in the list has the same vregion. */
 278                 if (region->list.next != &group->region_list &&
 279                     list_next_entry(region, list)->vregion == region->vregion)
 280                         multi = true;
 281                 mlxsw_reg_pagt_acl_id_pack(pagt_pl, acl_index++,
 282                                            region->id, multi);
 283         }
 284         mlxsw_reg_pagt_size_set(pagt_pl, acl_index);
 285         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pagt), pagt_pl);
 286 }
 287
 288 static int
 289 mlxsw_sp_acl_tcam_group_add(struct mlxsw_sp_acl_tcam *tcam,
 290                             struct mlxsw_sp_acl_tcam_group *group)
 291 {
 292         int err;
 293
 294         group->tcam = tcam;
 295         INIT_LIST_HEAD(&group->region_list);
 296
 297         err = mlxsw_sp_acl_tcam_group_id_get(tcam, &group->id);
 298         if (err)
 299                 return err;
 300
 301         mutex_init(&group->lock);
 302
 303         return 0;
 304 }
 305
 306 static void mlxsw_sp_acl_tcam_group_del(struct mlxsw_sp_acl_tcam_group *group)
 307 {
 308         struct mlxsw_sp_acl_tcam *tcam = group->tcam;
 309
 310         mutex_destroy(&group->lock);
 311         mlxsw_sp_acl_tcam_group_id_put(tcam, group->id);
 312         WARN_ON(!list_empty(&group->region_list));
 313 }
 314
 315 static int
 316 mlxsw_sp_acl_tcam_vgroup_add(struct mlxsw_sp *mlxsw_sp,
 317                              struct mlxsw_sp_acl_tcam *tcam,
 318                              struct mlxsw_sp_acl_tcam_vgroup *vgroup,
 319                              const struct mlxsw_sp_acl_tcam_pattern *patterns,
 320                              unsigned int patterns_count,
 321                              struct mlxsw_afk_element_usage *tmplt_elusage,
 322                              bool vregion_rehash_enabled,
 323                              unsigned int *p_min_prio,
 324                              unsigned int *p_max_prio)
 325 {
 326         int err;
 327
 328         vgroup->patterns = patterns;
 329         vgroup->patterns_count = patterns_count;
 330         vgroup->vregion_rehash_enabled = vregion_rehash_enabled;
 331         vgroup->p_min_prio = p_min_prio;
 332         vgroup->p_max_prio = p_max_prio;
 333
 334         if (tmplt_elusage) {
 335                 vgroup->tmplt_elusage_set = true;
 336                 memcpy(&vgroup->tmplt_elusage, tmplt_elusage,
 337                        sizeof(vgroup->tmplt_elusage));
 338         }
 339         INIT_LIST_HEAD(&vgroup->vregion_list);
 340
 341         err = mlxsw_sp_acl_tcam_group_add(tcam, &vgroup->group);
 342         if (err)
 343                 return err;
 344
 345         err = rhashtable_init(&vgroup->vchunk_ht,
 346                               &mlxsw_sp_acl_tcam_vchunk_ht_params);
 347         if (err)
 348                 goto err_rhashtable_init;
 349
 350         return 0;
 351
 352 err_rhashtable_init:
 353         mlxsw_sp_acl_tcam_group_del(&vgroup->group);
 354         return err;
 355 }
 356
 357 static void
 358 mlxsw_sp_acl_tcam_vgroup_del(struct mlxsw_sp_acl_tcam_vgroup *vgroup)
 359 {
 360         rhashtable_destroy(&vgroup->vchunk_ht);
 361         mlxsw_sp_acl_tcam_group_del(&vgroup->group);
 362         WARN_ON(!list_empty(&vgroup->vregion_list));
 363 }
 364
 365 static int
 366 mlxsw_sp_acl_tcam_group_bind(struct mlxsw_sp *mlxsw_sp,
 367                              struct mlxsw_sp_acl_tcam_group *group,
 368                              struct mlxsw_sp_port *mlxsw_sp_port,
 369                              bool ingress)
 370 {
 371         char ppbt_pl[MLXSW_REG_PPBT_LEN];
 372
 373         mlxsw_reg_ppbt_pack(ppbt_pl, ingress ? MLXSW_REG_PXBT_E_IACL :
 374                                                MLXSW_REG_PXBT_E_EACL,
 375                             MLXSW_REG_PXBT_OP_BIND, mlxsw_sp_port->local_port,
 376                             group->id);
 377         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ppbt), ppbt_pl);
 378 }
 379
 380 static void
 381 mlxsw_sp_acl_tcam_group_unbind(struct mlxsw_sp *mlxsw_sp,
 382                                struct mlxsw_sp_acl_tcam_group *group,
 383                                struct mlxsw_sp_port *mlxsw_sp_port,
 384                                bool ingress)
 385 {
 386         char ppbt_pl[MLXSW_REG_PPBT_LEN];
 387
 388         mlxsw_reg_ppbt_pack(ppbt_pl, ingress ? MLXSW_REG_PXBT_E_IACL :
 389                                                MLXSW_REG_PXBT_E_EACL,
 390                             MLXSW_REG_PXBT_OP_UNBIND, mlxsw_sp_port->local_port,
 391                             group->id);
 392         mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ppbt), ppbt_pl);
 393 }
 394
 395 static u16
 396 mlxsw_sp_acl_tcam_group_id(struct mlxsw_sp_acl_tcam_group *group)
 397 {
 398         return group->id;
 399 }
 400
 401 static unsigned int
 402 mlxsw_sp_acl_tcam_vregion_prio(struct mlxsw_sp_acl_tcam_vregion *vregion)
 403 {
 404         struct mlxsw_sp_acl_tcam_vchunk *vchunk;
 405
 406         if (list_empty(&vregion->vchunk_list))
 407                 return 0;
 408         /* As a priority of a vregion, return priority of the first vchunk */
 409         vchunk = list_first_entry(&vregion->vchunk_list,
 410                                   typeof(*vchunk), list);
 411         return vchunk->priority;
 412 }
 413
 414 static unsigned int
 415 mlxsw_sp_acl_tcam_vregion_max_prio(struct mlxsw_sp_acl_tcam_vregion *vregion)
 416 {
 417         struct mlxsw_sp_acl_tcam_vchunk *vchunk;
 418
 419         if (list_empty(&vregion->vchunk_list))
 420                 return 0;
 421         vchunk = list_last_entry(&vregion->vchunk_list,
 422                                  typeof(*vchunk), list);
 423         return vchunk->priority;
 424 }
 425
 426 static void
 427 mlxsw_sp_acl_tcam_vgroup_prio_update(struct mlxsw_sp_acl_tcam_vgroup *vgroup)
 428 {
 429         struct mlxsw_sp_acl_tcam_vregion *vregion;
 430
 431         if (list_empty(&vgroup->vregion_list))
 432                 return;
 433         vregion = list_first_entry(&vgroup->vregion_list,
 434                                    typeof(*vregion), list);
 435         *vgroup->p_min_prio = mlxsw_sp_acl_tcam_vregion_prio(vregion);
 436         vregion = list_last_entry(&vgroup->vregion_list,
 437                                   typeof(*vregion), list);
 438         *vgroup->p_max_prio = mlxsw_sp_acl_tcam_vregion_max_prio(vregion);
 439 }
 440
 441 static int
 442 mlxsw_sp_acl_tcam_group_region_attach(struct mlxsw_sp *mlxsw_sp,
 443                                       struct mlxsw_sp_acl_tcam_group *group,
 444                                       struct mlxsw_sp_acl_tcam_region *region,
 445                                       unsigned int priority,
 446                                       struct mlxsw_sp_acl_tcam_region *next_region)
 447 {
 448         struct mlxsw_sp_acl_tcam_region *region2;
 449         struct list_head *pos;
 450         int err;
 451
 452         mutex_lock(&group->lock);
 453         if (group->region_count == group->tcam->max_group_size) {
 454                 err = -ENOBUFS;
 455                 goto err_region_count_check;
 456         }
 457
 458         if (next_region) {
 459                 /* If the next region is defined, place the new one
 460                  * before it. The next one is a sibling.
 461                  */
 462                 pos = &next_region->list;
 463         } else {
 464                 /* Position the region inside the list according to priority */
 465                 list_for_each(pos, &group->region_list) {
 466                         region2 = list_entry(pos, typeof(*region2), list);
 467                         if (mlxsw_sp_acl_tcam_vregion_prio(region2->vregion) >
 468                             priority)
 469                                 break;
 470                 }
 471         }
 472         list_add_tail(&region->list, pos);
 473         region->group = group;
 474
 475         err = mlxsw_sp_acl_tcam_group_update(mlxsw_sp, group);
 476         if (err)
 477                 goto err_group_update;
 478
 479         group->region_count++;
 480         mutex_unlock(&group->lock);
 481         return 0;
 482
 483 err_group_update:
 484         list_del(&region->list);
 485 err_region_count_check:
 486         mutex_unlock(&group->lock);
 487         return err;
 488 }
 489
 490 static void
 491 mlxsw_sp_acl_tcam_group_region_detach(struct mlxsw_sp *mlxsw_sp,
 492                                       struct mlxsw_sp_acl_tcam_region *region)
 493 {
 494         struct mlxsw_sp_acl_tcam_group *group = region->group;
 495
 496         mutex_lock(&group->lock);
 497         list_del(&region->list);
 498         group->region_count--;
 499         mlxsw_sp_acl_tcam_group_update(mlxsw_sp, group);
 500         mutex_unlock(&group->lock);
 501 }
 502
 503 static int
 504 mlxsw_sp_acl_tcam_vgroup_vregion_attach(struct mlxsw_sp *mlxsw_sp,
 505                                         struct mlxsw_sp_acl_tcam_vgroup *vgroup,
 506                                         struct mlxsw_sp_acl_tcam_vregion *vregion,
 507                                         unsigned int priority)
 508 {
 509         struct mlxsw_sp_acl_tcam_vregion *vregion2;
 510         struct list_head *pos;
 511         int err;
 512
 513         /* Position the vregion inside the list according to priority */
 514         list_for_each(pos, &vgroup->vregion_list) {
 515                 vregion2 = list_entry(pos, typeof(*vregion2), list);
 516                 if (mlxsw_sp_acl_tcam_vregion_prio(vregion2) > priority)
 517                         break;
 518         }
 519         list_add_tail(&vregion->list, pos);
 520
 521         err = mlxsw_sp_acl_tcam_group_region_attach(mlxsw_sp, &vgroup->group,
 522                                                     vregion->region,
 523                                                     priority, NULL);
 524         if (err)
 525                 goto err_region_attach;
 526
 527         return 0;
 528
 529 err_region_attach:
 530         list_del(&vregion->list);
 531         return err;
 532 }
 533
 534 static void
 535 mlxsw_sp_acl_tcam_vgroup_vregion_detach(struct mlxsw_sp *mlxsw_sp,
 536                                         struct mlxsw_sp_acl_tcam_vregion *vregion)
 537 {
 538         list_del(&vregion->list);
 539         if (vregion->region2)
 540                 mlxsw_sp_acl_tcam_group_region_detach(mlxsw_sp,
 541                                                       vregion->region2);
 542         mlxsw_sp_acl_tcam_group_region_detach(mlxsw_sp, vregion->region);
 543 }
 544
 545 static struct mlxsw_sp_acl_tcam_vregion *
 546 mlxsw_sp_acl_tcam_vgroup_vregion_find(struct mlxsw_sp_acl_tcam_vgroup *vgroup,
 547                                       unsigned int priority,
 548                                       struct mlxsw_afk_element_usage *elusage,
 549                                       bool *p_need_split)
 550 {
 551         struct mlxsw_sp_acl_tcam_vregion *vregion, *vregion2;
 552         struct list_head *pos;
 553         bool issubset;
 554
 555         list_for_each(pos, &vgroup->vregion_list) {
 556                 vregion = list_entry(pos, typeof(*vregion), list);
 557
 558                 /* First, check if the requested priority does not rather belong
 559                  * under some of the next vregions.
 560                  */
 561                 if (pos->next != &vgroup->vregion_list) { /* not last */
 562                         vregion2 = list_entry(pos->next, typeof(*vregion2),
 563                                               list);
 564                         if (priority >=
 565                             mlxsw_sp_acl_tcam_vregion_prio(vregion2))
 566                                 continue;
 567                 }
 568
 569                 issubset = mlxsw_afk_key_info_subset(vregion->key_info,
 570                                                      elusage);
 571
 572                 /* If requested element usage would not fit and the priority
 573                  * is lower than the currently inspected vregion we cannot
 574                  * use this region, so return NULL to indicate new vregion has
 575                  * to be created.
 576                  */
 577                 if (!issubset &&
 578                     priority < mlxsw_sp_acl_tcam_vregion_prio(vregion))
 579                         return NULL;
 580
 581                 /* If requested element usage would not fit and the priority
 582                  * is higher than the currently inspected vregion we cannot
 583                  * use this vregion. There is still some hope that the next
 584                  * vregion would be the fit. So let it be processed and
 585                  * eventually break at the check right above this.
 586                  */
 587                 if (!issubset &&
 588                     priority > mlxsw_sp_acl_tcam_vregion_max_prio(vregion))
 589                         continue;
 590
 591                 /* Indicate if the vregion needs to be split in order to add
 592                  * the requested priority. Split is needed when requested
 593                  * element usage won't fit into the found vregion.
 594                  */
 595                 *p_need_split = !issubset;
 596                 return vregion;
 597         }
 598         return NULL; /* New vregion has to be created. */
 599 }
 600
 601 static void
 602 mlxsw_sp_acl_tcam_vgroup_use_patterns(struct mlxsw_sp_acl_tcam_vgroup *vgroup,
 603                                       struct mlxsw_afk_element_usage *elusage,
 604                                       struct mlxsw_afk_element_usage *out)
 605 {
 606         const struct mlxsw_sp_acl_tcam_pattern *pattern;
 607         int i;
 608
 609         /* In case the template is set, we don't have to look up the pattern
 610          * and just use the template.
 611          */
 612         if (vgroup->tmplt_elusage_set) {
 613                 memcpy(out, &vgroup->tmplt_elusage, sizeof(*out));
 614                 WARN_ON(!mlxsw_afk_element_usage_subset(elusage, out));
 615                 return;
 616         }
 617
 618         for (i = 0; i < vgroup->patterns_count; i++) {
 619                 pattern = &vgroup->patterns[i];
 620                 mlxsw_afk_element_usage_fill(out, pattern->elements,
 621                                              pattern->elements_count);
 622                 if (mlxsw_afk_element_usage_subset(elusage, out))
 623                         return;
 624         }
 625         memcpy(out, elusage, sizeof(*out));
 626 }
 627
 628 static int
 629 mlxsw_sp_acl_tcam_region_alloc(struct mlxsw_sp *mlxsw_sp,
 630                                struct mlxsw_sp_acl_tcam_region *region)
 631 {
 632         struct mlxsw_afk_key_info *key_info = region->key_info;
 633         char ptar_pl[MLXSW_REG_PTAR_LEN];
 634         unsigned int encodings_count;
 635         int i;
 636         int err;
 637
 638         mlxsw_reg_ptar_pack(ptar_pl, MLXSW_REG_PTAR_OP_ALLOC,
 639                             region->key_type,
 640                             MLXSW_SP_ACL_TCAM_REGION_BASE_COUNT,
 641                             region->id, region->tcam_region_info);
 642         encodings_count = mlxsw_afk_key_info_blocks_count_get(key_info);
 643         for (i = 0; i < encodings_count; i++) {
 644                 u16 encoding;
 645
 646                 encoding = mlxsw_afk_key_info_block_encoding_get(key_info, i);
 647                 mlxsw_reg_ptar_key_id_pack(ptar_pl, i, encoding);
 648         }
 649         err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ptar), ptar_pl);
 650         if (err)
 651                 return err;
 652         mlxsw_reg_ptar_unpack(ptar_pl, region->tcam_region_info);
 653         return 0;
 654 }
 655
 656 static void
 657 mlxsw_sp_acl_tcam_region_free(struct mlxsw_sp *mlxsw_sp,
 658                               struct mlxsw_sp_acl_tcam_region *region)
 659 {
 660         char ptar_pl[MLXSW_REG_PTAR_LEN];
 661
 662         mlxsw_reg_ptar_pack(ptar_pl, MLXSW_REG_PTAR_OP_FREE,
 663                             region->key_type, 0, region->id,
 664                             region->tcam_region_info);
 665         mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ptar), ptar_pl);
 666 }
 667
 668 static int
 669 mlxsw_sp_acl_tcam_region_enable(struct mlxsw_sp *mlxsw_sp,
 670                                 struct mlxsw_sp_acl_tcam_region *region)
 671 {
 672         char pacl_pl[MLXSW_REG_PACL_LEN];
 673
 674         mlxsw_reg_pacl_pack(pacl_pl, region->id, true,
 675                             region->tcam_region_info);
 676         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pacl), pacl_pl);
 677 }
 678
 679 static void
 680 mlxsw_sp_acl_tcam_region_disable(struct mlxsw_sp *mlxsw_sp,
 681                                  struct mlxsw_sp_acl_tcam_region *region)
 682 {
 683         char pacl_pl[MLXSW_REG_PACL_LEN];
 684
 685         mlxsw_reg_pacl_pack(pacl_pl, region->id, false,
 686                             region->tcam_region_info);
 687         mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pacl), pacl_pl);
 688 }
 689
 690 static struct mlxsw_sp_acl_tcam_region *
 691 mlxsw_sp_acl_tcam_region_create(struct mlxsw_sp *mlxsw_sp,
 692                                 struct mlxsw_sp_acl_tcam *tcam,
 693                                 struct mlxsw_sp_acl_tcam_vregion *vregion,
 694                                 void *hints_priv)
 695 {
 696         const struct mlxsw_sp_acl_tcam_ops *ops = mlxsw_sp->acl_tcam_ops;
 697         struct mlxsw_sp_acl_tcam_region *region;
 698         int err;
 699
 700         region = kzalloc(sizeof(*region) + ops->region_priv_size, GFP_KERNEL);
 701         if (!region)
 702                 return ERR_PTR(-ENOMEM);
 703         region->mlxsw_sp = mlxsw_sp;
 704         region->vregion = vregion;
 705         region->key_info = vregion->key_info;
 706
 707         err = mlxsw_sp_acl_tcam_region_id_get(tcam, &region->id);
 708         if (err)
 709                 goto err_region_id_get;
 710
 711         err = ops->region_associate(mlxsw_sp, region);
 712         if (err)
 713                 goto err_tcam_region_associate;
 714
 715         region->key_type = ops->key_type;
 716         err = mlxsw_sp_acl_tcam_region_alloc(mlxsw_sp, region);
 717         if (err)
 718                 goto err_tcam_region_alloc;
 719
 720         err = mlxsw_sp_acl_tcam_region_enable(mlxsw_sp, region);
 721         if (err)
 722                 goto err_tcam_region_enable;
 723
 724         err = ops->region_init(mlxsw_sp, region->priv, tcam->priv,
 725                                region, hints_priv);
 726         if (err)
 727                 goto err_tcam_region_init;
 728
 729         return region;
 730
 731 err_tcam_region_init:
 732         mlxsw_sp_acl_tcam_region_disable(mlxsw_sp, region);
 733 err_tcam_region_enable:
 734         mlxsw_sp_acl_tcam_region_free(mlxsw_sp, region);
 735 err_tcam_region_alloc:
 736 err_tcam_region_associate:
 737         mlxsw_sp_acl_tcam_region_id_put(tcam, region->id);
 738 err_region_id_get:
 739         kfree(region);
 740         return ERR_PTR(err);
 741 }
 742
 743 static void
 744 mlxsw_sp_acl_tcam_region_destroy(struct mlxsw_sp *mlxsw_sp,
 745                                  struct mlxsw_sp_acl_tcam_region *region)
 746 {
 747         const struct mlxsw_sp_acl_tcam_ops *ops = mlxsw_sp->acl_tcam_ops;
 748
 749         ops->region_fini(mlxsw_sp, region->priv);
 750         mlxsw_sp_acl_tcam_region_disable(mlxsw_sp, region);
 751         mlxsw_sp_acl_tcam_region_free(mlxsw_sp, region);
 752         mlxsw_sp_acl_tcam_region_id_put(region->group->tcam,
 753                                         region->id);
 754         kfree(region);
 755 }
 756
 757 static void
 758 mlxsw_sp_acl_tcam_vregion_rehash_work_schedule(struct mlxsw_sp_acl_tcam_vregion *vregion)
 759 {
 760         unsigned long interval = vregion->tcam->vregion_rehash_intrvl;
 761
 762         if (!interval)
 763                 return;
 764         mlxsw_core_schedule_dw(&vregion->rehash.dw,
 765                                msecs_to_jiffies(interval));
 766 }
 767
 768 static void
 769 mlxsw_sp_acl_tcam_vregion_rehash(struct mlxsw_sp *mlxsw_sp,
 770                                  struct mlxsw_sp_acl_tcam_vregion *vregion,
 771                                  int *credits);
 772
 773 static void mlxsw_sp_acl_tcam_vregion_rehash_work(struct work_struct *work)
 774 {
 775         struct mlxsw_sp_acl_tcam_vregion *vregion =
 776                 container_of(work, struct mlxsw_sp_acl_tcam_vregion,
 777                              rehash.dw.work);
 778         int credits = MLXSW_SP_ACL_TCAM_VREGION_REHASH_CREDITS;
 779
 780         mlxsw_sp_acl_tcam_vregion_rehash(vregion->mlxsw_sp, vregion, &credits);
 781         if (credits < 0)
 782                 /* Rehash gone out of credits so it was interrupted.
 783                  * Schedule the work as soon as possible to continue.
 784                  */
 785                 mlxsw_core_schedule_dw(&vregion->rehash.dw, 0);
 786         else
 787                 mlxsw_sp_acl_tcam_vregion_rehash_work_schedule(vregion);
 788 }
 789
 790 static void
 791 mlxsw_sp_acl_tcam_rehash_ctx_vchunk_changed(struct mlxsw_sp_acl_tcam_vchunk *vchunk)
 792 {
 793         struct mlxsw_sp_acl_tcam_vregion *vregion = vchunk->vregion;
 794
 795         /* If a rule was added or deleted from vchunk which is currently
 796          * under rehash migration, we have to reset the ventry pointers
 797          * to make sure all rules are properly migrated.
 798          */
 799         if (vregion->rehash.ctx.current_vchunk == vchunk) {
 800                 vregion->rehash.ctx.start_ventry = NULL;
 801                 vregion->rehash.ctx.stop_ventry = NULL;
 802         }
 803 }
 804
 805 static void
 806 mlxsw_sp_acl_tcam_rehash_ctx_vregion_changed(struct mlxsw_sp_acl_tcam_vregion *vregion)
 807 {
 808         /* If a chunk was added or deleted from vregion we have to reset
 809          * the current chunk pointer to make sure all chunks
 810          * are properly migrated.
 811          */
 812         vregion->rehash.ctx.current_vchunk = NULL;
 813 }
 814
 815 static struct mlxsw_sp_acl_tcam_vregion *
 816 mlxsw_sp_acl_tcam_vregion_create(struct mlxsw_sp *mlxsw_sp,
 817                                  struct mlxsw_sp_acl_tcam_vgroup *vgroup,
 818                                  unsigned int priority,
 819                                  struct mlxsw_afk_element_usage *elusage)
 820 {
 821         const struct mlxsw_sp_acl_tcam_ops *ops = mlxsw_sp->acl_tcam_ops;
 822         struct mlxsw_afk *afk = mlxsw_sp_acl_afk(mlxsw_sp->acl);
 823         struct mlxsw_sp_acl_tcam *tcam = vgroup->group.tcam;
 824         struct mlxsw_sp_acl_tcam_vregion *vregion;
 825         int err;
 826
 827         vregion = kzalloc(sizeof(*vregion), GFP_KERNEL);
 828         if (!vregion)
 829                 return ERR_PTR(-ENOMEM);
 830         INIT_LIST_HEAD(&vregion->vchunk_list);
 831         mutex_init(&vregion->lock);
 832         vregion->tcam = tcam;
 833         vregion->mlxsw_sp = mlxsw_sp;
 834         vregion->vgroup = vgroup;
 835         vregion->ref_count = 1;
 836
 837         vregion->key_info = mlxsw_afk_key_info_get(afk, elusage);
 838         if (IS_ERR(vregion->key_info)) {
 839                 err = PTR_ERR(vregion->key_info);
 840                 goto err_key_info_get;
 841         }
 842
 843         vregion->region = mlxsw_sp_acl_tcam_region_create(mlxsw_sp, tcam,
 844                                                           vregion, NULL);
 845         if (IS_ERR(vregion->region)) {
 846                 err = PTR_ERR(vregion->region);
 847                 goto err_region_create;
 848         }
 849
 850         err = mlxsw_sp_acl_tcam_vgroup_vregion_attach(mlxsw_sp, vgroup, vregion,
 851                                                       priority);
 852         if (err)
 853                 goto err_vgroup_vregion_attach;
 854
 855         if (vgroup->vregion_rehash_enabled && ops->region_rehash_hints_get) {
 856                 /* Create the delayed work for vregion periodic rehash */
 857                 INIT_DELAYED_WORK(&vregion->rehash.dw,
 858                                   mlxsw_sp_acl_tcam_vregion_rehash_work);
 859                 mlxsw_sp_acl_tcam_vregion_rehash_work_schedule(vregion);
 860                 mutex_lock(&tcam->lock);
 861                 list_add_tail(&vregion->tlist, &tcam->vregion_list);
 862                 mutex_unlock(&tcam->lock);
 863         }
 864
 865         return vregion;
 866
 867 err_vgroup_vregion_attach:
 868         mlxsw_sp_acl_tcam_region_destroy(mlxsw_sp, vregion->region);
 869 err_region_create:
 870         mlxsw_afk_key_info_put(vregion->key_info);
 871 err_key_info_get:
 872         kfree(vregion);
 873         return ERR_PTR(err);
 874 }
 875
 876 static void
 877 mlxsw_sp_acl_tcam_vregion_destroy(struct mlxsw_sp *mlxsw_sp,
 878                                   struct mlxsw_sp_acl_tcam_vregion *vregion)
 879 {
 880         const struct mlxsw_sp_acl_tcam_ops *ops = mlxsw_sp->acl_tcam_ops;
 881         struct mlxsw_sp_acl_tcam_vgroup *vgroup = vregion->vgroup;
 882         struct mlxsw_sp_acl_tcam *tcam = vregion->tcam;
 883
 884         if (vgroup->vregion_rehash_enabled && ops->region_rehash_hints_get) {
 885                 mutex_lock(&tcam->lock);
 886                 list_del(&vregion->tlist);
 887                 mutex_unlock(&tcam->lock);
 888                 cancel_delayed_work_sync(&vregion->rehash.dw);
 889         }
 890         mlxsw_sp_acl_tcam_vgroup_vregion_detach(mlxsw_sp, vregion);
 891         if (vregion->region2)
 892                 mlxsw_sp_acl_tcam_region_destroy(mlxsw_sp, vregion->region2);
 893         mlxsw_sp_acl_tcam_region_destroy(mlxsw_sp, vregion->region);
 894         mlxsw_afk_key_info_put(vregion->key_info);
 895         mutex_destroy(&vregion->lock);
 896         kfree(vregion);
 897 }
 898
 899 u32 mlxsw_sp_acl_tcam_vregion_rehash_intrvl_get(struct mlxsw_sp *mlxsw_sp,
 900                                                 struct mlxsw_sp_acl_tcam *tcam)
 901 {
 902         const struct mlxsw_sp_acl_tcam_ops *ops = mlxsw_sp->acl_tcam_ops;
 903         u32 vregion_rehash_intrvl;
 904
 905         if (WARN_ON(!ops->region_rehash_hints_get))
 906                 return 0;
 907         vregion_rehash_intrvl = tcam->vregion_rehash_intrvl;
 908         return vregion_rehash_intrvl;
 909 }
 910
 911 int mlxsw_sp_acl_tcam_vregion_rehash_intrvl_set(struct mlxsw_sp *mlxsw_sp,
 912                                                 struct mlxsw_sp_acl_tcam *tcam,
 913                                                 u32 val)
 914 {
 915         const struct mlxsw_sp_acl_tcam_ops *ops = mlxsw_sp->acl_tcam_ops;
 916         struct mlxsw_sp_acl_tcam_vregion *vregion;
 917
 918         if (val < MLXSW_SP_ACL_TCAM_VREGION_REHASH_INTRVL_MIN && val)
 919                 return -EINVAL;
 920         if (WARN_ON(!ops->region_rehash_hints_get))
 921                 return -EOPNOTSUPP;
 922         tcam->vregion_rehash_intrvl = val;
 923         mutex_lock(&tcam->lock);
 924         list_for_each_entry(vregion, &tcam->vregion_list, tlist) {
 925                 if (val)
 926                         mlxsw_core_schedule_dw(&vregion->rehash.dw, 0);
 927                 else
 928                         cancel_delayed_work_sync(&vregion->rehash.dw);
 929         }
 930         mutex_unlock(&tcam->lock);
 931         return 0;
 932 }
 933
 934 static struct mlxsw_sp_acl_tcam_vregion *
 935 mlxsw_sp_acl_tcam_vregion_get(struct mlxsw_sp *mlxsw_sp,
 936                               struct mlxsw_sp_acl_tcam_vgroup *vgroup,
 937                               unsigned int priority,
 938                               struct mlxsw_afk_element_usage *elusage)
 939 {
 940         struct mlxsw_afk_element_usage vregion_elusage;
 941         struct mlxsw_sp_acl_tcam_vregion *vregion;
 942         bool need_split;
 943
 944         vregion = mlxsw_sp_acl_tcam_vgroup_vregion_find(vgroup, priority,
 945                                                         elusage, &need_split);
 946         if (vregion) {
 947                 if (need_split) {
 948                         /* According to priority, new vchunk should belong to
 949                          * an existing vregion. However, this vchunk needs
 950                          * elements that vregion does not contain. We need
 951                          * to split the existing vregion into two and create
 952                          * a new vregion for the new vchunk in between.
 953                          * This is not supported now.
 954                          */
 955                         return ERR_PTR(-EOPNOTSUPP);
 956                 }
 957                 vregion->ref_count++;
 958                 return vregion;
 959         }
 960
 961         mlxsw_sp_acl_tcam_vgroup_use_patterns(vgroup, elusage,
 962                                               &vregion_elusage);
 963
 964         return mlxsw_sp_acl_tcam_vregion_create(mlxsw_sp, vgroup, priority,
 965                                                 &vregion_elusage);
 966 }
 967
 968 static void
 969 mlxsw_sp_acl_tcam_vregion_put(struct mlxsw_sp *mlxsw_sp,
 970                               struct mlxsw_sp_acl_tcam_vregion *vregion)
 971 {
 972         if (--vregion->ref_count)
 973                 return;
 974         mlxsw_sp_acl_tcam_vregion_destroy(mlxsw_sp, vregion);
 975 }
 976
 977 static struct mlxsw_sp_acl_tcam_chunk *
 978 mlxsw_sp_acl_tcam_chunk_create(struct mlxsw_sp *mlxsw_sp,
 979                                struct mlxsw_sp_acl_tcam_vchunk *vchunk,
 980                                struct mlxsw_sp_acl_tcam_region *region)
 981 {
 982         const struct mlxsw_sp_acl_tcam_ops *ops = mlxsw_sp->acl_tcam_ops;
 983         struct mlxsw_sp_acl_tcam_chunk *chunk;
 984
 985         chunk = kzalloc(sizeof(*chunk) + ops->chunk_priv_size, GFP_KERNEL);
 986         if (!chunk)
 987                 return ERR_PTR(-ENOMEM);
 988         chunk->vchunk = vchunk;
 989         chunk->region = region;
 990
 991         ops->chunk_init(region->priv, chunk->priv, vchunk->priority);
 992         return chunk;
 993 }
 994
 995 static void
 996 mlxsw_sp_acl_tcam_chunk_destroy(struct mlxsw_sp *mlxsw_sp,
 997                                 struct mlxsw_sp_acl_tcam_chunk *chunk)
 998 {
 999         const struct mlxsw_sp_acl_tcam_ops *ops = mlxsw_sp->acl_tcam_ops;
1000
1001         ops->chunk_fini(chunk->priv);
1002         kfree(chunk);
1003 }
1004
1005 static struct mlxsw_sp_acl_tcam_vchunk *
1006 mlxsw_sp_acl_tcam_vchunk_create(struct mlxsw_sp *mlxsw_sp,
1007                                 struct mlxsw_sp_acl_tcam_vgroup *vgroup,
1008                                 unsigned int priority,
1009                                 struct mlxsw_afk_element_usage *elusage)
1010 {
1011         struct mlxsw_sp_acl_tcam_vchunk *vchunk, *vchunk2;
1012         struct mlxsw_sp_acl_tcam_vregion *vregion;
1013         struct list_head *pos;
1014         int err;
1015
1016         if (priority == MLXSW_SP_ACL_TCAM_CATCHALL_PRIO)
1017                 return ERR_PTR(-EINVAL);
1018
1019         vchunk = kzalloc(sizeof(*vchunk), GFP_KERNEL);
1020         if (!vchunk)
1021                 return ERR_PTR(-ENOMEM);
1022         INIT_LIST_HEAD(&vchunk->ventry_list);
1023         vchunk->priority = priority;
1024         vchunk->vgroup = vgroup;
1025         vchunk->ref_count = 1;
1026
1027         vregion = mlxsw_sp_acl_tcam_vregion_get(mlxsw_sp, vgroup,
1028                                                 priority, elusage);
1029         if (IS_ERR(vregion)) {
1030                 err = PTR_ERR(vregion);
1031                 goto err_vregion_get;
1032         }
1033
1034         vchunk->vregion = vregion;
1035
1036         err = rhashtable_insert_fast(&vgroup->vchunk_ht, &vchunk->ht_node,
1037                                      mlxsw_sp_acl_tcam_vchunk_ht_params);
1038         if (err)
1039                 goto err_rhashtable_insert;
1040
1041         mutex_lock(&vregion->lock);
1042         vchunk->chunk = mlxsw_sp_acl_tcam_chunk_create(mlxsw_sp, vchunk,
1043                                                        vchunk->vregion->region);
1044         if (IS_ERR(vchunk->chunk)) {
1045                 mutex_unlock(&vregion->lock);
1046                 err = PTR_ERR(vchunk->chunk);
1047                 goto err_chunk_create;
1048         }
1049
1050         mlxsw_sp_acl_tcam_rehash_ctx_vregion_changed(vregion);
1051
1052         /* Position the vchunk inside the list according to priority */
1053         list_for_each(pos, &vregion->vchunk_list) {
1054                 vchunk2 = list_entry(pos, typeof(*vchunk2), list);
1055                 if (vchunk2->priority > priority)
1056                         break;
1057         }
1058         list_add_tail(&vchunk->list, pos);
1059         mutex_unlock(&vregion->lock);
1060         mlxsw_sp_acl_tcam_vgroup_prio_update(vgroup);
1061
1062         return vchunk;
1063
1064 err_chunk_create:
1065         rhashtable_remove_fast(&vgroup->vchunk_ht, &vchunk->ht_node,
1066                                mlxsw_sp_acl_tcam_vchunk_ht_params);
1067 err_rhashtable_insert:
1068         mlxsw_sp_acl_tcam_vregion_put(mlxsw_sp, vregion);
1069 err_vregion_get:
1070         kfree(vchunk);
1071         return ERR_PTR(err);
1072 }
1073
1074 static void
1075 mlxsw_sp_acl_tcam_vchunk_destroy(struct mlxsw_sp *mlxsw_sp,
1076                                  struct mlxsw_sp_acl_tcam_vchunk *vchunk)
1077 {
1078         struct mlxsw_sp_acl_tcam_vregion *vregion = vchunk->vregion;
1079         struct mlxsw_sp_acl_tcam_vgroup *vgroup = vchunk->vgroup;
1080
1081         mutex_lock(&vregion->lock);
1082         mlxsw_sp_acl_tcam_rehash_ctx_vregion_changed(vregion);
1083         list_del(&vchunk->list);
1084         if (vchunk->chunk2)
1085                 mlxsw_sp_acl_tcam_chunk_destroy(mlxsw_sp, vchunk->chunk2);
1086         mlxsw_sp_acl_tcam_chunk_destroy(mlxsw_sp, vchunk->chunk);
1087         mutex_unlock(&vregion->lock);
1088         rhashtable_remove_fast(&vgroup->vchunk_ht, &vchunk->ht_node,
1089                                mlxsw_sp_acl_tcam_vchunk_ht_params);
1090         mlxsw_sp_acl_tcam_vregion_put(mlxsw_sp, vchunk->vregion);
1091         kfree(vchunk);
1092         mlxsw_sp_acl_tcam_vgroup_prio_update(vgroup);
1093 }
1094
1095 static struct mlxsw_sp_acl_tcam_vchunk *
1096 mlxsw_sp_acl_tcam_vchunk_get(struct mlxsw_sp *mlxsw_sp,
1097                              struct mlxsw_sp_acl_tcam_vgroup *vgroup,
1098                              unsigned int priority,
1099                              struct mlxsw_afk_element_usage *elusage)
1100 {
1101         struct mlxsw_sp_acl_tcam_vchunk *vchunk;
1102
1103         vchunk = rhashtable_lookup_fast(&vgroup->vchunk_ht, &priority,
1104                                         mlxsw_sp_acl_tcam_vchunk_ht_params);
1105         if (vchunk) {
1106                 if (WARN_ON(!mlxsw_afk_key_info_subset(vchunk->vregion->key_info,
1107                                                        elusage)))
1108                         return ERR_PTR(-EINVAL);
1109                 vchunk->ref_count++;
1110                 return vchunk;
1111         }
1112         return mlxsw_sp_acl_tcam_vchunk_create(mlxsw_sp, vgroup,
1113                                                priority, elusage);
1114 }
1115
1116 static void
1117 mlxsw_sp_acl_tcam_vchunk_put(struct mlxsw_sp *mlxsw_sp,
1118                              struct mlxsw_sp_acl_tcam_vchunk *vchunk)
1119 {
1120         if (--vchunk->ref_count)
1121                 return;
1122         mlxsw_sp_acl_tcam_vchunk_destroy(mlxsw_sp, vchunk);
1123 }
1124
1125 static struct mlxsw_sp_acl_tcam_entry *
1126 mlxsw_sp_acl_tcam_entry_create(struct mlxsw_sp *mlxsw_sp,
1127                                struct mlxsw_sp_acl_tcam_ventry *ventry,
1128                                struct mlxsw_sp_acl_tcam_chunk *chunk)
1129 {
1130         const struct mlxsw_sp_acl_tcam_ops *ops = mlxsw_sp->acl_tcam_ops;
1131         struct mlxsw_sp_acl_tcam_entry *entry;
1132         int err;
1133
1134         entry = kzalloc(sizeof(*entry) + ops->entry_priv_size, GFP_KERNEL);
1135         if (!entry)
1136                 return ERR_PTR(-ENOMEM);
1137         entry->ventry = ventry;
1138         entry->chunk = chunk;
1139
1140         err = ops->entry_add(mlxsw_sp, chunk->region->priv, chunk->priv,
1141                              entry->priv, ventry->rulei);
1142         if (err)
1143                 goto err_entry_add;
1144
1145         return entry;
1146
1147 err_entry_add:
1148         kfree(entry);
1149         return ERR_PTR(err);
1150 }
1151
1152 static void mlxsw_sp_acl_tcam_entry_destroy(struct mlxsw_sp *mlxsw_sp,
1153                                             struct mlxsw_sp_acl_tcam_entry *entry)
1154 {
1155         const struct mlxsw_sp_acl_tcam_ops *ops = mlxsw_sp->acl_tcam_ops;
1156
1157         ops->entry_del(mlxsw_sp, entry->chunk->region->priv,
1158                        entry->chunk->priv, entry->priv);
1159         kfree(entry);
1160 }
1161
1162 static int
1163 mlxsw_sp_acl_tcam_entry_action_replace(struct mlxsw_sp *mlxsw_sp,
1164                                        struct mlxsw_sp_acl_tcam_region *region,
1165                                        struct mlxsw_sp_acl_tcam_entry *entry,
1166                                        struct mlxsw_sp_acl_rule_info *rulei)
1167 {
1168         const struct mlxsw_sp_acl_tcam_ops *ops = mlxsw_sp->acl_tcam_ops;
1169
1170         return ops->entry_action_replace(mlxsw_sp, region->priv,
1171                                          entry->priv, rulei);
1172 }
1173
1174 static int
1175 mlxsw_sp_acl_tcam_entry_activity_get(struct mlxsw_sp *mlxsw_sp,
1176                                      struct mlxsw_sp_acl_tcam_entry *entry,
1177                                      bool *activity)
1178 {
1179         const struct mlxsw_sp_acl_tcam_ops *ops = mlxsw_sp->acl_tcam_ops;
1180
1181         return ops->entry_activity_get(mlxsw_sp, entry->chunk->region->priv,
1182                                        entry->priv, activity);
1183 }
1184
1185 static int mlxsw_sp_acl_tcam_ventry_add(struct mlxsw_sp *mlxsw_sp,
1186                                         struct mlxsw_sp_acl_tcam_vgroup *vgroup,
1187                                         struct mlxsw_sp_acl_tcam_ventry *ventry,
1188                                         struct mlxsw_sp_acl_rule_info *rulei)
1189 {
1190         struct mlxsw_sp_acl_tcam_vregion *vregion;
1191         struct mlxsw_sp_acl_tcam_vchunk *vchunk;
1192         int err;
1193
1194         vchunk = mlxsw_sp_acl_tcam_vchunk_get(mlxsw_sp, vgroup, rulei->priority,
1195                                               &rulei->values.elusage);
1196         if (IS_ERR(vchunk))
1197                 return PTR_ERR(vchunk);
1198
1199         ventry->vchunk = vchunk;
1200         ventry->rulei = rulei;
1201         vregion = vchunk->vregion;
1202
1203         mutex_lock(&vregion->lock);
1204         ventry->entry = mlxsw_sp_acl_tcam_entry_create(mlxsw_sp, ventry,
1205                                                        vchunk->chunk);
1206         if (IS_ERR(ventry->entry)) {
1207                 mutex_unlock(&vregion->lock);
1208                 err = PTR_ERR(ventry->entry);
1209                 goto err_entry_create;
1210         }
1211
1212         list_add_tail(&ventry->list, &vchunk->ventry_list);
1213         mlxsw_sp_acl_tcam_rehash_ctx_vchunk_changed(vchunk);
1214         mutex_unlock(&vregion->lock);
1215
1216         return 0;
1217
1218 err_entry_create:
1219         mlxsw_sp_acl_tcam_vchunk_put(mlxsw_sp, vchunk);
1220         return err;
1221 }
1222
1223 static void mlxsw_sp_acl_tcam_ventry_del(struct mlxsw_sp *mlxsw_sp,
1224                                          struct mlxsw_sp_acl_tcam_ventry *ventry)
1225 {
1226         struct mlxsw_sp_acl_tcam_vchunk *vchunk = ventry->vchunk;
1227         struct mlxsw_sp_acl_tcam_vregion *vregion = vchunk->vregion;
1228
1229         mutex_lock(&vregion->lock);
1230         mlxsw_sp_acl_tcam_rehash_ctx_vchunk_changed(vchunk);
1231         list_del(&ventry->list);
1232         mlxsw_sp_acl_tcam_entry_destroy(mlxsw_sp, ventry->entry);
1233         mutex_unlock(&vregion->lock);
1234         mlxsw_sp_acl_tcam_vchunk_put(mlxsw_sp, vchunk);
1235 }
1236
1237 static int
1238 mlxsw_sp_acl_tcam_ventry_action_replace(struct mlxsw_sp *mlxsw_sp,
1239                                         struct mlxsw_sp_acl_tcam_ventry *ventry,
1240                                         struct mlxsw_sp_acl_rule_info *rulei)
1241 {
1242         struct mlxsw_sp_acl_tcam_vchunk *vchunk = ventry->vchunk;
1243
1244         return mlxsw_sp_acl_tcam_entry_action_replace(mlxsw_sp,
1245                                                       vchunk->vregion->region,
1246                                                       ventry->entry, rulei);
1247 }
1248
1249 static int
1250 mlxsw_sp_acl_tcam_ventry_activity_get(struct mlxsw_sp *mlxsw_sp,
1251                                       struct mlxsw_sp_acl_tcam_ventry *ventry,
1252                                       bool *activity)
1253 {
1254         return mlxsw_sp_acl_tcam_entry_activity_get(mlxsw_sp,
1255                                                     ventry->entry, activity);
1256 }
1257
1258 static int
1259 mlxsw_sp_acl_tcam_ventry_migrate(struct mlxsw_sp *mlxsw_sp,
1260                                  struct mlxsw_sp_acl_tcam_ventry *ventry,
1261                                  struct mlxsw_sp_acl_tcam_chunk *chunk,
1262                                  int *credits)
1263 {
1264         struct mlxsw_sp_acl_tcam_entry *new_entry;
1265
1266         /* First check if the entry is not already where we want it to be. */
1267         if (ventry->entry->chunk == chunk)
1268                 return 0;
1269
1270         if (--(*credits) < 0)
1271                 return 0;
1272
1273         new_entry = mlxsw_sp_acl_tcam_entry_create(mlxsw_sp, ventry, chunk);
1274         if (IS_ERR(new_entry))
1275                 return PTR_ERR(new_entry);
1276         mlxsw_sp_acl_tcam_entry_destroy(mlxsw_sp, ventry->entry);
1277         ventry->entry = new_entry;
1278         return 0;
1279 }
1280
1281 static int
1282 mlxsw_sp_acl_tcam_vchunk_migrate_start(struct mlxsw_sp *mlxsw_sp,
1283                                        struct mlxsw_sp_acl_tcam_vchunk *vchunk,
1284                                        struct mlxsw_sp_acl_tcam_region *region,
1285                                        struct mlxsw_sp_acl_tcam_rehash_ctx *ctx)
1286 {
1287         struct mlxsw_sp_acl_tcam_chunk *new_chunk;
1288
1289         new_chunk = mlxsw_sp_acl_tcam_chunk_create(mlxsw_sp, vchunk, region);
1290         if (IS_ERR(new_chunk))
1291                 return PTR_ERR(new_chunk);
1292         vchunk->chunk2 = vchunk->chunk;
1293         vchunk->chunk = new_chunk;
1294         ctx->current_vchunk = vchunk;
1295         ctx->start_ventry = NULL;
1296         ctx->stop_ventry = NULL;
1297         return 0;
1298 }
1299
1300 static void
1301 mlxsw_sp_acl_tcam_vchunk_migrate_end(struct mlxsw_sp *mlxsw_sp,
1302                                      struct mlxsw_sp_acl_tcam_vchunk *vchunk,
1303                                      struct mlxsw_sp_acl_tcam_rehash_ctx *ctx)
1304 {
1305         mlxsw_sp_acl_tcam_chunk_destroy(mlxsw_sp, vchunk->chunk2);
1306         vchunk->chunk2 = NULL;
1307         ctx->current_vchunk = NULL;
1308 }
1309
1310 static int
1311 mlxsw_sp_acl_tcam_vchunk_migrate_one(struct mlxsw_sp *mlxsw_sp,
1312                                      struct mlxsw_sp_acl_tcam_vchunk *vchunk,
1313                                      struct mlxsw_sp_acl_tcam_region *region,
1314                                      struct mlxsw_sp_acl_tcam_rehash_ctx *ctx,
1315                                      int *credits)
1316 {
1317         struct mlxsw_sp_acl_tcam_ventry *ventry;
1318         int err;
1319
1320         if (vchunk->chunk->region != region) {
1321                 err = mlxsw_sp_acl_tcam_vchunk_migrate_start(mlxsw_sp, vchunk,
1322                                                              region, ctx);
1323                 if (err)
1324                         return err;
1325         } else if (!vchunk->chunk2) {
1326                 /* The chunk is already as it should be, nothing to do. */
1327                 return 0;
1328         }
1329
1330         /* If the migration got interrupted, we have the ventry to start from
1331          * stored in context.
1332          */
1333         if (ctx->start_ventry)
1334                 ventry = ctx->start_ventry;
1335         else
1336                 ventry = list_first_entry(&vchunk->ventry_list,
1337                                           typeof(*ventry), list);
1338
1339         list_for_each_entry_from(ventry, &vchunk->ventry_list, list) {
1340                 /* During rollback, once we reach the ventry that failed
1341                  * to migrate, we are done.
1342                  */
1343                 if (ventry == ctx->stop_ventry)
1344                         break;
1345
1346                 err = mlxsw_sp_acl_tcam_ventry_migrate(mlxsw_sp, ventry,
1347                                                        vchunk->chunk, credits);
1348                 if (err) {
1349                         if (ctx->this_is_rollback) {
1350                                 /* Save the ventry which we ended with and try
1351                                  * to continue later on.
1352                                  */
1353                                 ctx->start_ventry = ventry;
1354                                 return err;
1355                         }
1356                         /* Swap the chunk and chunk2 pointers so the follow-up
1357                          * rollback call will see the original chunk pointer
1358                          * in vchunk->chunk.
1359                          */
1360                         swap(vchunk->chunk, vchunk->chunk2);
1361                         /* The rollback has to be done from beginning of the
1362                          * chunk, that is why we have to null the start_ventry.
1363                          * However, we know where to stop the rollback,
1364                          * at the current ventry.
1365                          */
1366                         ctx->start_ventry = NULL;
1367                         ctx->stop_ventry = ventry;
1368                         return err;
1369                 } else if (*credits < 0) {
1370                         /* We are out of credits, the rest of the ventries
1371                          * will be migrated later. Save the ventry
1372                          * which we ended with.
1373                          */
1374                         ctx->start_ventry = ventry;
1375                         return 0;
1376                 }
1377         }
1378
1379         mlxsw_sp_acl_tcam_vchunk_migrate_end(mlxsw_sp, vchunk, ctx);
1380         return 0;
1381 }
1382
1383 static int
1384 mlxsw_sp_acl_tcam_vchunk_migrate_all(struct mlxsw_sp *mlxsw_sp,
1385                                      struct mlxsw_sp_acl_tcam_vregion *vregion,
1386                                      struct mlxsw_sp_acl_tcam_rehash_ctx *ctx,
1387                                      int *credits)
1388 {
1389         struct mlxsw_sp_acl_tcam_vchunk *vchunk;
1390         int err;
1391
1392         /* If the migration got interrupted, we have the vchunk
1393          * we are working on stored in context.
1394          */
1395         if (ctx->current_vchunk)
1396                 vchunk = ctx->current_vchunk;
1397         else
1398                 vchunk = list_first_entry(&vregion->vchunk_list,
1399                                           typeof(*vchunk), list);
1400
1401         list_for_each_entry_from(vchunk, &vregion->vchunk_list, list) {
1402                 err = mlxsw_sp_acl_tcam_vchunk_migrate_one(mlxsw_sp, vchunk,
1403                                                            vregion->region,
1404                                                            ctx, credits);
1405                 if (err || *credits < 0)
1406                         return err;
1407         }
1408         return 0;
1409 }
1410
1411 static int
1412 mlxsw_sp_acl_tcam_vregion_migrate(struct mlxsw_sp *mlxsw_sp,
1413                                   struct mlxsw_sp_acl_tcam_vregion *vregion,
1414                                   struct mlxsw_sp_acl_tcam_rehash_ctx *ctx,
1415                                   int *credits)
1416 {
1417         int err, err2;
1418
1419         trace_mlxsw_sp_acl_tcam_vregion_migrate(mlxsw_sp, vregion);
1420         mutex_lock(&vregion->lock);
1421         err = mlxsw_sp_acl_tcam_vchunk_migrate_all(mlxsw_sp, vregion,
1422                                                    ctx, credits);
1423         if (err) {
1424                 /* In case migration was not successful, we need to swap
1425                  * so the original region pointer is assigned again
1426                  * to vregion->region.
1427                  */
1428                 swap(vregion->region, vregion->region2);
1429                 ctx->current_vchunk = NULL;
1430                 ctx->this_is_rollback = true;
1431                 err2 = mlxsw_sp_acl_tcam_vchunk_migrate_all(mlxsw_sp, vregion,
1432                                                             ctx, credits);
1433                 if (err2) {
1434                         trace_mlxsw_sp_acl_tcam_vregion_rehash_rollback_failed(mlxsw_sp,
1435                                                                                vregion);
1436                         dev_err(mlxsw_sp->bus_info->dev, "Failed to rollback during vregion migration fail\n");
1437                         /* Let the rollback to be continued later on. */
1438                 }
1439         }
1440         mutex_unlock(&vregion->lock);
1441         trace_mlxsw_sp_acl_tcam_vregion_migrate_end(mlxsw_sp, vregion);
1442         return err;
1443 }
1444
1445 static bool
1446 mlxsw_sp_acl_tcam_vregion_rehash_in_progress(const struct mlxsw_sp_acl_tcam_rehash_ctx *ctx)
1447 {
1448         return ctx->hints_priv;
1449 }
1450
1451 static int
1452 mlxsw_sp_acl_tcam_vregion_rehash_start(struct mlxsw_sp *mlxsw_sp,
1453                                        struct mlxsw_sp_acl_tcam_vregion *vregion,
1454                                        struct mlxsw_sp_acl_tcam_rehash_ctx *ctx)
1455 {
1456         const struct mlxsw_sp_acl_tcam_ops *ops = mlxsw_sp->acl_tcam_ops;
1457         unsigned int priority = mlxsw_sp_acl_tcam_vregion_prio(vregion);
1458         struct mlxsw_sp_acl_tcam_region *new_region;
1459         void *hints_priv;
1460         int err;
1461
1462         trace_mlxsw_sp_acl_tcam_vregion_rehash(mlxsw_sp, vregion);
1463
1464         hints_priv = ops->region_rehash_hints_get(vregion->region->priv);
1465         if (IS_ERR(hints_priv))
1466                 return PTR_ERR(hints_priv);
1467
1468         new_region = mlxsw_sp_acl_tcam_region_create(mlxsw_sp, vregion->tcam,
1469                                                      vregion, hints_priv);
1470         if (IS_ERR(new_region)) {
1471                 err = PTR_ERR(new_region);
1472                 goto err_region_create;
1473         }
1474
1475         /* vregion->region contains the pointer to the new region
1476          * we are going to migrate to.
1477          */
1478         vregion->region2 = vregion->region;
1479         vregion->region = new_region;
1480         err = mlxsw_sp_acl_tcam_group_region_attach(mlxsw_sp,
1481                                                     vregion->region2->group,
1482                                                     new_region, priority,
1483                                                     vregion->region2);
1484         if (err)
1485                 goto err_group_region_attach;
1486
1487         ctx->hints_priv = hints_priv;
1488         ctx->this_is_rollback = false;
1489
1490         return 0;
1491
1492 err_group_region_attach:
1493         vregion->region = vregion->region2;
1494         vregion->region2 = NULL;
1495         mlxsw_sp_acl_tcam_region_destroy(mlxsw_sp, new_region);
1496 err_region_create:
1497         ops->region_rehash_hints_put(hints_priv);
1498         return err;
1499 }
1500
1501 static void
1502 mlxsw_sp_acl_tcam_vregion_rehash_end(struct mlxsw_sp *mlxsw_sp,
1503                                      struct mlxsw_sp_acl_tcam_vregion *vregion,
1504                                      struct mlxsw_sp_acl_tcam_rehash_ctx *ctx)
1505 {
1506         struct mlxsw_sp_acl_tcam_region *unused_region = vregion->region2;
1507         const struct mlxsw_sp_acl_tcam_ops *ops = mlxsw_sp->acl_tcam_ops;
1508
1509         vregion->region2 = NULL;
1510         mlxsw_sp_acl_tcam_group_region_detach(mlxsw_sp, unused_region);
1511         mlxsw_sp_acl_tcam_region_destroy(mlxsw_sp, unused_region);
1512         ops->region_rehash_hints_put(ctx->hints_priv);
1513         ctx->hints_priv = NULL;
1514 }
1515
1516 static void
1517 mlxsw_sp_acl_tcam_vregion_rehash(struct mlxsw_sp *mlxsw_sp,
1518                                  struct mlxsw_sp_acl_tcam_vregion *vregion,
1519                                  int *credits)
1520 {
1521         struct mlxsw_sp_acl_tcam_rehash_ctx *ctx = &vregion->rehash.ctx;
1522         int err;
1523
1524         /* Check if the previous rehash work was interrupted
1525          * which means we have to continue it now.
1526          * If not, start a new rehash.
1527          */
1528         if (!mlxsw_sp_acl_tcam_vregion_rehash_in_progress(ctx)) {
1529                 err = mlxsw_sp_acl_tcam_vregion_rehash_start(mlxsw_sp,
1530                                                              vregion, ctx);
1531                 if (err) {
1532                         if (err != -EAGAIN)
1533                                 dev_err(mlxsw_sp->bus_info->dev, "Failed get rehash hints\n");
1534                         return;
1535                 }
1536         }
1537
1538         err = mlxsw_sp_acl_tcam_vregion_migrate(mlxsw_sp, vregion,
1539                                                 ctx, credits);
1540         if (err) {
1541                 dev_err(mlxsw_sp->bus_info->dev, "Failed to migrate vregion\n");
1542         }
1543
1544         if (*credits >= 0)
1545                 mlxsw_sp_acl_tcam_vregion_rehash_end(mlxsw_sp, vregion, ctx);
1546 }
1547
1548 static const enum mlxsw_afk_element mlxsw_sp_acl_tcam_pattern_ipv4[] = {
1549         MLXSW_AFK_ELEMENT_SRC_SYS_PORT,
1550         MLXSW_AFK_ELEMENT_DMAC_32_47,
1551         MLXSW_AFK_ELEMENT_DMAC_0_31,
1552         MLXSW_AFK_ELEMENT_SMAC_32_47,
1553         MLXSW_AFK_ELEMENT_SMAC_0_31,
1554         MLXSW_AFK_ELEMENT_ETHERTYPE,
1555         MLXSW_AFK_ELEMENT_IP_PROTO,
1556         MLXSW_AFK_ELEMENT_SRC_IP_0_31,
1557         MLXSW_AFK_ELEMENT_DST_IP_0_31,
1558         MLXSW_AFK_ELEMENT_DST_L4_PORT,
1559         MLXSW_AFK_ELEMENT_SRC_L4_PORT,
1560         MLXSW_AFK_ELEMENT_VID,
1561         MLXSW_AFK_ELEMENT_PCP,
1562         MLXSW_AFK_ELEMENT_TCP_FLAGS,
1563         MLXSW_AFK_ELEMENT_IP_TTL_,
1564         MLXSW_AFK_ELEMENT_IP_ECN,
1565         MLXSW_AFK_ELEMENT_IP_DSCP,
1566 };
1567
1568 static const enum mlxsw_afk_element mlxsw_sp_acl_tcam_pattern_ipv6[] = {
1569         MLXSW_AFK_ELEMENT_ETHERTYPE,
1570         MLXSW_AFK_ELEMENT_IP_PROTO,
1571         MLXSW_AFK_ELEMENT_SRC_IP_96_127,
1572         MLXSW_AFK_ELEMENT_SRC_IP_64_95,
1573         MLXSW_AFK_ELEMENT_SRC_IP_32_63,
1574         MLXSW_AFK_ELEMENT_SRC_IP_0_31,
1575         MLXSW_AFK_ELEMENT_DST_IP_96_127,
1576         MLXSW_AFK_ELEMENT_DST_IP_64_95,
1577         MLXSW_AFK_ELEMENT_DST_IP_32_63,
1578         MLXSW_AFK_ELEMENT_DST_IP_0_31,
1579         MLXSW_AFK_ELEMENT_DST_L4_PORT,
1580         MLXSW_AFK_ELEMENT_SRC_L4_PORT,
1581 };
1582
1583 static const struct mlxsw_sp_acl_tcam_pattern mlxsw_sp_acl_tcam_patterns[] = {
1584         {
1585                 .elements = mlxsw_sp_acl_tcam_pattern_ipv4,
1586                 .elements_count = ARRAY_SIZE(mlxsw_sp_acl_tcam_pattern_ipv4),
1587         },
1588         {
1589                 .elements = mlxsw_sp_acl_tcam_pattern_ipv6,
1590                 .elements_count = ARRAY_SIZE(mlxsw_sp_acl_tcam_pattern_ipv6),
1591         },
1592 };
1593
1594 #define MLXSW_SP_ACL_TCAM_PATTERNS_COUNT \
1595         ARRAY_SIZE(mlxsw_sp_acl_tcam_patterns)
1596
1597 struct mlxsw_sp_acl_tcam_flower_ruleset {
1598         struct mlxsw_sp_acl_tcam_vgroup vgroup;
1599 };
1600
1601 struct mlxsw_sp_acl_tcam_flower_rule {
1602         struct mlxsw_sp_acl_tcam_ventry ventry;
1603 };
1604
1605 static int
1606 mlxsw_sp_acl_tcam_flower_ruleset_add(struct mlxsw_sp *mlxsw_sp,
1607                                      struct mlxsw_sp_acl_tcam *tcam,
1608                                      void *ruleset_priv,
1609                                      struct mlxsw_afk_element_usage *tmplt_elusage,
1610                                      unsigned int *p_min_prio,
1611                                      unsigned int *p_max_prio)
1612 {
1613         struct mlxsw_sp_acl_tcam_flower_ruleset *ruleset = ruleset_priv;
1614
1615         return mlxsw_sp_acl_tcam_vgroup_add(mlxsw_sp, tcam, &ruleset->vgroup,
1616                                             mlxsw_sp_acl_tcam_patterns,
1617                                             MLXSW_SP_ACL_TCAM_PATTERNS_COUNT,
1618                                             tmplt_elusage, true,
1619                                             p_min_prio, p_max_prio);
1620 }
1621
1622 static void
1623 mlxsw_sp_acl_tcam_flower_ruleset_del(struct mlxsw_sp *mlxsw_sp,
1624                                      void *ruleset_priv)
1625 {
1626         struct mlxsw_sp_acl_tcam_flower_ruleset *ruleset = ruleset_priv;
1627
1628         mlxsw_sp_acl_tcam_vgroup_del(&ruleset->vgroup);
1629 }
1630
1631 static int
1632 mlxsw_sp_acl_tcam_flower_ruleset_bind(struct mlxsw_sp *mlxsw_sp,
1633                                       void *ruleset_priv,
1634                                       struct mlxsw_sp_port *mlxsw_sp_port,
1635                                       bool ingress)
1636 {
1637         struct mlxsw_sp_acl_tcam_flower_ruleset *ruleset = ruleset_priv;
1638
1639         return mlxsw_sp_acl_tcam_group_bind(mlxsw_sp, &ruleset->vgroup.group,
1640                                             mlxsw_sp_port, ingress);
1641 }
1642
1643 static void
1644 mlxsw_sp_acl_tcam_flower_ruleset_unbind(struct mlxsw_sp *mlxsw_sp,
1645                                         void *ruleset_priv,
1646                                         struct mlxsw_sp_port *mlxsw_sp_port,
1647                                         bool ingress)
1648 {
1649         struct mlxsw_sp_acl_tcam_flower_ruleset *ruleset = ruleset_priv;
1650
1651         mlxsw_sp_acl_tcam_group_unbind(mlxsw_sp, &ruleset->vgroup.group,
1652                                        mlxsw_sp_port, ingress);
1653 }
1654
1655 static u16
1656 mlxsw_sp_acl_tcam_flower_ruleset_group_id(void *ruleset_priv)
1657 {
1658         struct mlxsw_sp_acl_tcam_flower_ruleset *ruleset = ruleset_priv;
1659
1660         return mlxsw_sp_acl_tcam_group_id(&ruleset->vgroup.group);
1661 }
1662
1663 static int
1664 mlxsw_sp_acl_tcam_flower_rule_add(struct mlxsw_sp *mlxsw_sp,
1665                                   void *ruleset_priv, void *rule_priv,
1666                                   struct mlxsw_sp_acl_rule_info *rulei)
1667 {
1668         struct mlxsw_sp_acl_tcam_flower_ruleset *ruleset = ruleset_priv;
1669         struct mlxsw_sp_acl_tcam_flower_rule *rule = rule_priv;
1670
1671         return mlxsw_sp_acl_tcam_ventry_add(mlxsw_sp, &ruleset->vgroup,
1672                                             &rule->ventry, rulei);
1673 }
1674
1675 static void
1676 mlxsw_sp_acl_tcam_flower_rule_del(struct mlxsw_sp *mlxsw_sp, void *rule_priv)
1677 {
1678         struct mlxsw_sp_acl_tcam_flower_rule *rule = rule_priv;
1679
1680         mlxsw_sp_acl_tcam_ventry_del(mlxsw_sp, &rule->ventry);
1681 }
1682
1683 static int
1684 mlxsw_sp_acl_tcam_flower_rule_action_replace(struct mlxsw_sp *mlxsw_sp,
1685                                              void *rule_priv,
1686                                              struct mlxsw_sp_acl_rule_info *rulei)
1687 {
1688         return -EOPNOTSUPP;
1689 }
1690
1691 static int
1692 mlxsw_sp_acl_tcam_flower_rule_activity_get(struct mlxsw_sp *mlxsw_sp,
1693                                            void *rule_priv, bool *activity)
1694 {
1695         struct mlxsw_sp_acl_tcam_flower_rule *rule = rule_priv;
1696
1697         return mlxsw_sp_acl_tcam_ventry_activity_get(mlxsw_sp, &rule->ventry,
1698                                                      activity);
1699 }
1700
1701 static const struct mlxsw_sp_acl_profile_ops mlxsw_sp_acl_tcam_flower_ops = {
1702         .ruleset_priv_size      = sizeof(struct mlxsw_sp_acl_tcam_flower_ruleset),
1703         .ruleset_add            = mlxsw_sp_acl_tcam_flower_ruleset_add,
1704         .ruleset_del            = mlxsw_sp_acl_tcam_flower_ruleset_del,
1705         .ruleset_bind           = mlxsw_sp_acl_tcam_flower_ruleset_bind,
1706         .ruleset_unbind         = mlxsw_sp_acl_tcam_flower_ruleset_unbind,
1707         .ruleset_group_id       = mlxsw_sp_acl_tcam_flower_ruleset_group_id,
1708         .rule_priv_size         = sizeof(struct mlxsw_sp_acl_tcam_flower_rule),
1709         .rule_add               = mlxsw_sp_acl_tcam_flower_rule_add,
1710         .rule_del               = mlxsw_sp_acl_tcam_flower_rule_del,
1711         .rule_action_replace    = mlxsw_sp_acl_tcam_flower_rule_action_replace,
1712         .rule_activity_get      = mlxsw_sp_acl_tcam_flower_rule_activity_get,
1713 };
1714
1715 struct mlxsw_sp_acl_tcam_mr_ruleset {
1716         struct mlxsw_sp_acl_tcam_vchunk *vchunk;
1717         struct mlxsw_sp_acl_tcam_vgroup vgroup;
1718 };
1719
1720 struct mlxsw_sp_acl_tcam_mr_rule {
1721         struct mlxsw_sp_acl_tcam_ventry ventry;
1722 };
1723
1724 static int
1725 mlxsw_sp_acl_tcam_mr_ruleset_add(struct mlxsw_sp *mlxsw_sp,
1726                                  struct mlxsw_sp_acl_tcam *tcam,
1727                                  void *ruleset_priv,
1728                                  struct mlxsw_afk_element_usage *tmplt_elusage,
1729                                  unsigned int *p_min_prio,
1730                                  unsigned int *p_max_prio)
1731 {
1732         struct mlxsw_sp_acl_tcam_mr_ruleset *ruleset = ruleset_priv;
1733         int err;
1734
1735         err = mlxsw_sp_acl_tcam_vgroup_add(mlxsw_sp, tcam, &ruleset->vgroup,
1736                                            mlxsw_sp_acl_tcam_patterns,
1737                                            MLXSW_SP_ACL_TCAM_PATTERNS_COUNT,
1738                                            tmplt_elusage, false,
1739                                            p_min_prio, p_max_prio);
1740         if (err)
1741                 return err;
1742
1743         /* For most of the TCAM clients it would make sense to take a tcam chunk
1744          * only when the first rule is written. This is not the case for
1745          * multicast router as it is required to bind the multicast router to a
1746          * specific ACL Group ID which must exist in HW before multicast router
1747          * is initialized.
1748          */
1749         ruleset->vchunk = mlxsw_sp_acl_tcam_vchunk_get(mlxsw_sp,
1750                                                        &ruleset->vgroup, 1,
1751                                                        tmplt_elusage);
1752         if (IS_ERR(ruleset->vchunk)) {
1753                 err = PTR_ERR(ruleset->vchunk);
1754                 goto err_chunk_get;
1755         }
1756
1757         return 0;
1758
1759 err_chunk_get:
1760         mlxsw_sp_acl_tcam_vgroup_del(&ruleset->vgroup);
1761         return err;
1762 }
1763
1764 static void
1765 mlxsw_sp_acl_tcam_mr_ruleset_del(struct mlxsw_sp *mlxsw_sp, void *ruleset_priv)
1766 {
1767         struct mlxsw_sp_acl_tcam_mr_ruleset *ruleset = ruleset_priv;
1768
1769         mlxsw_sp_acl_tcam_vchunk_put(mlxsw_sp, ruleset->vchunk);
1770         mlxsw_sp_acl_tcam_vgroup_del(&ruleset->vgroup);
1771 }
1772
1773 static int
1774 mlxsw_sp_acl_tcam_mr_ruleset_bind(struct mlxsw_sp *mlxsw_sp, void *ruleset_priv,
1775                                   struct mlxsw_sp_port *mlxsw_sp_port,
1776                                   bool ingress)
1777 {
1778         /* Binding is done when initializing multicast router */
1779         return 0;
1780 }
1781
1782 static void
1783 mlxsw_sp_acl_tcam_mr_ruleset_unbind(struct mlxsw_sp *mlxsw_sp,
1784                                     void *ruleset_priv,
1785                                     struct mlxsw_sp_port *mlxsw_sp_port,
1786                                     bool ingress)
1787 {
1788 }
1789
1790 static u16
1791 mlxsw_sp_acl_tcam_mr_ruleset_group_id(void *ruleset_priv)
1792 {
1793         struct mlxsw_sp_acl_tcam_mr_ruleset *ruleset = ruleset_priv;
1794
1795         return mlxsw_sp_acl_tcam_group_id(&ruleset->vgroup.group);
1796 }
1797
1798 static int
1799 mlxsw_sp_acl_tcam_mr_rule_add(struct mlxsw_sp *mlxsw_sp, void *ruleset_priv,
1800                               void *rule_priv,
1801                               struct mlxsw_sp_acl_rule_info *rulei)
1802 {
1803         struct mlxsw_sp_acl_tcam_mr_ruleset *ruleset = ruleset_priv;
1804         struct mlxsw_sp_acl_tcam_mr_rule *rule = rule_priv;
1805
1806         return mlxsw_sp_acl_tcam_ventry_add(mlxsw_sp, &ruleset->vgroup,
1807                                            &rule->ventry, rulei);
1808 }
1809
1810 static void
1811 mlxsw_sp_acl_tcam_mr_rule_del(struct mlxsw_sp *mlxsw_sp, void *rule_priv)
1812 {
1813         struct mlxsw_sp_acl_tcam_mr_rule *rule = rule_priv;
1814
1815         mlxsw_sp_acl_tcam_ventry_del(mlxsw_sp, &rule->ventry);
1816 }
1817
1818 static int
1819 mlxsw_sp_acl_tcam_mr_rule_action_replace(struct mlxsw_sp *mlxsw_sp,
1820                                          void *rule_priv,
1821                                          struct mlxsw_sp_acl_rule_info *rulei)
1822 {
1823         struct mlxsw_sp_acl_tcam_mr_rule *rule = rule_priv;
1824
1825         return mlxsw_sp_acl_tcam_ventry_action_replace(mlxsw_sp, &rule->ventry,
1826                                                        rulei);
1827 }
1828
1829 static int
1830 mlxsw_sp_acl_tcam_mr_rule_activity_get(struct mlxsw_sp *mlxsw_sp,
1831                                        void *rule_priv, bool *activity)
1832 {
1833         struct mlxsw_sp_acl_tcam_mr_rule *rule = rule_priv;
1834
1835         return mlxsw_sp_acl_tcam_ventry_activity_get(mlxsw_sp, &rule->ventry,
1836                                                      activity);
1837 }
1838
1839 static const struct mlxsw_sp_acl_profile_ops mlxsw_sp_acl_tcam_mr_ops = {
1840         .ruleset_priv_size      = sizeof(struct mlxsw_sp_acl_tcam_mr_ruleset),
1841         .ruleset_add            = mlxsw_sp_acl_tcam_mr_ruleset_add,
1842         .ruleset_del            = mlxsw_sp_acl_tcam_mr_ruleset_del,
1843         .ruleset_bind           = mlxsw_sp_acl_tcam_mr_ruleset_bind,
1844         .ruleset_unbind         = mlxsw_sp_acl_tcam_mr_ruleset_unbind,
1845         .ruleset_group_id       = mlxsw_sp_acl_tcam_mr_ruleset_group_id,
1846         .rule_priv_size         = sizeof(struct mlxsw_sp_acl_tcam_mr_rule),
1847         .rule_add               = mlxsw_sp_acl_tcam_mr_rule_add,
1848         .rule_del               = mlxsw_sp_acl_tcam_mr_rule_del,
1849         .rule_action_replace    = mlxsw_sp_acl_tcam_mr_rule_action_replace,
1850         .rule_activity_get      = mlxsw_sp_acl_tcam_mr_rule_activity_get,
1851 };
1852
1853 static const struct mlxsw_sp_acl_profile_ops *
1854 mlxsw_sp_acl_tcam_profile_ops_arr[] = {
1855         [MLXSW_SP_ACL_PROFILE_FLOWER] = &mlxsw_sp_acl_tcam_flower_ops,
1856         [MLXSW_SP_ACL_PROFILE_MR] = &mlxsw_sp_acl_tcam_mr_ops,
1857 };
1858
1859 const struct mlxsw_sp_acl_profile_ops *
1860 mlxsw_sp_acl_tcam_profile_ops(struct mlxsw_sp *mlxsw_sp,
1861                               enum mlxsw_sp_acl_profile profile)
1862 {
1863         const struct mlxsw_sp_acl_profile_ops *ops;
1864
1865         if (WARN_ON(profile >= ARRAY_SIZE(mlxsw_sp_acl_tcam_profile_ops_arr)))
1866                 return NULL;
1867         ops = mlxsw_sp_acl_tcam_profile_ops_arr[profile];
1868         if (WARN_ON(!ops))
1869                 return NULL;
1870         return ops;
1871 }