1 // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0
2 /* Copyright (c) 2018 Mellanox Technologies. All rights reserved */
4 #include <linux/bitmap.h>
5 #include <linux/errno.h>
6 #include <linux/genalloc.h>
8 #include <linux/kernel.h>
9 #include <linux/list.h>
10 #include <linux/mutex.h>
11 #include <linux/objagg.h>
12 #include <linux/rtnetlink.h>
13 #include <linux/slab.h>
18 #include "spectrum_acl_tcam.h"
20 /* gen_pool_alloc() returns 0 when allocation fails, so use an offset */
21 #define MLXSW_SP_ACL_ERP_GENALLOC_OFFSET 0x100
22 #define MLXSW_SP_ACL_ERP_MAX_PER_REGION 16
24 struct mlxsw_sp_acl_erp_core {
25 unsigned int erpt_entries_size[MLXSW_SP_ACL_ATCAM_REGION_TYPE_MAX + 1];
26 struct gen_pool *erp_tables;
27 struct mlxsw_sp *mlxsw_sp;
28 struct mlxsw_sp_acl_bf *bf;
29 unsigned int num_erp_banks;
32 struct mlxsw_sp_acl_erp_key {
33 char mask[MLXSW_REG_PTCEX_FLEX_KEY_BLOCKS_LEN];
34 #define __MASK_LEN 0x38
35 #define __MASK_IDX(i) (__MASK_LEN - (i) - 1)
39 struct mlxsw_sp_acl_erp {
40 struct mlxsw_sp_acl_erp_key key;
43 DECLARE_BITMAP(mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN);
44 struct list_head list;
45 struct mlxsw_sp_acl_erp_table *erp_table;
48 struct mlxsw_sp_acl_erp_master_mask {
49 DECLARE_BITMAP(bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN);
50 unsigned int count[MLXSW_SP_ACL_TCAM_MASK_LEN];
53 struct mlxsw_sp_acl_erp_table {
54 struct mlxsw_sp_acl_erp_master_mask master_mask;
55 DECLARE_BITMAP(erp_id_bitmap, MLXSW_SP_ACL_ERP_MAX_PER_REGION);
56 DECLARE_BITMAP(erp_index_bitmap, MLXSW_SP_ACL_ERP_MAX_PER_REGION);
57 struct list_head atcam_erps_list;
58 struct mlxsw_sp_acl_erp_core *erp_core;
59 struct mlxsw_sp_acl_atcam_region *aregion;
60 const struct mlxsw_sp_acl_erp_table_ops *ops;
61 unsigned long base_index;
62 unsigned int num_atcam_erps;
63 unsigned int num_max_atcam_erps;
64 unsigned int num_ctcam_erps;
65 unsigned int num_deltas;
66 struct objagg *objagg;
67 struct mutex objagg_lock; /* guards objagg manipulation */
70 struct mlxsw_sp_acl_erp_table_ops {
71 struct mlxsw_sp_acl_erp *
72 (*erp_create)(struct mlxsw_sp_acl_erp_table *erp_table,
73 struct mlxsw_sp_acl_erp_key *key);
74 void (*erp_destroy)(struct mlxsw_sp_acl_erp_table *erp_table,
75 struct mlxsw_sp_acl_erp *erp);
78 static struct mlxsw_sp_acl_erp *
79 mlxsw_sp_acl_erp_mask_create(struct mlxsw_sp_acl_erp_table *erp_table,
80 struct mlxsw_sp_acl_erp_key *key);
82 mlxsw_sp_acl_erp_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table,
83 struct mlxsw_sp_acl_erp *erp);
84 static struct mlxsw_sp_acl_erp *
85 mlxsw_sp_acl_erp_second_mask_create(struct mlxsw_sp_acl_erp_table *erp_table,
86 struct mlxsw_sp_acl_erp_key *key);
88 mlxsw_sp_acl_erp_second_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table,
89 struct mlxsw_sp_acl_erp *erp);
90 static struct mlxsw_sp_acl_erp *
91 mlxsw_sp_acl_erp_first_mask_create(struct mlxsw_sp_acl_erp_table *erp_table,
92 struct mlxsw_sp_acl_erp_key *key);
94 mlxsw_sp_acl_erp_first_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table,
95 struct mlxsw_sp_acl_erp *erp);
97 mlxsw_sp_acl_erp_no_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table,
98 struct mlxsw_sp_acl_erp *erp);
100 static const struct mlxsw_sp_acl_erp_table_ops erp_multiple_masks_ops = {
101 .erp_create = mlxsw_sp_acl_erp_mask_create,
102 .erp_destroy = mlxsw_sp_acl_erp_mask_destroy,
105 static const struct mlxsw_sp_acl_erp_table_ops erp_two_masks_ops = {
106 .erp_create = mlxsw_sp_acl_erp_mask_create,
107 .erp_destroy = mlxsw_sp_acl_erp_second_mask_destroy,
110 static const struct mlxsw_sp_acl_erp_table_ops erp_single_mask_ops = {
111 .erp_create = mlxsw_sp_acl_erp_second_mask_create,
112 .erp_destroy = mlxsw_sp_acl_erp_first_mask_destroy,
115 static const struct mlxsw_sp_acl_erp_table_ops erp_no_mask_ops = {
116 .erp_create = mlxsw_sp_acl_erp_first_mask_create,
117 .erp_destroy = mlxsw_sp_acl_erp_no_mask_destroy,
121 mlxsw_sp_acl_erp_table_is_used(const struct mlxsw_sp_acl_erp_table *erp_table)
123 return erp_table->ops != &erp_single_mask_ops &&
124 erp_table->ops != &erp_no_mask_ops;
128 mlxsw_sp_acl_erp_bank_get(const struct mlxsw_sp_acl_erp *erp)
130 return erp->index % erp->erp_table->erp_core->num_erp_banks;
134 mlxsw_sp_acl_erp_table_entry_size(const struct mlxsw_sp_acl_erp_table *erp_table)
136 struct mlxsw_sp_acl_atcam_region *aregion = erp_table->aregion;
137 struct mlxsw_sp_acl_erp_core *erp_core = erp_table->erp_core;
139 return erp_core->erpt_entries_size[aregion->type];
142 static int mlxsw_sp_acl_erp_id_get(struct mlxsw_sp_acl_erp_table *erp_table,
147 id = find_first_zero_bit(erp_table->erp_id_bitmap,
148 MLXSW_SP_ACL_ERP_MAX_PER_REGION);
149 if (id < MLXSW_SP_ACL_ERP_MAX_PER_REGION) {
150 __set_bit(id, erp_table->erp_id_bitmap);
158 static void mlxsw_sp_acl_erp_id_put(struct mlxsw_sp_acl_erp_table *erp_table,
161 __clear_bit(id, erp_table->erp_id_bitmap);
165 mlxsw_sp_acl_erp_master_mask_bit_set(unsigned long bit,
166 struct mlxsw_sp_acl_erp_master_mask *mask)
168 if (mask->count[bit]++ == 0)
169 __set_bit(bit, mask->bitmap);
173 mlxsw_sp_acl_erp_master_mask_bit_clear(unsigned long bit,
174 struct mlxsw_sp_acl_erp_master_mask *mask)
176 if (--mask->count[bit] == 0)
177 __clear_bit(bit, mask->bitmap);
181 mlxsw_sp_acl_erp_master_mask_update(struct mlxsw_sp_acl_erp_table *erp_table)
183 struct mlxsw_sp_acl_tcam_region *region = erp_table->aregion->region;
184 struct mlxsw_sp *mlxsw_sp = region->mlxsw_sp;
185 char percr_pl[MLXSW_REG_PERCR_LEN];
188 mlxsw_reg_percr_pack(percr_pl, region->id);
189 master_mask = mlxsw_reg_percr_master_mask_data(percr_pl);
190 bitmap_to_arr32((u32 *) master_mask, erp_table->master_mask.bitmap,
191 MLXSW_SP_ACL_TCAM_MASK_LEN);
193 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(percr), percr_pl);
197 mlxsw_sp_acl_erp_master_mask_set(struct mlxsw_sp_acl_erp_table *erp_table,
198 struct mlxsw_sp_acl_erp_key *key)
200 DECLARE_BITMAP(mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN);
204 bitmap_from_arr32(mask_bitmap, (u32 *) key->mask,
205 MLXSW_SP_ACL_TCAM_MASK_LEN);
206 for_each_set_bit(bit, mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN)
207 mlxsw_sp_acl_erp_master_mask_bit_set(bit,
208 &erp_table->master_mask);
210 err = mlxsw_sp_acl_erp_master_mask_update(erp_table);
212 goto err_master_mask_update;
216 err_master_mask_update:
217 for_each_set_bit(bit, mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN)
218 mlxsw_sp_acl_erp_master_mask_bit_clear(bit,
219 &erp_table->master_mask);
224 mlxsw_sp_acl_erp_master_mask_clear(struct mlxsw_sp_acl_erp_table *erp_table,
225 struct mlxsw_sp_acl_erp_key *key)
227 DECLARE_BITMAP(mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN);
231 bitmap_from_arr32(mask_bitmap, (u32 *) key->mask,
232 MLXSW_SP_ACL_TCAM_MASK_LEN);
233 for_each_set_bit(bit, mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN)
234 mlxsw_sp_acl_erp_master_mask_bit_clear(bit,
235 &erp_table->master_mask);
237 err = mlxsw_sp_acl_erp_master_mask_update(erp_table);
239 goto err_master_mask_update;
243 err_master_mask_update:
244 for_each_set_bit(bit, mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN)
245 mlxsw_sp_acl_erp_master_mask_bit_set(bit,
246 &erp_table->master_mask);
250 static struct mlxsw_sp_acl_erp *
251 mlxsw_sp_acl_erp_generic_create(struct mlxsw_sp_acl_erp_table *erp_table,
252 struct mlxsw_sp_acl_erp_key *key)
254 struct mlxsw_sp_acl_erp *erp;
257 erp = kzalloc(sizeof(*erp), GFP_KERNEL);
259 return ERR_PTR(-ENOMEM);
261 err = mlxsw_sp_acl_erp_id_get(erp_table, &erp->id);
265 memcpy(&erp->key, key, sizeof(*key));
266 list_add(&erp->list, &erp_table->atcam_erps_list);
267 erp_table->num_atcam_erps++;
268 erp->erp_table = erp_table;
270 err = mlxsw_sp_acl_erp_master_mask_set(erp_table, &erp->key);
272 goto err_master_mask_set;
277 erp_table->num_atcam_erps--;
278 list_del(&erp->list);
279 mlxsw_sp_acl_erp_id_put(erp_table, erp->id);
286 mlxsw_sp_acl_erp_generic_destroy(struct mlxsw_sp_acl_erp *erp)
288 struct mlxsw_sp_acl_erp_table *erp_table = erp->erp_table;
290 mlxsw_sp_acl_erp_master_mask_clear(erp_table, &erp->key);
291 erp_table->num_atcam_erps--;
292 list_del(&erp->list);
293 mlxsw_sp_acl_erp_id_put(erp_table, erp->id);
298 mlxsw_sp_acl_erp_table_alloc(struct mlxsw_sp_acl_erp_core *erp_core,
299 unsigned int num_erps,
300 enum mlxsw_sp_acl_atcam_region_type region_type,
301 unsigned long *p_index)
303 unsigned int num_rows, entry_size;
306 /* We only allow allocations of entire rows */
307 if (num_erps % erp_core->num_erp_banks != 0)
310 entry_size = erp_core->erpt_entries_size[region_type];
311 num_rows = num_erps / erp_core->num_erp_banks;
313 index = gen_pool_alloc(erp_core->erp_tables, num_rows * entry_size);
317 *p_index = index - MLXSW_SP_ACL_ERP_GENALLOC_OFFSET;
323 mlxsw_sp_acl_erp_table_free(struct mlxsw_sp_acl_erp_core *erp_core,
324 unsigned int num_erps,
325 enum mlxsw_sp_acl_atcam_region_type region_type,
328 unsigned long base_index;
329 unsigned int entry_size;
332 entry_size = erp_core->erpt_entries_size[region_type];
333 base_index = index + MLXSW_SP_ACL_ERP_GENALLOC_OFFSET;
334 size = num_erps / erp_core->num_erp_banks * entry_size;
335 gen_pool_free(erp_core->erp_tables, base_index, size);
338 static struct mlxsw_sp_acl_erp *
339 mlxsw_sp_acl_erp_table_master_rp(struct mlxsw_sp_acl_erp_table *erp_table)
341 if (!list_is_singular(&erp_table->atcam_erps_list))
344 return list_first_entry(&erp_table->atcam_erps_list,
345 struct mlxsw_sp_acl_erp, list);
348 static int mlxsw_sp_acl_erp_index_get(struct mlxsw_sp_acl_erp_table *erp_table,
353 index = find_first_zero_bit(erp_table->erp_index_bitmap,
354 erp_table->num_max_atcam_erps);
355 if (index < erp_table->num_max_atcam_erps) {
356 __set_bit(index, erp_table->erp_index_bitmap);
364 static void mlxsw_sp_acl_erp_index_put(struct mlxsw_sp_acl_erp_table *erp_table,
367 __clear_bit(index, erp_table->erp_index_bitmap);
371 mlxsw_sp_acl_erp_table_locate(const struct mlxsw_sp_acl_erp_table *erp_table,
372 const struct mlxsw_sp_acl_erp *erp,
373 u8 *p_erpt_bank, u8 *p_erpt_index)
375 unsigned int entry_size = mlxsw_sp_acl_erp_table_entry_size(erp_table);
376 struct mlxsw_sp_acl_erp_core *erp_core = erp_table->erp_core;
379 *p_erpt_bank = erp->index % erp_core->num_erp_banks;
380 row = erp->index / erp_core->num_erp_banks;
381 *p_erpt_index = erp_table->base_index + row * entry_size;
385 mlxsw_sp_acl_erp_table_erp_add(struct mlxsw_sp_acl_erp_table *erp_table,
386 struct mlxsw_sp_acl_erp *erp)
388 struct mlxsw_sp *mlxsw_sp = erp_table->erp_core->mlxsw_sp;
389 enum mlxsw_reg_perpt_key_size key_size;
390 char perpt_pl[MLXSW_REG_PERPT_LEN];
391 u8 erpt_bank, erpt_index;
393 mlxsw_sp_acl_erp_table_locate(erp_table, erp, &erpt_bank, &erpt_index);
394 key_size = (enum mlxsw_reg_perpt_key_size) erp_table->aregion->type;
395 mlxsw_reg_perpt_pack(perpt_pl, erpt_bank, erpt_index, key_size, erp->id,
396 0, erp_table->base_index, erp->index,
398 mlxsw_reg_perpt_erp_vector_pack(perpt_pl, erp_table->erp_index_bitmap,
399 MLXSW_SP_ACL_ERP_MAX_PER_REGION);
400 mlxsw_reg_perpt_erp_vector_set(perpt_pl, erp->index, true);
401 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(perpt), perpt_pl);
404 static void mlxsw_sp_acl_erp_table_erp_del(struct mlxsw_sp_acl_erp *erp)
406 char empty_mask[MLXSW_REG_PTCEX_FLEX_KEY_BLOCKS_LEN] = { 0 };
407 struct mlxsw_sp_acl_erp_table *erp_table = erp->erp_table;
408 struct mlxsw_sp *mlxsw_sp = erp_table->erp_core->mlxsw_sp;
409 enum mlxsw_reg_perpt_key_size key_size;
410 char perpt_pl[MLXSW_REG_PERPT_LEN];
411 u8 erpt_bank, erpt_index;
413 mlxsw_sp_acl_erp_table_locate(erp_table, erp, &erpt_bank, &erpt_index);
414 key_size = (enum mlxsw_reg_perpt_key_size) erp_table->aregion->type;
415 mlxsw_reg_perpt_pack(perpt_pl, erpt_bank, erpt_index, key_size, erp->id,
416 0, erp_table->base_index, erp->index, empty_mask);
417 mlxsw_reg_perpt_erp_vector_pack(perpt_pl, erp_table->erp_index_bitmap,
418 MLXSW_SP_ACL_ERP_MAX_PER_REGION);
419 mlxsw_reg_perpt_erp_vector_set(perpt_pl, erp->index, false);
420 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(perpt), perpt_pl);
424 mlxsw_sp_acl_erp_table_enable(struct mlxsw_sp_acl_erp_table *erp_table,
427 struct mlxsw_sp_acl_tcam_region *region = erp_table->aregion->region;
428 struct mlxsw_sp *mlxsw_sp = erp_table->erp_core->mlxsw_sp;
429 char pererp_pl[MLXSW_REG_PERERP_LEN];
431 mlxsw_reg_pererp_pack(pererp_pl, region->id, ctcam_le, true, 0,
432 erp_table->base_index, 0);
433 mlxsw_reg_pererp_erp_vector_pack(pererp_pl, erp_table->erp_index_bitmap,
434 MLXSW_SP_ACL_ERP_MAX_PER_REGION);
436 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pererp), pererp_pl);
440 mlxsw_sp_acl_erp_table_disable(struct mlxsw_sp_acl_erp_table *erp_table)
442 struct mlxsw_sp_acl_tcam_region *region = erp_table->aregion->region;
443 struct mlxsw_sp *mlxsw_sp = erp_table->erp_core->mlxsw_sp;
444 char pererp_pl[MLXSW_REG_PERERP_LEN];
445 struct mlxsw_sp_acl_erp *master_rp;
447 master_rp = mlxsw_sp_acl_erp_table_master_rp(erp_table);
448 /* It is possible we do not have a master RP when we disable the
449 * table when there are no rules in the A-TCAM and the last C-TCAM
452 mlxsw_reg_pererp_pack(pererp_pl, region->id, false, false, 0, 0,
453 master_rp ? master_rp->id : 0);
454 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pererp), pererp_pl);
458 mlxsw_sp_acl_erp_table_relocate(struct mlxsw_sp_acl_erp_table *erp_table)
460 struct mlxsw_sp_acl_erp *erp;
463 list_for_each_entry(erp, &erp_table->atcam_erps_list, list) {
464 err = mlxsw_sp_acl_erp_table_erp_add(erp_table, erp);
466 goto err_table_erp_add;
472 list_for_each_entry_continue_reverse(erp, &erp_table->atcam_erps_list,
474 mlxsw_sp_acl_erp_table_erp_del(erp);
479 mlxsw_sp_acl_erp_table_expand(struct mlxsw_sp_acl_erp_table *erp_table)
481 unsigned int num_erps, old_num_erps = erp_table->num_max_atcam_erps;
482 struct mlxsw_sp_acl_erp_core *erp_core = erp_table->erp_core;
483 unsigned long old_base_index = erp_table->base_index;
484 bool ctcam_le = erp_table->num_ctcam_erps > 0;
487 if (erp_table->num_atcam_erps < erp_table->num_max_atcam_erps)
490 if (erp_table->num_max_atcam_erps == MLXSW_SP_ACL_ERP_MAX_PER_REGION)
493 num_erps = old_num_erps + erp_core->num_erp_banks;
494 err = mlxsw_sp_acl_erp_table_alloc(erp_core, num_erps,
495 erp_table->aregion->type,
496 &erp_table->base_index);
499 erp_table->num_max_atcam_erps = num_erps;
501 err = mlxsw_sp_acl_erp_table_relocate(erp_table);
503 goto err_table_relocate;
505 err = mlxsw_sp_acl_erp_table_enable(erp_table, ctcam_le);
507 goto err_table_enable;
509 mlxsw_sp_acl_erp_table_free(erp_core, old_num_erps,
510 erp_table->aregion->type, old_base_index);
516 erp_table->num_max_atcam_erps = old_num_erps;
517 mlxsw_sp_acl_erp_table_free(erp_core, num_erps,
518 erp_table->aregion->type,
519 erp_table->base_index);
520 erp_table->base_index = old_base_index;
525 mlxsw_acl_erp_table_bf_add(struct mlxsw_sp_acl_erp_table *erp_table,
526 struct mlxsw_sp_acl_erp *erp)
528 struct mlxsw_sp_acl_atcam_region *aregion = erp_table->aregion;
529 unsigned int erp_bank = mlxsw_sp_acl_erp_bank_get(erp);
530 struct mlxsw_sp_acl_atcam_entry *aentry;
533 list_for_each_entry(aentry, &aregion->entries_list, list) {
534 err = mlxsw_sp_acl_bf_entry_add(aregion->region->mlxsw_sp,
535 erp_table->erp_core->bf,
536 aregion, erp_bank, aentry);
538 goto bf_entry_add_err;
544 list_for_each_entry_continue_reverse(aentry, &aregion->entries_list,
546 mlxsw_sp_acl_bf_entry_del(aregion->region->mlxsw_sp,
547 erp_table->erp_core->bf,
548 aregion, erp_bank, aentry);
553 mlxsw_acl_erp_table_bf_del(struct mlxsw_sp_acl_erp_table *erp_table,
554 struct mlxsw_sp_acl_erp *erp)
556 struct mlxsw_sp_acl_atcam_region *aregion = erp_table->aregion;
557 unsigned int erp_bank = mlxsw_sp_acl_erp_bank_get(erp);
558 struct mlxsw_sp_acl_atcam_entry *aentry;
560 list_for_each_entry_reverse(aentry, &aregion->entries_list, list)
561 mlxsw_sp_acl_bf_entry_del(aregion->region->mlxsw_sp,
562 erp_table->erp_core->bf,
563 aregion, erp_bank, aentry);
567 mlxsw_sp_acl_erp_region_table_trans(struct mlxsw_sp_acl_erp_table *erp_table)
569 struct mlxsw_sp_acl_erp_core *erp_core = erp_table->erp_core;
570 struct mlxsw_sp_acl_erp *master_rp;
573 /* Initially, allocate a single eRP row. Expand later as needed */
574 err = mlxsw_sp_acl_erp_table_alloc(erp_core, erp_core->num_erp_banks,
575 erp_table->aregion->type,
576 &erp_table->base_index);
579 erp_table->num_max_atcam_erps = erp_core->num_erp_banks;
581 /* Transition the sole RP currently configured (the master RP)
584 master_rp = mlxsw_sp_acl_erp_table_master_rp(erp_table);
587 goto err_table_master_rp;
590 /* Make sure the master RP is using a valid index, as
591 * only a single eRP row is currently allocated.
593 master_rp->index = 0;
594 __set_bit(master_rp->index, erp_table->erp_index_bitmap);
596 err = mlxsw_sp_acl_erp_table_erp_add(erp_table, master_rp);
598 goto err_table_master_rp_add;
600 /* Update Bloom filter before enabling eRP table, as rules
601 * on the master RP were not set to Bloom filter up to this
604 err = mlxsw_acl_erp_table_bf_add(erp_table, master_rp);
606 goto err_table_bf_add;
608 err = mlxsw_sp_acl_erp_table_enable(erp_table, false);
610 goto err_table_enable;
615 mlxsw_acl_erp_table_bf_del(erp_table, master_rp);
617 mlxsw_sp_acl_erp_table_erp_del(master_rp);
618 err_table_master_rp_add:
619 __clear_bit(master_rp->index, erp_table->erp_index_bitmap);
621 mlxsw_sp_acl_erp_table_free(erp_core, erp_table->num_max_atcam_erps,
622 erp_table->aregion->type,
623 erp_table->base_index);
628 mlxsw_sp_acl_erp_region_master_mask_trans(struct mlxsw_sp_acl_erp_table *erp_table)
630 struct mlxsw_sp_acl_erp_core *erp_core = erp_table->erp_core;
631 struct mlxsw_sp_acl_erp *master_rp;
633 mlxsw_sp_acl_erp_table_disable(erp_table);
634 master_rp = mlxsw_sp_acl_erp_table_master_rp(erp_table);
637 mlxsw_acl_erp_table_bf_del(erp_table, master_rp);
638 mlxsw_sp_acl_erp_table_erp_del(master_rp);
639 __clear_bit(master_rp->index, erp_table->erp_index_bitmap);
640 mlxsw_sp_acl_erp_table_free(erp_core, erp_table->num_max_atcam_erps,
641 erp_table->aregion->type,
642 erp_table->base_index);
646 mlxsw_sp_acl_erp_region_erp_add(struct mlxsw_sp_acl_erp_table *erp_table,
647 struct mlxsw_sp_acl_erp *erp)
649 struct mlxsw_sp_acl_tcam_region *region = erp_table->aregion->region;
650 struct mlxsw_sp *mlxsw_sp = erp_table->erp_core->mlxsw_sp;
651 bool ctcam_le = erp_table->num_ctcam_erps > 0;
652 char pererp_pl[MLXSW_REG_PERERP_LEN];
654 mlxsw_reg_pererp_pack(pererp_pl, region->id, ctcam_le, true, 0,
655 erp_table->base_index, 0);
656 mlxsw_reg_pererp_erp_vector_pack(pererp_pl, erp_table->erp_index_bitmap,
657 MLXSW_SP_ACL_ERP_MAX_PER_REGION);
658 mlxsw_reg_pererp_erpt_vector_set(pererp_pl, erp->index, true);
660 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pererp), pererp_pl);
663 static void mlxsw_sp_acl_erp_region_erp_del(struct mlxsw_sp_acl_erp *erp)
665 struct mlxsw_sp_acl_erp_table *erp_table = erp->erp_table;
666 struct mlxsw_sp_acl_tcam_region *region = erp_table->aregion->region;
667 struct mlxsw_sp *mlxsw_sp = erp_table->erp_core->mlxsw_sp;
668 bool ctcam_le = erp_table->num_ctcam_erps > 0;
669 char pererp_pl[MLXSW_REG_PERERP_LEN];
671 mlxsw_reg_pererp_pack(pererp_pl, region->id, ctcam_le, true, 0,
672 erp_table->base_index, 0);
673 mlxsw_reg_pererp_erp_vector_pack(pererp_pl, erp_table->erp_index_bitmap,
674 MLXSW_SP_ACL_ERP_MAX_PER_REGION);
675 mlxsw_reg_pererp_erpt_vector_set(pererp_pl, erp->index, false);
677 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pererp), pererp_pl);
681 mlxsw_sp_acl_erp_region_ctcam_enable(struct mlxsw_sp_acl_erp_table *erp_table)
683 /* No need to re-enable lookup in the C-TCAM */
684 if (erp_table->num_ctcam_erps > 1)
687 return mlxsw_sp_acl_erp_table_enable(erp_table, true);
691 mlxsw_sp_acl_erp_region_ctcam_disable(struct mlxsw_sp_acl_erp_table *erp_table)
693 /* Only disable C-TCAM lookup when last C-TCAM eRP is deleted */
694 if (erp_table->num_ctcam_erps > 1)
697 mlxsw_sp_acl_erp_table_enable(erp_table, false);
701 __mlxsw_sp_acl_erp_table_other_inc(struct mlxsw_sp_acl_erp_table *erp_table,
702 unsigned int *inc_num)
706 /* If there are C-TCAM eRP or deltas in use we need to transition
707 * the region to use eRP table, if it is not already done
709 if (!mlxsw_sp_acl_erp_table_is_used(erp_table)) {
710 err = mlxsw_sp_acl_erp_region_table_trans(erp_table);
715 /* When C-TCAM or deltas are used, the eRP table must be used */
716 if (erp_table->ops != &erp_multiple_masks_ops)
717 erp_table->ops = &erp_multiple_masks_ops;
724 static int mlxsw_sp_acl_erp_ctcam_inc(struct mlxsw_sp_acl_erp_table *erp_table)
726 return __mlxsw_sp_acl_erp_table_other_inc(erp_table,
727 &erp_table->num_ctcam_erps);
730 static int mlxsw_sp_acl_erp_delta_inc(struct mlxsw_sp_acl_erp_table *erp_table)
732 return __mlxsw_sp_acl_erp_table_other_inc(erp_table,
733 &erp_table->num_deltas);
737 __mlxsw_sp_acl_erp_table_other_dec(struct mlxsw_sp_acl_erp_table *erp_table,
738 unsigned int *dec_num)
742 /* If there are no C-TCAM eRP or deltas in use, the state we
743 * transition to depends on the number of A-TCAM eRPs currently
746 if (erp_table->num_ctcam_erps > 0 || erp_table->num_deltas > 0)
749 switch (erp_table->num_atcam_erps) {
751 /* Keep using the eRP table, but correctly set the
752 * operations pointer so that when an A-TCAM eRP is
753 * deleted we will transition to use the master mask
755 erp_table->ops = &erp_two_masks_ops;
758 /* We only kept the eRP table because we had C-TCAM
759 * eRPs in use. Now that the last C-TCAM eRP is gone we
760 * can stop using the table and transition to use the
763 mlxsw_sp_acl_erp_region_master_mask_trans(erp_table);
764 erp_table->ops = &erp_single_mask_ops;
767 /* There are no more eRPs of any kind used by the region
768 * so free its eRP table and transition to initial state
770 mlxsw_sp_acl_erp_table_disable(erp_table);
771 mlxsw_sp_acl_erp_table_free(erp_table->erp_core,
772 erp_table->num_max_atcam_erps,
773 erp_table->aregion->type,
774 erp_table->base_index);
775 erp_table->ops = &erp_no_mask_ops;
782 static void mlxsw_sp_acl_erp_ctcam_dec(struct mlxsw_sp_acl_erp_table *erp_table)
784 __mlxsw_sp_acl_erp_table_other_dec(erp_table,
785 &erp_table->num_ctcam_erps);
788 static void mlxsw_sp_acl_erp_delta_dec(struct mlxsw_sp_acl_erp_table *erp_table)
790 __mlxsw_sp_acl_erp_table_other_dec(erp_table,
791 &erp_table->num_deltas);
794 static struct mlxsw_sp_acl_erp *
795 mlxsw_sp_acl_erp_ctcam_mask_create(struct mlxsw_sp_acl_erp_table *erp_table,
796 struct mlxsw_sp_acl_erp_key *key)
798 struct mlxsw_sp_acl_erp *erp;
801 erp = kzalloc(sizeof(*erp), GFP_KERNEL);
803 return ERR_PTR(-ENOMEM);
805 memcpy(&erp->key, key, sizeof(*key));
806 bitmap_from_arr32(erp->mask_bitmap, (u32 *) key->mask,
807 MLXSW_SP_ACL_TCAM_MASK_LEN);
809 err = mlxsw_sp_acl_erp_ctcam_inc(erp_table);
811 goto err_erp_ctcam_inc;
813 erp->erp_table = erp_table;
815 err = mlxsw_sp_acl_erp_master_mask_set(erp_table, &erp->key);
817 goto err_master_mask_set;
819 err = mlxsw_sp_acl_erp_region_ctcam_enable(erp_table);
821 goto err_erp_region_ctcam_enable;
825 err_erp_region_ctcam_enable:
826 mlxsw_sp_acl_erp_master_mask_clear(erp_table, &erp->key);
828 mlxsw_sp_acl_erp_ctcam_dec(erp_table);
835 mlxsw_sp_acl_erp_ctcam_mask_destroy(struct mlxsw_sp_acl_erp *erp)
837 struct mlxsw_sp_acl_erp_table *erp_table = erp->erp_table;
839 mlxsw_sp_acl_erp_region_ctcam_disable(erp_table);
840 mlxsw_sp_acl_erp_master_mask_clear(erp_table, &erp->key);
841 mlxsw_sp_acl_erp_ctcam_dec(erp_table);
845 static struct mlxsw_sp_acl_erp *
846 mlxsw_sp_acl_erp_mask_create(struct mlxsw_sp_acl_erp_table *erp_table,
847 struct mlxsw_sp_acl_erp_key *key)
849 struct mlxsw_sp_acl_erp *erp;
853 return mlxsw_sp_acl_erp_ctcam_mask_create(erp_table, key);
855 /* Expand the eRP table for the new eRP, if needed */
856 err = mlxsw_sp_acl_erp_table_expand(erp_table);
860 erp = mlxsw_sp_acl_erp_generic_create(erp_table, key);
864 err = mlxsw_sp_acl_erp_index_get(erp_table, &erp->index);
866 goto err_erp_index_get;
868 err = mlxsw_sp_acl_erp_table_erp_add(erp_table, erp);
870 goto err_table_erp_add;
872 err = mlxsw_sp_acl_erp_region_erp_add(erp_table, erp);
874 goto err_region_erp_add;
876 erp_table->ops = &erp_multiple_masks_ops;
881 mlxsw_sp_acl_erp_table_erp_del(erp);
883 mlxsw_sp_acl_erp_index_put(erp_table, erp->index);
885 mlxsw_sp_acl_erp_generic_destroy(erp);
890 mlxsw_sp_acl_erp_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table,
891 struct mlxsw_sp_acl_erp *erp)
894 return mlxsw_sp_acl_erp_ctcam_mask_destroy(erp);
896 mlxsw_sp_acl_erp_region_erp_del(erp);
897 mlxsw_sp_acl_erp_table_erp_del(erp);
898 mlxsw_sp_acl_erp_index_put(erp_table, erp->index);
899 mlxsw_sp_acl_erp_generic_destroy(erp);
901 if (erp_table->num_atcam_erps == 2 && erp_table->num_ctcam_erps == 0 &&
902 erp_table->num_deltas == 0)
903 erp_table->ops = &erp_two_masks_ops;
906 static struct mlxsw_sp_acl_erp *
907 mlxsw_sp_acl_erp_second_mask_create(struct mlxsw_sp_acl_erp_table *erp_table,
908 struct mlxsw_sp_acl_erp_key *key)
910 struct mlxsw_sp_acl_erp *erp;
914 return mlxsw_sp_acl_erp_ctcam_mask_create(erp_table, key);
916 /* Transition to use eRP table instead of master mask */
917 err = mlxsw_sp_acl_erp_region_table_trans(erp_table);
921 erp = mlxsw_sp_acl_erp_generic_create(erp_table, key);
927 err = mlxsw_sp_acl_erp_index_get(erp_table, &erp->index);
929 goto err_erp_index_get;
931 err = mlxsw_sp_acl_erp_table_erp_add(erp_table, erp);
933 goto err_table_erp_add;
935 err = mlxsw_sp_acl_erp_region_erp_add(erp_table, erp);
937 goto err_region_erp_add;
939 erp_table->ops = &erp_two_masks_ops;
944 mlxsw_sp_acl_erp_table_erp_del(erp);
946 mlxsw_sp_acl_erp_index_put(erp_table, erp->index);
948 mlxsw_sp_acl_erp_generic_destroy(erp);
950 mlxsw_sp_acl_erp_region_master_mask_trans(erp_table);
955 mlxsw_sp_acl_erp_second_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table,
956 struct mlxsw_sp_acl_erp *erp)
959 return mlxsw_sp_acl_erp_ctcam_mask_destroy(erp);
961 mlxsw_sp_acl_erp_region_erp_del(erp);
962 mlxsw_sp_acl_erp_table_erp_del(erp);
963 mlxsw_sp_acl_erp_index_put(erp_table, erp->index);
964 mlxsw_sp_acl_erp_generic_destroy(erp);
965 /* Transition to use master mask instead of eRP table */
966 mlxsw_sp_acl_erp_region_master_mask_trans(erp_table);
968 erp_table->ops = &erp_single_mask_ops;
971 static struct mlxsw_sp_acl_erp *
972 mlxsw_sp_acl_erp_first_mask_create(struct mlxsw_sp_acl_erp_table *erp_table,
973 struct mlxsw_sp_acl_erp_key *key)
975 struct mlxsw_sp_acl_erp *erp;
978 return ERR_PTR(-EINVAL);
980 erp = mlxsw_sp_acl_erp_generic_create(erp_table, key);
984 erp_table->ops = &erp_single_mask_ops;
990 mlxsw_sp_acl_erp_first_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table,
991 struct mlxsw_sp_acl_erp *erp)
993 mlxsw_sp_acl_erp_generic_destroy(erp);
994 erp_table->ops = &erp_no_mask_ops;
998 mlxsw_sp_acl_erp_no_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table,
999 struct mlxsw_sp_acl_erp *erp)
1004 struct mlxsw_sp_acl_erp_mask *
1005 mlxsw_sp_acl_erp_mask_get(struct mlxsw_sp_acl_atcam_region *aregion,
1006 const char *mask, bool ctcam)
1008 struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table;
1009 struct mlxsw_sp_acl_erp_key key;
1010 struct objagg_obj *objagg_obj;
1012 memcpy(key.mask, mask, MLXSW_REG_PTCEX_FLEX_KEY_BLOCKS_LEN);
1014 mutex_lock(&erp_table->objagg_lock);
1015 objagg_obj = objagg_obj_get(erp_table->objagg, &key);
1016 mutex_unlock(&erp_table->objagg_lock);
1017 if (IS_ERR(objagg_obj))
1018 return ERR_CAST(objagg_obj);
1019 return (struct mlxsw_sp_acl_erp_mask *) objagg_obj;
1022 void mlxsw_sp_acl_erp_mask_put(struct mlxsw_sp_acl_atcam_region *aregion,
1023 struct mlxsw_sp_acl_erp_mask *erp_mask)
1025 struct objagg_obj *objagg_obj = (struct objagg_obj *) erp_mask;
1026 struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table;
1028 mutex_lock(&erp_table->objagg_lock);
1029 objagg_obj_put(erp_table->objagg, objagg_obj);
1030 mutex_unlock(&erp_table->objagg_lock);
1033 int mlxsw_sp_acl_erp_bf_insert(struct mlxsw_sp *mlxsw_sp,
1034 struct mlxsw_sp_acl_atcam_region *aregion,
1035 struct mlxsw_sp_acl_erp_mask *erp_mask,
1036 struct mlxsw_sp_acl_atcam_entry *aentry)
1038 struct objagg_obj *objagg_obj = (struct objagg_obj *) erp_mask;
1039 const struct mlxsw_sp_acl_erp *erp = objagg_obj_root_priv(objagg_obj);
1040 unsigned int erp_bank;
1042 if (!mlxsw_sp_acl_erp_table_is_used(erp->erp_table))
1045 erp_bank = mlxsw_sp_acl_erp_bank_get(erp);
1046 return mlxsw_sp_acl_bf_entry_add(mlxsw_sp,
1047 erp->erp_table->erp_core->bf,
1048 aregion, erp_bank, aentry);
1051 void mlxsw_sp_acl_erp_bf_remove(struct mlxsw_sp *mlxsw_sp,
1052 struct mlxsw_sp_acl_atcam_region *aregion,
1053 struct mlxsw_sp_acl_erp_mask *erp_mask,
1054 struct mlxsw_sp_acl_atcam_entry *aentry)
1056 struct objagg_obj *objagg_obj = (struct objagg_obj *) erp_mask;
1057 const struct mlxsw_sp_acl_erp *erp = objagg_obj_root_priv(objagg_obj);
1058 unsigned int erp_bank;
1060 if (!mlxsw_sp_acl_erp_table_is_used(erp->erp_table))
1063 erp_bank = mlxsw_sp_acl_erp_bank_get(erp);
1064 mlxsw_sp_acl_bf_entry_del(mlxsw_sp,
1065 erp->erp_table->erp_core->bf,
1066 aregion, erp_bank, aentry);
1070 mlxsw_sp_acl_erp_mask_is_ctcam(const struct mlxsw_sp_acl_erp_mask *erp_mask)
1072 struct objagg_obj *objagg_obj = (struct objagg_obj *) erp_mask;
1073 const struct mlxsw_sp_acl_erp_key *key = objagg_obj_raw(objagg_obj);
1078 u8 mlxsw_sp_acl_erp_mask_erp_id(const struct mlxsw_sp_acl_erp_mask *erp_mask)
1080 struct objagg_obj *objagg_obj = (struct objagg_obj *) erp_mask;
1081 const struct mlxsw_sp_acl_erp *erp = objagg_obj_root_priv(objagg_obj);
1086 struct mlxsw_sp_acl_erp_delta {
1087 struct mlxsw_sp_acl_erp_key key;
1092 u16 mlxsw_sp_acl_erp_delta_start(const struct mlxsw_sp_acl_erp_delta *delta)
1094 return delta->start;
1097 u8 mlxsw_sp_acl_erp_delta_mask(const struct mlxsw_sp_acl_erp_delta *delta)
1102 u8 mlxsw_sp_acl_erp_delta_value(const struct mlxsw_sp_acl_erp_delta *delta,
1103 const char *enc_key)
1105 u16 start = delta->start;
1106 u8 mask = delta->mask;
1112 tmp = (unsigned char) enc_key[__MASK_IDX(start / 8)];
1113 if (start / 8 + 1 < __MASK_LEN)
1114 tmp |= (unsigned char) enc_key[__MASK_IDX(start / 8 + 1)] << 8;
1120 void mlxsw_sp_acl_erp_delta_clear(const struct mlxsw_sp_acl_erp_delta *delta,
1121 const char *enc_key)
1123 u16 start = delta->start;
1124 u8 mask = delta->mask;
1125 unsigned char *byte;
1132 byte = (unsigned char *) &enc_key[__MASK_IDX(start / 8)];
1133 *byte &= tmp & 0xff;
1134 if (start / 8 + 1 < __MASK_LEN) {
1135 byte = (unsigned char *) &enc_key[__MASK_IDX(start / 8 + 1)];
1136 *byte &= (tmp >> 8) & 0xff;
1140 static const struct mlxsw_sp_acl_erp_delta
1141 mlxsw_sp_acl_erp_delta_default = {};
1143 const struct mlxsw_sp_acl_erp_delta *
1144 mlxsw_sp_acl_erp_delta(const struct mlxsw_sp_acl_erp_mask *erp_mask)
1146 struct objagg_obj *objagg_obj = (struct objagg_obj *) erp_mask;
1147 const struct mlxsw_sp_acl_erp_delta *delta;
1149 delta = objagg_obj_delta_priv(objagg_obj);
1151 delta = &mlxsw_sp_acl_erp_delta_default;
1156 mlxsw_sp_acl_erp_delta_fill(const struct mlxsw_sp_acl_erp_key *parent_key,
1157 const struct mlxsw_sp_acl_erp_key *key,
1158 u16 *delta_start, u8 *delta_mask)
1166 /* The difference between 2 masks can be up to 8 consecutive bits. */
1167 for (i = 0; i < __MASK_LEN; i++) {
1168 if (parent_key->mask[__MASK_IDX(i)] == key->mask[__MASK_IDX(i)])
1172 else if (si != i - 1)
1176 /* The masks are the same, this can happen in case eRPs with
1177 * the same mask were created in both A-TCAM and C-TCAM.
1178 * The only possible condition under which this can happen
1179 * is identical rule insertion. Delta is not possible here.
1183 pmask = (unsigned char) parent_key->mask[__MASK_IDX(si)];
1184 mask = (unsigned char) key->mask[__MASK_IDX(si)];
1185 if (si + 1 < __MASK_LEN) {
1186 pmask |= (unsigned char) parent_key->mask[__MASK_IDX(si + 1)] << 8;
1187 mask |= (unsigned char) key->mask[__MASK_IDX(si + 1)] << 8;
1190 if ((pmask ^ mask) & pmask)
1193 while (!(mask & (1 << offset)))
1200 *delta_start = si * 8 + offset;
1206 static bool mlxsw_sp_acl_erp_delta_check(void *priv, const void *parent_obj,
1209 const struct mlxsw_sp_acl_erp_key *parent_key = parent_obj;
1210 const struct mlxsw_sp_acl_erp_key *key = obj;
1215 err = mlxsw_sp_acl_erp_delta_fill(parent_key, key,
1216 &delta_start, &delta_mask);
1217 return err ? false : true;
1220 static int mlxsw_sp_acl_erp_hints_obj_cmp(const void *obj1, const void *obj2)
1222 const struct mlxsw_sp_acl_erp_key *key1 = obj1;
1223 const struct mlxsw_sp_acl_erp_key *key2 = obj2;
1225 /* For hints purposes, two objects are considered equal
1226 * in case the masks are the same. Does not matter what
1227 * the "ctcam" value is.
1229 return memcmp(key1->mask, key2->mask, sizeof(key1->mask));
1232 static void *mlxsw_sp_acl_erp_delta_create(void *priv, void *parent_obj,
1235 struct mlxsw_sp_acl_erp_key *parent_key = parent_obj;
1236 struct mlxsw_sp_acl_atcam_region *aregion = priv;
1237 struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table;
1238 struct mlxsw_sp_acl_erp_key *key = obj;
1239 struct mlxsw_sp_acl_erp_delta *delta;
1244 if (parent_key->ctcam || key->ctcam)
1245 return ERR_PTR(-EINVAL);
1246 err = mlxsw_sp_acl_erp_delta_fill(parent_key, key,
1247 &delta_start, &delta_mask);
1249 return ERR_PTR(-EINVAL);
1251 delta = kzalloc(sizeof(*delta), GFP_KERNEL);
1253 return ERR_PTR(-ENOMEM);
1254 delta->start = delta_start;
1255 delta->mask = delta_mask;
1257 err = mlxsw_sp_acl_erp_delta_inc(erp_table);
1259 goto err_erp_delta_inc;
1261 memcpy(&delta->key, key, sizeof(*key));
1262 err = mlxsw_sp_acl_erp_master_mask_set(erp_table, &delta->key);
1264 goto err_master_mask_set;
1268 err_master_mask_set:
1269 mlxsw_sp_acl_erp_delta_dec(erp_table);
1272 return ERR_PTR(err);
1275 static void mlxsw_sp_acl_erp_delta_destroy(void *priv, void *delta_priv)
1277 struct mlxsw_sp_acl_erp_delta *delta = delta_priv;
1278 struct mlxsw_sp_acl_atcam_region *aregion = priv;
1279 struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table;
1281 mlxsw_sp_acl_erp_master_mask_clear(erp_table, &delta->key);
1282 mlxsw_sp_acl_erp_delta_dec(erp_table);
1286 static void *mlxsw_sp_acl_erp_root_create(void *priv, void *obj,
1287 unsigned int root_id)
1289 struct mlxsw_sp_acl_atcam_region *aregion = priv;
1290 struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table;
1291 struct mlxsw_sp_acl_erp_key *key = obj;
1294 root_id != OBJAGG_OBJ_ROOT_ID_INVALID &&
1295 root_id >= MLXSW_SP_ACL_ERP_MAX_PER_REGION)
1296 return ERR_PTR(-ENOBUFS);
1297 return erp_table->ops->erp_create(erp_table, key);
1300 static void mlxsw_sp_acl_erp_root_destroy(void *priv, void *root_priv)
1302 struct mlxsw_sp_acl_atcam_region *aregion = priv;
1303 struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table;
1305 erp_table->ops->erp_destroy(erp_table, root_priv);
1308 static const struct objagg_ops mlxsw_sp_acl_erp_objagg_ops = {
1309 .obj_size = sizeof(struct mlxsw_sp_acl_erp_key),
1310 .delta_check = mlxsw_sp_acl_erp_delta_check,
1311 .hints_obj_cmp = mlxsw_sp_acl_erp_hints_obj_cmp,
1312 .delta_create = mlxsw_sp_acl_erp_delta_create,
1313 .delta_destroy = mlxsw_sp_acl_erp_delta_destroy,
1314 .root_create = mlxsw_sp_acl_erp_root_create,
1315 .root_destroy = mlxsw_sp_acl_erp_root_destroy,
1318 static struct mlxsw_sp_acl_erp_table *
1319 mlxsw_sp_acl_erp_table_create(struct mlxsw_sp_acl_atcam_region *aregion,
1320 struct objagg_hints *hints)
1322 struct mlxsw_sp_acl_erp_table *erp_table;
1325 erp_table = kzalloc(sizeof(*erp_table), GFP_KERNEL);
1327 return ERR_PTR(-ENOMEM);
1329 erp_table->objagg = objagg_create(&mlxsw_sp_acl_erp_objagg_ops,
1331 if (IS_ERR(erp_table->objagg)) {
1332 err = PTR_ERR(erp_table->objagg);
1333 goto err_objagg_create;
1336 erp_table->erp_core = aregion->atcam->erp_core;
1337 erp_table->ops = &erp_no_mask_ops;
1338 INIT_LIST_HEAD(&erp_table->atcam_erps_list);
1339 erp_table->aregion = aregion;
1340 mutex_init(&erp_table->objagg_lock);
1346 return ERR_PTR(err);
1350 mlxsw_sp_acl_erp_table_destroy(struct mlxsw_sp_acl_erp_table *erp_table)
1352 WARN_ON(!list_empty(&erp_table->atcam_erps_list));
1353 mutex_destroy(&erp_table->objagg_lock);
1354 objagg_destroy(erp_table->objagg);
1359 mlxsw_sp_acl_erp_master_mask_init(struct mlxsw_sp_acl_atcam_region *aregion)
1361 struct mlxsw_sp *mlxsw_sp = aregion->region->mlxsw_sp;
1362 char percr_pl[MLXSW_REG_PERCR_LEN];
1364 mlxsw_reg_percr_pack(percr_pl, aregion->region->id);
1365 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(percr), percr_pl);
1369 mlxsw_sp_acl_erp_region_param_init(struct mlxsw_sp_acl_atcam_region *aregion)
1371 struct mlxsw_sp *mlxsw_sp = aregion->region->mlxsw_sp;
1372 char pererp_pl[MLXSW_REG_PERERP_LEN];
1374 mlxsw_reg_pererp_pack(pererp_pl, aregion->region->id, false, false, 0,
1376 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pererp), pererp_pl);
1380 mlxsw_sp_acl_erp_hints_check(struct mlxsw_sp *mlxsw_sp,
1381 struct mlxsw_sp_acl_atcam_region *aregion,
1382 struct objagg_hints *hints, bool *p_rehash_needed)
1384 struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table;
1385 const struct objagg_stats *ostats;
1386 const struct objagg_stats *hstats;
1389 *p_rehash_needed = false;
1391 mutex_lock(&erp_table->objagg_lock);
1392 ostats = objagg_stats_get(erp_table->objagg);
1393 mutex_unlock(&erp_table->objagg_lock);
1394 if (IS_ERR(ostats)) {
1395 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to get ERP stats\n");
1396 return PTR_ERR(ostats);
1399 hstats = objagg_hints_stats_get(hints);
1400 if (IS_ERR(hstats)) {
1401 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to get ERP hints stats\n");
1402 err = PTR_ERR(hstats);
1403 goto err_hints_stats_get;
1406 /* Very basic criterion for now. */
1407 if (hstats->root_count < ostats->root_count)
1408 *p_rehash_needed = true;
1412 objagg_stats_put(hstats);
1413 err_hints_stats_get:
1414 objagg_stats_put(ostats);
1419 mlxsw_sp_acl_erp_rehash_hints_get(struct mlxsw_sp_acl_atcam_region *aregion)
1421 struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table;
1422 struct mlxsw_sp *mlxsw_sp = aregion->region->mlxsw_sp;
1423 struct objagg_hints *hints;
1427 mutex_lock(&erp_table->objagg_lock);
1428 hints = objagg_hints_get(erp_table->objagg,
1429 OBJAGG_OPT_ALGO_SIMPLE_GREEDY);
1430 mutex_unlock(&erp_table->objagg_lock);
1431 if (IS_ERR(hints)) {
1432 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to create ERP hints\n");
1433 return ERR_CAST(hints);
1435 err = mlxsw_sp_acl_erp_hints_check(mlxsw_sp, aregion, hints,
1440 if (!rehash_needed) {
1447 objagg_hints_put(hints);
1448 return ERR_PTR(err);
1451 void mlxsw_sp_acl_erp_rehash_hints_put(void *hints_priv)
1453 struct objagg_hints *hints = hints_priv;
1455 objagg_hints_put(hints);
1458 int mlxsw_sp_acl_erp_region_init(struct mlxsw_sp_acl_atcam_region *aregion,
1461 struct mlxsw_sp_acl_erp_table *erp_table;
1462 struct objagg_hints *hints = hints_priv;
1465 erp_table = mlxsw_sp_acl_erp_table_create(aregion, hints);
1466 if (IS_ERR(erp_table))
1467 return PTR_ERR(erp_table);
1468 aregion->erp_table = erp_table;
1470 /* Initialize the region's master mask to all zeroes */
1471 err = mlxsw_sp_acl_erp_master_mask_init(aregion);
1473 goto err_erp_master_mask_init;
1475 /* Initialize the region to not use the eRP table */
1476 err = mlxsw_sp_acl_erp_region_param_init(aregion);
1478 goto err_erp_region_param_init;
1482 err_erp_region_param_init:
1483 err_erp_master_mask_init:
1484 mlxsw_sp_acl_erp_table_destroy(erp_table);
1488 void mlxsw_sp_acl_erp_region_fini(struct mlxsw_sp_acl_atcam_region *aregion)
1490 mlxsw_sp_acl_erp_table_destroy(aregion->erp_table);
1494 mlxsw_sp_acl_erp_tables_sizes_query(struct mlxsw_sp *mlxsw_sp,
1495 struct mlxsw_sp_acl_erp_core *erp_core)
1499 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_ERPT_ENTRIES_2KB) ||
1500 !MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_ERPT_ENTRIES_4KB) ||
1501 !MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_ERPT_ENTRIES_8KB) ||
1502 !MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_ERPT_ENTRIES_12KB))
1505 size = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_ERPT_ENTRIES_2KB);
1506 erp_core->erpt_entries_size[MLXSW_SP_ACL_ATCAM_REGION_TYPE_2KB] = size;
1508 size = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_ERPT_ENTRIES_4KB);
1509 erp_core->erpt_entries_size[MLXSW_SP_ACL_ATCAM_REGION_TYPE_4KB] = size;
1511 size = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_ERPT_ENTRIES_8KB);
1512 erp_core->erpt_entries_size[MLXSW_SP_ACL_ATCAM_REGION_TYPE_8KB] = size;
1514 size = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_ERPT_ENTRIES_12KB);
1515 erp_core->erpt_entries_size[MLXSW_SP_ACL_ATCAM_REGION_TYPE_12KB] = size;
1520 static int mlxsw_sp_acl_erp_tables_init(struct mlxsw_sp *mlxsw_sp,
1521 struct mlxsw_sp_acl_erp_core *erp_core)
1523 unsigned int erpt_bank_size;
1526 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_MAX_ERPT_BANK_SIZE) ||
1527 !MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_MAX_ERPT_BANKS))
1529 erpt_bank_size = MLXSW_CORE_RES_GET(mlxsw_sp->core,
1530 ACL_MAX_ERPT_BANK_SIZE);
1531 erp_core->num_erp_banks = MLXSW_CORE_RES_GET(mlxsw_sp->core,
1532 ACL_MAX_ERPT_BANKS);
1534 erp_core->erp_tables = gen_pool_create(0, -1);
1535 if (!erp_core->erp_tables)
1537 gen_pool_set_algo(erp_core->erp_tables, gen_pool_best_fit, NULL);
1539 err = gen_pool_add(erp_core->erp_tables,
1540 MLXSW_SP_ACL_ERP_GENALLOC_OFFSET, erpt_bank_size,
1543 goto err_gen_pool_add;
1545 erp_core->bf = mlxsw_sp_acl_bf_init(mlxsw_sp, erp_core->num_erp_banks);
1546 if (IS_ERR(erp_core->bf)) {
1547 err = PTR_ERR(erp_core->bf);
1551 /* Different regions require masks of different sizes */
1552 err = mlxsw_sp_acl_erp_tables_sizes_query(mlxsw_sp, erp_core);
1554 goto err_erp_tables_sizes_query;
1558 err_erp_tables_sizes_query:
1559 mlxsw_sp_acl_bf_fini(erp_core->bf);
1562 gen_pool_destroy(erp_core->erp_tables);
1566 static void mlxsw_sp_acl_erp_tables_fini(struct mlxsw_sp *mlxsw_sp,
1567 struct mlxsw_sp_acl_erp_core *erp_core)
1569 mlxsw_sp_acl_bf_fini(erp_core->bf);
1570 gen_pool_destroy(erp_core->erp_tables);
1573 int mlxsw_sp_acl_erps_init(struct mlxsw_sp *mlxsw_sp,
1574 struct mlxsw_sp_acl_atcam *atcam)
1576 struct mlxsw_sp_acl_erp_core *erp_core;
1579 erp_core = kzalloc(sizeof(*erp_core), GFP_KERNEL);
1582 erp_core->mlxsw_sp = mlxsw_sp;
1583 atcam->erp_core = erp_core;
1585 err = mlxsw_sp_acl_erp_tables_init(mlxsw_sp, erp_core);
1587 goto err_erp_tables_init;
1591 err_erp_tables_init:
1596 void mlxsw_sp_acl_erps_fini(struct mlxsw_sp *mlxsw_sp,
1597 struct mlxsw_sp_acl_atcam *atcam)
1599 mlxsw_sp_acl_erp_tables_fini(mlxsw_sp, atcam->erp_core);
1600 kfree(atcam->erp_core);