drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_keys.c

   1 // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0
   2 /* Copyright (c) 2017-2018 Mellanox Technologies. All rights reserved */
   3
   4 #include <linux/kernel.h>
   5 #include <linux/slab.h>
   6 #include <linux/list.h>
   7 #include <linux/errno.h>
   8
   9 #include "item.h"
  10 #include "core_acl_flex_keys.h"
  11
  12 /* For the purpose of the driver, define an internal storage scratchpad
  13  * that will be used to store key/mask values. For each defined element type
  14  * define an internal storage geometry.
  15  *
  16  * When adding new elements, MLXSW_AFK_ELEMENT_STORAGE_SIZE must be increased
  17  * accordingly.
  18  */
  19 static const struct mlxsw_afk_element_info mlxsw_afk_element_infos[] = {
  20         MLXSW_AFK_ELEMENT_INFO_U32(SRC_SYS_PORT, 0x00, 16, 16),
  21         MLXSW_AFK_ELEMENT_INFO_BUF(DMAC_32_47, 0x04, 2),
  22         MLXSW_AFK_ELEMENT_INFO_BUF(DMAC_0_31, 0x06, 4),
  23         MLXSW_AFK_ELEMENT_INFO_BUF(SMAC_32_47, 0x0A, 2),
  24         MLXSW_AFK_ELEMENT_INFO_BUF(SMAC_0_31, 0x0C, 4),
  25         MLXSW_AFK_ELEMENT_INFO_U32(ETHERTYPE, 0x00, 0, 16),
  26         MLXSW_AFK_ELEMENT_INFO_U32(IP_PROTO, 0x10, 0, 8),
  27         MLXSW_AFK_ELEMENT_INFO_U32(VID, 0x10, 8, 12),
  28         MLXSW_AFK_ELEMENT_INFO_U32(PCP, 0x10, 20, 3),
  29         MLXSW_AFK_ELEMENT_INFO_U32(TCP_FLAGS, 0x10, 23, 9),
  30         MLXSW_AFK_ELEMENT_INFO_U32(DST_L4_PORT, 0x14, 0, 16),
  31         MLXSW_AFK_ELEMENT_INFO_U32(SRC_L4_PORT, 0x14, 16, 16),
  32         MLXSW_AFK_ELEMENT_INFO_U32(IP_TTL_, 0x18, 0, 8),
  33         MLXSW_AFK_ELEMENT_INFO_U32(IP_ECN, 0x18, 9, 2),
  34         MLXSW_AFK_ELEMENT_INFO_U32(IP_DSCP, 0x18, 11, 6),
  35         MLXSW_AFK_ELEMENT_INFO_U32(VIRT_ROUTER_8_10, 0x18, 17, 3),
  36         MLXSW_AFK_ELEMENT_INFO_U32(VIRT_ROUTER_0_7, 0x18, 20, 8),
  37         MLXSW_AFK_ELEMENT_INFO_BUF(SRC_IP_96_127, 0x20, 4),
  38         MLXSW_AFK_ELEMENT_INFO_BUF(SRC_IP_64_95, 0x24, 4),
  39         MLXSW_AFK_ELEMENT_INFO_BUF(SRC_IP_32_63, 0x28, 4),
  40         MLXSW_AFK_ELEMENT_INFO_BUF(SRC_IP_0_31, 0x2C, 4),
  41         MLXSW_AFK_ELEMENT_INFO_BUF(DST_IP_96_127, 0x30, 4),
  42         MLXSW_AFK_ELEMENT_INFO_BUF(DST_IP_64_95, 0x34, 4),
  43         MLXSW_AFK_ELEMENT_INFO_BUF(DST_IP_32_63, 0x38, 4),
  44         MLXSW_AFK_ELEMENT_INFO_BUF(DST_IP_0_31, 0x3C, 4),
  45 };
  46
  47 struct mlxsw_afk {
  48         struct list_head key_info_list;
  49         unsigned int max_blocks;
  50         const struct mlxsw_afk_ops *ops;
  51         const struct mlxsw_afk_block *blocks;
  52         unsigned int blocks_count;
  53 };
  54
  55 static bool mlxsw_afk_blocks_check(struct mlxsw_afk *mlxsw_afk)
  56 {
  57         int i;
  58         int j;
  59
  60         for (i = 0; i < mlxsw_afk->blocks_count; i++) {
  61                 const struct mlxsw_afk_block *block = &mlxsw_afk->blocks[i];
  62
  63                 for (j = 0; j < block->instances_count; j++) {
  64                         const struct mlxsw_afk_element_info *elinfo;
  65                         struct mlxsw_afk_element_inst *elinst;
  66
  67                         elinst = &block->instances[j];
  68                         elinfo = &mlxsw_afk_element_infos[elinst->element];
  69                         if (elinst->type != elinfo->type ||
  70                             (!elinst->avoid_size_check &&
  71                              elinst->item.size.bits !=
  72                              elinfo->item.size.bits))
  73                                 return false;
  74                 }
  75         }
  76         return true;
  77 }
  78
  79 struct mlxsw_afk *mlxsw_afk_create(unsigned int max_blocks,
  80                                    const struct mlxsw_afk_ops *ops)
  81 {
  82         struct mlxsw_afk *mlxsw_afk;
  83
  84         mlxsw_afk = kzalloc(sizeof(*mlxsw_afk), GFP_KERNEL);
  85         if (!mlxsw_afk)
  86                 return NULL;
  87         INIT_LIST_HEAD(&mlxsw_afk->key_info_list);
  88         mlxsw_afk->max_blocks = max_blocks;
  89         mlxsw_afk->ops = ops;
  90         mlxsw_afk->blocks = ops->blocks;
  91         mlxsw_afk->blocks_count = ops->blocks_count;
  92         WARN_ON(!mlxsw_afk_blocks_check(mlxsw_afk));
  93         return mlxsw_afk;
  94 }
  95 EXPORT_SYMBOL(mlxsw_afk_create);
  96
  97 void mlxsw_afk_destroy(struct mlxsw_afk *mlxsw_afk)
  98 {
  99         WARN_ON(!list_empty(&mlxsw_afk->key_info_list));
 100         kfree(mlxsw_afk);
 101 }
 102 EXPORT_SYMBOL(mlxsw_afk_destroy);
 103
 104 struct mlxsw_afk_key_info {
 105         struct list_head list;
 106         unsigned int ref_count;
 107         unsigned int blocks_count;
 108         int element_to_block[MLXSW_AFK_ELEMENT_MAX]; /* index is element, value
 109                                                       * is index inside "blocks"
 110                                                       */
 111         struct mlxsw_afk_element_usage elusage;
 112         const struct mlxsw_afk_block *blocks[];
 113 };
 114
 115 static bool
 116 mlxsw_afk_key_info_elements_eq(struct mlxsw_afk_key_info *key_info,
 117                                struct mlxsw_afk_element_usage *elusage)
 118 {
 119         return memcmp(&key_info->elusage, elusage, sizeof(*elusage)) == 0;
 120 }
 121
 122 static struct mlxsw_afk_key_info *
 123 mlxsw_afk_key_info_find(struct mlxsw_afk *mlxsw_afk,
 124                         struct mlxsw_afk_element_usage *elusage)
 125 {
 126         struct mlxsw_afk_key_info *key_info;
 127
 128         list_for_each_entry(key_info, &mlxsw_afk->key_info_list, list) {
 129                 if (mlxsw_afk_key_info_elements_eq(key_info, elusage))
 130                         return key_info;
 131         }
 132         return NULL;
 133 }
 134
 135 struct mlxsw_afk_picker {
 136         DECLARE_BITMAP(element, MLXSW_AFK_ELEMENT_MAX);
 137         unsigned int total;
 138 };
 139
 140 static void mlxsw_afk_picker_count_hits(struct mlxsw_afk *mlxsw_afk,
 141                                         struct mlxsw_afk_picker *picker,
 142                                         enum mlxsw_afk_element element)
 143 {
 144         int i;
 145         int j;
 146
 147         for (i = 0; i < mlxsw_afk->blocks_count; i++) {
 148                 const struct mlxsw_afk_block *block = &mlxsw_afk->blocks[i];
 149
 150                 for (j = 0; j < block->instances_count; j++) {
 151                         struct mlxsw_afk_element_inst *elinst;
 152
 153                         elinst = &block->instances[j];
 154                         if (elinst->element == element) {
 155                                 __set_bit(element, picker[i].element);
 156                                 picker[i].total++;
 157                         }
 158                 }
 159         }
 160 }
 161
 162 static void mlxsw_afk_picker_subtract_hits(struct mlxsw_afk *mlxsw_afk,
 163                                            struct mlxsw_afk_picker *picker,
 164                                            int block_index)
 165 {
 166         DECLARE_BITMAP(hits_element, MLXSW_AFK_ELEMENT_MAX);
 167         int i;
 168         int j;
 169
 170         memcpy(&hits_element, &picker[block_index].element,
 171                sizeof(hits_element));
 172
 173         for (i = 0; i < mlxsw_afk->blocks_count; i++) {
 174                 for_each_set_bit(j, hits_element, MLXSW_AFK_ELEMENT_MAX) {
 175                         if (__test_and_clear_bit(j, picker[i].element))
 176                                 picker[i].total--;
 177                 }
 178         }
 179 }
 180
 181 static int mlxsw_afk_picker_most_hits_get(struct mlxsw_afk *mlxsw_afk,
 182                                           struct mlxsw_afk_picker *picker)
 183 {
 184         int most_index = -EINVAL; /* Should never happen to return this */
 185         int most_hits = 0;
 186         int i;
 187
 188         for (i = 0; i < mlxsw_afk->blocks_count; i++) {
 189                 if (picker[i].total > most_hits) {
 190                         most_hits = picker[i].total;
 191                         most_index = i;
 192                 }
 193         }
 194         return most_index;
 195 }
 196
 197 static int mlxsw_afk_picker_key_info_add(struct mlxsw_afk *mlxsw_afk,
 198                                          struct mlxsw_afk_picker *picker,
 199                                          int block_index,
 200                                          struct mlxsw_afk_key_info *key_info)
 201 {
 202         enum mlxsw_afk_element element;
 203
 204         if (key_info->blocks_count == mlxsw_afk->max_blocks)
 205                 return -EINVAL;
 206
 207         for_each_set_bit(element, picker[block_index].element,
 208                          MLXSW_AFK_ELEMENT_MAX) {
 209                 key_info->element_to_block[element] = key_info->blocks_count;
 210                 mlxsw_afk_element_usage_add(&key_info->elusage, element);
 211         }
 212
 213         key_info->blocks[key_info->blocks_count] =
 214                                         &mlxsw_afk->blocks[block_index];
 215         key_info->blocks_count++;
 216         return 0;
 217 }
 218
 219 static int mlxsw_afk_picker(struct mlxsw_afk *mlxsw_afk,
 220                             struct mlxsw_afk_key_info *key_info,
 221                             struct mlxsw_afk_element_usage *elusage)
 222 {
 223         struct mlxsw_afk_picker *picker;
 224         enum mlxsw_afk_element element;
 225         int err;
 226
 227         picker = kcalloc(mlxsw_afk->blocks_count, sizeof(*picker), GFP_KERNEL);
 228         if (!picker)
 229                 return -ENOMEM;
 230
 231         /* Since the same elements could be present in multiple blocks,
 232          * we must find out optimal block list in order to make the
 233          * block count as low as possible.
 234          *
 235          * First, we count hits. We go over all available blocks and count
 236          * how many of requested elements are covered by each.
 237          *
 238          * Then in loop, we find block with most hits and add it to
 239          * output key_info. Then we have to subtract this block hits so
 240          * the next iteration will find most suitable block for
 241          * the rest of requested elements.
 242          */
 243
 244         mlxsw_afk_element_usage_for_each(element, elusage)
 245                 mlxsw_afk_picker_count_hits(mlxsw_afk, picker, element);
 246
 247         do {
 248                 int block_index;
 249
 250                 block_index = mlxsw_afk_picker_most_hits_get(mlxsw_afk, picker);
 251                 if (block_index < 0) {
 252                         err = block_index;
 253                         goto out;
 254                 }
 255                 err = mlxsw_afk_picker_key_info_add(mlxsw_afk, picker,
 256                                                     block_index, key_info);
 257                 if (err)
 258                         goto out;
 259                 mlxsw_afk_picker_subtract_hits(mlxsw_afk, picker, block_index);
 260         } while (!mlxsw_afk_key_info_elements_eq(key_info, elusage));
 261
 262         err = 0;
 263 out:
 264         kfree(picker);
 265         return err;
 266 }
 267
 268 static struct mlxsw_afk_key_info *
 269 mlxsw_afk_key_info_create(struct mlxsw_afk *mlxsw_afk,
 270                           struct mlxsw_afk_element_usage *elusage)
 271 {
 272         struct mlxsw_afk_key_info *key_info;
 273         int err;
 274
 275         key_info = kzalloc(struct_size(key_info, blocks, mlxsw_afk->max_blocks),
 276                            GFP_KERNEL);
 277         if (!key_info)
 278                 return ERR_PTR(-ENOMEM);
 279         err = mlxsw_afk_picker(mlxsw_afk, key_info, elusage);
 280         if (err)
 281                 goto err_picker;
 282         list_add(&key_info->list, &mlxsw_afk->key_info_list);
 283         key_info->ref_count = 1;
 284         return key_info;
 285
 286 err_picker:
 287         kfree(key_info);
 288         return ERR_PTR(err);
 289 }
 290
 291 static void mlxsw_afk_key_info_destroy(struct mlxsw_afk_key_info *key_info)
 292 {
 293         list_del(&key_info->list);
 294         kfree(key_info);
 295 }
 296
 297 struct mlxsw_afk_key_info *
 298 mlxsw_afk_key_info_get(struct mlxsw_afk *mlxsw_afk,
 299                        struct mlxsw_afk_element_usage *elusage)
 300 {
 301         struct mlxsw_afk_key_info *key_info;
 302
 303         key_info = mlxsw_afk_key_info_find(mlxsw_afk, elusage);
 304         if (key_info) {
 305                 key_info->ref_count++;
 306                 return key_info;
 307         }
 308         return mlxsw_afk_key_info_create(mlxsw_afk, elusage);
 309 }
 310 EXPORT_SYMBOL(mlxsw_afk_key_info_get);
 311
 312 void mlxsw_afk_key_info_put(struct mlxsw_afk_key_info *key_info)
 313 {
 314         if (--key_info->ref_count)
 315                 return;
 316         mlxsw_afk_key_info_destroy(key_info);
 317 }
 318 EXPORT_SYMBOL(mlxsw_afk_key_info_put);
 319
 320 bool mlxsw_afk_key_info_subset(struct mlxsw_afk_key_info *key_info,
 321                                struct mlxsw_afk_element_usage *elusage)
 322 {
 323         return mlxsw_afk_element_usage_subset(elusage, &key_info->elusage);
 324 }
 325 EXPORT_SYMBOL(mlxsw_afk_key_info_subset);
 326
 327 static const struct mlxsw_afk_element_inst *
 328 mlxsw_afk_block_elinst_get(const struct mlxsw_afk_block *block,
 329                            enum mlxsw_afk_element element)
 330 {
 331         int i;
 332
 333         for (i = 0; i < block->instances_count; i++) {
 334                 struct mlxsw_afk_element_inst *elinst;
 335
 336                 elinst = &block->instances[i];
 337                 if (elinst->element == element)
 338                         return elinst;
 339         }
 340         return NULL;
 341 }
 342
 343 static const struct mlxsw_afk_element_inst *
 344 mlxsw_afk_key_info_elinst_get(struct mlxsw_afk_key_info *key_info,
 345                               enum mlxsw_afk_element element,
 346                               int *p_block_index)
 347 {
 348         const struct mlxsw_afk_element_inst *elinst;
 349         const struct mlxsw_afk_block *block;
 350         int block_index;
 351
 352         if (WARN_ON(!test_bit(element, key_info->elusage.usage)))
 353                 return NULL;
 354         block_index = key_info->element_to_block[element];
 355         block = key_info->blocks[block_index];
 356
 357         elinst = mlxsw_afk_block_elinst_get(block, element);
 358         if (WARN_ON(!elinst))
 359                 return NULL;
 360
 361         *p_block_index = block_index;
 362         return elinst;
 363 }
 364
 365 u16
 366 mlxsw_afk_key_info_block_encoding_get(const struct mlxsw_afk_key_info *key_info,
 367                                       int block_index)
 368 {
 369         return key_info->blocks[block_index]->encoding;
 370 }
 371 EXPORT_SYMBOL(mlxsw_afk_key_info_block_encoding_get);
 372
 373 unsigned int
 374 mlxsw_afk_key_info_blocks_count_get(const struct mlxsw_afk_key_info *key_info)
 375 {
 376         return key_info->blocks_count;
 377 }
 378 EXPORT_SYMBOL(mlxsw_afk_key_info_blocks_count_get);
 379
 380 void mlxsw_afk_values_add_u32(struct mlxsw_afk_element_values *values,
 381                               enum mlxsw_afk_element element,
 382                               u32 key_value, u32 mask_value)
 383 {
 384         const struct mlxsw_afk_element_info *elinfo =
 385                                 &mlxsw_afk_element_infos[element];
 386         const struct mlxsw_item *storage_item = &elinfo->item;
 387
 388         if (!mask_value)
 389                 return;
 390         if (WARN_ON(elinfo->type != MLXSW_AFK_ELEMENT_TYPE_U32))
 391                 return;
 392         __mlxsw_item_set32(values->storage.key, storage_item, 0, key_value);
 393         __mlxsw_item_set32(values->storage.mask, storage_item, 0, mask_value);
 394         mlxsw_afk_element_usage_add(&values->elusage, element);
 395 }
 396 EXPORT_SYMBOL(mlxsw_afk_values_add_u32);
 397
 398 void mlxsw_afk_values_add_buf(struct mlxsw_afk_element_values *values,
 399                               enum mlxsw_afk_element element,
 400                               const char *key_value, const char *mask_value,
 401                               unsigned int len)
 402 {
 403         const struct mlxsw_afk_element_info *elinfo =
 404                                 &mlxsw_afk_element_infos[element];
 405         const struct mlxsw_item *storage_item = &elinfo->item;
 406
 407         if (!memchr_inv(mask_value, 0, len)) /* If mask is zero */
 408                 return;
 409         if (WARN_ON(elinfo->type != MLXSW_AFK_ELEMENT_TYPE_BUF) ||
 410             WARN_ON(elinfo->item.size.bytes != len))
 411                 return;
 412         __mlxsw_item_memcpy_to(values->storage.key, key_value,
 413                                storage_item, 0);
 414         __mlxsw_item_memcpy_to(values->storage.mask, mask_value,
 415                                storage_item, 0);
 416         mlxsw_afk_element_usage_add(&values->elusage, element);
 417 }
 418 EXPORT_SYMBOL(mlxsw_afk_values_add_buf);
 419
 420 static void mlxsw_sp_afk_encode_u32(const struct mlxsw_item *storage_item,
 421                                     const struct mlxsw_item *output_item,
 422                                     char *storage, char *output, int diff)
 423 {
 424         u32 value;
 425
 426         value = __mlxsw_item_get32(storage, storage_item, 0);
 427         __mlxsw_item_set32(output, output_item, 0, value + diff);
 428 }
 429
 430 static void mlxsw_sp_afk_encode_buf(const struct mlxsw_item *storage_item,
 431                                     const struct mlxsw_item *output_item,
 432                                     char *storage, char *output)
 433 {
 434         char *storage_data = __mlxsw_item_data(storage, storage_item, 0);
 435         char *output_data = __mlxsw_item_data(output, output_item, 0);
 436         size_t len = output_item->size.bytes;
 437
 438         memcpy(output_data, storage_data, len);
 439 }
 440
 441 static void
 442 mlxsw_sp_afk_encode_one(const struct mlxsw_afk_element_inst *elinst,
 443                         char *output, char *storage, int u32_diff)
 444 {
 445         const struct mlxsw_item *output_item = &elinst->item;
 446         const struct mlxsw_afk_element_info *elinfo;
 447         const struct mlxsw_item *storage_item;
 448
 449         elinfo = &mlxsw_afk_element_infos[elinst->element];
 450         storage_item = &elinfo->item;
 451         if (elinst->type == MLXSW_AFK_ELEMENT_TYPE_U32)
 452                 mlxsw_sp_afk_encode_u32(storage_item, output_item,
 453                                         storage, output, u32_diff);
 454         else if (elinst->type == MLXSW_AFK_ELEMENT_TYPE_BUF)
 455                 mlxsw_sp_afk_encode_buf(storage_item, output_item,
 456                                         storage, output);
 457 }
 458
 459 #define MLXSW_SP_AFK_KEY_BLOCK_MAX_SIZE 16
 460
 461 void mlxsw_afk_encode(struct mlxsw_afk *mlxsw_afk,
 462                       struct mlxsw_afk_key_info *key_info,
 463                       struct mlxsw_afk_element_values *values,
 464                       char *key, char *mask)
 465 {
 466         unsigned int blocks_count =
 467                         mlxsw_afk_key_info_blocks_count_get(key_info);
 468         char block_mask[MLXSW_SP_AFK_KEY_BLOCK_MAX_SIZE];
 469         char block_key[MLXSW_SP_AFK_KEY_BLOCK_MAX_SIZE];
 470         const struct mlxsw_afk_element_inst *elinst;
 471         enum mlxsw_afk_element element;
 472         int block_index, i;
 473
 474         for (i = 0; i < blocks_count; i++) {
 475                 memset(block_key, 0, MLXSW_SP_AFK_KEY_BLOCK_MAX_SIZE);
 476                 memset(block_mask, 0, MLXSW_SP_AFK_KEY_BLOCK_MAX_SIZE);
 477
 478                 mlxsw_afk_element_usage_for_each(element, &values->elusage) {
 479                         elinst = mlxsw_afk_key_info_elinst_get(key_info,
 480                                                                element,
 481                                                                &block_index);
 482                         if (!elinst || block_index != i)
 483                                 continue;
 484
 485                         mlxsw_sp_afk_encode_one(elinst, block_key,
 486                                                 values->storage.key,
 487                                                 elinst->u32_key_diff);
 488                         mlxsw_sp_afk_encode_one(elinst, block_mask,
 489                                                 values->storage.mask, 0);
 490                 }
 491
 492                 mlxsw_afk->ops->encode_block(key, i, block_key);
 493                 mlxsw_afk->ops->encode_block(mask, i, block_mask);
 494         }
 495 }
 496 EXPORT_SYMBOL(mlxsw_afk_encode);
 497
 498 void mlxsw_afk_clear(struct mlxsw_afk *mlxsw_afk, char *key,
 499                      int block_start, int block_end)
 500 {
 501         int i;
 502
 503         for (i = block_start; i <= block_end; i++)
 504                 mlxsw_afk->ops->clear_block(key, i);
 505 }
 506 EXPORT_SYMBOL(mlxsw_afk_clear);