net/netfilter/nf_tables_offload.c

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 #include <linux/init.h>
   3 #include <linux/module.h>
   4 #include <linux/netfilter.h>
   5 #include <net/flow_offload.h>
   6 #include <net/netfilter/nf_tables.h>
   7 #include <net/netfilter/nf_tables_offload.h>
   8 #include <net/pkt_cls.h>
   9
  10 static struct nft_flow_rule *nft_flow_rule_alloc(int num_actions)
  11 {
  12         struct nft_flow_rule *flow;
  13
  14         flow = kzalloc(sizeof(struct nft_flow_rule), GFP_KERNEL);
  15         if (!flow)
  16                 return NULL;
  17
  18         flow->rule = flow_rule_alloc(num_actions);
  19         if (!flow->rule) {
  20                 kfree(flow);
  21                 return NULL;
  22         }
  23
  24         flow->rule->match.dissector     = &flow->match.dissector;
  25         flow->rule->match.mask          = &flow->match.mask;
  26         flow->rule->match.key           = &flow->match.key;
  27
  28         return flow;
  29 }
  30
  31 struct nft_flow_rule *nft_flow_rule_create(struct net *net,
  32                                            const struct nft_rule *rule)
  33 {
  34         struct nft_offload_ctx *ctx;
  35         struct nft_flow_rule *flow;
  36         int num_actions = 0, err;
  37         struct nft_expr *expr;
  38
  39         expr = nft_expr_first(rule);
  40         while (expr->ops && expr != nft_expr_last(rule)) {
  41                 if (expr->ops->offload_flags & NFT_OFFLOAD_F_ACTION)
  42                         num_actions++;
  43
  44                 expr = nft_expr_next(expr);
  45         }
  46
  47         if (num_actions == 0)
  48                 return ERR_PTR(-EOPNOTSUPP);
  49
  50         flow = nft_flow_rule_alloc(num_actions);
  51         if (!flow)
  52                 return ERR_PTR(-ENOMEM);
  53
  54         expr = nft_expr_first(rule);
  55
  56         ctx = kzalloc(sizeof(struct nft_offload_ctx), GFP_KERNEL);
  57         if (!ctx) {
  58                 err = -ENOMEM;
  59                 goto err_out;
  60         }
  61         ctx->net = net;
  62         ctx->dep.type = NFT_OFFLOAD_DEP_UNSPEC;
  63
  64         while (expr->ops && expr != nft_expr_last(rule)) {
  65                 if (!expr->ops->offload) {
  66                         err = -EOPNOTSUPP;
  67                         goto err_out;
  68                 }
  69                 err = expr->ops->offload(ctx, flow, expr);
  70                 if (err < 0)
  71                         goto err_out;
  72
  73                 expr = nft_expr_next(expr);
  74         }
  75         flow->proto = ctx->dep.l3num;
  76         kfree(ctx);
  77
  78         return flow;
  79 err_out:
  80         kfree(ctx);
  81         nft_flow_rule_destroy(flow);
  82
  83         return ERR_PTR(err);
  84 }
  85
  86 void nft_flow_rule_destroy(struct nft_flow_rule *flow)
  87 {
  88         struct flow_action_entry *entry;
  89         int i;
  90
  91         flow_action_for_each(i, entry, &flow->rule->action) {
  92                 switch (entry->id) {
  93                 case FLOW_ACTION_REDIRECT:
  94                 case FLOW_ACTION_MIRRED:
  95                         dev_put(entry->dev);
  96                         break;
  97                 default:
  98                         break;
  99                 }
 100         }
 101         kfree(flow->rule);
 102         kfree(flow);
 103 }
 104
 105 void nft_offload_set_dependency(struct nft_offload_ctx *ctx,
 106                                 enum nft_offload_dep_type type)
 107 {
 108         ctx->dep.type = type;
 109 }
 110
 111 void nft_offload_update_dependency(struct nft_offload_ctx *ctx,
 112                                    const void *data, u32 len)
 113 {
 114         switch (ctx->dep.type) {
 115         case NFT_OFFLOAD_DEP_NETWORK:
 116                 WARN_ON(len != sizeof(__u16));
 117                 memcpy(&ctx->dep.l3num, data, sizeof(__u16));
 118                 break;
 119         case NFT_OFFLOAD_DEP_TRANSPORT:
 120                 WARN_ON(len != sizeof(__u8));
 121                 memcpy(&ctx->dep.protonum, data, sizeof(__u8));
 122                 break;
 123         default:
 124                 break;
 125         }
 126         ctx->dep.type = NFT_OFFLOAD_DEP_UNSPEC;
 127 }
 128
 129 static void nft_flow_offload_common_init(struct flow_cls_common_offload *common,
 130                                          __be16 proto, int priority,
 131                                          struct netlink_ext_ack *extack)
 132 {
 133         common->protocol = proto;
 134         common->prio = priority;
 135         common->extack = extack;
 136 }
 137
 138 static int nft_setup_cb_call(enum tc_setup_type type, void *type_data,
 139                              struct list_head *cb_list)
 140 {
 141         struct flow_block_cb *block_cb;
 142         int err;
 143
 144         list_for_each_entry(block_cb, cb_list, list) {
 145                 err = block_cb->cb(type, type_data, block_cb->cb_priv);
 146                 if (err < 0)
 147                         return err;
 148         }
 149         return 0;
 150 }
 151
 152 int nft_chain_offload_priority(struct nft_base_chain *basechain)
 153 {
 154         if (basechain->ops.priority <= 0 ||
 155             basechain->ops.priority > USHRT_MAX)
 156                 return -1;
 157
 158         return 0;
 159 }
 160
 161 static void nft_flow_cls_offload_setup(struct flow_cls_offload *cls_flow,
 162                                        const struct nft_base_chain *basechain,
 163                                        const struct nft_rule *rule,
 164                                        const struct nft_flow_rule *flow,
 165                                        struct netlink_ext_ack *extack,
 166                                        enum flow_cls_command command)
 167 {
 168         __be16 proto = ETH_P_ALL;
 169
 170         memset(cls_flow, 0, sizeof(*cls_flow));
 171
 172         if (flow)
 173                 proto = flow->proto;
 174
 175         nft_flow_offload_common_init(&cls_flow->common, proto,
 176                                      basechain->ops.priority, extack);
 177         cls_flow->command = command;
 178         cls_flow->cookie = (unsigned long) rule;
 179         if (flow)
 180                 cls_flow->rule = flow->rule;
 181 }
 182
 183 static int nft_flow_offload_rule(struct nft_chain *chain,
 184                                  struct nft_rule *rule,
 185                                  struct nft_flow_rule *flow,
 186                                  enum flow_cls_command command)
 187 {
 188         struct netlink_ext_ack extack = {};
 189         struct flow_cls_offload cls_flow;
 190         struct nft_base_chain *basechain;
 191
 192         if (!nft_is_base_chain(chain))
 193                 return -EOPNOTSUPP;
 194
 195         basechain = nft_base_chain(chain);
 196         nft_flow_cls_offload_setup(&cls_flow, basechain, rule, flow, &extack,
 197                                    command);
 198
 199         return nft_setup_cb_call(TC_SETUP_CLSFLOWER, &cls_flow,
 200                                  &basechain->flow_block.cb_list);
 201 }
 202
 203 static int nft_flow_offload_bind(struct flow_block_offload *bo,
 204                                  struct nft_base_chain *basechain)
 205 {
 206         list_splice(&bo->cb_list, &basechain->flow_block.cb_list);
 207         return 0;
 208 }
 209
 210 static int nft_flow_offload_unbind(struct flow_block_offload *bo,
 211                                    struct nft_base_chain *basechain)
 212 {
 213         struct flow_block_cb *block_cb, *next;
 214         struct flow_cls_offload cls_flow;
 215         struct netlink_ext_ack extack;
 216         struct nft_chain *chain;
 217         struct nft_rule *rule;
 218
 219         chain = &basechain->chain;
 220         list_for_each_entry(rule, &chain->rules, list) {
 221                 memset(&extack, 0, sizeof(extack));
 222                 nft_flow_cls_offload_setup(&cls_flow, basechain, rule, NULL,
 223                                            &extack, FLOW_CLS_DESTROY);
 224                 nft_setup_cb_call(TC_SETUP_CLSFLOWER, &cls_flow, &bo->cb_list);
 225         }
 226
 227         list_for_each_entry_safe(block_cb, next, &bo->cb_list, list) {
 228                 list_del(&block_cb->list);
 229                 flow_block_cb_free(block_cb);
 230         }
 231
 232         return 0;
 233 }
 234
 235 static int nft_block_setup(struct nft_base_chain *basechain,
 236                            struct flow_block_offload *bo,
 237                            enum flow_block_command cmd)
 238 {
 239         int err;
 240
 241         switch (cmd) {
 242         case FLOW_BLOCK_BIND:
 243                 err = nft_flow_offload_bind(bo, basechain);
 244                 break;
 245         case FLOW_BLOCK_UNBIND:
 246                 err = nft_flow_offload_unbind(bo, basechain);
 247                 break;
 248         default:
 249                 WARN_ON_ONCE(1);
 250                 err = -EOPNOTSUPP;
 251         }
 252
 253         return err;
 254 }
 255
 256 static void nft_flow_block_offload_init(struct flow_block_offload *bo,
 257                                         struct net *net,
 258                                         enum flow_block_command cmd,
 259                                         struct nft_base_chain *basechain,
 260                                         struct netlink_ext_ack *extack)
 261 {
 262         memset(bo, 0, sizeof(*bo));
 263         bo->net         = net;
 264         bo->block       = &basechain->flow_block;
 265         bo->command     = cmd;
 266         bo->binder_type = FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS;
 267         bo->extack      = extack;
 268         INIT_LIST_HEAD(&bo->cb_list);
 269 }
 270
 271 static int nft_block_offload_cmd(struct nft_base_chain *chain,
 272                                  struct net_device *dev,
 273                                  enum flow_block_command cmd)
 274 {
 275         struct netlink_ext_ack extack = {};
 276         struct flow_block_offload bo;
 277         int err;
 278
 279         nft_flow_block_offload_init(&bo, dev_net(dev), cmd, chain, &extack);
 280
 281         err = dev->netdev_ops->ndo_setup_tc(dev, TC_SETUP_BLOCK, &bo);
 282         if (err < 0)
 283                 return err;
 284
 285         return nft_block_setup(chain, &bo, cmd);
 286 }
 287
 288 static void nft_indr_block_cleanup(struct flow_block_cb *block_cb)
 289 {
 290         struct nft_base_chain *basechain = block_cb->indr.data;
 291         struct net_device *dev = block_cb->indr.dev;
 292         struct netlink_ext_ack extack = {};
 293         struct net *net = dev_net(dev);
 294         struct flow_block_offload bo;
 295
 296         nft_flow_block_offload_init(&bo, dev_net(dev), FLOW_BLOCK_UNBIND,
 297                                     basechain, &extack);
 298         mutex_lock(&net->nft.commit_mutex);
 299         list_move(&block_cb->list, &bo.cb_list);
 300         nft_flow_offload_unbind(&bo, basechain);
 301         mutex_unlock(&net->nft.commit_mutex);
 302 }
 303
 304 static int nft_indr_block_offload_cmd(struct nft_base_chain *basechain,
 305                                       struct net_device *dev,
 306                                       enum flow_block_command cmd)
 307 {
 308         struct netlink_ext_ack extack = {};
 309         struct flow_block_offload bo;
 310         int err;
 311
 312         nft_flow_block_offload_init(&bo, dev_net(dev), cmd, basechain, &extack);
 313
 314         err = flow_indr_dev_setup_offload(dev, TC_SETUP_BLOCK, basechain, &bo,
 315                                           nft_indr_block_cleanup);
 316         if (err < 0)
 317                 return err;
 318
 319         if (list_empty(&bo.cb_list))
 320                 return -EOPNOTSUPP;
 321
 322         return nft_block_setup(basechain, &bo, cmd);
 323 }
 324
 325 #define FLOW_SETUP_BLOCK TC_SETUP_BLOCK
 326
 327 static int nft_chain_offload_cmd(struct nft_base_chain *basechain,
 328                                  struct net_device *dev,
 329                                  enum flow_block_command cmd)
 330 {
 331         int err;
 332
 333         if (dev->netdev_ops->ndo_setup_tc)
 334                 err = nft_block_offload_cmd(basechain, dev, cmd);
 335         else
 336                 err = nft_indr_block_offload_cmd(basechain, dev, cmd);
 337
 338         return err;
 339 }
 340
 341 static int nft_flow_block_chain(struct nft_base_chain *basechain,
 342                                 const struct net_device *this_dev,
 343                                 enum flow_block_command cmd)
 344 {
 345         struct net_device *dev;
 346         struct nft_hook *hook;
 347         int err, i = 0;
 348
 349         list_for_each_entry(hook, &basechain->hook_list, list) {
 350                 dev = hook->ops.dev;
 351                 if (this_dev && this_dev != dev)
 352                         continue;
 353
 354                 err = nft_chain_offload_cmd(basechain, dev, cmd);
 355                 if (err < 0 && cmd == FLOW_BLOCK_BIND) {
 356                         if (!this_dev)
 357                                 goto err_flow_block;
 358
 359                         return err;
 360                 }
 361                 i++;
 362         }
 363
 364         return 0;
 365
 366 err_flow_block:
 367         list_for_each_entry(hook, &basechain->hook_list, list) {
 368                 if (i-- <= 0)
 369                         break;
 370
 371                 dev = hook->ops.dev;
 372                 nft_chain_offload_cmd(basechain, dev, FLOW_BLOCK_UNBIND);
 373         }
 374         return err;
 375 }
 376
 377 static int nft_flow_offload_chain(struct nft_chain *chain, u8 *ppolicy,
 378                                   enum flow_block_command cmd)
 379 {
 380         struct nft_base_chain *basechain;
 381         u8 policy;
 382
 383         if (!nft_is_base_chain(chain))
 384                 return -EOPNOTSUPP;
 385
 386         basechain = nft_base_chain(chain);
 387         policy = ppolicy ? *ppolicy : basechain->policy;
 388
 389         /* Only default policy to accept is supported for now. */
 390         if (cmd == FLOW_BLOCK_BIND && policy == NF_DROP)
 391                 return -EOPNOTSUPP;
 392
 393         return nft_flow_block_chain(basechain, NULL, cmd);
 394 }
 395
 396 static void nft_flow_rule_offload_abort(struct net *net,
 397                                         struct nft_trans *trans)
 398 {
 399         int err = 0;
 400
 401         list_for_each_entry_continue_reverse(trans, &net->nft.commit_list, list) {
 402                 if (trans->ctx.family != NFPROTO_NETDEV)
 403                         continue;
 404
 405                 switch (trans->msg_type) {
 406                 case NFT_MSG_NEWCHAIN:
 407                         if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD) ||
 408                             nft_trans_chain_update(trans))
 409                                 continue;
 410
 411                         err = nft_flow_offload_chain(trans->ctx.chain, NULL,
 412                                                      FLOW_BLOCK_UNBIND);
 413                         break;
 414                 case NFT_MSG_DELCHAIN:
 415                         if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
 416                                 continue;
 417
 418                         err = nft_flow_offload_chain(trans->ctx.chain, NULL,
 419                                                      FLOW_BLOCK_BIND);
 420                         break;
 421                 case NFT_MSG_NEWRULE:
 422                         if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
 423                                 continue;
 424
 425                         err = nft_flow_offload_rule(trans->ctx.chain,
 426                                                     nft_trans_rule(trans),
 427                                                     NULL, FLOW_CLS_DESTROY);
 428                         break;
 429                 case NFT_MSG_DELRULE:
 430                         if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
 431                                 continue;
 432
 433                         err = nft_flow_offload_rule(trans->ctx.chain,
 434                                                     nft_trans_rule(trans),
 435                                                     nft_trans_flow_rule(trans),
 436                                                     FLOW_CLS_REPLACE);
 437                         break;
 438                 }
 439
 440                 if (WARN_ON_ONCE(err))
 441                         break;
 442         }
 443 }
 444
 445 int nft_flow_rule_offload_commit(struct net *net)
 446 {
 447         struct nft_trans *trans;
 448         int err = 0;
 449         u8 policy;
 450
 451         list_for_each_entry(trans, &net->nft.commit_list, list) {
 452                 if (trans->ctx.family != NFPROTO_NETDEV)
 453                         continue;
 454
 455                 switch (trans->msg_type) {
 456                 case NFT_MSG_NEWCHAIN:
 457                         if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD) ||
 458                             nft_trans_chain_update(trans))
 459                                 continue;
 460
 461                         policy = nft_trans_chain_policy(trans);
 462                         err = nft_flow_offload_chain(trans->ctx.chain, &policy,
 463                                                      FLOW_BLOCK_BIND);
 464                         break;
 465                 case NFT_MSG_DELCHAIN:
 466                         if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
 467                                 continue;
 468
 469                         policy = nft_trans_chain_policy(trans);
 470                         err = nft_flow_offload_chain(trans->ctx.chain, &policy,
 471                                                      FLOW_BLOCK_UNBIND);
 472                         break;
 473                 case NFT_MSG_NEWRULE:
 474                         if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
 475                                 continue;
 476
 477                         if (trans->ctx.flags & NLM_F_REPLACE ||
 478                             !(trans->ctx.flags & NLM_F_APPEND)) {
 479                                 err = -EOPNOTSUPP;
 480                                 break;
 481                         }
 482                         err = nft_flow_offload_rule(trans->ctx.chain,
 483                                                     nft_trans_rule(trans),
 484                                                     nft_trans_flow_rule(trans),
 485                                                     FLOW_CLS_REPLACE);
 486                         break;
 487                 case NFT_MSG_DELRULE:
 488                         if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
 489                                 continue;
 490
 491                         err = nft_flow_offload_rule(trans->ctx.chain,
 492                                                     nft_trans_rule(trans),
 493                                                     NULL, FLOW_CLS_DESTROY);
 494                         break;
 495                 }
 496
 497                 if (err) {
 498                         nft_flow_rule_offload_abort(net, trans);
 499                         break;
 500                 }
 501         }
 502
 503         list_for_each_entry(trans, &net->nft.commit_list, list) {
 504                 if (trans->ctx.family != NFPROTO_NETDEV)
 505                         continue;
 506
 507                 switch (trans->msg_type) {
 508                 case NFT_MSG_NEWRULE:
 509                 case NFT_MSG_DELRULE:
 510                         if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
 511                                 continue;
 512
 513                         nft_flow_rule_destroy(nft_trans_flow_rule(trans));
 514                         break;
 515                 default:
 516                         break;
 517                 }
 518         }
 519
 520         return err;
 521 }
 522
 523 static struct nft_chain *__nft_offload_get_chain(struct net_device *dev)
 524 {
 525         struct nft_base_chain *basechain;
 526         struct net *net = dev_net(dev);
 527         struct nft_hook *hook, *found;
 528         const struct nft_table *table;
 529         struct nft_chain *chain;
 530
 531         list_for_each_entry(table, &net->nft.tables, list) {
 532                 if (table->family != NFPROTO_NETDEV)
 533                         continue;
 534
 535                 list_for_each_entry(chain, &table->chains, list) {
 536                         if (!nft_is_base_chain(chain) ||
 537                             !(chain->flags & NFT_CHAIN_HW_OFFLOAD))
 538                                 continue;
 539
 540                         found = NULL;
 541                         basechain = nft_base_chain(chain);
 542                         list_for_each_entry(hook, &basechain->hook_list, list) {
 543                                 if (hook->ops.dev != dev)
 544                                         continue;
 545
 546                                 found = hook;
 547                                 break;
 548                         }
 549                         if (!found)
 550                                 continue;
 551
 552                         return chain;
 553                 }
 554         }
 555
 556         return NULL;
 557 }
 558
 559 static int nft_offload_netdev_event(struct notifier_block *this,
 560                                     unsigned long event, void *ptr)
 561 {
 562         struct net_device *dev = netdev_notifier_info_to_dev(ptr);
 563         struct net *net = dev_net(dev);
 564         struct nft_chain *chain;
 565
 566         if (event != NETDEV_UNREGISTER)
 567                 return NOTIFY_DONE;
 568
 569         mutex_lock(&net->nft.commit_mutex);
 570         chain = __nft_offload_get_chain(dev);
 571         if (chain)
 572                 nft_flow_block_chain(nft_base_chain(chain), dev,
 573                                      FLOW_BLOCK_UNBIND);
 574
 575         mutex_unlock(&net->nft.commit_mutex);
 576
 577         return NOTIFY_DONE;
 578 }
 579
 580 static struct notifier_block nft_offload_netdev_notifier = {
 581         .notifier_call  = nft_offload_netdev_event,
 582 };
 583
 584 int nft_offload_init(void)
 585 {
 586         return register_netdevice_notifier(&nft_offload_netdev_notifier);
 587 }
 588
 589 void nft_offload_exit(void)
 590 {
 591         unregister_netdevice_notifier(&nft_offload_netdev_notifier);
 592 }