net/netfilter/nf_tables_offload.c

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 #include <linux/init.h>
   3 #include <linux/module.h>
   4 #include <linux/netfilter.h>
   5 #include <net/flow_offload.h>
   6 #include <net/netfilter/nf_tables.h>
   7 #include <net/netfilter/nf_tables_offload.h>
   8 #include <net/pkt_cls.h>
   9
  10 static struct nft_flow_rule *nft_flow_rule_alloc(int num_actions)
  11 {
  12         struct nft_flow_rule *flow;
  13
  14         flow = kzalloc(sizeof(struct nft_flow_rule), GFP_KERNEL);
  15         if (!flow)
  16                 return NULL;
  17
  18         flow->rule = flow_rule_alloc(num_actions);
  19         if (!flow->rule) {
  20                 kfree(flow);
  21                 return NULL;
  22         }
  23
  24         flow->rule->match.dissector     = &flow->match.dissector;
  25         flow->rule->match.mask          = &flow->match.mask;
  26         flow->rule->match.key           = &flow->match.key;
  27
  28         return flow;
  29 }
  30
  31 struct nft_flow_rule *nft_flow_rule_create(struct net *net,
  32                                            const struct nft_rule *rule)
  33 {
  34         struct nft_offload_ctx *ctx;
  35         struct nft_flow_rule *flow;
  36         int num_actions = 0, err;
  37         struct nft_expr *expr;
  38
  39         expr = nft_expr_first(rule);
  40         while (expr->ops && expr != nft_expr_last(rule)) {
  41                 if (expr->ops->offload_flags & NFT_OFFLOAD_F_ACTION)
  42                         num_actions++;
  43
  44                 expr = nft_expr_next(expr);
  45         }
  46
  47         if (num_actions == 0)
  48                 return ERR_PTR(-EOPNOTSUPP);
  49
  50         flow = nft_flow_rule_alloc(num_actions);
  51         if (!flow)
  52                 return ERR_PTR(-ENOMEM);
  53
  54         expr = nft_expr_first(rule);
  55
  56         ctx = kzalloc(sizeof(struct nft_offload_ctx), GFP_KERNEL);
  57         if (!ctx) {
  58                 err = -ENOMEM;
  59                 goto err_out;
  60         }
  61         ctx->net = net;
  62         ctx->dep.type = NFT_OFFLOAD_DEP_UNSPEC;
  63
  64         while (expr->ops && expr != nft_expr_last(rule)) {
  65                 if (!expr->ops->offload) {
  66                         err = -EOPNOTSUPP;
  67                         goto err_out;
  68                 }
  69                 err = expr->ops->offload(ctx, flow, expr);
  70                 if (err < 0)
  71                         goto err_out;
  72
  73                 expr = nft_expr_next(expr);
  74         }
  75         flow->proto = ctx->dep.l3num;
  76         kfree(ctx);
  77
  78         return flow;
  79 err_out:
  80         kfree(ctx);
  81         nft_flow_rule_destroy(flow);
  82
  83         return ERR_PTR(err);
  84 }
  85
  86 void nft_flow_rule_destroy(struct nft_flow_rule *flow)
  87 {
  88         struct flow_action_entry *entry;
  89         int i;
  90
  91         flow_action_for_each(i, entry, &flow->rule->action) {
  92                 switch (entry->id) {
  93                 case FLOW_ACTION_REDIRECT:
  94                 case FLOW_ACTION_MIRRED:
  95                         dev_put(entry->dev);
  96                         break;
  97                 default:
  98                         break;
  99                 }
 100         }
 101         kfree(flow->rule);
 102         kfree(flow);
 103 }
 104
 105 void nft_offload_set_dependency(struct nft_offload_ctx *ctx,
 106                                 enum nft_offload_dep_type type)
 107 {
 108         ctx->dep.type = type;
 109 }
 110
 111 void nft_offload_update_dependency(struct nft_offload_ctx *ctx,
 112                                    const void *data, u32 len)
 113 {
 114         switch (ctx->dep.type) {
 115         case NFT_OFFLOAD_DEP_NETWORK:
 116                 WARN_ON(len != sizeof(__u16));
 117                 memcpy(&ctx->dep.l3num, data, sizeof(__u16));
 118                 break;
 119         case NFT_OFFLOAD_DEP_TRANSPORT:
 120                 WARN_ON(len != sizeof(__u8));
 121                 memcpy(&ctx->dep.protonum, data, sizeof(__u8));
 122                 break;
 123         default:
 124                 break;
 125         }
 126         ctx->dep.type = NFT_OFFLOAD_DEP_UNSPEC;
 127 }
 128
 129 static void nft_flow_offload_common_init(struct flow_cls_common_offload *common,
 130                                          __be16 proto, int priority,
 131                                          struct netlink_ext_ack *extack)
 132 {
 133         common->protocol = proto;
 134         common->prio = priority;
 135         common->extack = extack;
 136 }
 137
 138 static int nft_setup_cb_call(enum tc_setup_type type, void *type_data,
 139                              struct list_head *cb_list)
 140 {
 141         struct flow_block_cb *block_cb;
 142         int err;
 143
 144         list_for_each_entry(block_cb, cb_list, list) {
 145                 err = block_cb->cb(type, type_data, block_cb->cb_priv);
 146                 if (err < 0)
 147                         return err;
 148         }
 149         return 0;
 150 }
 151
 152 int nft_chain_offload_priority(struct nft_base_chain *basechain)
 153 {
 154         if (basechain->ops.priority <= 0 ||
 155             basechain->ops.priority > USHRT_MAX)
 156                 return -1;
 157
 158         return 0;
 159 }
 160
 161 static void nft_flow_cls_offload_setup(struct flow_cls_offload *cls_flow,
 162                                        const struct nft_base_chain *basechain,
 163                                        const struct nft_rule *rule,
 164                                        const struct nft_flow_rule *flow,
 165                                        struct netlink_ext_ack *extack,
 166                                        enum flow_cls_command command)
 167 {
 168         __be16 proto = ETH_P_ALL;
 169
 170         memset(cls_flow, 0, sizeof(*cls_flow));
 171
 172         if (flow)
 173                 proto = flow->proto;
 174
 175         nft_flow_offload_common_init(&cls_flow->common, proto,
 176                                      basechain->ops.priority, extack);
 177         cls_flow->command = command;
 178         cls_flow->cookie = (unsigned long) rule;
 179         if (flow)
 180                 cls_flow->rule = flow->rule;
 181 }
 182
 183 static int nft_flow_offload_rule(struct nft_chain *chain,
 184                                  struct nft_rule *rule,
 185                                  struct nft_flow_rule *flow,
 186                                  enum flow_cls_command command)
 187 {
 188         struct netlink_ext_ack extack = {};
 189         struct flow_cls_offload cls_flow;
 190         struct nft_base_chain *basechain;
 191
 192         if (!nft_is_base_chain(chain))
 193                 return -EOPNOTSUPP;
 194
 195         basechain = nft_base_chain(chain);
 196         nft_flow_cls_offload_setup(&cls_flow, basechain, rule, flow, &extack,
 197                                    command);
 198
 199         return nft_setup_cb_call(TC_SETUP_CLSFLOWER, &cls_flow,
 200                                  &basechain->flow_block.cb_list);
 201 }
 202
 203 static int nft_flow_offload_bind(struct flow_block_offload *bo,
 204                                  struct nft_base_chain *basechain)
 205 {
 206         list_splice(&bo->cb_list, &basechain->flow_block.cb_list);
 207         return 0;
 208 }
 209
 210 static int nft_flow_offload_unbind(struct flow_block_offload *bo,
 211                                    struct nft_base_chain *basechain)
 212 {
 213         struct flow_block_cb *block_cb, *next;
 214         struct flow_cls_offload cls_flow;
 215         struct netlink_ext_ack extack;
 216         struct nft_chain *chain;
 217         struct nft_rule *rule;
 218
 219         chain = &basechain->chain;
 220         list_for_each_entry(rule, &chain->rules, list) {
 221                 memset(&extack, 0, sizeof(extack));
 222                 nft_flow_cls_offload_setup(&cls_flow, basechain, rule, NULL,
 223                                            &extack, FLOW_CLS_DESTROY);
 224                 nft_setup_cb_call(TC_SETUP_CLSFLOWER, &cls_flow, &bo->cb_list);
 225         }
 226
 227         list_for_each_entry_safe(block_cb, next, &bo->cb_list, list) {
 228                 list_del(&block_cb->list);
 229                 flow_block_cb_free(block_cb);
 230         }
 231
 232         return 0;
 233 }
 234
 235 static int nft_block_setup(struct nft_base_chain *basechain,
 236                            struct flow_block_offload *bo,
 237                            enum flow_block_command cmd)
 238 {
 239         int err;
 240
 241         switch (cmd) {
 242         case FLOW_BLOCK_BIND:
 243                 err = nft_flow_offload_bind(bo, basechain);
 244                 break;
 245         case FLOW_BLOCK_UNBIND:
 246                 err = nft_flow_offload_unbind(bo, basechain);
 247                 break;
 248         default:
 249                 WARN_ON_ONCE(1);
 250                 err = -EOPNOTSUPP;
 251         }
 252
 253         return err;
 254 }
 255
 256 static void nft_flow_block_offload_init(struct flow_block_offload *bo,
 257                                         struct net *net,
 258                                         enum flow_block_command cmd,
 259                                         struct nft_base_chain *basechain,
 260                                         struct netlink_ext_ack *extack)
 261 {
 262         memset(bo, 0, sizeof(*bo));
 263         bo->net         = net;
 264         bo->block       = &basechain->flow_block;
 265         bo->command     = cmd;
 266         bo->binder_type = FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS;
 267         bo->extack      = extack;
 268         INIT_LIST_HEAD(&bo->cb_list);
 269 }
 270
 271 static int nft_block_offload_cmd(struct nft_base_chain *chain,
 272                                  struct net_device *dev,
 273                                  enum flow_block_command cmd)
 274 {
 275         struct netlink_ext_ack extack = {};
 276         struct flow_block_offload bo;
 277         int err;
 278
 279         nft_flow_block_offload_init(&bo, dev_net(dev), cmd, chain, &extack);
 280
 281         err = dev->netdev_ops->ndo_setup_tc(dev, TC_SETUP_BLOCK, &bo);
 282         if (err < 0)
 283                 return err;
 284
 285         return nft_block_setup(chain, &bo, cmd);
 286 }
 287
 288 static void nft_indr_block_cleanup(struct flow_block_cb *block_cb)
 289 {
 290         struct nft_base_chain *basechain = block_cb->indr.data;
 291         struct net_device *dev = block_cb->indr.dev;
 292         struct netlink_ext_ack extack = {};
 293         struct net *net = dev_net(dev);
 294         struct flow_block_offload bo;
 295
 296         nft_flow_block_offload_init(&bo, dev_net(dev), FLOW_BLOCK_UNBIND,
 297                                     basechain, &extack);
 298         mutex_lock(&net->nft.commit_mutex);
 299         list_del(&block_cb->driver_list);
 300         list_move(&block_cb->list, &bo.cb_list);
 301         nft_flow_offload_unbind(&bo, basechain);
 302         mutex_unlock(&net->nft.commit_mutex);
 303 }
 304
 305 static int nft_indr_block_offload_cmd(struct nft_base_chain *basechain,
 306                                       struct net_device *dev,
 307                                       enum flow_block_command cmd)
 308 {
 309         struct netlink_ext_ack extack = {};
 310         struct flow_block_offload bo;
 311         int err;
 312
 313         nft_flow_block_offload_init(&bo, dev_net(dev), cmd, basechain, &extack);
 314
 315         err = flow_indr_dev_setup_offload(dev, NULL, TC_SETUP_BLOCK, basechain, &bo,
 316                                           nft_indr_block_cleanup);
 317         if (err < 0)
 318                 return err;
 319
 320         if (list_empty(&bo.cb_list))
 321                 return -EOPNOTSUPP;
 322
 323         return nft_block_setup(basechain, &bo, cmd);
 324 }
 325
 326 #define FLOW_SETUP_BLOCK TC_SETUP_BLOCK
 327
 328 static int nft_chain_offload_cmd(struct nft_base_chain *basechain,
 329                                  struct net_device *dev,
 330                                  enum flow_block_command cmd)
 331 {
 332         int err;
 333
 334         if (dev->netdev_ops->ndo_setup_tc)
 335                 err = nft_block_offload_cmd(basechain, dev, cmd);
 336         else
 337                 err = nft_indr_block_offload_cmd(basechain, dev, cmd);
 338
 339         return err;
 340 }
 341
 342 static int nft_flow_block_chain(struct nft_base_chain *basechain,
 343                                 const struct net_device *this_dev,
 344                                 enum flow_block_command cmd)
 345 {
 346         struct net_device *dev;
 347         struct nft_hook *hook;
 348         int err, i = 0;
 349
 350         list_for_each_entry(hook, &basechain->hook_list, list) {
 351                 dev = hook->ops.dev;
 352                 if (this_dev && this_dev != dev)
 353                         continue;
 354
 355                 err = nft_chain_offload_cmd(basechain, dev, cmd);
 356                 if (err < 0 && cmd == FLOW_BLOCK_BIND) {
 357                         if (!this_dev)
 358                                 goto err_flow_block;
 359
 360                         return err;
 361                 }
 362                 i++;
 363         }
 364
 365         return 0;
 366
 367 err_flow_block:
 368         list_for_each_entry(hook, &basechain->hook_list, list) {
 369                 if (i-- <= 0)
 370                         break;
 371
 372                 dev = hook->ops.dev;
 373                 nft_chain_offload_cmd(basechain, dev, FLOW_BLOCK_UNBIND);
 374         }
 375         return err;
 376 }
 377
 378 static int nft_flow_offload_chain(struct nft_chain *chain, u8 *ppolicy,
 379                                   enum flow_block_command cmd)
 380 {
 381         struct nft_base_chain *basechain;
 382         u8 policy;
 383
 384         if (!nft_is_base_chain(chain))
 385                 return -EOPNOTSUPP;
 386
 387         basechain = nft_base_chain(chain);
 388         policy = ppolicy ? *ppolicy : basechain->policy;
 389
 390         /* Only default policy to accept is supported for now. */
 391         if (cmd == FLOW_BLOCK_BIND && policy == NF_DROP)
 392                 return -EOPNOTSUPP;
 393
 394         return nft_flow_block_chain(basechain, NULL, cmd);
 395 }
 396
 397 static void nft_flow_rule_offload_abort(struct net *net,
 398                                         struct nft_trans *trans)
 399 {
 400         int err = 0;
 401
 402         list_for_each_entry_continue_reverse(trans, &net->nft.commit_list, list) {
 403                 if (trans->ctx.family != NFPROTO_NETDEV)
 404                         continue;
 405
 406                 switch (trans->msg_type) {
 407                 case NFT_MSG_NEWCHAIN:
 408                         if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD) ||
 409                             nft_trans_chain_update(trans))
 410                                 continue;
 411
 412                         err = nft_flow_offload_chain(trans->ctx.chain, NULL,
 413                                                      FLOW_BLOCK_UNBIND);
 414                         break;
 415                 case NFT_MSG_DELCHAIN:
 416                         if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
 417                                 continue;
 418
 419                         err = nft_flow_offload_chain(trans->ctx.chain, NULL,
 420                                                      FLOW_BLOCK_BIND);
 421                         break;
 422                 case NFT_MSG_NEWRULE:
 423                         if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
 424                                 continue;
 425
 426                         err = nft_flow_offload_rule(trans->ctx.chain,
 427                                                     nft_trans_rule(trans),
 428                                                     NULL, FLOW_CLS_DESTROY);
 429                         break;
 430                 case NFT_MSG_DELRULE:
 431                         if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
 432                                 continue;
 433
 434                         err = nft_flow_offload_rule(trans->ctx.chain,
 435                                                     nft_trans_rule(trans),
 436                                                     nft_trans_flow_rule(trans),
 437                                                     FLOW_CLS_REPLACE);
 438                         break;
 439                 }
 440
 441                 if (WARN_ON_ONCE(err))
 442                         break;
 443         }
 444 }
 445
 446 int nft_flow_rule_offload_commit(struct net *net)
 447 {
 448         struct nft_trans *trans;
 449         int err = 0;
 450         u8 policy;
 451
 452         list_for_each_entry(trans, &net->nft.commit_list, list) {
 453                 if (trans->ctx.family != NFPROTO_NETDEV)
 454                         continue;
 455
 456                 switch (trans->msg_type) {
 457                 case NFT_MSG_NEWCHAIN:
 458                         if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD) ||
 459                             nft_trans_chain_update(trans))
 460                                 continue;
 461
 462                         policy = nft_trans_chain_policy(trans);
 463                         err = nft_flow_offload_chain(trans->ctx.chain, &policy,
 464                                                      FLOW_BLOCK_BIND);
 465                         break;
 466                 case NFT_MSG_DELCHAIN:
 467                         if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
 468                                 continue;
 469
 470                         policy = nft_trans_chain_policy(trans);
 471                         err = nft_flow_offload_chain(trans->ctx.chain, &policy,
 472                                                      FLOW_BLOCK_UNBIND);
 473                         break;
 474                 case NFT_MSG_NEWRULE:
 475                         if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
 476                                 continue;
 477
 478                         if (trans->ctx.flags & NLM_F_REPLACE ||
 479                             !(trans->ctx.flags & NLM_F_APPEND)) {
 480                                 err = -EOPNOTSUPP;
 481                                 break;
 482                         }
 483                         err = nft_flow_offload_rule(trans->ctx.chain,
 484                                                     nft_trans_rule(trans),
 485                                                     nft_trans_flow_rule(trans),
 486                                                     FLOW_CLS_REPLACE);
 487                         break;
 488                 case NFT_MSG_DELRULE:
 489                         if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
 490                                 continue;
 491
 492                         err = nft_flow_offload_rule(trans->ctx.chain,
 493                                                     nft_trans_rule(trans),
 494                                                     NULL, FLOW_CLS_DESTROY);
 495                         break;
 496                 }
 497
 498                 if (err) {
 499                         nft_flow_rule_offload_abort(net, trans);
 500                         break;
 501                 }
 502         }
 503
 504         list_for_each_entry(trans, &net->nft.commit_list, list) {
 505                 if (trans->ctx.family != NFPROTO_NETDEV)
 506                         continue;
 507
 508                 switch (trans->msg_type) {
 509                 case NFT_MSG_NEWRULE:
 510                 case NFT_MSG_DELRULE:
 511                         if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
 512                                 continue;
 513
 514                         nft_flow_rule_destroy(nft_trans_flow_rule(trans));
 515                         break;
 516                 default:
 517                         break;
 518                 }
 519         }
 520
 521         return err;
 522 }
 523
 524 static struct nft_chain *__nft_offload_get_chain(struct net_device *dev)
 525 {
 526         struct nft_base_chain *basechain;
 527         struct net *net = dev_net(dev);
 528         struct nft_hook *hook, *found;
 529         const struct nft_table *table;
 530         struct nft_chain *chain;
 531
 532         list_for_each_entry(table, &net->nft.tables, list) {
 533                 if (table->family != NFPROTO_NETDEV)
 534                         continue;
 535
 536                 list_for_each_entry(chain, &table->chains, list) {
 537                         if (!nft_is_base_chain(chain) ||
 538                             !(chain->flags & NFT_CHAIN_HW_OFFLOAD))
 539                                 continue;
 540
 541                         found = NULL;
 542                         basechain = nft_base_chain(chain);
 543                         list_for_each_entry(hook, &basechain->hook_list, list) {
 544                                 if (hook->ops.dev != dev)
 545                                         continue;
 546
 547                                 found = hook;
 548                                 break;
 549                         }
 550                         if (!found)
 551                                 continue;
 552
 553                         return chain;
 554                 }
 555         }
 556
 557         return NULL;
 558 }
 559
 560 static int nft_offload_netdev_event(struct notifier_block *this,
 561                                     unsigned long event, void *ptr)
 562 {
 563         struct net_device *dev = netdev_notifier_info_to_dev(ptr);
 564         struct net *net = dev_net(dev);
 565         struct nft_chain *chain;
 566
 567         if (event != NETDEV_UNREGISTER)
 568                 return NOTIFY_DONE;
 569
 570         mutex_lock(&net->nft.commit_mutex);
 571         chain = __nft_offload_get_chain(dev);
 572         if (chain)
 573                 nft_flow_block_chain(nft_base_chain(chain), dev,
 574                                      FLOW_BLOCK_UNBIND);
 575
 576         mutex_unlock(&net->nft.commit_mutex);
 577
 578         return NOTIFY_DONE;
 579 }
 580
 581 static struct notifier_block nft_offload_netdev_notifier = {
 582         .notifier_call  = nft_offload_netdev_event,
 583 };
 584
 585 int nft_offload_init(void)
 586 {
 587         return register_netdevice_notifier(&nft_offload_netdev_notifier);
 588 }
 589
 590 void nft_offload_exit(void)
 591 {
 592         unregister_netdevice_notifier(&nft_offload_netdev_notifier);
 593 }