net/netfilter/nf_tables_offload.c

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 #include <linux/init.h>
   3 #include <linux/module.h>
   4 #include <linux/netfilter.h>
   5 #include <net/flow_offload.h>
   6 #include <net/netfilter/nf_tables.h>
   7 #include <net/netfilter/nf_tables_offload.h>
   8 #include <net/pkt_cls.h>
   9
  10 static struct nft_flow_rule *nft_flow_rule_alloc(int num_actions)
  11 {
  12         struct nft_flow_rule *flow;
  13
  14         flow = kzalloc(sizeof(struct nft_flow_rule), GFP_KERNEL);
  15         if (!flow)
  16                 return NULL;
  17
  18         flow->rule = flow_rule_alloc(num_actions);
  19         if (!flow->rule) {
  20                 kfree(flow);
  21                 return NULL;
  22         }
  23
  24         flow->rule->match.dissector     = &flow->match.dissector;
  25         flow->rule->match.mask          = &flow->match.mask;
  26         flow->rule->match.key           = &flow->match.key;
  27
  28         return flow;
  29 }
  30
  31 struct nft_flow_rule *nft_flow_rule_create(struct net *net,
  32                                            const struct nft_rule *rule)
  33 {
  34         struct nft_offload_ctx *ctx;
  35         struct nft_flow_rule *flow;
  36         int num_actions = 0, err;
  37         struct nft_expr *expr;
  38
  39         expr = nft_expr_first(rule);
  40         while (expr->ops && expr != nft_expr_last(rule)) {
  41                 if (expr->ops->offload_flags & NFT_OFFLOAD_F_ACTION)
  42                         num_actions++;
  43
  44                 expr = nft_expr_next(expr);
  45         }
  46
  47         flow = nft_flow_rule_alloc(num_actions);
  48         if (!flow)
  49                 return ERR_PTR(-ENOMEM);
  50
  51         expr = nft_expr_first(rule);
  52
  53         ctx = kzalloc(sizeof(struct nft_offload_ctx), GFP_KERNEL);
  54         if (!ctx) {
  55                 err = -ENOMEM;
  56                 goto err_out;
  57         }
  58         ctx->net = net;
  59         ctx->dep.type = NFT_OFFLOAD_DEP_UNSPEC;
  60
  61         while (expr->ops && expr != nft_expr_last(rule)) {
  62                 if (!expr->ops->offload) {
  63                         err = -EOPNOTSUPP;
  64                         goto err_out;
  65                 }
  66                 err = expr->ops->offload(ctx, flow, expr);
  67                 if (err < 0)
  68                         goto err_out;
  69
  70                 expr = nft_expr_next(expr);
  71         }
  72         flow->proto = ctx->dep.l3num;
  73         kfree(ctx);
  74
  75         return flow;
  76 err_out:
  77         kfree(ctx);
  78         nft_flow_rule_destroy(flow);
  79
  80         return ERR_PTR(err);
  81 }
  82
  83 void nft_flow_rule_destroy(struct nft_flow_rule *flow)
  84 {
  85         struct flow_action_entry *entry;
  86         int i;
  87
  88         flow_action_for_each(i, entry, &flow->rule->action) {
  89                 switch (entry->id) {
  90                 case FLOW_ACTION_REDIRECT:
  91                 case FLOW_ACTION_MIRRED:
  92                         dev_put(entry->dev);
  93                         break;
  94                 default:
  95                         break;
  96                 }
  97         }
  98         kfree(flow->rule);
  99         kfree(flow);
 100 }
 101
 102 void nft_offload_set_dependency(struct nft_offload_ctx *ctx,
 103                                 enum nft_offload_dep_type type)
 104 {
 105         ctx->dep.type = type;
 106 }
 107
 108 void nft_offload_update_dependency(struct nft_offload_ctx *ctx,
 109                                    const void *data, u32 len)
 110 {
 111         switch (ctx->dep.type) {
 112         case NFT_OFFLOAD_DEP_NETWORK:
 113                 WARN_ON(len != sizeof(__u16));
 114                 memcpy(&ctx->dep.l3num, data, sizeof(__u16));
 115                 break;
 116         case NFT_OFFLOAD_DEP_TRANSPORT:
 117                 WARN_ON(len != sizeof(__u8));
 118                 memcpy(&ctx->dep.protonum, data, sizeof(__u8));
 119                 break;
 120         default:
 121                 break;
 122         }
 123         ctx->dep.type = NFT_OFFLOAD_DEP_UNSPEC;
 124 }
 125
 126 static void nft_flow_offload_common_init(struct flow_cls_common_offload *common,
 127                                          __be16 proto, int priority,
 128                                          struct netlink_ext_ack *extack)
 129 {
 130         common->protocol = proto;
 131         common->prio = priority;
 132         common->extack = extack;
 133 }
 134
 135 static int nft_setup_cb_call(enum tc_setup_type type, void *type_data,
 136                              struct list_head *cb_list)
 137 {
 138         struct flow_block_cb *block_cb;
 139         int err;
 140
 141         list_for_each_entry(block_cb, cb_list, list) {
 142                 err = block_cb->cb(type, type_data, block_cb->cb_priv);
 143                 if (err < 0)
 144                         return err;
 145         }
 146         return 0;
 147 }
 148
 149 int nft_chain_offload_priority(struct nft_base_chain *basechain)
 150 {
 151         if (basechain->ops.priority <= 0 ||
 152             basechain->ops.priority > USHRT_MAX)
 153                 return -1;
 154
 155         return 0;
 156 }
 157
 158 static void nft_flow_cls_offload_setup(struct flow_cls_offload *cls_flow,
 159                                        const struct nft_base_chain *basechain,
 160                                        const struct nft_rule *rule,
 161                                        const struct nft_flow_rule *flow,
 162                                        struct netlink_ext_ack *extack,
 163                                        enum flow_cls_command command)
 164 {
 165         __be16 proto = ETH_P_ALL;
 166
 167         memset(cls_flow, 0, sizeof(*cls_flow));
 168
 169         if (flow)
 170                 proto = flow->proto;
 171
 172         nft_flow_offload_common_init(&cls_flow->common, proto,
 173                                      basechain->ops.priority, extack);
 174         cls_flow->command = command;
 175         cls_flow->cookie = (unsigned long) rule;
 176         if (flow)
 177                 cls_flow->rule = flow->rule;
 178 }
 179
 180 static int nft_flow_offload_rule(struct nft_chain *chain,
 181                                  struct nft_rule *rule,
 182                                  struct nft_flow_rule *flow,
 183                                  enum flow_cls_command command)
 184 {
 185         struct netlink_ext_ack extack = {};
 186         struct flow_cls_offload cls_flow;
 187         struct nft_base_chain *basechain;
 188
 189         if (!nft_is_base_chain(chain))
 190                 return -EOPNOTSUPP;
 191
 192         basechain = nft_base_chain(chain);
 193         nft_flow_cls_offload_setup(&cls_flow, basechain, rule, flow, &extack,
 194                                    command);
 195
 196         return nft_setup_cb_call(TC_SETUP_CLSFLOWER, &cls_flow,
 197                                  &basechain->flow_block.cb_list);
 198 }
 199
 200 static int nft_flow_offload_bind(struct flow_block_offload *bo,
 201                                  struct nft_base_chain *basechain)
 202 {
 203         list_splice(&bo->cb_list, &basechain->flow_block.cb_list);
 204         return 0;
 205 }
 206
 207 static int nft_flow_offload_unbind(struct flow_block_offload *bo,
 208                                    struct nft_base_chain *basechain)
 209 {
 210         struct flow_block_cb *block_cb, *next;
 211         struct flow_cls_offload cls_flow;
 212         struct netlink_ext_ack extack;
 213         struct nft_chain *chain;
 214         struct nft_rule *rule;
 215
 216         chain = &basechain->chain;
 217         list_for_each_entry(rule, &chain->rules, list) {
 218                 memset(&extack, 0, sizeof(extack));
 219                 nft_flow_cls_offload_setup(&cls_flow, basechain, rule, NULL,
 220                                            &extack, FLOW_CLS_DESTROY);
 221                 nft_setup_cb_call(TC_SETUP_CLSFLOWER, &cls_flow, &bo->cb_list);
 222         }
 223
 224         list_for_each_entry_safe(block_cb, next, &bo->cb_list, list) {
 225                 list_del(&block_cb->list);
 226                 flow_block_cb_free(block_cb);
 227         }
 228
 229         return 0;
 230 }
 231
 232 static int nft_block_setup(struct nft_base_chain *basechain,
 233                            struct flow_block_offload *bo,
 234                            enum flow_block_command cmd)
 235 {
 236         int err;
 237
 238         switch (cmd) {
 239         case FLOW_BLOCK_BIND:
 240                 err = nft_flow_offload_bind(bo, basechain);
 241                 break;
 242         case FLOW_BLOCK_UNBIND:
 243                 err = nft_flow_offload_unbind(bo, basechain);
 244                 break;
 245         default:
 246                 WARN_ON_ONCE(1);
 247                 err = -EOPNOTSUPP;
 248         }
 249
 250         return err;
 251 }
 252
 253 static void nft_flow_block_offload_init(struct flow_block_offload *bo,
 254                                         struct net *net,
 255                                         enum flow_block_command cmd,
 256                                         struct nft_base_chain *basechain,
 257                                         struct netlink_ext_ack *extack)
 258 {
 259         memset(bo, 0, sizeof(*bo));
 260         bo->net         = net;
 261         bo->block       = &basechain->flow_block;
 262         bo->command     = cmd;
 263         bo->binder_type = FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS;
 264         bo->extack      = extack;
 265         INIT_LIST_HEAD(&bo->cb_list);
 266 }
 267
 268 static int nft_block_offload_cmd(struct nft_base_chain *chain,
 269                                  struct net_device *dev,
 270                                  enum flow_block_command cmd)
 271 {
 272         struct netlink_ext_ack extack = {};
 273         struct flow_block_offload bo;
 274         int err;
 275
 276         nft_flow_block_offload_init(&bo, dev_net(dev), cmd, chain, &extack);
 277
 278         err = dev->netdev_ops->ndo_setup_tc(dev, TC_SETUP_BLOCK, &bo);
 279         if (err < 0)
 280                 return err;
 281
 282         return nft_block_setup(chain, &bo, cmd);
 283 }
 284
 285 static void nft_indr_block_ing_cmd(struct net_device *dev,
 286                                    struct nft_base_chain *chain,
 287                                    flow_indr_block_bind_cb_t *cb,
 288                                    void *cb_priv,
 289                                    enum flow_block_command cmd)
 290 {
 291         struct netlink_ext_ack extack = {};
 292         struct flow_block_offload bo;
 293
 294         if (!chain)
 295                 return;
 296
 297         nft_flow_block_offload_init(&bo, dev_net(dev), cmd, chain, &extack);
 298
 299         cb(dev, cb_priv, TC_SETUP_BLOCK, &bo);
 300
 301         nft_block_setup(chain, &bo, cmd);
 302 }
 303
 304 static int nft_indr_block_offload_cmd(struct nft_base_chain *chain,
 305                                       struct net_device *dev,
 306                                       enum flow_block_command cmd)
 307 {
 308         struct netlink_ext_ack extack = {};
 309         struct flow_block_offload bo;
 310
 311         nft_flow_block_offload_init(&bo, dev_net(dev), cmd, chain, &extack);
 312
 313         flow_indr_block_call(dev, &bo, cmd);
 314
 315         if (list_empty(&bo.cb_list))
 316                 return -EOPNOTSUPP;
 317
 318         return nft_block_setup(chain, &bo, cmd);
 319 }
 320
 321 #define FLOW_SETUP_BLOCK TC_SETUP_BLOCK
 322
 323 static int nft_chain_offload_cmd(struct nft_base_chain *basechain,
 324                                  struct net_device *dev,
 325                                  enum flow_block_command cmd)
 326 {
 327         int err;
 328
 329         if (dev->netdev_ops->ndo_setup_tc)
 330                 err = nft_block_offload_cmd(basechain, dev, cmd);
 331         else
 332                 err = nft_indr_block_offload_cmd(basechain, dev, cmd);
 333
 334         return err;
 335 }
 336
 337 static int nft_flow_block_chain(struct nft_base_chain *basechain,
 338                                 const struct net_device *this_dev,
 339                                 enum flow_block_command cmd)
 340 {
 341         struct net_device *dev;
 342         struct nft_hook *hook;
 343         int err, i = 0;
 344
 345         list_for_each_entry(hook, &basechain->hook_list, list) {
 346                 dev = hook->ops.dev;
 347                 if (this_dev && this_dev != dev)
 348                         continue;
 349
 350                 err = nft_chain_offload_cmd(basechain, dev, cmd);
 351                 if (err < 0 && cmd == FLOW_BLOCK_BIND) {
 352                         if (!this_dev)
 353                                 goto err_flow_block;
 354
 355                         return err;
 356                 }
 357                 i++;
 358         }
 359
 360         return 0;
 361
 362 err_flow_block:
 363         list_for_each_entry(hook, &basechain->hook_list, list) {
 364                 if (i-- <= 0)
 365                         break;
 366
 367                 dev = hook->ops.dev;
 368                 nft_chain_offload_cmd(basechain, dev, FLOW_BLOCK_UNBIND);
 369         }
 370         return err;
 371 }
 372
 373 static int nft_flow_offload_chain(struct nft_chain *chain, u8 *ppolicy,
 374                                   enum flow_block_command cmd)
 375 {
 376         struct nft_base_chain *basechain;
 377         u8 policy;
 378
 379         if (!nft_is_base_chain(chain))
 380                 return -EOPNOTSUPP;
 381
 382         basechain = nft_base_chain(chain);
 383         policy = ppolicy ? *ppolicy : basechain->policy;
 384
 385         /* Only default policy to accept is supported for now. */
 386         if (cmd == FLOW_BLOCK_BIND && policy == NF_DROP)
 387                 return -EOPNOTSUPP;
 388
 389         return nft_flow_block_chain(basechain, NULL, cmd);
 390 }
 391
 392 static void nft_flow_rule_offload_abort(struct net *net,
 393                                         struct nft_trans *trans)
 394 {
 395         int err = 0;
 396
 397         list_for_each_entry_continue_reverse(trans, &net->nft.commit_list, list) {
 398                 if (trans->ctx.family != NFPROTO_NETDEV)
 399                         continue;
 400
 401                 switch (trans->msg_type) {
 402                 case NFT_MSG_NEWCHAIN:
 403                         if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD) ||
 404                             nft_trans_chain_update(trans))
 405                                 continue;
 406
 407                         err = nft_flow_offload_chain(trans->ctx.chain, NULL,
 408                                                      FLOW_BLOCK_UNBIND);
 409                         break;
 410                 case NFT_MSG_DELCHAIN:
 411                         if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
 412                                 continue;
 413
 414                         err = nft_flow_offload_chain(trans->ctx.chain, NULL,
 415                                                      FLOW_BLOCK_BIND);
 416                         break;
 417                 case NFT_MSG_NEWRULE:
 418                         if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
 419                                 continue;
 420
 421                         err = nft_flow_offload_rule(trans->ctx.chain,
 422                                                     nft_trans_rule(trans),
 423                                                     NULL, FLOW_CLS_DESTROY);
 424                         break;
 425                 case NFT_MSG_DELRULE:
 426                         if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
 427                                 continue;
 428
 429                         err = nft_flow_offload_rule(trans->ctx.chain,
 430                                                     nft_trans_rule(trans),
 431                                                     nft_trans_flow_rule(trans),
 432                                                     FLOW_CLS_REPLACE);
 433                         break;
 434                 }
 435
 436                 if (WARN_ON_ONCE(err))
 437                         break;
 438         }
 439 }
 440
 441 int nft_flow_rule_offload_commit(struct net *net)
 442 {
 443         struct nft_trans *trans;
 444         int err = 0;
 445         u8 policy;
 446
 447         list_for_each_entry(trans, &net->nft.commit_list, list) {
 448                 if (trans->ctx.family != NFPROTO_NETDEV)
 449                         continue;
 450
 451                 switch (trans->msg_type) {
 452                 case NFT_MSG_NEWCHAIN:
 453                         if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD) ||
 454                             nft_trans_chain_update(trans))
 455                                 continue;
 456
 457                         policy = nft_trans_chain_policy(trans);
 458                         err = nft_flow_offload_chain(trans->ctx.chain, &policy,
 459                                                      FLOW_BLOCK_BIND);
 460                         break;
 461                 case NFT_MSG_DELCHAIN:
 462                         if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
 463                                 continue;
 464
 465                         policy = nft_trans_chain_policy(trans);
 466                         err = nft_flow_offload_chain(trans->ctx.chain, &policy,
 467                                                      FLOW_BLOCK_UNBIND);
 468                         break;
 469                 case NFT_MSG_NEWRULE:
 470                         if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
 471                                 continue;
 472
 473                         if (trans->ctx.flags & NLM_F_REPLACE ||
 474                             !(trans->ctx.flags & NLM_F_APPEND)) {
 475                                 err = -EOPNOTSUPP;
 476                                 break;
 477                         }
 478                         err = nft_flow_offload_rule(trans->ctx.chain,
 479                                                     nft_trans_rule(trans),
 480                                                     nft_trans_flow_rule(trans),
 481                                                     FLOW_CLS_REPLACE);
 482                         break;
 483                 case NFT_MSG_DELRULE:
 484                         if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
 485                                 continue;
 486
 487                         err = nft_flow_offload_rule(trans->ctx.chain,
 488                                                     nft_trans_rule(trans),
 489                                                     NULL, FLOW_CLS_DESTROY);
 490                         break;
 491                 }
 492
 493                 if (err) {
 494                         nft_flow_rule_offload_abort(net, trans);
 495                         break;
 496                 }
 497         }
 498
 499         list_for_each_entry(trans, &net->nft.commit_list, list) {
 500                 if (trans->ctx.family != NFPROTO_NETDEV)
 501                         continue;
 502
 503                 switch (trans->msg_type) {
 504                 case NFT_MSG_NEWRULE:
 505                 case NFT_MSG_DELRULE:
 506                         if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD))
 507                                 continue;
 508
 509                         nft_flow_rule_destroy(nft_trans_flow_rule(trans));
 510                         break;
 511                 default:
 512                         break;
 513                 }
 514         }
 515
 516         return err;
 517 }
 518
 519 static struct nft_chain *__nft_offload_get_chain(struct net_device *dev)
 520 {
 521         struct nft_base_chain *basechain;
 522         struct net *net = dev_net(dev);
 523         struct nft_hook *hook, *found;
 524         const struct nft_table *table;
 525         struct nft_chain *chain;
 526
 527         list_for_each_entry(table, &net->nft.tables, list) {
 528                 if (table->family != NFPROTO_NETDEV)
 529                         continue;
 530
 531                 list_for_each_entry(chain, &table->chains, list) {
 532                         if (!nft_is_base_chain(chain) ||
 533                             !(chain->flags & NFT_CHAIN_HW_OFFLOAD))
 534                                 continue;
 535
 536                         found = NULL;
 537                         basechain = nft_base_chain(chain);
 538                         list_for_each_entry(hook, &basechain->hook_list, list) {
 539                                 if (hook->ops.dev != dev)
 540                                         continue;
 541
 542                                 found = hook;
 543                                 break;
 544                         }
 545                         if (!found)
 546                                 continue;
 547
 548                         return chain;
 549                 }
 550         }
 551
 552         return NULL;
 553 }
 554
 555 static void nft_indr_block_cb(struct net_device *dev,
 556                               flow_indr_block_bind_cb_t *cb, void *cb_priv,
 557                               enum flow_block_command cmd)
 558 {
 559         struct net *net = dev_net(dev);
 560         struct nft_chain *chain;
 561
 562         mutex_lock(&net->nft.commit_mutex);
 563         chain = __nft_offload_get_chain(dev);
 564         if (chain) {
 565                 struct nft_base_chain *basechain;
 566
 567                 basechain = nft_base_chain(chain);
 568                 nft_indr_block_ing_cmd(dev, basechain, cb, cb_priv, cmd);
 569         }
 570         mutex_unlock(&net->nft.commit_mutex);
 571 }
 572
 573 static int nft_offload_netdev_event(struct notifier_block *this,
 574                                     unsigned long event, void *ptr)
 575 {
 576         struct net_device *dev = netdev_notifier_info_to_dev(ptr);
 577         struct net *net = dev_net(dev);
 578         struct nft_chain *chain;
 579
 580         mutex_lock(&net->nft.commit_mutex);
 581         chain = __nft_offload_get_chain(dev);
 582         if (chain)
 583                 nft_flow_block_chain(nft_base_chain(chain), dev,
 584                                      FLOW_BLOCK_UNBIND);
 585
 586         mutex_unlock(&net->nft.commit_mutex);
 587
 588         return NOTIFY_DONE;
 589 }
 590
 591 static struct flow_indr_block_entry block_ing_entry = {
 592         .cb     = nft_indr_block_cb,
 593         .list   = LIST_HEAD_INIT(block_ing_entry.list),
 594 };
 595
 596 static struct notifier_block nft_offload_netdev_notifier = {
 597         .notifier_call  = nft_offload_netdev_event,
 598 };
 599
 600 int nft_offload_init(void)
 601 {
 602         int err;
 603
 604         err = register_netdevice_notifier(&nft_offload_netdev_notifier);
 605         if (err < 0)
 606                 return err;
 607
 608         flow_indr_add_block_cb(&block_ing_entry);
 609
 610         return 0;
 611 }
 612
 613 void nft_offload_exit(void)
 614 {
 615         flow_indr_del_block_cb(&block_ing_entry);
 616         unregister_netdevice_notifier(&nft_offload_netdev_notifier);
 617 }