net/netfilter/nf_tables_api.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (c) 2007-2009 Patrick McHardy <kaber@trash.net>
   4  *
   5  * Development of this code funded by Astaro AG (http://www.astaro.com/)
   6  */
   7
   8 #include <linux/module.h>
   9 #include <linux/init.h>
  10 #include <linux/list.h>
  11 #include <linux/skbuff.h>
  12 #include <linux/netlink.h>
  13 #include <linux/vmalloc.h>
  14 #include <linux/rhashtable.h>
  15 #include <linux/audit.h>
  16 #include <linux/netfilter.h>
  17 #include <linux/netfilter/nfnetlink.h>
  18 #include <linux/netfilter/nf_tables.h>
  19 #include <net/netfilter/nf_flow_table.h>
  20 #include <net/netfilter/nf_tables_core.h>
  21 #include <net/netfilter/nf_tables.h>
  22 #include <net/netfilter/nf_tables_offload.h>
  23 #include <net/net_namespace.h>
  24 #include <net/sock.h>
  25
  26 #define NFT_MODULE_AUTOLOAD_LIMIT (MODULE_NAME_LEN - sizeof("nft-expr-255-"))
  27
  28 unsigned int nf_tables_net_id __read_mostly;
  29
  30 static LIST_HEAD(nf_tables_expressions);
  31 static LIST_HEAD(nf_tables_objects);
  32 static LIST_HEAD(nf_tables_flowtables);
  33 static LIST_HEAD(nf_tables_destroy_list);
  34 static DEFINE_SPINLOCK(nf_tables_destroy_list_lock);
  35 static u64 table_handle;
  36
  37 enum {
  38         NFT_VALIDATE_SKIP       = 0,
  39         NFT_VALIDATE_NEED,
  40         NFT_VALIDATE_DO,
  41 };
  42
  43 static struct rhltable nft_objname_ht;
  44
  45 static u32 nft_chain_hash(const void *data, u32 len, u32 seed);
  46 static u32 nft_chain_hash_obj(const void *data, u32 len, u32 seed);
  47 static int nft_chain_hash_cmp(struct rhashtable_compare_arg *, const void *);
  48
  49 static u32 nft_objname_hash(const void *data, u32 len, u32 seed);
  50 static u32 nft_objname_hash_obj(const void *data, u32 len, u32 seed);
  51 static int nft_objname_hash_cmp(struct rhashtable_compare_arg *, const void *);
  52
  53 static const struct rhashtable_params nft_chain_ht_params = {
  54         .head_offset            = offsetof(struct nft_chain, rhlhead),
  55         .key_offset             = offsetof(struct nft_chain, name),
  56         .hashfn                 = nft_chain_hash,
  57         .obj_hashfn             = nft_chain_hash_obj,
  58         .obj_cmpfn              = nft_chain_hash_cmp,
  59         .automatic_shrinking    = true,
  60 };
  61
  62 static const struct rhashtable_params nft_objname_ht_params = {
  63         .head_offset            = offsetof(struct nft_object, rhlhead),
  64         .key_offset             = offsetof(struct nft_object, key),
  65         .hashfn                 = nft_objname_hash,
  66         .obj_hashfn             = nft_objname_hash_obj,
  67         .obj_cmpfn              = nft_objname_hash_cmp,
  68         .automatic_shrinking    = true,
  69 };
  70
  71 struct nft_audit_data {
  72         struct nft_table *table;
  73         int entries;
  74         int op;
  75         struct list_head list;
  76 };
  77
  78 static const u8 nft2audit_op[NFT_MSG_MAX] = { // enum nf_tables_msg_types
  79         [NFT_MSG_NEWTABLE]      = AUDIT_NFT_OP_TABLE_REGISTER,
  80         [NFT_MSG_GETTABLE]      = AUDIT_NFT_OP_INVALID,
  81         [NFT_MSG_DELTABLE]      = AUDIT_NFT_OP_TABLE_UNREGISTER,
  82         [NFT_MSG_NEWCHAIN]      = AUDIT_NFT_OP_CHAIN_REGISTER,
  83         [NFT_MSG_GETCHAIN]      = AUDIT_NFT_OP_INVALID,
  84         [NFT_MSG_DELCHAIN]      = AUDIT_NFT_OP_CHAIN_UNREGISTER,
  85         [NFT_MSG_NEWRULE]       = AUDIT_NFT_OP_RULE_REGISTER,
  86         [NFT_MSG_GETRULE]       = AUDIT_NFT_OP_INVALID,
  87         [NFT_MSG_DELRULE]       = AUDIT_NFT_OP_RULE_UNREGISTER,
  88         [NFT_MSG_NEWSET]        = AUDIT_NFT_OP_SET_REGISTER,
  89         [NFT_MSG_GETSET]        = AUDIT_NFT_OP_INVALID,
  90         [NFT_MSG_DELSET]        = AUDIT_NFT_OP_SET_UNREGISTER,
  91         [NFT_MSG_NEWSETELEM]    = AUDIT_NFT_OP_SETELEM_REGISTER,
  92         [NFT_MSG_GETSETELEM]    = AUDIT_NFT_OP_INVALID,
  93         [NFT_MSG_DELSETELEM]    = AUDIT_NFT_OP_SETELEM_UNREGISTER,
  94         [NFT_MSG_NEWGEN]        = AUDIT_NFT_OP_GEN_REGISTER,
  95         [NFT_MSG_GETGEN]        = AUDIT_NFT_OP_INVALID,
  96         [NFT_MSG_TRACE]         = AUDIT_NFT_OP_INVALID,
  97         [NFT_MSG_NEWOBJ]        = AUDIT_NFT_OP_OBJ_REGISTER,
  98         [NFT_MSG_GETOBJ]        = AUDIT_NFT_OP_INVALID,
  99         [NFT_MSG_DELOBJ]        = AUDIT_NFT_OP_OBJ_UNREGISTER,
 100         [NFT_MSG_GETOBJ_RESET]  = AUDIT_NFT_OP_OBJ_RESET,
 101         [NFT_MSG_NEWFLOWTABLE]  = AUDIT_NFT_OP_FLOWTABLE_REGISTER,
 102         [NFT_MSG_GETFLOWTABLE]  = AUDIT_NFT_OP_INVALID,
 103         [NFT_MSG_DELFLOWTABLE]  = AUDIT_NFT_OP_FLOWTABLE_UNREGISTER,
 104 };
 105
 106 static void nft_validate_state_update(struct net *net, u8 new_validate_state)
 107 {
 108         struct nftables_pernet *nft_net = nft_pernet(net);
 109
 110         switch (nft_net->validate_state) {
 111         case NFT_VALIDATE_SKIP:
 112                 WARN_ON_ONCE(new_validate_state == NFT_VALIDATE_DO);
 113                 break;
 114         case NFT_VALIDATE_NEED:
 115                 break;
 116         case NFT_VALIDATE_DO:
 117                 if (new_validate_state == NFT_VALIDATE_NEED)
 118                         return;
 119         }
 120
 121         nft_net->validate_state = new_validate_state;
 122 }
 123 static void nf_tables_trans_destroy_work(struct work_struct *w);
 124 static DECLARE_WORK(trans_destroy_work, nf_tables_trans_destroy_work);
 125
 126 static void nft_ctx_init(struct nft_ctx *ctx,
 127                          struct net *net,
 128                          const struct sk_buff *skb,
 129                          const struct nlmsghdr *nlh,
 130                          u8 family,
 131                          struct nft_table *table,
 132                          struct nft_chain *chain,
 133                          const struct nlattr * const *nla)
 134 {
 135         ctx->net        = net;
 136         ctx->family     = family;
 137         ctx->level      = 0;
 138         ctx->table      = table;
 139         ctx->chain      = chain;
 140         ctx->nla        = nla;
 141         ctx->portid     = NETLINK_CB(skb).portid;
 142         ctx->report     = nlmsg_report(nlh);
 143         ctx->flags      = nlh->nlmsg_flags;
 144         ctx->seq        = nlh->nlmsg_seq;
 145 }
 146
 147 static struct nft_trans *nft_trans_alloc_gfp(const struct nft_ctx *ctx,
 148                                              int msg_type, u32 size, gfp_t gfp)
 149 {
 150         struct nft_trans *trans;
 151
 152         trans = kzalloc(sizeof(struct nft_trans) + size, gfp);
 153         if (trans == NULL)
 154                 return NULL;
 155
 156         trans->msg_type = msg_type;
 157         trans->ctx      = *ctx;
 158
 159         return trans;
 160 }
 161
 162 static struct nft_trans *nft_trans_alloc(const struct nft_ctx *ctx,
 163                                          int msg_type, u32 size)
 164 {
 165         return nft_trans_alloc_gfp(ctx, msg_type, size, GFP_KERNEL);
 166 }
 167
 168 static void nft_trans_destroy(struct nft_trans *trans)
 169 {
 170         list_del(&trans->list);
 171         kfree(trans);
 172 }
 173
 174 static void nft_set_trans_bind(const struct nft_ctx *ctx, struct nft_set *set)
 175 {
 176         struct nftables_pernet *nft_net;
 177         struct net *net = ctx->net;
 178         struct nft_trans *trans;
 179
 180         if (!nft_set_is_anonymous(set))
 181                 return;
 182
 183         nft_net = nft_pernet(net);
 184         list_for_each_entry_reverse(trans, &nft_net->commit_list, list) {
 185                 switch (trans->msg_type) {
 186                 case NFT_MSG_NEWSET:
 187                         if (nft_trans_set(trans) == set)
 188                                 nft_trans_set_bound(trans) = true;
 189                         break;
 190                 case NFT_MSG_NEWSETELEM:
 191                         if (nft_trans_elem_set(trans) == set)
 192                                 nft_trans_elem_set_bound(trans) = true;
 193                         break;
 194                 }
 195         }
 196 }
 197
 198 static int nft_netdev_register_hooks(struct net *net,
 199                                      struct list_head *hook_list)
 200 {
 201         struct nft_hook *hook;
 202         int err, j;
 203
 204         j = 0;
 205         list_for_each_entry(hook, hook_list, list) {
 206                 err = nf_register_net_hook(net, &hook->ops);
 207                 if (err < 0)
 208                         goto err_register;
 209
 210                 j++;
 211         }
 212         return 0;
 213
 214 err_register:
 215         list_for_each_entry(hook, hook_list, list) {
 216                 if (j-- <= 0)
 217                         break;
 218
 219                 nf_unregister_net_hook(net, &hook->ops);
 220         }
 221         return err;
 222 }
 223
 224 static void nft_netdev_unregister_hooks(struct net *net,
 225                                         struct list_head *hook_list)
 226 {
 227         struct nft_hook *hook;
 228
 229         list_for_each_entry(hook, hook_list, list)
 230                 nf_unregister_net_hook(net, &hook->ops);
 231 }
 232
 233 static int nf_tables_register_hook(struct net *net,
 234                                    const struct nft_table *table,
 235                                    struct nft_chain *chain)
 236 {
 237         struct nft_base_chain *basechain;
 238         const struct nf_hook_ops *ops;
 239
 240         if (table->flags & NFT_TABLE_F_DORMANT ||
 241             !nft_is_base_chain(chain))
 242                 return 0;
 243
 244         basechain = nft_base_chain(chain);
 245         ops = &basechain->ops;
 246
 247         if (basechain->type->ops_register)
 248                 return basechain->type->ops_register(net, ops);
 249
 250         if (nft_base_chain_netdev(table->family, basechain->ops.hooknum))
 251                 return nft_netdev_register_hooks(net, &basechain->hook_list);
 252
 253         return nf_register_net_hook(net, &basechain->ops);
 254 }
 255
 256 static void nf_tables_unregister_hook(struct net *net,
 257                                       const struct nft_table *table,
 258                                       struct nft_chain *chain)
 259 {
 260         struct nft_base_chain *basechain;
 261         const struct nf_hook_ops *ops;
 262
 263         if (table->flags & NFT_TABLE_F_DORMANT ||
 264             !nft_is_base_chain(chain))
 265                 return;
 266         basechain = nft_base_chain(chain);
 267         ops = &basechain->ops;
 268
 269         if (basechain->type->ops_unregister)
 270                 return basechain->type->ops_unregister(net, ops);
 271
 272         if (nft_base_chain_netdev(table->family, basechain->ops.hooknum))
 273                 nft_netdev_unregister_hooks(net, &basechain->hook_list);
 274         else
 275                 nf_unregister_net_hook(net, &basechain->ops);
 276 }
 277
 278 static void nft_trans_commit_list_add_tail(struct net *net, struct nft_trans *trans)
 279 {
 280         struct nftables_pernet *nft_net = nft_pernet(net);
 281
 282         list_add_tail(&trans->list, &nft_net->commit_list);
 283 }
 284
 285 static int nft_trans_table_add(struct nft_ctx *ctx, int msg_type)
 286 {
 287         struct nft_trans *trans;
 288
 289         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_table));
 290         if (trans == NULL)
 291                 return -ENOMEM;
 292
 293         if (msg_type == NFT_MSG_NEWTABLE)
 294                 nft_activate_next(ctx->net, ctx->table);
 295
 296         nft_trans_commit_list_add_tail(ctx->net, trans);
 297         return 0;
 298 }
 299
 300 static int nft_deltable(struct nft_ctx *ctx)
 301 {
 302         int err;
 303
 304         err = nft_trans_table_add(ctx, NFT_MSG_DELTABLE);
 305         if (err < 0)
 306                 return err;
 307
 308         nft_deactivate_next(ctx->net, ctx->table);
 309         return err;
 310 }
 311
 312 static struct nft_trans *nft_trans_chain_add(struct nft_ctx *ctx, int msg_type)
 313 {
 314         struct nft_trans *trans;
 315
 316         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_chain));
 317         if (trans == NULL)
 318                 return ERR_PTR(-ENOMEM);
 319
 320         if (msg_type == NFT_MSG_NEWCHAIN) {
 321                 nft_activate_next(ctx->net, ctx->chain);
 322
 323                 if (ctx->nla[NFTA_CHAIN_ID]) {
 324                         nft_trans_chain_id(trans) =
 325                                 ntohl(nla_get_be32(ctx->nla[NFTA_CHAIN_ID]));
 326                 }
 327         }
 328
 329         nft_trans_commit_list_add_tail(ctx->net, trans);
 330         return trans;
 331 }
 332
 333 static int nft_delchain(struct nft_ctx *ctx)
 334 {
 335         struct nft_trans *trans;
 336
 337         trans = nft_trans_chain_add(ctx, NFT_MSG_DELCHAIN);
 338         if (IS_ERR(trans))
 339                 return PTR_ERR(trans);
 340
 341         ctx->table->use--;
 342         nft_deactivate_next(ctx->net, ctx->chain);
 343
 344         return 0;
 345 }
 346
 347 static void nft_rule_expr_activate(const struct nft_ctx *ctx,
 348                                    struct nft_rule *rule)
 349 {
 350         struct nft_expr *expr;
 351
 352         expr = nft_expr_first(rule);
 353         while (nft_expr_more(rule, expr)) {
 354                 if (expr->ops->activate)
 355                         expr->ops->activate(ctx, expr);
 356
 357                 expr = nft_expr_next(expr);
 358         }
 359 }
 360
 361 static void nft_rule_expr_deactivate(const struct nft_ctx *ctx,
 362                                      struct nft_rule *rule,
 363                                      enum nft_trans_phase phase)
 364 {
 365         struct nft_expr *expr;
 366
 367         expr = nft_expr_first(rule);
 368         while (nft_expr_more(rule, expr)) {
 369                 if (expr->ops->deactivate)
 370                         expr->ops->deactivate(ctx, expr, phase);
 371
 372                 expr = nft_expr_next(expr);
 373         }
 374 }
 375
 376 static int
 377 nf_tables_delrule_deactivate(struct nft_ctx *ctx, struct nft_rule *rule)
 378 {
 379         /* You cannot delete the same rule twice */
 380         if (nft_is_active_next(ctx->net, rule)) {
 381                 nft_deactivate_next(ctx->net, rule);
 382                 ctx->chain->use--;
 383                 return 0;
 384         }
 385         return -ENOENT;
 386 }
 387
 388 static struct nft_trans *nft_trans_rule_add(struct nft_ctx *ctx, int msg_type,
 389                                             struct nft_rule *rule)
 390 {
 391         struct nft_trans *trans;
 392
 393         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_rule));
 394         if (trans == NULL)
 395                 return NULL;
 396
 397         if (msg_type == NFT_MSG_NEWRULE && ctx->nla[NFTA_RULE_ID] != NULL) {
 398                 nft_trans_rule_id(trans) =
 399                         ntohl(nla_get_be32(ctx->nla[NFTA_RULE_ID]));
 400         }
 401         nft_trans_rule(trans) = rule;
 402         nft_trans_commit_list_add_tail(ctx->net, trans);
 403
 404         return trans;
 405 }
 406
 407 static int nft_delrule(struct nft_ctx *ctx, struct nft_rule *rule)
 408 {
 409         struct nft_flow_rule *flow;
 410         struct nft_trans *trans;
 411         int err;
 412
 413         trans = nft_trans_rule_add(ctx, NFT_MSG_DELRULE, rule);
 414         if (trans == NULL)
 415                 return -ENOMEM;
 416
 417         if (ctx->chain->flags & NFT_CHAIN_HW_OFFLOAD) {
 418                 flow = nft_flow_rule_create(ctx->net, rule);
 419                 if (IS_ERR(flow)) {
 420                         nft_trans_destroy(trans);
 421                         return PTR_ERR(flow);
 422                 }
 423
 424                 nft_trans_flow_rule(trans) = flow;
 425         }
 426
 427         err = nf_tables_delrule_deactivate(ctx, rule);
 428         if (err < 0) {
 429                 nft_trans_destroy(trans);
 430                 return err;
 431         }
 432         nft_rule_expr_deactivate(ctx, rule, NFT_TRANS_PREPARE);
 433
 434         return 0;
 435 }
 436
 437 static int nft_delrule_by_chain(struct nft_ctx *ctx)
 438 {
 439         struct nft_rule *rule;
 440         int err;
 441
 442         list_for_each_entry(rule, &ctx->chain->rules, list) {
 443                 if (!nft_is_active_next(ctx->net, rule))
 444                         continue;
 445
 446                 err = nft_delrule(ctx, rule);
 447                 if (err < 0)
 448                         return err;
 449         }
 450         return 0;
 451 }
 452
 453 static int nft_trans_set_add(const struct nft_ctx *ctx, int msg_type,
 454                              struct nft_set *set)
 455 {
 456         struct nft_trans *trans;
 457
 458         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_set));
 459         if (trans == NULL)
 460                 return -ENOMEM;
 461
 462         if (msg_type == NFT_MSG_NEWSET && ctx->nla[NFTA_SET_ID] != NULL) {
 463                 nft_trans_set_id(trans) =
 464                         ntohl(nla_get_be32(ctx->nla[NFTA_SET_ID]));
 465                 nft_activate_next(ctx->net, set);
 466         }
 467         nft_trans_set(trans) = set;
 468         nft_trans_commit_list_add_tail(ctx->net, trans);
 469
 470         return 0;
 471 }
 472
 473 static int nft_delset(const struct nft_ctx *ctx, struct nft_set *set)
 474 {
 475         int err;
 476
 477         err = nft_trans_set_add(ctx, NFT_MSG_DELSET, set);
 478         if (err < 0)
 479                 return err;
 480
 481         nft_deactivate_next(ctx->net, set);
 482         ctx->table->use--;
 483
 484         return err;
 485 }
 486
 487 static int nft_trans_obj_add(struct nft_ctx *ctx, int msg_type,
 488                              struct nft_object *obj)
 489 {
 490         struct nft_trans *trans;
 491
 492         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_obj));
 493         if (trans == NULL)
 494                 return -ENOMEM;
 495
 496         if (msg_type == NFT_MSG_NEWOBJ)
 497                 nft_activate_next(ctx->net, obj);
 498
 499         nft_trans_obj(trans) = obj;
 500         nft_trans_commit_list_add_tail(ctx->net, trans);
 501
 502         return 0;
 503 }
 504
 505 static int nft_delobj(struct nft_ctx *ctx, struct nft_object *obj)
 506 {
 507         int err;
 508
 509         err = nft_trans_obj_add(ctx, NFT_MSG_DELOBJ, obj);
 510         if (err < 0)
 511                 return err;
 512
 513         nft_deactivate_next(ctx->net, obj);
 514         ctx->table->use--;
 515
 516         return err;
 517 }
 518
 519 static int nft_trans_flowtable_add(struct nft_ctx *ctx, int msg_type,
 520                                    struct nft_flowtable *flowtable)
 521 {
 522         struct nft_trans *trans;
 523
 524         trans = nft_trans_alloc(ctx, msg_type,
 525                                 sizeof(struct nft_trans_flowtable));
 526         if (trans == NULL)
 527                 return -ENOMEM;
 528
 529         if (msg_type == NFT_MSG_NEWFLOWTABLE)
 530                 nft_activate_next(ctx->net, flowtable);
 531
 532         nft_trans_flowtable(trans) = flowtable;
 533         nft_trans_commit_list_add_tail(ctx->net, trans);
 534
 535         return 0;
 536 }
 537
 538 static int nft_delflowtable(struct nft_ctx *ctx,
 539                             struct nft_flowtable *flowtable)
 540 {
 541         int err;
 542
 543         err = nft_trans_flowtable_add(ctx, NFT_MSG_DELFLOWTABLE, flowtable);
 544         if (err < 0)
 545                 return err;
 546
 547         nft_deactivate_next(ctx->net, flowtable);
 548         ctx->table->use--;
 549
 550         return err;
 551 }
 552
 553 /*
 554  * Tables
 555  */
 556
 557 static struct nft_table *nft_table_lookup(const struct net *net,
 558                                           const struct nlattr *nla,
 559                                           u8 family, u8 genmask, u32 nlpid)
 560 {
 561         struct nftables_pernet *nft_net;
 562         struct nft_table *table;
 563
 564         if (nla == NULL)
 565                 return ERR_PTR(-EINVAL);
 566
 567         nft_net = nft_pernet(net);
 568         list_for_each_entry_rcu(table, &nft_net->tables, list,
 569                                 lockdep_is_held(&nft_net->commit_mutex)) {
 570                 if (!nla_strcmp(nla, table->name) &&
 571                     table->family == family &&
 572                     nft_active_genmask(table, genmask)) {
 573                         if (nft_table_has_owner(table) &&
 574                             table->nlpid != nlpid)
 575                                 return ERR_PTR(-EPERM);
 576
 577                         return table;
 578                 }
 579         }
 580
 581         return ERR_PTR(-ENOENT);
 582 }
 583
 584 static struct nft_table *nft_table_lookup_byhandle(const struct net *net,
 585                                                    const struct nlattr *nla,
 586                                                    u8 genmask)
 587 {
 588         struct nftables_pernet *nft_net;
 589         struct nft_table *table;
 590
 591         nft_net = nft_pernet(net);
 592         list_for_each_entry(table, &nft_net->tables, list) {
 593                 if (be64_to_cpu(nla_get_be64(nla)) == table->handle &&
 594                     nft_active_genmask(table, genmask))
 595                         return table;
 596         }
 597
 598         return ERR_PTR(-ENOENT);
 599 }
 600
 601 static inline u64 nf_tables_alloc_handle(struct nft_table *table)
 602 {
 603         return ++table->hgenerator;
 604 }
 605
 606 static const struct nft_chain_type *chain_type[NFPROTO_NUMPROTO][NFT_CHAIN_T_MAX];
 607
 608 static const struct nft_chain_type *
 609 __nft_chain_type_get(u8 family, enum nft_chain_types type)
 610 {
 611         if (family >= NFPROTO_NUMPROTO ||
 612             type >= NFT_CHAIN_T_MAX)
 613                 return NULL;
 614
 615         return chain_type[family][type];
 616 }
 617
 618 static const struct nft_chain_type *
 619 __nf_tables_chain_type_lookup(const struct nlattr *nla, u8 family)
 620 {
 621         const struct nft_chain_type *type;
 622         int i;
 623
 624         for (i = 0; i < NFT_CHAIN_T_MAX; i++) {
 625                 type = __nft_chain_type_get(family, i);
 626                 if (!type)
 627                         continue;
 628                 if (!nla_strcmp(nla, type->name))
 629                         return type;
 630         }
 631         return NULL;
 632 }
 633
 634 struct nft_module_request {
 635         struct list_head        list;
 636         char                    module[MODULE_NAME_LEN];
 637         bool                    done;
 638 };
 639
 640 #ifdef CONFIG_MODULES
 641 __printf(2, 3) int nft_request_module(struct net *net, const char *fmt,
 642                                       ...)
 643 {
 644         char module_name[MODULE_NAME_LEN];
 645         struct nftables_pernet *nft_net;
 646         struct nft_module_request *req;
 647         va_list args;
 648         int ret;
 649
 650         va_start(args, fmt);
 651         ret = vsnprintf(module_name, MODULE_NAME_LEN, fmt, args);
 652         va_end(args);
 653         if (ret >= MODULE_NAME_LEN)
 654                 return 0;
 655
 656         nft_net = nft_pernet(net);
 657         list_for_each_entry(req, &nft_net->module_list, list) {
 658                 if (!strcmp(req->module, module_name)) {
 659                         if (req->done)
 660                                 return 0;
 661
 662                         /* A request to load this module already exists. */
 663                         return -EAGAIN;
 664                 }
 665         }
 666
 667         req = kmalloc(sizeof(*req), GFP_KERNEL);
 668         if (!req)
 669                 return -ENOMEM;
 670
 671         req->done = false;
 672         strlcpy(req->module, module_name, MODULE_NAME_LEN);
 673         list_add_tail(&req->list, &nft_net->module_list);
 674
 675         return -EAGAIN;
 676 }
 677 EXPORT_SYMBOL_GPL(nft_request_module);
 678 #endif
 679
 680 static void lockdep_nfnl_nft_mutex_not_held(void)
 681 {
 682 #ifdef CONFIG_PROVE_LOCKING
 683         if (debug_locks)
 684                 WARN_ON_ONCE(lockdep_nfnl_is_held(NFNL_SUBSYS_NFTABLES));
 685 #endif
 686 }
 687
 688 static const struct nft_chain_type *
 689 nf_tables_chain_type_lookup(struct net *net, const struct nlattr *nla,
 690                             u8 family, bool autoload)
 691 {
 692         const struct nft_chain_type *type;
 693
 694         type = __nf_tables_chain_type_lookup(nla, family);
 695         if (type != NULL)
 696                 return type;
 697
 698         lockdep_nfnl_nft_mutex_not_held();
 699 #ifdef CONFIG_MODULES
 700         if (autoload) {
 701                 if (nft_request_module(net, "nft-chain-%u-%.*s", family,
 702                                        nla_len(nla),
 703                                        (const char *)nla_data(nla)) == -EAGAIN)
 704                         return ERR_PTR(-EAGAIN);
 705         }
 706 #endif
 707         return ERR_PTR(-ENOENT);
 708 }
 709
 710 static __be16 nft_base_seq(const struct net *net)
 711 {
 712         struct nftables_pernet *nft_net = nft_pernet(net);
 713
 714         return htons(nft_net->base_seq & 0xffff);
 715 }
 716
 717 static const struct nla_policy nft_table_policy[NFTA_TABLE_MAX + 1] = {
 718         [NFTA_TABLE_NAME]       = { .type = NLA_STRING,
 719                                     .len = NFT_TABLE_MAXNAMELEN - 1 },
 720         [NFTA_TABLE_FLAGS]      = { .type = NLA_U32 },
 721         [NFTA_TABLE_HANDLE]     = { .type = NLA_U64 },
 722         [NFTA_TABLE_USERDATA]   = { .type = NLA_BINARY,
 723                                     .len = NFT_USERDATA_MAXLEN }
 724 };
 725
 726 static int nf_tables_fill_table_info(struct sk_buff *skb, struct net *net,
 727                                      u32 portid, u32 seq, int event, u32 flags,
 728                                      int family, const struct nft_table *table)
 729 {
 730         struct nlmsghdr *nlh;
 731
 732         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
 733         nlh = nfnl_msg_put(skb, portid, seq, event, flags, family,
 734                            NFNETLINK_V0, nft_base_seq(net));
 735         if (!nlh)
 736                 goto nla_put_failure;
 737
 738         if (nla_put_string(skb, NFTA_TABLE_NAME, table->name) ||
 739             nla_put_be32(skb, NFTA_TABLE_FLAGS,
 740                          htonl(table->flags & NFT_TABLE_F_MASK)) ||
 741             nla_put_be32(skb, NFTA_TABLE_USE, htonl(table->use)) ||
 742             nla_put_be64(skb, NFTA_TABLE_HANDLE, cpu_to_be64(table->handle),
 743                          NFTA_TABLE_PAD))
 744                 goto nla_put_failure;
 745         if (nft_table_has_owner(table) &&
 746             nla_put_be32(skb, NFTA_TABLE_OWNER, htonl(table->nlpid)))
 747                 goto nla_put_failure;
 748
 749         if (table->udata) {
 750                 if (nla_put(skb, NFTA_TABLE_USERDATA, table->udlen, table->udata))
 751                         goto nla_put_failure;
 752         }
 753
 754         nlmsg_end(skb, nlh);
 755         return 0;
 756
 757 nla_put_failure:
 758         nlmsg_trim(skb, nlh);
 759         return -1;
 760 }
 761
 762 struct nftnl_skb_parms {
 763         bool report;
 764 };
 765 #define NFT_CB(skb)     (*(struct nftnl_skb_parms*)&((skb)->cb))
 766
 767 static void nft_notify_enqueue(struct sk_buff *skb, bool report,
 768                                struct list_head *notify_list)
 769 {
 770         NFT_CB(skb).report = report;
 771         list_add_tail(&skb->list, notify_list);
 772 }
 773
 774 static void nf_tables_table_notify(const struct nft_ctx *ctx, int event)
 775 {
 776         struct nftables_pernet *nft_net;
 777         struct sk_buff *skb;
 778         int err;
 779
 780         if (!ctx->report &&
 781             !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
 782                 return;
 783
 784         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
 785         if (skb == NULL)
 786                 goto err;
 787
 788         err = nf_tables_fill_table_info(skb, ctx->net, ctx->portid, ctx->seq,
 789                                         event, 0, ctx->family, ctx->table);
 790         if (err < 0) {
 791                 kfree_skb(skb);
 792                 goto err;
 793         }
 794
 795         nft_net = nft_pernet(ctx->net);
 796         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
 797         return;
 798 err:
 799         nfnetlink_set_err(ctx->net, ctx->portid, NFNLGRP_NFTABLES, -ENOBUFS);
 800 }
 801
 802 static int nf_tables_dump_tables(struct sk_buff *skb,
 803                                  struct netlink_callback *cb)
 804 {
 805         const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
 806         struct nftables_pernet *nft_net;
 807         const struct nft_table *table;
 808         unsigned int idx = 0, s_idx = cb->args[0];
 809         struct net *net = sock_net(skb->sk);
 810         int family = nfmsg->nfgen_family;
 811
 812         rcu_read_lock();
 813         nft_net = nft_pernet(net);
 814         cb->seq = nft_net->base_seq;
 815
 816         list_for_each_entry_rcu(table, &nft_net->tables, list) {
 817                 if (family != NFPROTO_UNSPEC && family != table->family)
 818                         continue;
 819
 820                 if (idx < s_idx)
 821                         goto cont;
 822                 if (idx > s_idx)
 823                         memset(&cb->args[1], 0,
 824                                sizeof(cb->args) - sizeof(cb->args[0]));
 825                 if (!nft_is_active(net, table))
 826                         continue;
 827                 if (nf_tables_fill_table_info(skb, net,
 828                                               NETLINK_CB(cb->skb).portid,
 829                                               cb->nlh->nlmsg_seq,
 830                                               NFT_MSG_NEWTABLE, NLM_F_MULTI,
 831                                               table->family, table) < 0)
 832                         goto done;
 833
 834                 nl_dump_check_consistent(cb, nlmsg_hdr(skb));
 835 cont:
 836                 idx++;
 837         }
 838 done:
 839         rcu_read_unlock();
 840         cb->args[0] = idx;
 841         return skb->len;
 842 }
 843
 844 static int nft_netlink_dump_start_rcu(struct sock *nlsk, struct sk_buff *skb,
 845                                       const struct nlmsghdr *nlh,
 846                                       struct netlink_dump_control *c)
 847 {
 848         int err;
 849
 850         if (!try_module_get(THIS_MODULE))
 851                 return -EINVAL;
 852
 853         rcu_read_unlock();
 854         err = netlink_dump_start(nlsk, skb, nlh, c);
 855         rcu_read_lock();
 856         module_put(THIS_MODULE);
 857
 858         return err;
 859 }
 860
 861 /* called with rcu_read_lock held */
 862 static int nf_tables_gettable(struct sk_buff *skb, const struct nfnl_info *info,
 863                               const struct nlattr * const nla[])
 864 {
 865         struct netlink_ext_ack *extack = info->extack;
 866         u8 genmask = nft_genmask_cur(info->net);
 867         u8 family = info->nfmsg->nfgen_family;
 868         const struct nft_table *table;
 869         struct net *net = info->net;
 870         struct sk_buff *skb2;
 871         int err;
 872
 873         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
 874                 struct netlink_dump_control c = {
 875                         .dump = nf_tables_dump_tables,
 876                         .module = THIS_MODULE,
 877                 };
 878
 879                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
 880         }
 881
 882         table = nft_table_lookup(net, nla[NFTA_TABLE_NAME], family, genmask, 0);
 883         if (IS_ERR(table)) {
 884                 NL_SET_BAD_ATTR(extack, nla[NFTA_TABLE_NAME]);
 885                 return PTR_ERR(table);
 886         }
 887
 888         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
 889         if (!skb2)
 890                 return -ENOMEM;
 891
 892         err = nf_tables_fill_table_info(skb2, net, NETLINK_CB(skb).portid,
 893                                         info->nlh->nlmsg_seq, NFT_MSG_NEWTABLE,
 894                                         0, family, table);
 895         if (err < 0)
 896                 goto err_fill_table_info;
 897
 898         return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
 899
 900 err_fill_table_info:
 901         kfree_skb(skb2);
 902         return err;
 903 }
 904
 905 static void nft_table_disable(struct net *net, struct nft_table *table, u32 cnt)
 906 {
 907         struct nft_chain *chain;
 908         u32 i = 0;
 909
 910         list_for_each_entry(chain, &table->chains, list) {
 911                 if (!nft_is_active_next(net, chain))
 912                         continue;
 913                 if (!nft_is_base_chain(chain))
 914                         continue;
 915
 916                 if (cnt && i++ == cnt)
 917                         break;
 918
 919                 nf_tables_unregister_hook(net, table, chain);
 920         }
 921 }
 922
 923 static int nf_tables_table_enable(struct net *net, struct nft_table *table)
 924 {
 925         struct nft_chain *chain;
 926         int err, i = 0;
 927
 928         list_for_each_entry(chain, &table->chains, list) {
 929                 if (!nft_is_active_next(net, chain))
 930                         continue;
 931                 if (!nft_is_base_chain(chain))
 932                         continue;
 933
 934                 err = nf_tables_register_hook(net, table, chain);
 935                 if (err < 0)
 936                         goto err_register_hooks;
 937
 938                 i++;
 939         }
 940         return 0;
 941
 942 err_register_hooks:
 943         if (i)
 944                 nft_table_disable(net, table, i);
 945         return err;
 946 }
 947
 948 static void nf_tables_table_disable(struct net *net, struct nft_table *table)
 949 {
 950         table->flags &= ~NFT_TABLE_F_DORMANT;
 951         nft_table_disable(net, table, 0);
 952         table->flags |= NFT_TABLE_F_DORMANT;
 953 }
 954
 955 #define __NFT_TABLE_F_INTERNAL          (NFT_TABLE_F_MASK + 1)
 956 #define __NFT_TABLE_F_WAS_DORMANT       (__NFT_TABLE_F_INTERNAL << 0)
 957 #define __NFT_TABLE_F_WAS_AWAKEN        (__NFT_TABLE_F_INTERNAL << 1)
 958 #define __NFT_TABLE_F_UPDATE            (__NFT_TABLE_F_WAS_DORMANT | \
 959                                          __NFT_TABLE_F_WAS_AWAKEN)
 960
 961 static int nf_tables_updtable(struct nft_ctx *ctx)
 962 {
 963         struct nft_trans *trans;
 964         u32 flags;
 965         int ret;
 966
 967         if (!ctx->nla[NFTA_TABLE_FLAGS])
 968                 return 0;
 969
 970         flags = ntohl(nla_get_be32(ctx->nla[NFTA_TABLE_FLAGS]));
 971         if (flags & ~NFT_TABLE_F_MASK)
 972                 return -EOPNOTSUPP;
 973
 974         if (flags == ctx->table->flags)
 975                 return 0;
 976
 977         if ((nft_table_has_owner(ctx->table) &&
 978              !(flags & NFT_TABLE_F_OWNER)) ||
 979             (!nft_table_has_owner(ctx->table) &&
 980              flags & NFT_TABLE_F_OWNER))
 981                 return -EOPNOTSUPP;
 982
 983         trans = nft_trans_alloc(ctx, NFT_MSG_NEWTABLE,
 984                                 sizeof(struct nft_trans_table));
 985         if (trans == NULL)
 986                 return -ENOMEM;
 987
 988         if ((flags & NFT_TABLE_F_DORMANT) &&
 989             !(ctx->table->flags & NFT_TABLE_F_DORMANT)) {
 990                 ctx->table->flags |= NFT_TABLE_F_DORMANT;
 991                 if (!(ctx->table->flags & __NFT_TABLE_F_UPDATE))
 992                         ctx->table->flags |= __NFT_TABLE_F_WAS_AWAKEN;
 993         } else if (!(flags & NFT_TABLE_F_DORMANT) &&
 994                    ctx->table->flags & NFT_TABLE_F_DORMANT) {
 995                 ctx->table->flags &= ~NFT_TABLE_F_DORMANT;
 996                 if (!(ctx->table->flags & __NFT_TABLE_F_UPDATE)) {
 997                         ret = nf_tables_table_enable(ctx->net, ctx->table);
 998                         if (ret < 0)
 999                                 goto err_register_hooks;
1000
1001                         ctx->table->flags |= __NFT_TABLE_F_WAS_DORMANT;
1002                 }
1003         }
1004
1005         nft_trans_table_update(trans) = true;
1006         nft_trans_commit_list_add_tail(ctx->net, trans);
1007
1008         return 0;
1009
1010 err_register_hooks:
1011         nft_trans_destroy(trans);
1012         return ret;
1013 }
1014
1015 static u32 nft_chain_hash(const void *data, u32 len, u32 seed)
1016 {
1017         const char *name = data;
1018
1019         return jhash(name, strlen(name), seed);
1020 }
1021
1022 static u32 nft_chain_hash_obj(const void *data, u32 len, u32 seed)
1023 {
1024         const struct nft_chain *chain = data;
1025
1026         return nft_chain_hash(chain->name, 0, seed);
1027 }
1028
1029 static int nft_chain_hash_cmp(struct rhashtable_compare_arg *arg,
1030                               const void *ptr)
1031 {
1032         const struct nft_chain *chain = ptr;
1033         const char *name = arg->key;
1034
1035         return strcmp(chain->name, name);
1036 }
1037
1038 static u32 nft_objname_hash(const void *data, u32 len, u32 seed)
1039 {
1040         const struct nft_object_hash_key *k = data;
1041
1042         seed ^= hash_ptr(k->table, 32);
1043
1044         return jhash(k->name, strlen(k->name), seed);
1045 }
1046
1047 static u32 nft_objname_hash_obj(const void *data, u32 len, u32 seed)
1048 {
1049         const struct nft_object *obj = data;
1050
1051         return nft_objname_hash(&obj->key, 0, seed);
1052 }
1053
1054 static int nft_objname_hash_cmp(struct rhashtable_compare_arg *arg,
1055                                 const void *ptr)
1056 {
1057         const struct nft_object_hash_key *k = arg->key;
1058         const struct nft_object *obj = ptr;
1059
1060         if (obj->key.table != k->table)
1061                 return -1;
1062
1063         return strcmp(obj->key.name, k->name);
1064 }
1065
1066 static int nf_tables_newtable(struct sk_buff *skb, const struct nfnl_info *info,
1067                               const struct nlattr * const nla[])
1068 {
1069         struct nftables_pernet *nft_net = nft_pernet(info->net);
1070         struct netlink_ext_ack *extack = info->extack;
1071         u8 genmask = nft_genmask_next(info->net);
1072         u8 family = info->nfmsg->nfgen_family;
1073         struct net *net = info->net;
1074         const struct nlattr *attr;
1075         struct nft_table *table;
1076         struct nft_ctx ctx;
1077         u32 flags = 0;
1078         int err;
1079
1080         lockdep_assert_held(&nft_net->commit_mutex);
1081         attr = nla[NFTA_TABLE_NAME];
1082         table = nft_table_lookup(net, attr, family, genmask,
1083                                  NETLINK_CB(skb).portid);
1084         if (IS_ERR(table)) {
1085                 if (PTR_ERR(table) != -ENOENT)
1086                         return PTR_ERR(table);
1087         } else {
1088                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
1089                         NL_SET_BAD_ATTR(extack, attr);
1090                         return -EEXIST;
1091                 }
1092                 if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
1093                         return -EOPNOTSUPP;
1094
1095                 nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
1096
1097                 return nf_tables_updtable(&ctx);
1098         }
1099
1100         if (nla[NFTA_TABLE_FLAGS]) {
1101                 flags = ntohl(nla_get_be32(nla[NFTA_TABLE_FLAGS]));
1102                 if (flags & ~NFT_TABLE_F_MASK)
1103                         return -EOPNOTSUPP;
1104         }
1105
1106         err = -ENOMEM;
1107         table = kzalloc(sizeof(*table), GFP_KERNEL);
1108         if (table == NULL)
1109                 goto err_kzalloc;
1110
1111         table->name = nla_strdup(attr, GFP_KERNEL);
1112         if (table->name == NULL)
1113                 goto err_strdup;
1114
1115         if (nla[NFTA_TABLE_USERDATA]) {
1116                 table->udata = nla_memdup(nla[NFTA_TABLE_USERDATA], GFP_KERNEL);
1117                 if (table->udata == NULL)
1118                         goto err_table_udata;
1119
1120                 table->udlen = nla_len(nla[NFTA_TABLE_USERDATA]);
1121         }
1122
1123         err = rhltable_init(&table->chains_ht, &nft_chain_ht_params);
1124         if (err)
1125                 goto err_chain_ht;
1126
1127         INIT_LIST_HEAD(&table->chains);
1128         INIT_LIST_HEAD(&table->sets);
1129         INIT_LIST_HEAD(&table->objects);
1130         INIT_LIST_HEAD(&table->flowtables);
1131         table->family = family;
1132         table->flags = flags;
1133         table->handle = ++table_handle;
1134         if (table->flags & NFT_TABLE_F_OWNER)
1135                 table->nlpid = NETLINK_CB(skb).portid;
1136
1137         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
1138         err = nft_trans_table_add(&ctx, NFT_MSG_NEWTABLE);
1139         if (err < 0)
1140                 goto err_trans;
1141
1142         list_add_tail_rcu(&table->list, &nft_net->tables);
1143         return 0;
1144 err_trans:
1145         rhltable_destroy(&table->chains_ht);
1146 err_chain_ht:
1147         kfree(table->udata);
1148 err_table_udata:
1149         kfree(table->name);
1150 err_strdup:
1151         kfree(table);
1152 err_kzalloc:
1153         return err;
1154 }
1155
1156 static int nft_flush_table(struct nft_ctx *ctx)
1157 {
1158         struct nft_flowtable *flowtable, *nft;
1159         struct nft_chain *chain, *nc;
1160         struct nft_object *obj, *ne;
1161         struct nft_set *set, *ns;
1162         int err;
1163
1164         list_for_each_entry(chain, &ctx->table->chains, list) {
1165                 if (!nft_is_active_next(ctx->net, chain))
1166                         continue;
1167
1168                 if (nft_chain_is_bound(chain))
1169                         continue;
1170
1171                 ctx->chain = chain;
1172
1173                 err = nft_delrule_by_chain(ctx);
1174                 if (err < 0)
1175                         goto out;
1176         }
1177
1178         list_for_each_entry_safe(set, ns, &ctx->table->sets, list) {
1179                 if (!nft_is_active_next(ctx->net, set))
1180                         continue;
1181
1182                 if (nft_set_is_anonymous(set) &&
1183                     !list_empty(&set->bindings))
1184                         continue;
1185
1186                 err = nft_delset(ctx, set);
1187                 if (err < 0)
1188                         goto out;
1189         }
1190
1191         list_for_each_entry_safe(flowtable, nft, &ctx->table->flowtables, list) {
1192                 if (!nft_is_active_next(ctx->net, flowtable))
1193                         continue;
1194
1195                 err = nft_delflowtable(ctx, flowtable);
1196                 if (err < 0)
1197                         goto out;
1198         }
1199
1200         list_for_each_entry_safe(obj, ne, &ctx->table->objects, list) {
1201                 if (!nft_is_active_next(ctx->net, obj))
1202                         continue;
1203
1204                 err = nft_delobj(ctx, obj);
1205                 if (err < 0)
1206                         goto out;
1207         }
1208
1209         list_for_each_entry_safe(chain, nc, &ctx->table->chains, list) {
1210                 if (!nft_is_active_next(ctx->net, chain))
1211                         continue;
1212
1213                 if (nft_chain_is_bound(chain))
1214                         continue;
1215
1216                 ctx->chain = chain;
1217
1218                 err = nft_delchain(ctx);
1219                 if (err < 0)
1220                         goto out;
1221         }
1222
1223         err = nft_deltable(ctx);
1224 out:
1225         return err;
1226 }
1227
1228 static int nft_flush(struct nft_ctx *ctx, int family)
1229 {
1230         struct nftables_pernet *nft_net = nft_pernet(ctx->net);
1231         const struct nlattr * const *nla = ctx->nla;
1232         struct nft_table *table, *nt;
1233         int err = 0;
1234
1235         list_for_each_entry_safe(table, nt, &nft_net->tables, list) {
1236                 if (family != AF_UNSPEC && table->family != family)
1237                         continue;
1238
1239                 ctx->family = table->family;
1240
1241                 if (!nft_is_active_next(ctx->net, table))
1242                         continue;
1243
1244                 if (nft_table_has_owner(table) && table->nlpid != ctx->portid)
1245                         continue;
1246
1247                 if (nla[NFTA_TABLE_NAME] &&
1248                     nla_strcmp(nla[NFTA_TABLE_NAME], table->name) != 0)
1249                         continue;
1250
1251                 ctx->table = table;
1252
1253                 err = nft_flush_table(ctx);
1254                 if (err < 0)
1255                         goto out;
1256         }
1257 out:
1258         return err;
1259 }
1260
1261 static int nf_tables_deltable(struct sk_buff *skb, const struct nfnl_info *info,
1262                               const struct nlattr * const nla[])
1263 {
1264         struct netlink_ext_ack *extack = info->extack;
1265         u8 genmask = nft_genmask_next(info->net);
1266         u8 family = info->nfmsg->nfgen_family;
1267         struct net *net = info->net;
1268         const struct nlattr *attr;
1269         struct nft_table *table;
1270         struct nft_ctx ctx;
1271
1272         nft_ctx_init(&ctx, net, skb, info->nlh, 0, NULL, NULL, nla);
1273         if (family == AF_UNSPEC ||
1274             (!nla[NFTA_TABLE_NAME] && !nla[NFTA_TABLE_HANDLE]))
1275                 return nft_flush(&ctx, family);
1276
1277         if (nla[NFTA_TABLE_HANDLE]) {
1278                 attr = nla[NFTA_TABLE_HANDLE];
1279                 table = nft_table_lookup_byhandle(net, attr, genmask);
1280         } else {
1281                 attr = nla[NFTA_TABLE_NAME];
1282                 table = nft_table_lookup(net, attr, family, genmask,
1283                                          NETLINK_CB(skb).portid);
1284         }
1285
1286         if (IS_ERR(table)) {
1287                 NL_SET_BAD_ATTR(extack, attr);
1288                 return PTR_ERR(table);
1289         }
1290
1291         if (info->nlh->nlmsg_flags & NLM_F_NONREC &&
1292             table->use > 0)
1293                 return -EBUSY;
1294
1295         ctx.family = family;
1296         ctx.table = table;
1297
1298         return nft_flush_table(&ctx);
1299 }
1300
1301 static void nf_tables_table_destroy(struct nft_ctx *ctx)
1302 {
1303         if (WARN_ON(ctx->table->use > 0))
1304                 return;
1305
1306         rhltable_destroy(&ctx->table->chains_ht);
1307         kfree(ctx->table->name);
1308         kfree(ctx->table->udata);
1309         kfree(ctx->table);
1310 }
1311
1312 void nft_register_chain_type(const struct nft_chain_type *ctype)
1313 {
1314         nfnl_lock(NFNL_SUBSYS_NFTABLES);
1315         if (WARN_ON(__nft_chain_type_get(ctype->family, ctype->type))) {
1316                 nfnl_unlock(NFNL_SUBSYS_NFTABLES);
1317                 return;
1318         }
1319         chain_type[ctype->family][ctype->type] = ctype;
1320         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
1321 }
1322 EXPORT_SYMBOL_GPL(nft_register_chain_type);
1323
1324 void nft_unregister_chain_type(const struct nft_chain_type *ctype)
1325 {
1326         nfnl_lock(NFNL_SUBSYS_NFTABLES);
1327         chain_type[ctype->family][ctype->type] = NULL;
1328         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
1329 }
1330 EXPORT_SYMBOL_GPL(nft_unregister_chain_type);
1331
1332 /*
1333  * Chains
1334  */
1335
1336 static struct nft_chain *
1337 nft_chain_lookup_byhandle(const struct nft_table *table, u64 handle, u8 genmask)
1338 {
1339         struct nft_chain *chain;
1340
1341         list_for_each_entry(chain, &table->chains, list) {
1342                 if (chain->handle == handle &&
1343                     nft_active_genmask(chain, genmask))
1344                         return chain;
1345         }
1346
1347         return ERR_PTR(-ENOENT);
1348 }
1349
1350 static bool lockdep_commit_lock_is_held(const struct net *net)
1351 {
1352 #ifdef CONFIG_PROVE_LOCKING
1353         struct nftables_pernet *nft_net = nft_pernet(net);
1354
1355         return lockdep_is_held(&nft_net->commit_mutex);
1356 #else
1357         return true;
1358 #endif
1359 }
1360
1361 static struct nft_chain *nft_chain_lookup(struct net *net,
1362                                           struct nft_table *table,
1363                                           const struct nlattr *nla, u8 genmask)
1364 {
1365         char search[NFT_CHAIN_MAXNAMELEN + 1];
1366         struct rhlist_head *tmp, *list;
1367         struct nft_chain *chain;
1368
1369         if (nla == NULL)
1370                 return ERR_PTR(-EINVAL);
1371
1372         nla_strscpy(search, nla, sizeof(search));
1373
1374         WARN_ON(!rcu_read_lock_held() &&
1375                 !lockdep_commit_lock_is_held(net));
1376
1377         chain = ERR_PTR(-ENOENT);
1378         rcu_read_lock();
1379         list = rhltable_lookup(&table->chains_ht, search, nft_chain_ht_params);
1380         if (!list)
1381                 goto out_unlock;
1382
1383         rhl_for_each_entry_rcu(chain, tmp, list, rhlhead) {
1384                 if (nft_active_genmask(chain, genmask))
1385                         goto out_unlock;
1386         }
1387         chain = ERR_PTR(-ENOENT);
1388 out_unlock:
1389         rcu_read_unlock();
1390         return chain;
1391 }
1392
1393 static const struct nla_policy nft_chain_policy[NFTA_CHAIN_MAX + 1] = {
1394         [NFTA_CHAIN_TABLE]      = { .type = NLA_STRING,
1395                                     .len = NFT_TABLE_MAXNAMELEN - 1 },
1396         [NFTA_CHAIN_HANDLE]     = { .type = NLA_U64 },
1397         [NFTA_CHAIN_NAME]       = { .type = NLA_STRING,
1398                                     .len = NFT_CHAIN_MAXNAMELEN - 1 },
1399         [NFTA_CHAIN_HOOK]       = { .type = NLA_NESTED },
1400         [NFTA_CHAIN_POLICY]     = { .type = NLA_U32 },
1401         [NFTA_CHAIN_TYPE]       = { .type = NLA_STRING,
1402                                     .len = NFT_MODULE_AUTOLOAD_LIMIT },
1403         [NFTA_CHAIN_COUNTERS]   = { .type = NLA_NESTED },
1404         [NFTA_CHAIN_FLAGS]      = { .type = NLA_U32 },
1405         [NFTA_CHAIN_ID]         = { .type = NLA_U32 },
1406         [NFTA_CHAIN_USERDATA]   = { .type = NLA_BINARY,
1407                                     .len = NFT_USERDATA_MAXLEN },
1408 };
1409
1410 static const struct nla_policy nft_hook_policy[NFTA_HOOK_MAX + 1] = {
1411         [NFTA_HOOK_HOOKNUM]     = { .type = NLA_U32 },
1412         [NFTA_HOOK_PRIORITY]    = { .type = NLA_U32 },
1413         [NFTA_HOOK_DEV]         = { .type = NLA_STRING,
1414                                     .len = IFNAMSIZ - 1 },
1415 };
1416
1417 static int nft_dump_stats(struct sk_buff *skb, struct nft_stats __percpu *stats)
1418 {
1419         struct nft_stats *cpu_stats, total;
1420         struct nlattr *nest;
1421         unsigned int seq;
1422         u64 pkts, bytes;
1423         int cpu;
1424
1425         if (!stats)
1426                 return 0;
1427
1428         memset(&total, 0, sizeof(total));
1429         for_each_possible_cpu(cpu) {
1430                 cpu_stats = per_cpu_ptr(stats, cpu);
1431                 do {
1432                         seq = u64_stats_fetch_begin_irq(&cpu_stats->syncp);
1433                         pkts = cpu_stats->pkts;
1434                         bytes = cpu_stats->bytes;
1435                 } while (u64_stats_fetch_retry_irq(&cpu_stats->syncp, seq));
1436                 total.pkts += pkts;
1437                 total.bytes += bytes;
1438         }
1439         nest = nla_nest_start_noflag(skb, NFTA_CHAIN_COUNTERS);
1440         if (nest == NULL)
1441                 goto nla_put_failure;
1442
1443         if (nla_put_be64(skb, NFTA_COUNTER_PACKETS, cpu_to_be64(total.pkts),
1444                          NFTA_COUNTER_PAD) ||
1445             nla_put_be64(skb, NFTA_COUNTER_BYTES, cpu_to_be64(total.bytes),
1446                          NFTA_COUNTER_PAD))
1447                 goto nla_put_failure;
1448
1449         nla_nest_end(skb, nest);
1450         return 0;
1451
1452 nla_put_failure:
1453         return -ENOSPC;
1454 }
1455
1456 static int nft_dump_basechain_hook(struct sk_buff *skb, int family,
1457                                    const struct nft_base_chain *basechain)
1458 {
1459         const struct nf_hook_ops *ops = &basechain->ops;
1460         struct nft_hook *hook, *first = NULL;
1461         struct nlattr *nest, *nest_devs;
1462         int n = 0;
1463
1464         nest = nla_nest_start_noflag(skb, NFTA_CHAIN_HOOK);
1465         if (nest == NULL)
1466                 goto nla_put_failure;
1467         if (nla_put_be32(skb, NFTA_HOOK_HOOKNUM, htonl(ops->hooknum)))
1468                 goto nla_put_failure;
1469         if (nla_put_be32(skb, NFTA_HOOK_PRIORITY, htonl(ops->priority)))
1470                 goto nla_put_failure;
1471
1472         if (nft_base_chain_netdev(family, ops->hooknum)) {
1473                 nest_devs = nla_nest_start_noflag(skb, NFTA_HOOK_DEVS);
1474                 list_for_each_entry(hook, &basechain->hook_list, list) {
1475                         if (!first)
1476                                 first = hook;
1477
1478                         if (nla_put_string(skb, NFTA_DEVICE_NAME,
1479                                            hook->ops.dev->name))
1480                                 goto nla_put_failure;
1481                         n++;
1482                 }
1483                 nla_nest_end(skb, nest_devs);
1484
1485                 if (n == 1 &&
1486                     nla_put_string(skb, NFTA_HOOK_DEV, first->ops.dev->name))
1487                         goto nla_put_failure;
1488         }
1489         nla_nest_end(skb, nest);
1490
1491         return 0;
1492 nla_put_failure:
1493         return -1;
1494 }
1495
1496 static int nf_tables_fill_chain_info(struct sk_buff *skb, struct net *net,
1497                                      u32 portid, u32 seq, int event, u32 flags,
1498                                      int family, const struct nft_table *table,
1499                                      const struct nft_chain *chain)
1500 {
1501         struct nlmsghdr *nlh;
1502
1503         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
1504         nlh = nfnl_msg_put(skb, portid, seq, event, flags, family,
1505                            NFNETLINK_V0, nft_base_seq(net));
1506         if (!nlh)
1507                 goto nla_put_failure;
1508
1509         if (nla_put_string(skb, NFTA_CHAIN_TABLE, table->name))
1510                 goto nla_put_failure;
1511         if (nla_put_be64(skb, NFTA_CHAIN_HANDLE, cpu_to_be64(chain->handle),
1512                          NFTA_CHAIN_PAD))
1513                 goto nla_put_failure;
1514         if (nla_put_string(skb, NFTA_CHAIN_NAME, chain->name))
1515                 goto nla_put_failure;
1516
1517         if (nft_is_base_chain(chain)) {
1518                 const struct nft_base_chain *basechain = nft_base_chain(chain);
1519                 struct nft_stats __percpu *stats;
1520
1521                 if (nft_dump_basechain_hook(skb, family, basechain))
1522                         goto nla_put_failure;
1523
1524                 if (nla_put_be32(skb, NFTA_CHAIN_POLICY,
1525                                  htonl(basechain->policy)))
1526                         goto nla_put_failure;
1527
1528                 if (nla_put_string(skb, NFTA_CHAIN_TYPE, basechain->type->name))
1529                         goto nla_put_failure;
1530
1531                 stats = rcu_dereference_check(basechain->stats,
1532                                               lockdep_commit_lock_is_held(net));
1533                 if (nft_dump_stats(skb, stats))
1534                         goto nla_put_failure;
1535         }
1536
1537         if (chain->flags &&
1538             nla_put_be32(skb, NFTA_CHAIN_FLAGS, htonl(chain->flags)))
1539                 goto nla_put_failure;
1540
1541         if (nla_put_be32(skb, NFTA_CHAIN_USE, htonl(chain->use)))
1542                 goto nla_put_failure;
1543
1544         if (chain->udata &&
1545             nla_put(skb, NFTA_CHAIN_USERDATA, chain->udlen, chain->udata))
1546                 goto nla_put_failure;
1547
1548         nlmsg_end(skb, nlh);
1549         return 0;
1550
1551 nla_put_failure:
1552         nlmsg_trim(skb, nlh);
1553         return -1;
1554 }
1555
1556 static void nf_tables_chain_notify(const struct nft_ctx *ctx, int event)
1557 {
1558         struct nftables_pernet *nft_net;
1559         struct sk_buff *skb;
1560         int err;
1561
1562         if (!ctx->report &&
1563             !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
1564                 return;
1565
1566         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
1567         if (skb == NULL)
1568                 goto err;
1569
1570         err = nf_tables_fill_chain_info(skb, ctx->net, ctx->portid, ctx->seq,
1571                                         event, 0, ctx->family, ctx->table,
1572                                         ctx->chain);
1573         if (err < 0) {
1574                 kfree_skb(skb);
1575                 goto err;
1576         }
1577
1578         nft_net = nft_pernet(ctx->net);
1579         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
1580         return;
1581 err:
1582         nfnetlink_set_err(ctx->net, ctx->portid, NFNLGRP_NFTABLES, -ENOBUFS);
1583 }
1584
1585 static int nf_tables_dump_chains(struct sk_buff *skb,
1586                                  struct netlink_callback *cb)
1587 {
1588         const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
1589         unsigned int idx = 0, s_idx = cb->args[0];
1590         struct net *net = sock_net(skb->sk);
1591         int family = nfmsg->nfgen_family;
1592         struct nftables_pernet *nft_net;
1593         const struct nft_table *table;
1594         const struct nft_chain *chain;
1595
1596         rcu_read_lock();
1597         nft_net = nft_pernet(net);
1598         cb->seq = nft_net->base_seq;
1599
1600         list_for_each_entry_rcu(table, &nft_net->tables, list) {
1601                 if (family != NFPROTO_UNSPEC && family != table->family)
1602                         continue;
1603
1604                 list_for_each_entry_rcu(chain, &table->chains, list) {
1605                         if (idx < s_idx)
1606                                 goto cont;
1607                         if (idx > s_idx)
1608                                 memset(&cb->args[1], 0,
1609                                        sizeof(cb->args) - sizeof(cb->args[0]));
1610                         if (!nft_is_active(net, chain))
1611                                 continue;
1612                         if (nf_tables_fill_chain_info(skb, net,
1613                                                       NETLINK_CB(cb->skb).portid,
1614                                                       cb->nlh->nlmsg_seq,
1615                                                       NFT_MSG_NEWCHAIN,
1616                                                       NLM_F_MULTI,
1617                                                       table->family, table,
1618                                                       chain) < 0)
1619                                 goto done;
1620
1621                         nl_dump_check_consistent(cb, nlmsg_hdr(skb));
1622 cont:
1623                         idx++;
1624                 }
1625         }
1626 done:
1627         rcu_read_unlock();
1628         cb->args[0] = idx;
1629         return skb->len;
1630 }
1631
1632 /* called with rcu_read_lock held */
1633 static int nf_tables_getchain(struct sk_buff *skb, const struct nfnl_info *info,
1634                               const struct nlattr * const nla[])
1635 {
1636         struct netlink_ext_ack *extack = info->extack;
1637         u8 genmask = nft_genmask_cur(info->net);
1638         u8 family = info->nfmsg->nfgen_family;
1639         const struct nft_chain *chain;
1640         struct net *net = info->net;
1641         struct nft_table *table;
1642         struct sk_buff *skb2;
1643         int err;
1644
1645         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
1646                 struct netlink_dump_control c = {
1647                         .dump = nf_tables_dump_chains,
1648                         .module = THIS_MODULE,
1649                 };
1650
1651                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
1652         }
1653
1654         table = nft_table_lookup(net, nla[NFTA_CHAIN_TABLE], family, genmask, 0);
1655         if (IS_ERR(table)) {
1656                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TABLE]);
1657                 return PTR_ERR(table);
1658         }
1659
1660         chain = nft_chain_lookup(net, table, nla[NFTA_CHAIN_NAME], genmask);
1661         if (IS_ERR(chain)) {
1662                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_NAME]);
1663                 return PTR_ERR(chain);
1664         }
1665
1666         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
1667         if (!skb2)
1668                 return -ENOMEM;
1669
1670         err = nf_tables_fill_chain_info(skb2, net, NETLINK_CB(skb).portid,
1671                                         info->nlh->nlmsg_seq, NFT_MSG_NEWCHAIN,
1672                                         0, family, table, chain);
1673         if (err < 0)
1674                 goto err_fill_chain_info;
1675
1676         return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
1677
1678 err_fill_chain_info:
1679         kfree_skb(skb2);
1680         return err;
1681 }
1682
1683 static const struct nla_policy nft_counter_policy[NFTA_COUNTER_MAX + 1] = {
1684         [NFTA_COUNTER_PACKETS]  = { .type = NLA_U64 },
1685         [NFTA_COUNTER_BYTES]    = { .type = NLA_U64 },
1686 };
1687
1688 static struct nft_stats __percpu *nft_stats_alloc(const struct nlattr *attr)
1689 {
1690         struct nlattr *tb[NFTA_COUNTER_MAX+1];
1691         struct nft_stats __percpu *newstats;
1692         struct nft_stats *stats;
1693         int err;
1694
1695         err = nla_parse_nested_deprecated(tb, NFTA_COUNTER_MAX, attr,
1696                                           nft_counter_policy, NULL);
1697         if (err < 0)
1698                 return ERR_PTR(err);
1699
1700         if (!tb[NFTA_COUNTER_BYTES] || !tb[NFTA_COUNTER_PACKETS])
1701                 return ERR_PTR(-EINVAL);
1702
1703         newstats = netdev_alloc_pcpu_stats(struct nft_stats);
1704         if (newstats == NULL)
1705                 return ERR_PTR(-ENOMEM);
1706
1707         /* Restore old counters on this cpu, no problem. Per-cpu statistics
1708          * are not exposed to userspace.
1709          */
1710         preempt_disable();
1711         stats = this_cpu_ptr(newstats);
1712         stats->bytes = be64_to_cpu(nla_get_be64(tb[NFTA_COUNTER_BYTES]));
1713         stats->pkts = be64_to_cpu(nla_get_be64(tb[NFTA_COUNTER_PACKETS]));
1714         preempt_enable();
1715
1716         return newstats;
1717 }
1718
1719 static void nft_chain_stats_replace(struct nft_trans *trans)
1720 {
1721         struct nft_base_chain *chain = nft_base_chain(trans->ctx.chain);
1722
1723         if (!nft_trans_chain_stats(trans))
1724                 return;
1725
1726         nft_trans_chain_stats(trans) =
1727                 rcu_replace_pointer(chain->stats, nft_trans_chain_stats(trans),
1728                                     lockdep_commit_lock_is_held(trans->ctx.net));
1729
1730         if (!nft_trans_chain_stats(trans))
1731                 static_branch_inc(&nft_counters_enabled);
1732 }
1733
1734 static void nf_tables_chain_free_chain_rules(struct nft_chain *chain)
1735 {
1736         struct nft_rule **g0 = rcu_dereference_raw(chain->rules_gen_0);
1737         struct nft_rule **g1 = rcu_dereference_raw(chain->rules_gen_1);
1738
1739         if (g0 != g1)
1740                 kvfree(g1);
1741         kvfree(g0);
1742
1743         /* should be NULL either via abort or via successful commit */
1744         WARN_ON_ONCE(chain->rules_next);
1745         kvfree(chain->rules_next);
1746 }
1747
1748 void nf_tables_chain_destroy(struct nft_ctx *ctx)
1749 {
1750         struct nft_chain *chain = ctx->chain;
1751         struct nft_hook *hook, *next;
1752
1753         if (WARN_ON(chain->use > 0))
1754                 return;
1755
1756         /* no concurrent access possible anymore */
1757         nf_tables_chain_free_chain_rules(chain);
1758
1759         if (nft_is_base_chain(chain)) {
1760                 struct nft_base_chain *basechain = nft_base_chain(chain);
1761
1762                 if (nft_base_chain_netdev(ctx->family, basechain->ops.hooknum)) {
1763                         list_for_each_entry_safe(hook, next,
1764                                                  &basechain->hook_list, list) {
1765                                 list_del_rcu(&hook->list);
1766                                 kfree_rcu(hook, rcu);
1767                         }
1768                 }
1769                 module_put(basechain->type->owner);
1770                 if (rcu_access_pointer(basechain->stats)) {
1771                         static_branch_dec(&nft_counters_enabled);
1772                         free_percpu(rcu_dereference_raw(basechain->stats));
1773                 }
1774                 kfree(chain->name);
1775                 kfree(chain->udata);
1776                 kfree(basechain);
1777         } else {
1778                 kfree(chain->name);
1779                 kfree(chain->udata);
1780                 kfree(chain);
1781         }
1782 }
1783
1784 static struct nft_hook *nft_netdev_hook_alloc(struct net *net,
1785                                               const struct nlattr *attr)
1786 {
1787         struct net_device *dev;
1788         char ifname[IFNAMSIZ];
1789         struct nft_hook *hook;
1790         int err;
1791
1792         hook = kmalloc(sizeof(struct nft_hook), GFP_KERNEL);
1793         if (!hook) {
1794                 err = -ENOMEM;
1795                 goto err_hook_alloc;
1796         }
1797
1798         nla_strscpy(ifname, attr, IFNAMSIZ);
1799         /* nf_tables_netdev_event() is called under rtnl_mutex, this is
1800          * indirectly serializing all the other holders of the commit_mutex with
1801          * the rtnl_mutex.
1802          */
1803         dev = __dev_get_by_name(net, ifname);
1804         if (!dev) {
1805                 err = -ENOENT;
1806                 goto err_hook_dev;
1807         }
1808         hook->ops.dev = dev;
1809         hook->inactive = false;
1810
1811         return hook;
1812
1813 err_hook_dev:
1814         kfree(hook);
1815 err_hook_alloc:
1816         return ERR_PTR(err);
1817 }
1818
1819 static struct nft_hook *nft_hook_list_find(struct list_head *hook_list,
1820                                            const struct nft_hook *this)
1821 {
1822         struct nft_hook *hook;
1823
1824         list_for_each_entry(hook, hook_list, list) {
1825                 if (this->ops.dev == hook->ops.dev)
1826                         return hook;
1827         }
1828
1829         return NULL;
1830 }
1831
1832 static int nf_tables_parse_netdev_hooks(struct net *net,
1833                                         const struct nlattr *attr,
1834                                         struct list_head *hook_list)
1835 {
1836         struct nft_hook *hook, *next;
1837         const struct nlattr *tmp;
1838         int rem, n = 0, err;
1839
1840         nla_for_each_nested(tmp, attr, rem) {
1841                 if (nla_type(tmp) != NFTA_DEVICE_NAME) {
1842                         err = -EINVAL;
1843                         goto err_hook;
1844                 }
1845
1846                 hook = nft_netdev_hook_alloc(net, tmp);
1847                 if (IS_ERR(hook)) {
1848                         err = PTR_ERR(hook);
1849                         goto err_hook;
1850                 }
1851                 if (nft_hook_list_find(hook_list, hook)) {
1852                         kfree(hook);
1853                         err = -EEXIST;
1854                         goto err_hook;
1855                 }
1856                 list_add_tail(&hook->list, hook_list);
1857                 n++;
1858
1859                 if (n == NFT_NETDEVICE_MAX) {
1860                         err = -EFBIG;
1861                         goto err_hook;
1862                 }
1863         }
1864
1865         return 0;
1866
1867 err_hook:
1868         list_for_each_entry_safe(hook, next, hook_list, list) {
1869                 list_del(&hook->list);
1870                 kfree(hook);
1871         }
1872         return err;
1873 }
1874
1875 struct nft_chain_hook {
1876         u32                             num;
1877         s32                             priority;
1878         const struct nft_chain_type     *type;
1879         struct list_head                list;
1880 };
1881
1882 static int nft_chain_parse_netdev(struct net *net,
1883                                   struct nlattr *tb[],
1884                                   struct list_head *hook_list)
1885 {
1886         struct nft_hook *hook;
1887         int err;
1888
1889         if (tb[NFTA_HOOK_DEV]) {
1890                 hook = nft_netdev_hook_alloc(net, tb[NFTA_HOOK_DEV]);
1891                 if (IS_ERR(hook))
1892                         return PTR_ERR(hook);
1893
1894                 list_add_tail(&hook->list, hook_list);
1895         } else if (tb[NFTA_HOOK_DEVS]) {
1896                 err = nf_tables_parse_netdev_hooks(net, tb[NFTA_HOOK_DEVS],
1897                                                    hook_list);
1898                 if (err < 0)
1899                         return err;
1900
1901                 if (list_empty(hook_list))
1902                         return -EINVAL;
1903         } else {
1904                 return -EINVAL;
1905         }
1906
1907         return 0;
1908 }
1909
1910 static int nft_chain_parse_hook(struct net *net,
1911                                 const struct nlattr * const nla[],
1912                                 struct nft_chain_hook *hook, u8 family,
1913                                 struct netlink_ext_ack *extack, bool autoload)
1914 {
1915         struct nftables_pernet *nft_net = nft_pernet(net);
1916         struct nlattr *ha[NFTA_HOOK_MAX + 1];
1917         const struct nft_chain_type *type;
1918         int err;
1919
1920         lockdep_assert_held(&nft_net->commit_mutex);
1921         lockdep_nfnl_nft_mutex_not_held();
1922
1923         err = nla_parse_nested_deprecated(ha, NFTA_HOOK_MAX,
1924                                           nla[NFTA_CHAIN_HOOK],
1925                                           nft_hook_policy, NULL);
1926         if (err < 0)
1927                 return err;
1928
1929         if (ha[NFTA_HOOK_HOOKNUM] == NULL ||
1930             ha[NFTA_HOOK_PRIORITY] == NULL)
1931                 return -EINVAL;
1932
1933         hook->num = ntohl(nla_get_be32(ha[NFTA_HOOK_HOOKNUM]));
1934         hook->priority = ntohl(nla_get_be32(ha[NFTA_HOOK_PRIORITY]));
1935
1936         type = __nft_chain_type_get(family, NFT_CHAIN_T_DEFAULT);
1937         if (!type)
1938                 return -EOPNOTSUPP;
1939
1940         if (nla[NFTA_CHAIN_TYPE]) {
1941                 type = nf_tables_chain_type_lookup(net, nla[NFTA_CHAIN_TYPE],
1942                                                    family, autoload);
1943                 if (IS_ERR(type)) {
1944                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TYPE]);
1945                         return PTR_ERR(type);
1946                 }
1947         }
1948         if (hook->num >= NFT_MAX_HOOKS || !(type->hook_mask & (1 << hook->num)))
1949                 return -EOPNOTSUPP;
1950
1951         if (type->type == NFT_CHAIN_T_NAT &&
1952             hook->priority <= NF_IP_PRI_CONNTRACK)
1953                 return -EOPNOTSUPP;
1954
1955         if (!try_module_get(type->owner)) {
1956                 if (nla[NFTA_CHAIN_TYPE])
1957                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TYPE]);
1958                 return -ENOENT;
1959         }
1960
1961         hook->type = type;
1962
1963         INIT_LIST_HEAD(&hook->list);
1964         if (nft_base_chain_netdev(family, hook->num)) {
1965                 err = nft_chain_parse_netdev(net, ha, &hook->list);
1966                 if (err < 0) {
1967                         module_put(type->owner);
1968                         return err;
1969                 }
1970         } else if (ha[NFTA_HOOK_DEV] || ha[NFTA_HOOK_DEVS]) {
1971                 module_put(type->owner);
1972                 return -EOPNOTSUPP;
1973         }
1974
1975         return 0;
1976 }
1977
1978 static void nft_chain_release_hook(struct nft_chain_hook *hook)
1979 {
1980         struct nft_hook *h, *next;
1981
1982         list_for_each_entry_safe(h, next, &hook->list, list) {
1983                 list_del(&h->list);
1984                 kfree(h);
1985         }
1986         module_put(hook->type->owner);
1987 }
1988
1989 struct nft_rules_old {
1990         struct rcu_head h;
1991         struct nft_rule **start;
1992 };
1993
1994 static struct nft_rule **nf_tables_chain_alloc_rules(const struct nft_chain *chain,
1995                                                      unsigned int alloc)
1996 {
1997         if (alloc > INT_MAX)
1998                 return NULL;
1999
2000         alloc += 1;     /* NULL, ends rules */
2001         if (sizeof(struct nft_rule *) > INT_MAX / alloc)
2002                 return NULL;
2003
2004         alloc *= sizeof(struct nft_rule *);
2005         alloc += sizeof(struct nft_rules_old);
2006
2007         return kvmalloc(alloc, GFP_KERNEL);
2008 }
2009
2010 static void nft_basechain_hook_init(struct nf_hook_ops *ops, u8 family,
2011                                     const struct nft_chain_hook *hook,
2012                                     struct nft_chain *chain)
2013 {
2014         ops->pf                 = family;
2015         ops->hooknum            = hook->num;
2016         ops->priority           = hook->priority;
2017         ops->priv               = chain;
2018         ops->hook               = hook->type->hooks[ops->hooknum];
2019         ops->hook_ops_type      = NF_HOOK_OP_NF_TABLES;
2020 }
2021
2022 static int nft_basechain_init(struct nft_base_chain *basechain, u8 family,
2023                               struct nft_chain_hook *hook, u32 flags)
2024 {
2025         struct nft_chain *chain;
2026         struct nft_hook *h;
2027
2028         basechain->type = hook->type;
2029         INIT_LIST_HEAD(&basechain->hook_list);
2030         chain = &basechain->chain;
2031
2032         if (nft_base_chain_netdev(family, hook->num)) {
2033                 list_splice_init(&hook->list, &basechain->hook_list);
2034                 list_for_each_entry(h, &basechain->hook_list, list)
2035                         nft_basechain_hook_init(&h->ops, family, hook, chain);
2036
2037                 basechain->ops.hooknum  = hook->num;
2038                 basechain->ops.priority = hook->priority;
2039         } else {
2040                 nft_basechain_hook_init(&basechain->ops, family, hook, chain);
2041         }
2042
2043         chain->flags |= NFT_CHAIN_BASE | flags;
2044         basechain->policy = NF_ACCEPT;
2045         if (chain->flags & NFT_CHAIN_HW_OFFLOAD &&
2046             nft_chain_offload_priority(basechain) < 0)
2047                 return -EOPNOTSUPP;
2048
2049         flow_block_init(&basechain->flow_block);
2050
2051         return 0;
2052 }
2053
2054 static int nft_chain_add(struct nft_table *table, struct nft_chain *chain)
2055 {
2056         int err;
2057
2058         err = rhltable_insert_key(&table->chains_ht, chain->name,
2059                                   &chain->rhlhead, nft_chain_ht_params);
2060         if (err)
2061                 return err;
2062
2063         list_add_tail_rcu(&chain->list, &table->chains);
2064
2065         return 0;
2066 }
2067
2068 static u64 chain_id;
2069
2070 static int nf_tables_addchain(struct nft_ctx *ctx, u8 family, u8 genmask,
2071                               u8 policy, u32 flags,
2072                               struct netlink_ext_ack *extack)
2073 {
2074         const struct nlattr * const *nla = ctx->nla;
2075         struct nft_table *table = ctx->table;
2076         struct nft_base_chain *basechain;
2077         struct nft_stats __percpu *stats;
2078         struct net *net = ctx->net;
2079         char name[NFT_NAME_MAXLEN];
2080         struct nft_trans *trans;
2081         struct nft_chain *chain;
2082         struct nft_rule **rules;
2083         int err;
2084
2085         if (table->use == UINT_MAX)
2086                 return -EOVERFLOW;
2087
2088         if (nla[NFTA_CHAIN_HOOK]) {
2089                 struct nft_chain_hook hook;
2090
2091                 if (flags & NFT_CHAIN_BINDING)
2092                         return -EOPNOTSUPP;
2093
2094                 err = nft_chain_parse_hook(net, nla, &hook, family, extack,
2095                                            true);
2096                 if (err < 0)
2097                         return err;
2098
2099                 basechain = kzalloc(sizeof(*basechain), GFP_KERNEL);
2100                 if (basechain == NULL) {
2101                         nft_chain_release_hook(&hook);
2102                         return -ENOMEM;
2103                 }
2104                 chain = &basechain->chain;
2105
2106                 if (nla[NFTA_CHAIN_COUNTERS]) {
2107                         stats = nft_stats_alloc(nla[NFTA_CHAIN_COUNTERS]);
2108                         if (IS_ERR(stats)) {
2109                                 nft_chain_release_hook(&hook);
2110                                 kfree(basechain);
2111                                 return PTR_ERR(stats);
2112                         }
2113                         rcu_assign_pointer(basechain->stats, stats);
2114                         static_branch_inc(&nft_counters_enabled);
2115                 }
2116
2117                 err = nft_basechain_init(basechain, family, &hook, flags);
2118                 if (err < 0) {
2119                         nft_chain_release_hook(&hook);
2120                         kfree(basechain);
2121                         return err;
2122                 }
2123         } else {
2124                 if (flags & NFT_CHAIN_BASE)
2125                         return -EINVAL;
2126                 if (flags & NFT_CHAIN_HW_OFFLOAD)
2127                         return -EOPNOTSUPP;
2128
2129                 chain = kzalloc(sizeof(*chain), GFP_KERNEL);
2130                 if (chain == NULL)
2131                         return -ENOMEM;
2132
2133                 chain->flags = flags;
2134         }
2135         ctx->chain = chain;
2136
2137         INIT_LIST_HEAD(&chain->rules);
2138         chain->handle = nf_tables_alloc_handle(table);
2139         chain->table = table;
2140
2141         if (nla[NFTA_CHAIN_NAME]) {
2142                 chain->name = nla_strdup(nla[NFTA_CHAIN_NAME], GFP_KERNEL);
2143         } else {
2144                 if (!(flags & NFT_CHAIN_BINDING)) {
2145                         err = -EINVAL;
2146                         goto err_destroy_chain;
2147                 }
2148
2149                 snprintf(name, sizeof(name), "__chain%llu", ++chain_id);
2150                 chain->name = kstrdup(name, GFP_KERNEL);
2151         }
2152
2153         if (!chain->name) {
2154                 err = -ENOMEM;
2155                 goto err_destroy_chain;
2156         }
2157
2158         if (nla[NFTA_CHAIN_USERDATA]) {
2159                 chain->udata = nla_memdup(nla[NFTA_CHAIN_USERDATA], GFP_KERNEL);
2160                 if (chain->udata == NULL) {
2161                         err = -ENOMEM;
2162                         goto err_destroy_chain;
2163                 }
2164                 chain->udlen = nla_len(nla[NFTA_CHAIN_USERDATA]);
2165         }
2166
2167         rules = nf_tables_chain_alloc_rules(chain, 0);
2168         if (!rules) {
2169                 err = -ENOMEM;
2170                 goto err_destroy_chain;
2171         }
2172
2173         *rules = NULL;
2174         rcu_assign_pointer(chain->rules_gen_0, rules);
2175         rcu_assign_pointer(chain->rules_gen_1, rules);
2176
2177         err = nf_tables_register_hook(net, table, chain);
2178         if (err < 0)
2179                 goto err_destroy_chain;
2180
2181         trans = nft_trans_chain_add(ctx, NFT_MSG_NEWCHAIN);
2182         if (IS_ERR(trans)) {
2183                 err = PTR_ERR(trans);
2184                 goto err_unregister_hook;
2185         }
2186
2187         nft_trans_chain_policy(trans) = NFT_CHAIN_POLICY_UNSET;
2188         if (nft_is_base_chain(chain))
2189                 nft_trans_chain_policy(trans) = policy;
2190
2191         err = nft_chain_add(table, chain);
2192         if (err < 0) {
2193                 nft_trans_destroy(trans);
2194                 goto err_unregister_hook;
2195         }
2196
2197         table->use++;
2198
2199         return 0;
2200 err_unregister_hook:
2201         nf_tables_unregister_hook(net, table, chain);
2202 err_destroy_chain:
2203         nf_tables_chain_destroy(ctx);
2204
2205         return err;
2206 }
2207
2208 static bool nft_hook_list_equal(struct list_head *hook_list1,
2209                                 struct list_head *hook_list2)
2210 {
2211         struct nft_hook *hook;
2212         int n = 0, m = 0;
2213
2214         n = 0;
2215         list_for_each_entry(hook, hook_list2, list) {
2216                 if (!nft_hook_list_find(hook_list1, hook))
2217                         return false;
2218
2219                 n++;
2220         }
2221         list_for_each_entry(hook, hook_list1, list)
2222                 m++;
2223
2224         return n == m;
2225 }
2226
2227 static int nf_tables_updchain(struct nft_ctx *ctx, u8 genmask, u8 policy,
2228                               u32 flags, const struct nlattr *attr,
2229                               struct netlink_ext_ack *extack)
2230 {
2231         const struct nlattr * const *nla = ctx->nla;
2232         struct nft_table *table = ctx->table;
2233         struct nft_chain *chain = ctx->chain;
2234         struct nft_base_chain *basechain;
2235         struct nft_stats *stats = NULL;
2236         struct nft_chain_hook hook;
2237         struct nf_hook_ops *ops;
2238         struct nft_trans *trans;
2239         int err;
2240
2241         if (chain->flags ^ flags)
2242                 return -EOPNOTSUPP;
2243
2244         if (nla[NFTA_CHAIN_HOOK]) {
2245                 if (!nft_is_base_chain(chain)) {
2246                         NL_SET_BAD_ATTR(extack, attr);
2247                         return -EEXIST;
2248                 }
2249                 err = nft_chain_parse_hook(ctx->net, nla, &hook, ctx->family,
2250                                            extack, false);
2251                 if (err < 0)
2252                         return err;
2253
2254                 basechain = nft_base_chain(chain);
2255                 if (basechain->type != hook.type) {
2256                         nft_chain_release_hook(&hook);
2257                         NL_SET_BAD_ATTR(extack, attr);
2258                         return -EEXIST;
2259                 }
2260
2261                 if (nft_base_chain_netdev(ctx->family, hook.num)) {
2262                         if (!nft_hook_list_equal(&basechain->hook_list,
2263                                                  &hook.list)) {
2264                                 nft_chain_release_hook(&hook);
2265                                 NL_SET_BAD_ATTR(extack, attr);
2266                                 return -EEXIST;
2267                         }
2268                 } else {
2269                         ops = &basechain->ops;
2270                         if (ops->hooknum != hook.num ||
2271                             ops->priority != hook.priority) {
2272                                 nft_chain_release_hook(&hook);
2273                                 NL_SET_BAD_ATTR(extack, attr);
2274                                 return -EEXIST;
2275                         }
2276                 }
2277                 nft_chain_release_hook(&hook);
2278         }
2279
2280         if (nla[NFTA_CHAIN_HANDLE] &&
2281             nla[NFTA_CHAIN_NAME]) {
2282                 struct nft_chain *chain2;
2283
2284                 chain2 = nft_chain_lookup(ctx->net, table,
2285                                           nla[NFTA_CHAIN_NAME], genmask);
2286                 if (!IS_ERR(chain2)) {
2287                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_NAME]);
2288                         return -EEXIST;
2289                 }
2290         }
2291
2292         if (nla[NFTA_CHAIN_COUNTERS]) {
2293                 if (!nft_is_base_chain(chain))
2294                         return -EOPNOTSUPP;
2295
2296                 stats = nft_stats_alloc(nla[NFTA_CHAIN_COUNTERS]);
2297                 if (IS_ERR(stats))
2298                         return PTR_ERR(stats);
2299         }
2300
2301         err = -ENOMEM;
2302         trans = nft_trans_alloc(ctx, NFT_MSG_NEWCHAIN,
2303                                 sizeof(struct nft_trans_chain));
2304         if (trans == NULL)
2305                 goto err;
2306
2307         nft_trans_chain_stats(trans) = stats;
2308         nft_trans_chain_update(trans) = true;
2309
2310         if (nla[NFTA_CHAIN_POLICY])
2311                 nft_trans_chain_policy(trans) = policy;
2312         else
2313                 nft_trans_chain_policy(trans) = -1;
2314
2315         if (nla[NFTA_CHAIN_HANDLE] &&
2316             nla[NFTA_CHAIN_NAME]) {
2317                 struct nftables_pernet *nft_net = nft_pernet(ctx->net);
2318                 struct nft_trans *tmp;
2319                 char *name;
2320
2321                 err = -ENOMEM;
2322                 name = nla_strdup(nla[NFTA_CHAIN_NAME], GFP_KERNEL);
2323                 if (!name)
2324                         goto err;
2325
2326                 err = -EEXIST;
2327                 list_for_each_entry(tmp, &nft_net->commit_list, list) {
2328                         if (tmp->msg_type == NFT_MSG_NEWCHAIN &&
2329                             tmp->ctx.table == table &&
2330                             nft_trans_chain_update(tmp) &&
2331                             nft_trans_chain_name(tmp) &&
2332                             strcmp(name, nft_trans_chain_name(tmp)) == 0) {
2333                                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_NAME]);
2334                                 kfree(name);
2335                                 goto err;
2336                         }
2337                 }
2338
2339                 nft_trans_chain_name(trans) = name;
2340         }
2341         nft_trans_commit_list_add_tail(ctx->net, trans);
2342
2343         return 0;
2344 err:
2345         free_percpu(stats);
2346         kfree(trans);
2347         return err;
2348 }
2349
2350 static struct nft_chain *nft_chain_lookup_byid(const struct net *net,
2351                                                const struct nlattr *nla)
2352 {
2353         struct nftables_pernet *nft_net = nft_pernet(net);
2354         u32 id = ntohl(nla_get_be32(nla));
2355         struct nft_trans *trans;
2356
2357         list_for_each_entry(trans, &nft_net->commit_list, list) {
2358                 struct nft_chain *chain = trans->ctx.chain;
2359
2360                 if (trans->msg_type == NFT_MSG_NEWCHAIN &&
2361                     id == nft_trans_chain_id(trans))
2362                         return chain;
2363         }
2364         return ERR_PTR(-ENOENT);
2365 }
2366
2367 static int nf_tables_newchain(struct sk_buff *skb, const struct nfnl_info *info,
2368                               const struct nlattr * const nla[])
2369 {
2370         struct nftables_pernet *nft_net = nft_pernet(info->net);
2371         struct netlink_ext_ack *extack = info->extack;
2372         u8 genmask = nft_genmask_next(info->net);
2373         u8 family = info->nfmsg->nfgen_family;
2374         struct nft_chain *chain = NULL;
2375         struct net *net = info->net;
2376         const struct nlattr *attr;
2377         struct nft_table *table;
2378         u8 policy = NF_ACCEPT;
2379         struct nft_ctx ctx;
2380         u64 handle = 0;
2381         u32 flags = 0;
2382
2383         lockdep_assert_held(&nft_net->commit_mutex);
2384
2385         table = nft_table_lookup(net, nla[NFTA_CHAIN_TABLE], family, genmask,
2386                                  NETLINK_CB(skb).portid);
2387         if (IS_ERR(table)) {
2388                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TABLE]);
2389                 return PTR_ERR(table);
2390         }
2391
2392         chain = NULL;
2393         attr = nla[NFTA_CHAIN_NAME];
2394
2395         if (nla[NFTA_CHAIN_HANDLE]) {
2396                 handle = be64_to_cpu(nla_get_be64(nla[NFTA_CHAIN_HANDLE]));
2397                 chain = nft_chain_lookup_byhandle(table, handle, genmask);
2398                 if (IS_ERR(chain)) {
2399                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_HANDLE]);
2400                         return PTR_ERR(chain);
2401                 }
2402                 attr = nla[NFTA_CHAIN_HANDLE];
2403         } else if (nla[NFTA_CHAIN_NAME]) {
2404                 chain = nft_chain_lookup(net, table, attr, genmask);
2405                 if (IS_ERR(chain)) {
2406                         if (PTR_ERR(chain) != -ENOENT) {
2407                                 NL_SET_BAD_ATTR(extack, attr);
2408                                 return PTR_ERR(chain);
2409                         }
2410                         chain = NULL;
2411                 }
2412         } else if (!nla[NFTA_CHAIN_ID]) {
2413                 return -EINVAL;
2414         }
2415
2416         if (nla[NFTA_CHAIN_POLICY]) {
2417                 if (chain != NULL &&
2418                     !nft_is_base_chain(chain)) {
2419                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_POLICY]);
2420                         return -EOPNOTSUPP;
2421                 }
2422
2423                 if (chain == NULL &&
2424                     nla[NFTA_CHAIN_HOOK] == NULL) {
2425                         NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_POLICY]);
2426                         return -EOPNOTSUPP;
2427                 }
2428
2429                 policy = ntohl(nla_get_be32(nla[NFTA_CHAIN_POLICY]));
2430                 switch (policy) {
2431                 case NF_DROP:
2432                 case NF_ACCEPT:
2433                         break;
2434                 default:
2435                         return -EINVAL;
2436                 }
2437         }
2438
2439         if (nla[NFTA_CHAIN_FLAGS])
2440                 flags = ntohl(nla_get_be32(nla[NFTA_CHAIN_FLAGS]));
2441         else if (chain)
2442                 flags = chain->flags;
2443
2444         if (flags & ~NFT_CHAIN_FLAGS)
2445                 return -EOPNOTSUPP;
2446
2447         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, chain, nla);
2448
2449         if (chain != NULL) {
2450                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
2451                         NL_SET_BAD_ATTR(extack, attr);
2452                         return -EEXIST;
2453                 }
2454                 if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
2455                         return -EOPNOTSUPP;
2456
2457                 flags |= chain->flags & NFT_CHAIN_BASE;
2458                 return nf_tables_updchain(&ctx, genmask, policy, flags, attr,
2459                                           extack);
2460         }
2461
2462         return nf_tables_addchain(&ctx, family, genmask, policy, flags, extack);
2463 }
2464
2465 static int nf_tables_delchain(struct sk_buff *skb, const struct nfnl_info *info,
2466                               const struct nlattr * const nla[])
2467 {
2468         struct netlink_ext_ack *extack = info->extack;
2469         u8 genmask = nft_genmask_next(info->net);
2470         u8 family = info->nfmsg->nfgen_family;
2471         struct net *net = info->net;
2472         const struct nlattr *attr;
2473         struct nft_table *table;
2474         struct nft_chain *chain;
2475         struct nft_rule *rule;
2476         struct nft_ctx ctx;
2477         u64 handle;
2478         u32 use;
2479         int err;
2480
2481         table = nft_table_lookup(net, nla[NFTA_CHAIN_TABLE], family, genmask,
2482                                  NETLINK_CB(skb).portid);
2483         if (IS_ERR(table)) {
2484                 NL_SET_BAD_ATTR(extack, nla[NFTA_CHAIN_TABLE]);
2485                 return PTR_ERR(table);
2486         }
2487
2488         if (nla[NFTA_CHAIN_HANDLE]) {
2489                 attr = nla[NFTA_CHAIN_HANDLE];
2490                 handle = be64_to_cpu(nla_get_be64(attr));
2491                 chain = nft_chain_lookup_byhandle(table, handle, genmask);
2492         } else {
2493                 attr = nla[NFTA_CHAIN_NAME];
2494                 chain = nft_chain_lookup(net, table, attr, genmask);
2495         }
2496         if (IS_ERR(chain)) {
2497                 NL_SET_BAD_ATTR(extack, attr);
2498                 return PTR_ERR(chain);
2499         }
2500
2501         if (info->nlh->nlmsg_flags & NLM_F_NONREC &&
2502             chain->use > 0)
2503                 return -EBUSY;
2504
2505         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, chain, nla);
2506
2507         use = chain->use;
2508         list_for_each_entry(rule, &chain->rules, list) {
2509                 if (!nft_is_active_next(net, rule))
2510                         continue;
2511                 use--;
2512
2513                 err = nft_delrule(&ctx, rule);
2514                 if (err < 0)
2515                         return err;
2516         }
2517
2518         /* There are rules and elements that are still holding references to us,
2519          * we cannot do a recursive removal in this case.
2520          */
2521         if (use > 0) {
2522                 NL_SET_BAD_ATTR(extack, attr);
2523                 return -EBUSY;
2524         }
2525
2526         return nft_delchain(&ctx);
2527 }
2528
2529 /*
2530  * Expressions
2531  */
2532
2533 /**
2534  *      nft_register_expr - register nf_tables expr type
2535  *      @type: expr type
2536  *
2537  *      Registers the expr type for use with nf_tables. Returns zero on
2538  *      success or a negative errno code otherwise.
2539  */
2540 int nft_register_expr(struct nft_expr_type *type)
2541 {
2542         nfnl_lock(NFNL_SUBSYS_NFTABLES);
2543         if (type->family == NFPROTO_UNSPEC)
2544                 list_add_tail_rcu(&type->list, &nf_tables_expressions);
2545         else
2546                 list_add_rcu(&type->list, &nf_tables_expressions);
2547         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
2548         return 0;
2549 }
2550 EXPORT_SYMBOL_GPL(nft_register_expr);
2551
2552 /**
2553  *      nft_unregister_expr - unregister nf_tables expr type
2554  *      @type: expr type
2555  *
2556  *      Unregisters the expr typefor use with nf_tables.
2557  */
2558 void nft_unregister_expr(struct nft_expr_type *type)
2559 {
2560         nfnl_lock(NFNL_SUBSYS_NFTABLES);
2561         list_del_rcu(&type->list);
2562         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
2563 }
2564 EXPORT_SYMBOL_GPL(nft_unregister_expr);
2565
2566 static const struct nft_expr_type *__nft_expr_type_get(u8 family,
2567                                                        struct nlattr *nla)
2568 {
2569         const struct nft_expr_type *type, *candidate = NULL;
2570
2571         list_for_each_entry(type, &nf_tables_expressions, list) {
2572                 if (!nla_strcmp(nla, type->name)) {
2573                         if (!type->family && !candidate)
2574                                 candidate = type;
2575                         else if (type->family == family)
2576                                 candidate = type;
2577                 }
2578         }
2579         return candidate;
2580 }
2581
2582 #ifdef CONFIG_MODULES
2583 static int nft_expr_type_request_module(struct net *net, u8 family,
2584                                         struct nlattr *nla)
2585 {
2586         if (nft_request_module(net, "nft-expr-%u-%.*s", family,
2587                                nla_len(nla), (char *)nla_data(nla)) == -EAGAIN)
2588                 return -EAGAIN;
2589
2590         return 0;
2591 }
2592 #endif
2593
2594 static const struct nft_expr_type *nft_expr_type_get(struct net *net,
2595                                                      u8 family,
2596                                                      struct nlattr *nla)
2597 {
2598         const struct nft_expr_type *type;
2599
2600         if (nla == NULL)
2601                 return ERR_PTR(-EINVAL);
2602
2603         type = __nft_expr_type_get(family, nla);
2604         if (type != NULL && try_module_get(type->owner))
2605                 return type;
2606
2607         lockdep_nfnl_nft_mutex_not_held();
2608 #ifdef CONFIG_MODULES
2609         if (type == NULL) {
2610                 if (nft_expr_type_request_module(net, family, nla) == -EAGAIN)
2611                         return ERR_PTR(-EAGAIN);
2612
2613                 if (nft_request_module(net, "nft-expr-%.*s",
2614                                        nla_len(nla),
2615                                        (char *)nla_data(nla)) == -EAGAIN)
2616                         return ERR_PTR(-EAGAIN);
2617         }
2618 #endif
2619         return ERR_PTR(-ENOENT);
2620 }
2621
2622 static const struct nla_policy nft_expr_policy[NFTA_EXPR_MAX + 1] = {
2623         [NFTA_EXPR_NAME]        = { .type = NLA_STRING,
2624                                     .len = NFT_MODULE_AUTOLOAD_LIMIT },
2625         [NFTA_EXPR_DATA]        = { .type = NLA_NESTED },
2626 };
2627
2628 static int nf_tables_fill_expr_info(struct sk_buff *skb,
2629                                     const struct nft_expr *expr)
2630 {
2631         if (nla_put_string(skb, NFTA_EXPR_NAME, expr->ops->type->name))
2632                 goto nla_put_failure;
2633
2634         if (expr->ops->dump) {
2635                 struct nlattr *data = nla_nest_start_noflag(skb,
2636                                                             NFTA_EXPR_DATA);
2637                 if (data == NULL)
2638                         goto nla_put_failure;
2639                 if (expr->ops->dump(skb, expr) < 0)
2640                         goto nla_put_failure;
2641                 nla_nest_end(skb, data);
2642         }
2643
2644         return skb->len;
2645
2646 nla_put_failure:
2647         return -1;
2648 };
2649
2650 int nft_expr_dump(struct sk_buff *skb, unsigned int attr,
2651                   const struct nft_expr *expr)
2652 {
2653         struct nlattr *nest;
2654
2655         nest = nla_nest_start_noflag(skb, attr);
2656         if (!nest)
2657                 goto nla_put_failure;
2658         if (nf_tables_fill_expr_info(skb, expr) < 0)
2659                 goto nla_put_failure;
2660         nla_nest_end(skb, nest);
2661         return 0;
2662
2663 nla_put_failure:
2664         return -1;
2665 }
2666
2667 struct nft_expr_info {
2668         const struct nft_expr_ops       *ops;
2669         const struct nlattr             *attr;
2670         struct nlattr                   *tb[NFT_EXPR_MAXATTR + 1];
2671 };
2672
2673 static int nf_tables_expr_parse(const struct nft_ctx *ctx,
2674                                 const struct nlattr *nla,
2675                                 struct nft_expr_info *info)
2676 {
2677         const struct nft_expr_type *type;
2678         const struct nft_expr_ops *ops;
2679         struct nlattr *tb[NFTA_EXPR_MAX + 1];
2680         int err;
2681
2682         err = nla_parse_nested_deprecated(tb, NFTA_EXPR_MAX, nla,
2683                                           nft_expr_policy, NULL);
2684         if (err < 0)
2685                 return err;
2686
2687         type = nft_expr_type_get(ctx->net, ctx->family, tb[NFTA_EXPR_NAME]);
2688         if (IS_ERR(type))
2689                 return PTR_ERR(type);
2690
2691         if (tb[NFTA_EXPR_DATA]) {
2692                 err = nla_parse_nested_deprecated(info->tb, type->maxattr,
2693                                                   tb[NFTA_EXPR_DATA],
2694                                                   type->policy, NULL);
2695                 if (err < 0)
2696                         goto err1;
2697         } else
2698                 memset(info->tb, 0, sizeof(info->tb[0]) * (type->maxattr + 1));
2699
2700         if (type->select_ops != NULL) {
2701                 ops = type->select_ops(ctx,
2702                                        (const struct nlattr * const *)info->tb);
2703                 if (IS_ERR(ops)) {
2704                         err = PTR_ERR(ops);
2705 #ifdef CONFIG_MODULES
2706                         if (err == -EAGAIN)
2707                                 if (nft_expr_type_request_module(ctx->net,
2708                                                                  ctx->family,
2709                                                                  tb[NFTA_EXPR_NAME]) != -EAGAIN)
2710                                         err = -ENOENT;
2711 #endif
2712                         goto err1;
2713                 }
2714         } else
2715                 ops = type->ops;
2716
2717         info->attr = nla;
2718         info->ops = ops;
2719
2720         return 0;
2721
2722 err1:
2723         module_put(type->owner);
2724         return err;
2725 }
2726
2727 static int nf_tables_newexpr(const struct nft_ctx *ctx,
2728                              const struct nft_expr_info *expr_info,
2729                              struct nft_expr *expr)
2730 {
2731         const struct nft_expr_ops *ops = expr_info->ops;
2732         int err;
2733
2734         expr->ops = ops;
2735         if (ops->init) {
2736                 err = ops->init(ctx, expr, (const struct nlattr **)expr_info->tb);
2737                 if (err < 0)
2738                         goto err1;
2739         }
2740
2741         return 0;
2742 err1:
2743         expr->ops = NULL;
2744         return err;
2745 }
2746
2747 static void nf_tables_expr_destroy(const struct nft_ctx *ctx,
2748                                    struct nft_expr *expr)
2749 {
2750         const struct nft_expr_type *type = expr->ops->type;
2751
2752         if (expr->ops->destroy)
2753                 expr->ops->destroy(ctx, expr);
2754         module_put(type->owner);
2755 }
2756
2757 static struct nft_expr *nft_expr_init(const struct nft_ctx *ctx,
2758                                       const struct nlattr *nla)
2759 {
2760         struct nft_expr_info expr_info;
2761         struct nft_expr *expr;
2762         struct module *owner;
2763         int err;
2764
2765         err = nf_tables_expr_parse(ctx, nla, &expr_info);
2766         if (err < 0)
2767                 goto err1;
2768
2769         err = -ENOMEM;
2770         expr = kzalloc(expr_info.ops->size, GFP_KERNEL);
2771         if (expr == NULL)
2772                 goto err2;
2773
2774         err = nf_tables_newexpr(ctx, &expr_info, expr);
2775         if (err < 0)
2776                 goto err3;
2777
2778         return expr;
2779 err3:
2780         kfree(expr);
2781 err2:
2782         owner = expr_info.ops->type->owner;
2783         if (expr_info.ops->type->release_ops)
2784                 expr_info.ops->type->release_ops(expr_info.ops);
2785
2786         module_put(owner);
2787 err1:
2788         return ERR_PTR(err);
2789 }
2790
2791 int nft_expr_clone(struct nft_expr *dst, struct nft_expr *src)
2792 {
2793         int err;
2794
2795         if (src->ops->clone) {
2796                 dst->ops = src->ops;
2797                 err = src->ops->clone(dst, src);
2798                 if (err < 0)
2799                         return err;
2800         } else {
2801                 memcpy(dst, src, src->ops->size);
2802         }
2803
2804         __module_get(src->ops->type->owner);
2805
2806         return 0;
2807 }
2808
2809 void nft_expr_destroy(const struct nft_ctx *ctx, struct nft_expr *expr)
2810 {
2811         nf_tables_expr_destroy(ctx, expr);
2812         kfree(expr);
2813 }
2814
2815 /*
2816  * Rules
2817  */
2818
2819 static struct nft_rule *__nft_rule_lookup(const struct nft_chain *chain,
2820                                           u64 handle)
2821 {
2822         struct nft_rule *rule;
2823
2824         // FIXME: this sucks
2825         list_for_each_entry_rcu(rule, &chain->rules, list) {
2826                 if (handle == rule->handle)
2827                         return rule;
2828         }
2829
2830         return ERR_PTR(-ENOENT);
2831 }
2832
2833 static struct nft_rule *nft_rule_lookup(const struct nft_chain *chain,
2834                                         const struct nlattr *nla)
2835 {
2836         if (nla == NULL)
2837                 return ERR_PTR(-EINVAL);
2838
2839         return __nft_rule_lookup(chain, be64_to_cpu(nla_get_be64(nla)));
2840 }
2841
2842 static const struct nla_policy nft_rule_policy[NFTA_RULE_MAX + 1] = {
2843         [NFTA_RULE_TABLE]       = { .type = NLA_STRING,
2844                                     .len = NFT_TABLE_MAXNAMELEN - 1 },
2845         [NFTA_RULE_CHAIN]       = { .type = NLA_STRING,
2846                                     .len = NFT_CHAIN_MAXNAMELEN - 1 },
2847         [NFTA_RULE_HANDLE]      = { .type = NLA_U64 },
2848         [NFTA_RULE_EXPRESSIONS] = { .type = NLA_NESTED },
2849         [NFTA_RULE_COMPAT]      = { .type = NLA_NESTED },
2850         [NFTA_RULE_POSITION]    = { .type = NLA_U64 },
2851         [NFTA_RULE_USERDATA]    = { .type = NLA_BINARY,
2852                                     .len = NFT_USERDATA_MAXLEN },
2853         [NFTA_RULE_ID]          = { .type = NLA_U32 },
2854         [NFTA_RULE_POSITION_ID] = { .type = NLA_U32 },
2855         [NFTA_RULE_CHAIN_ID]    = { .type = NLA_U32 },
2856 };
2857
2858 static int nf_tables_fill_rule_info(struct sk_buff *skb, struct net *net,
2859                                     u32 portid, u32 seq, int event,
2860                                     u32 flags, int family,
2861                                     const struct nft_table *table,
2862                                     const struct nft_chain *chain,
2863                                     const struct nft_rule *rule,
2864                                     const struct nft_rule *prule)
2865 {
2866         struct nlmsghdr *nlh;
2867         const struct nft_expr *expr, *next;
2868         struct nlattr *list;
2869         u16 type = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
2870
2871         nlh = nfnl_msg_put(skb, portid, seq, type, flags, family, NFNETLINK_V0,
2872                            nft_base_seq(net));
2873         if (!nlh)
2874                 goto nla_put_failure;
2875
2876         if (nla_put_string(skb, NFTA_RULE_TABLE, table->name))
2877                 goto nla_put_failure;
2878         if (nla_put_string(skb, NFTA_RULE_CHAIN, chain->name))
2879                 goto nla_put_failure;
2880         if (nla_put_be64(skb, NFTA_RULE_HANDLE, cpu_to_be64(rule->handle),
2881                          NFTA_RULE_PAD))
2882                 goto nla_put_failure;
2883
2884         if (event != NFT_MSG_DELRULE && prule) {
2885                 if (nla_put_be64(skb, NFTA_RULE_POSITION,
2886                                  cpu_to_be64(prule->handle),
2887                                  NFTA_RULE_PAD))
2888                         goto nla_put_failure;
2889         }
2890
2891         if (chain->flags & NFT_CHAIN_HW_OFFLOAD)
2892                 nft_flow_rule_stats(chain, rule);
2893
2894         list = nla_nest_start_noflag(skb, NFTA_RULE_EXPRESSIONS);
2895         if (list == NULL)
2896                 goto nla_put_failure;
2897         nft_rule_for_each_expr(expr, next, rule) {
2898                 if (nft_expr_dump(skb, NFTA_LIST_ELEM, expr) < 0)
2899                         goto nla_put_failure;
2900         }
2901         nla_nest_end(skb, list);
2902
2903         if (rule->udata) {
2904                 struct nft_userdata *udata = nft_userdata(rule);
2905                 if (nla_put(skb, NFTA_RULE_USERDATA, udata->len + 1,
2906                             udata->data) < 0)
2907                         goto nla_put_failure;
2908         }
2909
2910         nlmsg_end(skb, nlh);
2911         return 0;
2912
2913 nla_put_failure:
2914         nlmsg_trim(skb, nlh);
2915         return -1;
2916 }
2917
2918 static void nf_tables_rule_notify(const struct nft_ctx *ctx,
2919                                   const struct nft_rule *rule, int event)
2920 {
2921         struct nftables_pernet *nft_net = nft_pernet(ctx->net);
2922         struct sk_buff *skb;
2923         int err;
2924
2925         if (!ctx->report &&
2926             !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
2927                 return;
2928
2929         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
2930         if (skb == NULL)
2931                 goto err;
2932
2933         err = nf_tables_fill_rule_info(skb, ctx->net, ctx->portid, ctx->seq,
2934                                        event, 0, ctx->family, ctx->table,
2935                                        ctx->chain, rule, NULL);
2936         if (err < 0) {
2937                 kfree_skb(skb);
2938                 goto err;
2939         }
2940
2941         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
2942         return;
2943 err:
2944         nfnetlink_set_err(ctx->net, ctx->portid, NFNLGRP_NFTABLES, -ENOBUFS);
2945 }
2946
2947 struct nft_rule_dump_ctx {
2948         char *table;
2949         char *chain;
2950 };
2951
2952 static int __nf_tables_dump_rules(struct sk_buff *skb,
2953                                   unsigned int *idx,
2954                                   struct netlink_callback *cb,
2955                                   const struct nft_table *table,
2956                                   const struct nft_chain *chain)
2957 {
2958         struct net *net = sock_net(skb->sk);
2959         const struct nft_rule *rule, *prule;
2960         unsigned int s_idx = cb->args[0];
2961
2962         prule = NULL;
2963         list_for_each_entry_rcu(rule, &chain->rules, list) {
2964                 if (!nft_is_active(net, rule))
2965                         goto cont_skip;
2966                 if (*idx < s_idx)
2967                         goto cont;
2968                 if (*idx > s_idx) {
2969                         memset(&cb->args[1], 0,
2970                                         sizeof(cb->args) - sizeof(cb->args[0]));
2971                 }
2972                 if (nf_tables_fill_rule_info(skb, net, NETLINK_CB(cb->skb).portid,
2973                                         cb->nlh->nlmsg_seq,
2974                                         NFT_MSG_NEWRULE,
2975                                         NLM_F_MULTI | NLM_F_APPEND,
2976                                         table->family,
2977                                         table, chain, rule, prule) < 0)
2978                         return 1;
2979
2980                 nl_dump_check_consistent(cb, nlmsg_hdr(skb));
2981 cont:
2982                 prule = rule;
2983 cont_skip:
2984                 (*idx)++;
2985         }
2986         return 0;
2987 }
2988
2989 static int nf_tables_dump_rules(struct sk_buff *skb,
2990                                 struct netlink_callback *cb)
2991 {
2992         const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
2993         const struct nft_rule_dump_ctx *ctx = cb->data;
2994         struct nft_table *table;
2995         const struct nft_chain *chain;
2996         unsigned int idx = 0;
2997         struct net *net = sock_net(skb->sk);
2998         int family = nfmsg->nfgen_family;
2999         struct nftables_pernet *nft_net;
3000
3001         rcu_read_lock();
3002         nft_net = nft_pernet(net);
3003         cb->seq = nft_net->base_seq;
3004
3005         list_for_each_entry_rcu(table, &nft_net->tables, list) {
3006                 if (family != NFPROTO_UNSPEC && family != table->family)
3007                         continue;
3008
3009                 if (ctx && ctx->table && strcmp(ctx->table, table->name) != 0)
3010                         continue;
3011
3012                 if (ctx && ctx->table && ctx->chain) {
3013                         struct rhlist_head *list, *tmp;
3014
3015                         list = rhltable_lookup(&table->chains_ht, ctx->chain,
3016                                                nft_chain_ht_params);
3017                         if (!list)
3018                                 goto done;
3019
3020                         rhl_for_each_entry_rcu(chain, tmp, list, rhlhead) {
3021                                 if (!nft_is_active(net, chain))
3022                                         continue;
3023                                 __nf_tables_dump_rules(skb, &idx,
3024                                                        cb, table, chain);
3025                                 break;
3026                         }
3027                         goto done;
3028                 }
3029
3030                 list_for_each_entry_rcu(chain, &table->chains, list) {
3031                         if (__nf_tables_dump_rules(skb, &idx, cb, table, chain))
3032                                 goto done;
3033                 }
3034
3035                 if (ctx && ctx->table)
3036                         break;
3037         }
3038 done:
3039         rcu_read_unlock();
3040
3041         cb->args[0] = idx;
3042         return skb->len;
3043 }
3044
3045 static int nf_tables_dump_rules_start(struct netlink_callback *cb)
3046 {
3047         const struct nlattr * const *nla = cb->data;
3048         struct nft_rule_dump_ctx *ctx = NULL;
3049
3050         if (nla[NFTA_RULE_TABLE] || nla[NFTA_RULE_CHAIN]) {
3051                 ctx = kzalloc(sizeof(*ctx), GFP_ATOMIC);
3052                 if (!ctx)
3053                         return -ENOMEM;
3054
3055                 if (nla[NFTA_RULE_TABLE]) {
3056                         ctx->table = nla_strdup(nla[NFTA_RULE_TABLE],
3057                                                         GFP_ATOMIC);
3058                         if (!ctx->table) {
3059                                 kfree(ctx);
3060                                 return -ENOMEM;
3061                         }
3062                 }
3063                 if (nla[NFTA_RULE_CHAIN]) {
3064                         ctx->chain = nla_strdup(nla[NFTA_RULE_CHAIN],
3065                                                 GFP_ATOMIC);
3066                         if (!ctx->chain) {
3067                                 kfree(ctx->table);
3068                                 kfree(ctx);
3069                                 return -ENOMEM;
3070                         }
3071                 }
3072         }
3073
3074         cb->data = ctx;
3075         return 0;
3076 }
3077
3078 static int nf_tables_dump_rules_done(struct netlink_callback *cb)
3079 {
3080         struct nft_rule_dump_ctx *ctx = cb->data;
3081
3082         if (ctx) {
3083                 kfree(ctx->table);
3084                 kfree(ctx->chain);
3085                 kfree(ctx);
3086         }
3087         return 0;
3088 }
3089
3090 /* called with rcu_read_lock held */
3091 static int nf_tables_getrule(struct sk_buff *skb, const struct nfnl_info *info,
3092                              const struct nlattr * const nla[])
3093 {
3094         struct netlink_ext_ack *extack = info->extack;
3095         u8 genmask = nft_genmask_cur(info->net);
3096         u8 family = info->nfmsg->nfgen_family;
3097         const struct nft_chain *chain;
3098         const struct nft_rule *rule;
3099         struct net *net = info->net;
3100         struct nft_table *table;
3101         struct sk_buff *skb2;
3102         int err;
3103
3104         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
3105                 struct netlink_dump_control c = {
3106                         .start= nf_tables_dump_rules_start,
3107                         .dump = nf_tables_dump_rules,
3108                         .done = nf_tables_dump_rules_done,
3109                         .module = THIS_MODULE,
3110                         .data = (void *)nla,
3111                 };
3112
3113                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
3114         }
3115
3116         table = nft_table_lookup(net, nla[NFTA_RULE_TABLE], family, genmask, 0);
3117         if (IS_ERR(table)) {
3118                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_TABLE]);
3119                 return PTR_ERR(table);
3120         }
3121
3122         chain = nft_chain_lookup(net, table, nla[NFTA_RULE_CHAIN], genmask);
3123         if (IS_ERR(chain)) {
3124                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_CHAIN]);
3125                 return PTR_ERR(chain);
3126         }
3127
3128         rule = nft_rule_lookup(chain, nla[NFTA_RULE_HANDLE]);
3129         if (IS_ERR(rule)) {
3130                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_HANDLE]);
3131                 return PTR_ERR(rule);
3132         }
3133
3134         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
3135         if (!skb2)
3136                 return -ENOMEM;
3137
3138         err = nf_tables_fill_rule_info(skb2, net, NETLINK_CB(skb).portid,
3139                                        info->nlh->nlmsg_seq, NFT_MSG_NEWRULE, 0,
3140                                        family, table, chain, rule, NULL);
3141         if (err < 0)
3142                 goto err_fill_rule_info;
3143
3144         return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
3145
3146 err_fill_rule_info:
3147         kfree_skb(skb2);
3148         return err;
3149 }
3150
3151 static void nf_tables_rule_destroy(const struct nft_ctx *ctx,
3152                                    struct nft_rule *rule)
3153 {
3154         struct nft_expr *expr, *next;
3155
3156         /*
3157          * Careful: some expressions might not be initialized in case this
3158          * is called on error from nf_tables_newrule().
3159          */
3160         expr = nft_expr_first(rule);
3161         while (nft_expr_more(rule, expr)) {
3162                 next = nft_expr_next(expr);
3163                 nf_tables_expr_destroy(ctx, expr);
3164                 expr = next;
3165         }
3166         kfree(rule);
3167 }
3168
3169 void nf_tables_rule_release(const struct nft_ctx *ctx, struct nft_rule *rule)
3170 {
3171         nft_rule_expr_deactivate(ctx, rule, NFT_TRANS_RELEASE);
3172         nf_tables_rule_destroy(ctx, rule);
3173 }
3174
3175 int nft_chain_validate(const struct nft_ctx *ctx, const struct nft_chain *chain)
3176 {
3177         struct nft_expr *expr, *last;
3178         const struct nft_data *data;
3179         struct nft_rule *rule;
3180         int err;
3181
3182         if (ctx->level == NFT_JUMP_STACK_SIZE)
3183                 return -EMLINK;
3184
3185         list_for_each_entry(rule, &chain->rules, list) {
3186                 if (!nft_is_active_next(ctx->net, rule))
3187                         continue;
3188
3189                 nft_rule_for_each_expr(expr, last, rule) {
3190                         if (!expr->ops->validate)
3191                                 continue;
3192
3193                         err = expr->ops->validate(ctx, expr, &data);
3194                         if (err < 0)
3195                                 return err;
3196                 }
3197         }
3198
3199         return 0;
3200 }
3201 EXPORT_SYMBOL_GPL(nft_chain_validate);
3202
3203 static int nft_table_validate(struct net *net, const struct nft_table *table)
3204 {
3205         struct nft_chain *chain;
3206         struct nft_ctx ctx = {
3207                 .net    = net,
3208                 .family = table->family,
3209         };
3210         int err;
3211
3212         list_for_each_entry(chain, &table->chains, list) {
3213                 if (!nft_is_base_chain(chain))
3214                         continue;
3215
3216                 ctx.chain = chain;
3217                 err = nft_chain_validate(&ctx, chain);
3218                 if (err < 0)
3219                         return err;
3220         }
3221
3222         return 0;
3223 }
3224
3225 static struct nft_rule *nft_rule_lookup_byid(const struct net *net,
3226                                              const struct nlattr *nla);
3227
3228 #define NFT_RULE_MAXEXPRS       128
3229
3230 static int nf_tables_newrule(struct sk_buff *skb, const struct nfnl_info *info,
3231                              const struct nlattr * const nla[])
3232 {
3233         struct nftables_pernet *nft_net = nft_pernet(info->net);
3234         struct netlink_ext_ack *extack = info->extack;
3235         unsigned int size, i, n, ulen = 0, usize = 0;
3236         u8 genmask = nft_genmask_next(info->net);
3237         struct nft_rule *rule, *old_rule = NULL;
3238         struct nft_expr_info *expr_info = NULL;
3239         u8 family = info->nfmsg->nfgen_family;
3240         struct net *net = info->net;
3241         struct nft_flow_rule *flow;
3242         struct nft_userdata *udata;
3243         struct nft_table *table;
3244         struct nft_chain *chain;
3245         struct nft_trans *trans;
3246         u64 handle, pos_handle;
3247         struct nft_expr *expr;
3248         struct nft_ctx ctx;
3249         struct nlattr *tmp;
3250         int err, rem;
3251
3252         lockdep_assert_held(&nft_net->commit_mutex);
3253
3254         table = nft_table_lookup(net, nla[NFTA_RULE_TABLE], family, genmask,
3255                                  NETLINK_CB(skb).portid);
3256         if (IS_ERR(table)) {
3257                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_TABLE]);
3258                 return PTR_ERR(table);
3259         }
3260
3261         if (nla[NFTA_RULE_CHAIN]) {
3262                 chain = nft_chain_lookup(net, table, nla[NFTA_RULE_CHAIN],
3263                                          genmask);
3264                 if (IS_ERR(chain)) {
3265                         NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_CHAIN]);
3266                         return PTR_ERR(chain);
3267                 }
3268                 if (nft_chain_is_bound(chain))
3269                         return -EOPNOTSUPP;
3270
3271         } else if (nla[NFTA_RULE_CHAIN_ID]) {
3272                 chain = nft_chain_lookup_byid(net, nla[NFTA_RULE_CHAIN_ID]);
3273                 if (IS_ERR(chain)) {
3274                         NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_CHAIN_ID]);
3275                         return PTR_ERR(chain);
3276                 }
3277         } else {
3278                 return -EINVAL;
3279         }
3280
3281         if (nla[NFTA_RULE_HANDLE]) {
3282                 handle = be64_to_cpu(nla_get_be64(nla[NFTA_RULE_HANDLE]));
3283                 rule = __nft_rule_lookup(chain, handle);
3284                 if (IS_ERR(rule)) {
3285                         NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_HANDLE]);
3286                         return PTR_ERR(rule);
3287                 }
3288
3289                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
3290                         NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_HANDLE]);
3291                         return -EEXIST;
3292                 }
3293                 if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
3294                         old_rule = rule;
3295                 else
3296                         return -EOPNOTSUPP;
3297         } else {
3298                 if (!(info->nlh->nlmsg_flags & NLM_F_CREATE) ||
3299                     info->nlh->nlmsg_flags & NLM_F_REPLACE)
3300                         return -EINVAL;
3301                 handle = nf_tables_alloc_handle(table);
3302
3303                 if (chain->use == UINT_MAX)
3304                         return -EOVERFLOW;
3305
3306                 if (nla[NFTA_RULE_POSITION]) {
3307                         pos_handle = be64_to_cpu(nla_get_be64(nla[NFTA_RULE_POSITION]));
3308                         old_rule = __nft_rule_lookup(chain, pos_handle);
3309                         if (IS_ERR(old_rule)) {
3310                                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_POSITION]);
3311                                 return PTR_ERR(old_rule);
3312                         }
3313                 } else if (nla[NFTA_RULE_POSITION_ID]) {
3314                         old_rule = nft_rule_lookup_byid(net, nla[NFTA_RULE_POSITION_ID]);
3315                         if (IS_ERR(old_rule)) {
3316                                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_POSITION_ID]);
3317                                 return PTR_ERR(old_rule);
3318                         }
3319                 }
3320         }
3321
3322         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, chain, nla);
3323
3324         n = 0;
3325         size = 0;
3326         if (nla[NFTA_RULE_EXPRESSIONS]) {
3327                 expr_info = kvmalloc_array(NFT_RULE_MAXEXPRS,
3328                                            sizeof(struct nft_expr_info),
3329                                            GFP_KERNEL);
3330                 if (!expr_info)
3331                         return -ENOMEM;
3332
3333                 nla_for_each_nested(tmp, nla[NFTA_RULE_EXPRESSIONS], rem) {
3334                         err = -EINVAL;
3335                         if (nla_type(tmp) != NFTA_LIST_ELEM)
3336                                 goto err1;
3337                         if (n == NFT_RULE_MAXEXPRS)
3338                                 goto err1;
3339                         err = nf_tables_expr_parse(&ctx, tmp, &expr_info[n]);
3340                         if (err < 0) {
3341                                 NL_SET_BAD_ATTR(extack, tmp);
3342                                 goto err1;
3343                         }
3344                         size += expr_info[n].ops->size;
3345                         n++;
3346                 }
3347         }
3348         /* Check for overflow of dlen field */
3349         err = -EFBIG;
3350         if (size >= 1 << 12)
3351                 goto err1;
3352
3353         if (nla[NFTA_RULE_USERDATA]) {
3354                 ulen = nla_len(nla[NFTA_RULE_USERDATA]);
3355                 if (ulen > 0)
3356                         usize = sizeof(struct nft_userdata) + ulen;
3357         }
3358
3359         err = -ENOMEM;
3360         rule = kzalloc(sizeof(*rule) + size + usize, GFP_KERNEL);
3361         if (rule == NULL)
3362                 goto err1;
3363
3364         nft_activate_next(net, rule);
3365
3366         rule->handle = handle;
3367         rule->dlen   = size;
3368         rule->udata  = ulen ? 1 : 0;
3369
3370         if (ulen) {
3371                 udata = nft_userdata(rule);
3372                 udata->len = ulen - 1;
3373                 nla_memcpy(udata->data, nla[NFTA_RULE_USERDATA], ulen);
3374         }
3375
3376         expr = nft_expr_first(rule);
3377         for (i = 0; i < n; i++) {
3378                 err = nf_tables_newexpr(&ctx, &expr_info[i], expr);
3379                 if (err < 0) {
3380                         NL_SET_BAD_ATTR(extack, expr_info[i].attr);
3381                         goto err2;
3382                 }
3383
3384                 if (expr_info[i].ops->validate)
3385                         nft_validate_state_update(net, NFT_VALIDATE_NEED);
3386
3387                 expr_info[i].ops = NULL;
3388                 expr = nft_expr_next(expr);
3389         }
3390
3391         if (info->nlh->nlmsg_flags & NLM_F_REPLACE) {
3392                 trans = nft_trans_rule_add(&ctx, NFT_MSG_NEWRULE, rule);
3393                 if (trans == NULL) {
3394                         err = -ENOMEM;
3395                         goto err2;
3396                 }
3397                 err = nft_delrule(&ctx, old_rule);
3398                 if (err < 0) {
3399                         nft_trans_destroy(trans);
3400                         goto err2;
3401                 }
3402
3403                 list_add_tail_rcu(&rule->list, &old_rule->list);
3404         } else {
3405                 trans = nft_trans_rule_add(&ctx, NFT_MSG_NEWRULE, rule);
3406                 if (!trans) {
3407                         err = -ENOMEM;
3408                         goto err2;
3409                 }
3410
3411                 if (info->nlh->nlmsg_flags & NLM_F_APPEND) {
3412                         if (old_rule)
3413                                 list_add_rcu(&rule->list, &old_rule->list);
3414                         else
3415                                 list_add_tail_rcu(&rule->list, &chain->rules);
3416                  } else {
3417                         if (old_rule)
3418                                 list_add_tail_rcu(&rule->list, &old_rule->list);
3419                         else
3420                                 list_add_rcu(&rule->list, &chain->rules);
3421                 }
3422         }
3423         kvfree(expr_info);
3424         chain->use++;
3425
3426         if (nft_net->validate_state == NFT_VALIDATE_DO)
3427                 return nft_table_validate(net, table);
3428
3429         if (chain->flags & NFT_CHAIN_HW_OFFLOAD) {
3430                 flow = nft_flow_rule_create(net, rule);
3431                 if (IS_ERR(flow))
3432                         return PTR_ERR(flow);
3433
3434                 nft_trans_flow_rule(trans) = flow;
3435         }
3436
3437         return 0;
3438 err2:
3439         nf_tables_rule_release(&ctx, rule);
3440 err1:
3441         for (i = 0; i < n; i++) {
3442                 if (expr_info[i].ops) {
3443                         module_put(expr_info[i].ops->type->owner);
3444                         if (expr_info[i].ops->type->release_ops)
3445                                 expr_info[i].ops->type->release_ops(expr_info[i].ops);
3446                 }
3447         }
3448         kvfree(expr_info);
3449
3450         return err;
3451 }
3452
3453 static struct nft_rule *nft_rule_lookup_byid(const struct net *net,
3454                                              const struct nlattr *nla)
3455 {
3456         struct nftables_pernet *nft_net = nft_pernet(net);
3457         u32 id = ntohl(nla_get_be32(nla));
3458         struct nft_trans *trans;
3459
3460         list_for_each_entry(trans, &nft_net->commit_list, list) {
3461                 struct nft_rule *rule = nft_trans_rule(trans);
3462
3463                 if (trans->msg_type == NFT_MSG_NEWRULE &&
3464                     id == nft_trans_rule_id(trans))
3465                         return rule;
3466         }
3467         return ERR_PTR(-ENOENT);
3468 }
3469
3470 static int nf_tables_delrule(struct sk_buff *skb, const struct nfnl_info *info,
3471                              const struct nlattr * const nla[])
3472 {
3473         struct netlink_ext_ack *extack = info->extack;
3474         u8 genmask = nft_genmask_next(info->net);
3475         u8 family = info->nfmsg->nfgen_family;
3476         struct nft_chain *chain = NULL;
3477         struct net *net = info->net;
3478         struct nft_table *table;
3479         struct nft_rule *rule;
3480         struct nft_ctx ctx;
3481         int err = 0;
3482
3483         table = nft_table_lookup(net, nla[NFTA_RULE_TABLE], family, genmask,
3484                                  NETLINK_CB(skb).portid);
3485         if (IS_ERR(table)) {
3486                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_TABLE]);
3487                 return PTR_ERR(table);
3488         }
3489
3490         if (nla[NFTA_RULE_CHAIN]) {
3491                 chain = nft_chain_lookup(net, table, nla[NFTA_RULE_CHAIN],
3492                                          genmask);
3493                 if (IS_ERR(chain)) {
3494                         NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_CHAIN]);
3495                         return PTR_ERR(chain);
3496                 }
3497                 if (nft_chain_is_bound(chain))
3498                         return -EOPNOTSUPP;
3499         }
3500
3501         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, chain, nla);
3502
3503         if (chain) {
3504                 if (nla[NFTA_RULE_HANDLE]) {
3505                         rule = nft_rule_lookup(chain, nla[NFTA_RULE_HANDLE]);
3506                         if (IS_ERR(rule)) {
3507                                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_HANDLE]);
3508                                 return PTR_ERR(rule);
3509                         }
3510
3511                         err = nft_delrule(&ctx, rule);
3512                 } else if (nla[NFTA_RULE_ID]) {
3513                         rule = nft_rule_lookup_byid(net, nla[NFTA_RULE_ID]);
3514                         if (IS_ERR(rule)) {
3515                                 NL_SET_BAD_ATTR(extack, nla[NFTA_RULE_ID]);
3516                                 return PTR_ERR(rule);
3517                         }
3518
3519                         err = nft_delrule(&ctx, rule);
3520                 } else {
3521                         err = nft_delrule_by_chain(&ctx);
3522                 }
3523         } else {
3524                 list_for_each_entry(chain, &table->chains, list) {
3525                         if (!nft_is_active_next(net, chain))
3526                                 continue;
3527
3528                         ctx.chain = chain;
3529                         err = nft_delrule_by_chain(&ctx);
3530                         if (err < 0)
3531                                 break;
3532                 }
3533         }
3534
3535         return err;
3536 }
3537
3538 /*
3539  * Sets
3540  */
3541 static const struct nft_set_type *nft_set_types[] = {
3542         &nft_set_hash_fast_type,
3543         &nft_set_hash_type,
3544         &nft_set_rhash_type,
3545         &nft_set_bitmap_type,
3546         &nft_set_rbtree_type,
3547 #if defined(CONFIG_X86_64) && !defined(CONFIG_UML)
3548         &nft_set_pipapo_avx2_type,
3549 #endif
3550         &nft_set_pipapo_type,
3551 };
3552
3553 #define NFT_SET_FEATURES        (NFT_SET_INTERVAL | NFT_SET_MAP | \
3554                                  NFT_SET_TIMEOUT | NFT_SET_OBJECT | \
3555                                  NFT_SET_EVAL)
3556
3557 static bool nft_set_ops_candidate(const struct nft_set_type *type, u32 flags)
3558 {
3559         return (flags & type->features) == (flags & NFT_SET_FEATURES);
3560 }
3561
3562 /*
3563  * Select a set implementation based on the data characteristics and the
3564  * given policy. The total memory use might not be known if no size is
3565  * given, in that case the amount of memory per element is used.
3566  */
3567 static const struct nft_set_ops *
3568 nft_select_set_ops(const struct nft_ctx *ctx,
3569                    const struct nlattr * const nla[],
3570                    const struct nft_set_desc *desc,
3571                    enum nft_set_policies policy)
3572 {
3573         struct nftables_pernet *nft_net = nft_pernet(ctx->net);
3574         const struct nft_set_ops *ops, *bops;
3575         struct nft_set_estimate est, best;
3576         const struct nft_set_type *type;
3577         u32 flags = 0;
3578         int i;
3579
3580         lockdep_assert_held(&nft_net->commit_mutex);
3581         lockdep_nfnl_nft_mutex_not_held();
3582
3583         if (nla[NFTA_SET_FLAGS] != NULL)
3584                 flags = ntohl(nla_get_be32(nla[NFTA_SET_FLAGS]));
3585
3586         bops        = NULL;
3587         best.size   = ~0;
3588         best.lookup = ~0;
3589         best.space  = ~0;
3590
3591         for (i = 0; i < ARRAY_SIZE(nft_set_types); i++) {
3592                 type = nft_set_types[i];
3593                 ops = &type->ops;
3594
3595                 if (!nft_set_ops_candidate(type, flags))
3596                         continue;
3597                 if (!ops->estimate(desc, flags, &est))
3598                         continue;
3599
3600                 switch (policy) {
3601                 case NFT_SET_POL_PERFORMANCE:
3602                         if (est.lookup < best.lookup)
3603                                 break;
3604                         if (est.lookup == best.lookup &&
3605                             est.space < best.space)
3606                                 break;
3607                         continue;
3608                 case NFT_SET_POL_MEMORY:
3609                         if (!desc->size) {
3610                                 if (est.space < best.space)
3611                                         break;
3612                                 if (est.space == best.space &&
3613                                     est.lookup < best.lookup)
3614                                         break;
3615                         } else if (est.size < best.size || !bops) {
3616                                 break;
3617                         }
3618                         continue;
3619                 default:
3620                         break;
3621                 }
3622
3623                 bops = ops;
3624                 best = est;
3625         }
3626
3627         if (bops != NULL)
3628                 return bops;
3629
3630         return ERR_PTR(-EOPNOTSUPP);
3631 }
3632
3633 static const struct nla_policy nft_set_policy[NFTA_SET_MAX + 1] = {
3634         [NFTA_SET_TABLE]                = { .type = NLA_STRING,
3635                                             .len = NFT_TABLE_MAXNAMELEN - 1 },
3636         [NFTA_SET_NAME]                 = { .type = NLA_STRING,
3637                                             .len = NFT_SET_MAXNAMELEN - 1 },
3638         [NFTA_SET_FLAGS]                = { .type = NLA_U32 },
3639         [NFTA_SET_KEY_TYPE]             = { .type = NLA_U32 },
3640         [NFTA_SET_KEY_LEN]              = { .type = NLA_U32 },
3641         [NFTA_SET_DATA_TYPE]            = { .type = NLA_U32 },
3642         [NFTA_SET_DATA_LEN]             = { .type = NLA_U32 },
3643         [NFTA_SET_POLICY]               = { .type = NLA_U32 },
3644         [NFTA_SET_DESC]                 = { .type = NLA_NESTED },
3645         [NFTA_SET_ID]                   = { .type = NLA_U32 },
3646         [NFTA_SET_TIMEOUT]              = { .type = NLA_U64 },
3647         [NFTA_SET_GC_INTERVAL]          = { .type = NLA_U32 },
3648         [NFTA_SET_USERDATA]             = { .type = NLA_BINARY,
3649                                             .len  = NFT_USERDATA_MAXLEN },
3650         [NFTA_SET_OBJ_TYPE]             = { .type = NLA_U32 },
3651         [NFTA_SET_HANDLE]               = { .type = NLA_U64 },
3652         [NFTA_SET_EXPR]                 = { .type = NLA_NESTED },
3653         [NFTA_SET_EXPRESSIONS]          = { .type = NLA_NESTED },
3654 };
3655
3656 static const struct nla_policy nft_set_desc_policy[NFTA_SET_DESC_MAX + 1] = {
3657         [NFTA_SET_DESC_SIZE]            = { .type = NLA_U32 },
3658         [NFTA_SET_DESC_CONCAT]          = { .type = NLA_NESTED },
3659 };
3660
3661 static struct nft_set *nft_set_lookup(const struct nft_table *table,
3662                                       const struct nlattr *nla, u8 genmask)
3663 {
3664         struct nft_set *set;
3665
3666         if (nla == NULL)
3667                 return ERR_PTR(-EINVAL);
3668
3669         list_for_each_entry_rcu(set, &table->sets, list) {
3670                 if (!nla_strcmp(nla, set->name) &&
3671                     nft_active_genmask(set, genmask))
3672                         return set;
3673         }
3674         return ERR_PTR(-ENOENT);
3675 }
3676
3677 static struct nft_set *nft_set_lookup_byhandle(const struct nft_table *table,
3678                                                const struct nlattr *nla,
3679                                                u8 genmask)
3680 {
3681         struct nft_set *set;
3682
3683         list_for_each_entry(set, &table->sets, list) {
3684                 if (be64_to_cpu(nla_get_be64(nla)) == set->handle &&
3685                     nft_active_genmask(set, genmask))
3686                         return set;
3687         }
3688         return ERR_PTR(-ENOENT);
3689 }
3690
3691 static struct nft_set *nft_set_lookup_byid(const struct net *net,
3692                                            const struct nlattr *nla, u8 genmask)
3693 {
3694         struct nftables_pernet *nft_net = nft_pernet(net);
3695         u32 id = ntohl(nla_get_be32(nla));
3696         struct nft_trans *trans;
3697
3698         list_for_each_entry(trans, &nft_net->commit_list, list) {
3699                 if (trans->msg_type == NFT_MSG_NEWSET) {
3700                         struct nft_set *set = nft_trans_set(trans);
3701
3702                         if (id == nft_trans_set_id(trans) &&
3703                             nft_active_genmask(set, genmask))
3704                                 return set;
3705                 }
3706         }
3707         return ERR_PTR(-ENOENT);
3708 }
3709
3710 struct nft_set *nft_set_lookup_global(const struct net *net,
3711                                       const struct nft_table *table,
3712                                       const struct nlattr *nla_set_name,
3713                                       const struct nlattr *nla_set_id,
3714                                       u8 genmask)
3715 {
3716         struct nft_set *set;
3717
3718         set = nft_set_lookup(table, nla_set_name, genmask);
3719         if (IS_ERR(set)) {
3720                 if (!nla_set_id)
3721                         return set;
3722
3723                 set = nft_set_lookup_byid(net, nla_set_id, genmask);
3724         }
3725         return set;
3726 }
3727 EXPORT_SYMBOL_GPL(nft_set_lookup_global);
3728
3729 static int nf_tables_set_alloc_name(struct nft_ctx *ctx, struct nft_set *set,
3730                                     const char *name)
3731 {
3732         const struct nft_set *i;
3733         const char *p;
3734         unsigned long *inuse;
3735         unsigned int n = 0, min = 0;
3736
3737         p = strchr(name, '%');
3738         if (p != NULL) {
3739                 if (p[1] != 'd' || strchr(p + 2, '%'))
3740                         return -EINVAL;
3741
3742                 inuse = (unsigned long *)get_zeroed_page(GFP_KERNEL);
3743                 if (inuse == NULL)
3744                         return -ENOMEM;
3745 cont:
3746                 list_for_each_entry(i, &ctx->table->sets, list) {
3747                         int tmp;
3748
3749                         if (!nft_is_active_next(ctx->net, set))
3750                                 continue;
3751                         if (!sscanf(i->name, name, &tmp))
3752                                 continue;
3753                         if (tmp < min || tmp >= min + BITS_PER_BYTE * PAGE_SIZE)
3754                                 continue;
3755
3756                         set_bit(tmp - min, inuse);
3757                 }
3758
3759                 n = find_first_zero_bit(inuse, BITS_PER_BYTE * PAGE_SIZE);
3760                 if (n >= BITS_PER_BYTE * PAGE_SIZE) {
3761                         min += BITS_PER_BYTE * PAGE_SIZE;
3762                         memset(inuse, 0, PAGE_SIZE);
3763                         goto cont;
3764                 }
3765                 free_page((unsigned long)inuse);
3766         }
3767
3768         set->name = kasprintf(GFP_KERNEL, name, min + n);
3769         if (!set->name)
3770                 return -ENOMEM;
3771
3772         list_for_each_entry(i, &ctx->table->sets, list) {
3773                 if (!nft_is_active_next(ctx->net, i))
3774                         continue;
3775                 if (!strcmp(set->name, i->name)) {
3776                         kfree(set->name);
3777                         set->name = NULL;
3778                         return -ENFILE;
3779                 }
3780         }
3781         return 0;
3782 }
3783
3784 int nf_msecs_to_jiffies64(const struct nlattr *nla, u64 *result)
3785 {
3786         u64 ms = be64_to_cpu(nla_get_be64(nla));
3787         u64 max = (u64)(~((u64)0));
3788
3789         max = div_u64(max, NSEC_PER_MSEC);
3790         if (ms >= max)
3791                 return -ERANGE;
3792
3793         ms *= NSEC_PER_MSEC;
3794         *result = nsecs_to_jiffies64(ms);
3795         return 0;
3796 }
3797
3798 __be64 nf_jiffies64_to_msecs(u64 input)
3799 {
3800         return cpu_to_be64(jiffies64_to_msecs(input));
3801 }
3802
3803 static int nf_tables_fill_set_concat(struct sk_buff *skb,
3804                                      const struct nft_set *set)
3805 {
3806         struct nlattr *concat, *field;
3807         int i;
3808
3809         concat = nla_nest_start_noflag(skb, NFTA_SET_DESC_CONCAT);
3810         if (!concat)
3811                 return -ENOMEM;
3812
3813         for (i = 0; i < set->field_count; i++) {
3814                 field = nla_nest_start_noflag(skb, NFTA_LIST_ELEM);
3815                 if (!field)
3816                         return -ENOMEM;
3817
3818                 if (nla_put_be32(skb, NFTA_SET_FIELD_LEN,
3819                                  htonl(set->field_len[i])))
3820                         return -ENOMEM;
3821
3822                 nla_nest_end(skb, field);
3823         }
3824
3825         nla_nest_end(skb, concat);
3826
3827         return 0;
3828 }
3829
3830 static int nf_tables_fill_set(struct sk_buff *skb, const struct nft_ctx *ctx,
3831                               const struct nft_set *set, u16 event, u16 flags)
3832 {
3833         struct nlmsghdr *nlh;
3834         u32 portid = ctx->portid;
3835         struct nlattr *nest;
3836         u32 seq = ctx->seq;
3837         int i;
3838
3839         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
3840         nlh = nfnl_msg_put(skb, portid, seq, event, flags, ctx->family,
3841                            NFNETLINK_V0, nft_base_seq(ctx->net));
3842         if (!nlh)
3843                 goto nla_put_failure;
3844
3845         if (nla_put_string(skb, NFTA_SET_TABLE, ctx->table->name))
3846                 goto nla_put_failure;
3847         if (nla_put_string(skb, NFTA_SET_NAME, set->name))
3848                 goto nla_put_failure;
3849         if (nla_put_be64(skb, NFTA_SET_HANDLE, cpu_to_be64(set->handle),
3850                          NFTA_SET_PAD))
3851                 goto nla_put_failure;
3852         if (set->flags != 0)
3853                 if (nla_put_be32(skb, NFTA_SET_FLAGS, htonl(set->flags)))
3854                         goto nla_put_failure;
3855
3856         if (nla_put_be32(skb, NFTA_SET_KEY_TYPE, htonl(set->ktype)))
3857                 goto nla_put_failure;
3858         if (nla_put_be32(skb, NFTA_SET_KEY_LEN, htonl(set->klen)))
3859                 goto nla_put_failure;
3860         if (set->flags & NFT_SET_MAP) {
3861                 if (nla_put_be32(skb, NFTA_SET_DATA_TYPE, htonl(set->dtype)))
3862                         goto nla_put_failure;
3863                 if (nla_put_be32(skb, NFTA_SET_DATA_LEN, htonl(set->dlen)))
3864                         goto nla_put_failure;
3865         }
3866         if (set->flags & NFT_SET_OBJECT &&
3867             nla_put_be32(skb, NFTA_SET_OBJ_TYPE, htonl(set->objtype)))
3868                 goto nla_put_failure;
3869
3870         if (set->timeout &&
3871             nla_put_be64(skb, NFTA_SET_TIMEOUT,
3872                          nf_jiffies64_to_msecs(set->timeout),
3873                          NFTA_SET_PAD))
3874                 goto nla_put_failure;
3875         if (set->gc_int &&
3876             nla_put_be32(skb, NFTA_SET_GC_INTERVAL, htonl(set->gc_int)))
3877                 goto nla_put_failure;
3878
3879         if (set->policy != NFT_SET_POL_PERFORMANCE) {
3880                 if (nla_put_be32(skb, NFTA_SET_POLICY, htonl(set->policy)))
3881                         goto nla_put_failure;
3882         }
3883
3884         if (set->udata &&
3885             nla_put(skb, NFTA_SET_USERDATA, set->udlen, set->udata))
3886                 goto nla_put_failure;
3887
3888         nest = nla_nest_start_noflag(skb, NFTA_SET_DESC);
3889         if (!nest)
3890                 goto nla_put_failure;
3891         if (set->size &&
3892             nla_put_be32(skb, NFTA_SET_DESC_SIZE, htonl(set->size)))
3893                 goto nla_put_failure;
3894
3895         if (set->field_count > 1 &&
3896             nf_tables_fill_set_concat(skb, set))
3897                 goto nla_put_failure;
3898
3899         nla_nest_end(skb, nest);
3900
3901         if (set->num_exprs == 1) {
3902                 nest = nla_nest_start_noflag(skb, NFTA_SET_EXPR);
3903                 if (nf_tables_fill_expr_info(skb, set->exprs[0]) < 0)
3904                         goto nla_put_failure;
3905
3906                 nla_nest_end(skb, nest);
3907         } else if (set->num_exprs > 1) {
3908                 nest = nla_nest_start_noflag(skb, NFTA_SET_EXPRESSIONS);
3909                 if (nest == NULL)
3910                         goto nla_put_failure;
3911
3912                 for (i = 0; i < set->num_exprs; i++) {
3913                         if (nft_expr_dump(skb, NFTA_LIST_ELEM,
3914                                           set->exprs[i]) < 0)
3915                                 goto nla_put_failure;
3916                 }
3917                 nla_nest_end(skb, nest);
3918         }
3919
3920         nlmsg_end(skb, nlh);
3921         return 0;
3922
3923 nla_put_failure:
3924         nlmsg_trim(skb, nlh);
3925         return -1;
3926 }
3927
3928 static void nf_tables_set_notify(const struct nft_ctx *ctx,
3929                                  const struct nft_set *set, int event,
3930                                  gfp_t gfp_flags)
3931 {
3932         struct nftables_pernet *nft_net = nft_pernet(ctx->net);
3933         struct sk_buff *skb;
3934         u32 portid = ctx->portid;
3935         int err;
3936
3937         if (!ctx->report &&
3938             !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
3939                 return;
3940
3941         skb = nlmsg_new(NLMSG_GOODSIZE, gfp_flags);
3942         if (skb == NULL)
3943                 goto err;
3944
3945         err = nf_tables_fill_set(skb, ctx, set, event, 0);
3946         if (err < 0) {
3947                 kfree_skb(skb);
3948                 goto err;
3949         }
3950
3951         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
3952         return;
3953 err:
3954         nfnetlink_set_err(ctx->net, portid, NFNLGRP_NFTABLES, -ENOBUFS);
3955 }
3956
3957 static int nf_tables_dump_sets(struct sk_buff *skb, struct netlink_callback *cb)
3958 {
3959         const struct nft_set *set;
3960         unsigned int idx, s_idx = cb->args[0];
3961         struct nft_table *table, *cur_table = (struct nft_table *)cb->args[2];
3962         struct net *net = sock_net(skb->sk);
3963         struct nft_ctx *ctx = cb->data, ctx_set;
3964         struct nftables_pernet *nft_net;
3965
3966         if (cb->args[1])
3967                 return skb->len;
3968
3969         rcu_read_lock();
3970         nft_net = nft_pernet(net);
3971         cb->seq = nft_net->base_seq;
3972
3973         list_for_each_entry_rcu(table, &nft_net->tables, list) {
3974                 if (ctx->family != NFPROTO_UNSPEC &&
3975                     ctx->family != table->family)
3976                         continue;
3977
3978                 if (ctx->table && ctx->table != table)
3979                         continue;
3980
3981                 if (cur_table) {
3982                         if (cur_table != table)
3983                                 continue;
3984
3985                         cur_table = NULL;
3986                 }
3987                 idx = 0;
3988                 list_for_each_entry_rcu(set, &table->sets, list) {
3989                         if (idx < s_idx)
3990                                 goto cont;
3991                         if (!nft_is_active(net, set))
3992                                 goto cont;
3993
3994                         ctx_set = *ctx;
3995                         ctx_set.table = table;
3996                         ctx_set.family = table->family;
3997
3998                         if (nf_tables_fill_set(skb, &ctx_set, set,
3999                                                NFT_MSG_NEWSET,
4000                                                NLM_F_MULTI) < 0) {
4001                                 cb->args[0] = idx;
4002                                 cb->args[2] = (unsigned long) table;
4003                                 goto done;
4004                         }
4005                         nl_dump_check_consistent(cb, nlmsg_hdr(skb));
4006 cont:
4007                         idx++;
4008                 }
4009                 if (s_idx)
4010                         s_idx = 0;
4011         }
4012         cb->args[1] = 1;
4013 done:
4014         rcu_read_unlock();
4015         return skb->len;
4016 }
4017
4018 static int nf_tables_dump_sets_start(struct netlink_callback *cb)
4019 {
4020         struct nft_ctx *ctx_dump = NULL;
4021
4022         ctx_dump = kmemdup(cb->data, sizeof(*ctx_dump), GFP_ATOMIC);
4023         if (ctx_dump == NULL)
4024                 return -ENOMEM;
4025
4026         cb->data = ctx_dump;
4027         return 0;
4028 }
4029
4030 static int nf_tables_dump_sets_done(struct netlink_callback *cb)
4031 {
4032         kfree(cb->data);
4033         return 0;
4034 }
4035
4036 /* called with rcu_read_lock held */
4037 static int nf_tables_getset(struct sk_buff *skb, const struct nfnl_info *info,
4038                             const struct nlattr * const nla[])
4039 {
4040         struct netlink_ext_ack *extack = info->extack;
4041         u8 genmask = nft_genmask_cur(info->net);
4042         u8 family = info->nfmsg->nfgen_family;
4043         struct nft_table *table = NULL;
4044         struct net *net = info->net;
4045         const struct nft_set *set;
4046         struct sk_buff *skb2;
4047         struct nft_ctx ctx;
4048         int err;
4049
4050         if (nla[NFTA_SET_TABLE]) {
4051                 table = nft_table_lookup(net, nla[NFTA_SET_TABLE], family,
4052                                          genmask, 0);
4053                 if (IS_ERR(table)) {
4054                         NL_SET_BAD_ATTR(extack, nla[NFTA_SET_TABLE]);
4055                         return PTR_ERR(table);
4056                 }
4057         }
4058
4059         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
4060
4061         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
4062                 struct netlink_dump_control c = {
4063                         .start = nf_tables_dump_sets_start,
4064                         .dump = nf_tables_dump_sets,
4065                         .done = nf_tables_dump_sets_done,
4066                         .data = &ctx,
4067                         .module = THIS_MODULE,
4068                 };
4069
4070                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
4071         }
4072
4073         /* Only accept unspec with dump */
4074         if (info->nfmsg->nfgen_family == NFPROTO_UNSPEC)
4075                 return -EAFNOSUPPORT;
4076         if (!nla[NFTA_SET_TABLE])
4077                 return -EINVAL;
4078
4079         set = nft_set_lookup(table, nla[NFTA_SET_NAME], genmask);
4080         if (IS_ERR(set))
4081                 return PTR_ERR(set);
4082
4083         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
4084         if (skb2 == NULL)
4085                 return -ENOMEM;
4086
4087         err = nf_tables_fill_set(skb2, &ctx, set, NFT_MSG_NEWSET, 0);
4088         if (err < 0)
4089                 goto err_fill_set_info;
4090
4091         return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
4092
4093 err_fill_set_info:
4094         kfree_skb(skb2);
4095         return err;
4096 }
4097
4098 static const struct nla_policy nft_concat_policy[NFTA_SET_FIELD_MAX + 1] = {
4099         [NFTA_SET_FIELD_LEN]    = { .type = NLA_U32 },
4100 };
4101
4102 static int nft_set_desc_concat_parse(const struct nlattr *attr,
4103                                      struct nft_set_desc *desc)
4104 {
4105         struct nlattr *tb[NFTA_SET_FIELD_MAX + 1];
4106         u32 len;
4107         int err;
4108
4109         err = nla_parse_nested_deprecated(tb, NFTA_SET_FIELD_MAX, attr,
4110                                           nft_concat_policy, NULL);
4111         if (err < 0)
4112                 return err;
4113
4114         if (!tb[NFTA_SET_FIELD_LEN])
4115                 return -EINVAL;
4116
4117         len = ntohl(nla_get_be32(tb[NFTA_SET_FIELD_LEN]));
4118
4119         if (len * BITS_PER_BYTE / 32 > NFT_REG32_COUNT)
4120                 return -E2BIG;
4121
4122         desc->field_len[desc->field_count++] = len;
4123
4124         return 0;
4125 }
4126
4127 static int nft_set_desc_concat(struct nft_set_desc *desc,
4128                                const struct nlattr *nla)
4129 {
4130         struct nlattr *attr;
4131         int rem, err;
4132
4133         nla_for_each_nested(attr, nla, rem) {
4134                 if (nla_type(attr) != NFTA_LIST_ELEM)
4135                         return -EINVAL;
4136
4137                 err = nft_set_desc_concat_parse(attr, desc);
4138                 if (err < 0)
4139                         return err;
4140         }
4141
4142         return 0;
4143 }
4144
4145 static int nf_tables_set_desc_parse(struct nft_set_desc *desc,
4146                                     const struct nlattr *nla)
4147 {
4148         struct nlattr *da[NFTA_SET_DESC_MAX + 1];
4149         int err;
4150
4151         err = nla_parse_nested_deprecated(da, NFTA_SET_DESC_MAX, nla,
4152                                           nft_set_desc_policy, NULL);
4153         if (err < 0)
4154                 return err;
4155
4156         if (da[NFTA_SET_DESC_SIZE] != NULL)
4157                 desc->size = ntohl(nla_get_be32(da[NFTA_SET_DESC_SIZE]));
4158         if (da[NFTA_SET_DESC_CONCAT])
4159                 err = nft_set_desc_concat(desc, da[NFTA_SET_DESC_CONCAT]);
4160
4161         return err;
4162 }
4163
4164 static int nf_tables_newset(struct sk_buff *skb, const struct nfnl_info *info,
4165                             const struct nlattr * const nla[])
4166 {
4167         u32 ktype, dtype, flags, policy, gc_int, objtype;
4168         struct netlink_ext_ack *extack = info->extack;
4169         u8 genmask = nft_genmask_next(info->net);
4170         u8 family = info->nfmsg->nfgen_family;
4171         const struct nft_set_ops *ops;
4172         struct nft_expr *expr = NULL;
4173         struct net *net = info->net;
4174         struct nft_set_desc desc;
4175         struct nft_table *table;
4176         unsigned char *udata;
4177         struct nft_set *set;
4178         struct nft_ctx ctx;
4179         size_t alloc_size;
4180         u64 timeout;
4181         char *name;
4182         int err, i;
4183         u16 udlen;
4184         u64 size;
4185
4186         if (nla[NFTA_SET_TABLE] == NULL ||
4187             nla[NFTA_SET_NAME] == NULL ||
4188             nla[NFTA_SET_KEY_LEN] == NULL ||
4189             nla[NFTA_SET_ID] == NULL)
4190                 return -EINVAL;
4191
4192         memset(&desc, 0, sizeof(desc));
4193
4194         ktype = NFT_DATA_VALUE;
4195         if (nla[NFTA_SET_KEY_TYPE] != NULL) {
4196                 ktype = ntohl(nla_get_be32(nla[NFTA_SET_KEY_TYPE]));
4197                 if ((ktype & NFT_DATA_RESERVED_MASK) == NFT_DATA_RESERVED_MASK)
4198                         return -EINVAL;
4199         }
4200
4201         desc.klen = ntohl(nla_get_be32(nla[NFTA_SET_KEY_LEN]));
4202         if (desc.klen == 0 || desc.klen > NFT_DATA_VALUE_MAXLEN)
4203                 return -EINVAL;
4204
4205         flags = 0;
4206         if (nla[NFTA_SET_FLAGS] != NULL) {
4207                 flags = ntohl(nla_get_be32(nla[NFTA_SET_FLAGS]));
4208                 if (flags & ~(NFT_SET_ANONYMOUS | NFT_SET_CONSTANT |
4209                               NFT_SET_INTERVAL | NFT_SET_TIMEOUT |
4210                               NFT_SET_MAP | NFT_SET_EVAL |
4211                               NFT_SET_OBJECT | NFT_SET_CONCAT | NFT_SET_EXPR))
4212                         return -EOPNOTSUPP;
4213                 /* Only one of these operations is supported */
4214                 if ((flags & (NFT_SET_MAP | NFT_SET_OBJECT)) ==
4215                              (NFT_SET_MAP | NFT_SET_OBJECT))
4216                         return -EOPNOTSUPP;
4217                 if ((flags & (NFT_SET_EVAL | NFT_SET_OBJECT)) ==
4218                              (NFT_SET_EVAL | NFT_SET_OBJECT))
4219                         return -EOPNOTSUPP;
4220         }
4221
4222         dtype = 0;
4223         if (nla[NFTA_SET_DATA_TYPE] != NULL) {
4224                 if (!(flags & NFT_SET_MAP))
4225                         return -EINVAL;
4226
4227                 dtype = ntohl(nla_get_be32(nla[NFTA_SET_DATA_TYPE]));
4228                 if ((dtype & NFT_DATA_RESERVED_MASK) == NFT_DATA_RESERVED_MASK &&
4229                     dtype != NFT_DATA_VERDICT)
4230                         return -EINVAL;
4231
4232                 if (dtype != NFT_DATA_VERDICT) {
4233                         if (nla[NFTA_SET_DATA_LEN] == NULL)
4234                                 return -EINVAL;
4235                         desc.dlen = ntohl(nla_get_be32(nla[NFTA_SET_DATA_LEN]));
4236                         if (desc.dlen == 0 || desc.dlen > NFT_DATA_VALUE_MAXLEN)
4237                                 return -EINVAL;
4238                 } else
4239                         desc.dlen = sizeof(struct nft_verdict);
4240         } else if (flags & NFT_SET_MAP)
4241                 return -EINVAL;
4242
4243         if (nla[NFTA_SET_OBJ_TYPE] != NULL) {
4244                 if (!(flags & NFT_SET_OBJECT))
4245                         return -EINVAL;
4246
4247                 objtype = ntohl(nla_get_be32(nla[NFTA_SET_OBJ_TYPE]));
4248                 if (objtype == NFT_OBJECT_UNSPEC ||
4249                     objtype > NFT_OBJECT_MAX)
4250                         return -EOPNOTSUPP;
4251         } else if (flags & NFT_SET_OBJECT)
4252                 return -EINVAL;
4253         else
4254                 objtype = NFT_OBJECT_UNSPEC;
4255
4256         timeout = 0;
4257         if (nla[NFTA_SET_TIMEOUT] != NULL) {
4258                 if (!(flags & NFT_SET_TIMEOUT))
4259                         return -EINVAL;
4260
4261                 err = nf_msecs_to_jiffies64(nla[NFTA_SET_TIMEOUT], &timeout);
4262                 if (err)
4263                         return err;
4264         }
4265         gc_int = 0;
4266         if (nla[NFTA_SET_GC_INTERVAL] != NULL) {
4267                 if (!(flags & NFT_SET_TIMEOUT))
4268                         return -EINVAL;
4269                 gc_int = ntohl(nla_get_be32(nla[NFTA_SET_GC_INTERVAL]));
4270         }
4271
4272         policy = NFT_SET_POL_PERFORMANCE;
4273         if (nla[NFTA_SET_POLICY] != NULL)
4274                 policy = ntohl(nla_get_be32(nla[NFTA_SET_POLICY]));
4275
4276         if (nla[NFTA_SET_DESC] != NULL) {
4277                 err = nf_tables_set_desc_parse(&desc, nla[NFTA_SET_DESC]);
4278                 if (err < 0)
4279                         return err;
4280         }
4281
4282         if (nla[NFTA_SET_EXPR] || nla[NFTA_SET_EXPRESSIONS])
4283                 desc.expr = true;
4284
4285         table = nft_table_lookup(net, nla[NFTA_SET_TABLE], family, genmask,
4286                                  NETLINK_CB(skb).portid);
4287         if (IS_ERR(table)) {
4288                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_TABLE]);
4289                 return PTR_ERR(table);
4290         }
4291
4292         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
4293
4294         set = nft_set_lookup(table, nla[NFTA_SET_NAME], genmask);
4295         if (IS_ERR(set)) {
4296                 if (PTR_ERR(set) != -ENOENT) {
4297                         NL_SET_BAD_ATTR(extack, nla[NFTA_SET_NAME]);
4298                         return PTR_ERR(set);
4299                 }
4300         } else {
4301                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
4302                         NL_SET_BAD_ATTR(extack, nla[NFTA_SET_NAME]);
4303                         return -EEXIST;
4304                 }
4305                 if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
4306                         return -EOPNOTSUPP;
4307
4308                 return 0;
4309         }
4310
4311         if (!(info->nlh->nlmsg_flags & NLM_F_CREATE))
4312                 return -ENOENT;
4313
4314         ops = nft_select_set_ops(&ctx, nla, &desc, policy);
4315         if (IS_ERR(ops))
4316                 return PTR_ERR(ops);
4317
4318         udlen = 0;
4319         if (nla[NFTA_SET_USERDATA])
4320                 udlen = nla_len(nla[NFTA_SET_USERDATA]);
4321
4322         size = 0;
4323         if (ops->privsize != NULL)
4324                 size = ops->privsize(nla, &desc);
4325         alloc_size = sizeof(*set) + size + udlen;
4326         if (alloc_size < size)
4327                 return -ENOMEM;
4328         set = kvzalloc(alloc_size, GFP_KERNEL);
4329         if (!set)
4330                 return -ENOMEM;
4331
4332         name = nla_strdup(nla[NFTA_SET_NAME], GFP_KERNEL);
4333         if (!name) {
4334                 err = -ENOMEM;
4335                 goto err_set_name;
4336         }
4337
4338         err = nf_tables_set_alloc_name(&ctx, set, name);
4339         kfree(name);
4340         if (err < 0)
4341                 goto err_set_alloc_name;
4342
4343         if (nla[NFTA_SET_EXPR]) {
4344                 expr = nft_set_elem_expr_alloc(&ctx, set, nla[NFTA_SET_EXPR]);
4345                 if (IS_ERR(expr)) {
4346                         err = PTR_ERR(expr);
4347                         goto err_set_alloc_name;
4348                 }
4349                 set->exprs[0] = expr;
4350                 set->num_exprs++;
4351         } else if (nla[NFTA_SET_EXPRESSIONS]) {
4352                 struct nft_expr *expr;
4353                 struct nlattr *tmp;
4354                 int left;
4355
4356                 if (!(flags & NFT_SET_EXPR)) {
4357                         err = -EINVAL;
4358                         goto err_set_alloc_name;
4359                 }
4360                 i = 0;
4361                 nla_for_each_nested(tmp, nla[NFTA_SET_EXPRESSIONS], left) {
4362                         if (i == NFT_SET_EXPR_MAX) {
4363                                 err = -E2BIG;
4364                                 goto err_set_init;
4365                         }
4366                         if (nla_type(tmp) != NFTA_LIST_ELEM) {
4367                                 err = -EINVAL;
4368                                 goto err_set_init;
4369                         }
4370                         expr = nft_set_elem_expr_alloc(&ctx, set, tmp);
4371                         if (IS_ERR(expr)) {
4372                                 err = PTR_ERR(expr);
4373                                 goto err_set_init;
4374                         }
4375                         set->exprs[i++] = expr;
4376                         set->num_exprs++;
4377                 }
4378         }
4379
4380         udata = NULL;
4381         if (udlen) {
4382                 udata = set->data + size;
4383                 nla_memcpy(udata, nla[NFTA_SET_USERDATA], udlen);
4384         }
4385
4386         INIT_LIST_HEAD(&set->bindings);
4387         INIT_LIST_HEAD(&set->catchall_list);
4388         set->table = table;
4389         write_pnet(&set->net, net);
4390         set->ops   = ops;
4391         set->ktype = ktype;
4392         set->klen  = desc.klen;
4393         set->dtype = dtype;
4394         set->objtype = objtype;
4395         set->dlen  = desc.dlen;
4396         set->flags = flags;
4397         set->size  = desc.size;
4398         set->policy = policy;
4399         set->udlen  = udlen;
4400         set->udata  = udata;
4401         set->timeout = timeout;
4402         set->gc_int = gc_int;
4403         set->handle = nf_tables_alloc_handle(table);
4404
4405         set->field_count = desc.field_count;
4406         for (i = 0; i < desc.field_count; i++)
4407                 set->field_len[i] = desc.field_len[i];
4408
4409         err = ops->init(set, &desc, nla);
4410         if (err < 0)
4411                 goto err_set_init;
4412
4413         err = nft_trans_set_add(&ctx, NFT_MSG_NEWSET, set);
4414         if (err < 0)
4415                 goto err_set_trans;
4416
4417         list_add_tail_rcu(&set->list, &table->sets);
4418         table->use++;
4419         return 0;
4420
4421 err_set_trans:
4422         ops->destroy(set);
4423 err_set_init:
4424         for (i = 0; i < set->num_exprs; i++)
4425                 nft_expr_destroy(&ctx, set->exprs[i]);
4426 err_set_alloc_name:
4427         kfree(set->name);
4428 err_set_name:
4429         kvfree(set);
4430         return err;
4431 }
4432
4433 struct nft_set_elem_catchall {
4434         struct list_head        list;
4435         struct rcu_head         rcu;
4436         void                    *elem;
4437 };
4438
4439 static void nft_set_catchall_destroy(const struct nft_ctx *ctx,
4440                                      struct nft_set *set)
4441 {
4442         struct nft_set_elem_catchall *catchall;
4443
4444         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
4445                 list_del_rcu(&catchall->list);
4446                 nft_set_elem_destroy(set, catchall->elem, true);
4447                 kfree_rcu(catchall);
4448         }
4449 }
4450
4451 static void nft_set_destroy(const struct nft_ctx *ctx, struct nft_set *set)
4452 {
4453         int i;
4454
4455         if (WARN_ON(set->use > 0))
4456                 return;
4457
4458         for (i = 0; i < set->num_exprs; i++)
4459                 nft_expr_destroy(ctx, set->exprs[i]);
4460
4461         set->ops->destroy(set);
4462         nft_set_catchall_destroy(ctx, set);
4463         kfree(set->name);
4464         kvfree(set);
4465 }
4466
4467 static int nf_tables_delset(struct sk_buff *skb, const struct nfnl_info *info,
4468                             const struct nlattr * const nla[])
4469 {
4470         struct netlink_ext_ack *extack = info->extack;
4471         u8 genmask = nft_genmask_next(info->net);
4472         u8 family = info->nfmsg->nfgen_family;
4473         struct net *net = info->net;
4474         const struct nlattr *attr;
4475         struct nft_table *table;
4476         struct nft_set *set;
4477         struct nft_ctx ctx;
4478
4479         if (info->nfmsg->nfgen_family == NFPROTO_UNSPEC)
4480                 return -EAFNOSUPPORT;
4481
4482         table = nft_table_lookup(net, nla[NFTA_SET_TABLE], family,
4483                                  genmask, NETLINK_CB(skb).portid);
4484         if (IS_ERR(table)) {
4485                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_TABLE]);
4486                 return PTR_ERR(table);
4487         }
4488
4489         if (nla[NFTA_SET_HANDLE]) {
4490                 attr = nla[NFTA_SET_HANDLE];
4491                 set = nft_set_lookup_byhandle(table, attr, genmask);
4492         } else {
4493                 attr = nla[NFTA_SET_NAME];
4494                 set = nft_set_lookup(table, attr, genmask);
4495         }
4496
4497         if (IS_ERR(set)) {
4498                 NL_SET_BAD_ATTR(extack, attr);
4499                 return PTR_ERR(set);
4500         }
4501         if (set->use ||
4502             (info->nlh->nlmsg_flags & NLM_F_NONREC &&
4503              atomic_read(&set->nelems) > 0)) {
4504                 NL_SET_BAD_ATTR(extack, attr);
4505                 return -EBUSY;
4506         }
4507
4508         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
4509
4510         return nft_delset(&ctx, set);
4511 }
4512
4513 static int nft_validate_register_store(const struct nft_ctx *ctx,
4514                                        enum nft_registers reg,
4515                                        const struct nft_data *data,
4516                                        enum nft_data_types type,
4517                                        unsigned int len);
4518
4519 static int nft_setelem_data_validate(const struct nft_ctx *ctx,
4520                                      struct nft_set *set,
4521                                      struct nft_set_elem *elem)
4522 {
4523         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
4524         enum nft_registers dreg;
4525
4526         dreg = nft_type_to_reg(set->dtype);
4527         return nft_validate_register_store(ctx, dreg, nft_set_ext_data(ext),
4528                                            set->dtype == NFT_DATA_VERDICT ?
4529                                            NFT_DATA_VERDICT : NFT_DATA_VALUE,
4530                                            set->dlen);
4531 }
4532
4533 static int nf_tables_bind_check_setelem(const struct nft_ctx *ctx,
4534                                         struct nft_set *set,
4535                                         const struct nft_set_iter *iter,
4536                                         struct nft_set_elem *elem)
4537 {
4538         return nft_setelem_data_validate(ctx, set, elem);
4539 }
4540
4541 static int nft_set_catchall_bind_check(const struct nft_ctx *ctx,
4542                                        struct nft_set *set)
4543 {
4544         u8 genmask = nft_genmask_next(ctx->net);
4545         struct nft_set_elem_catchall *catchall;
4546         struct nft_set_elem elem;
4547         struct nft_set_ext *ext;
4548         int ret = 0;
4549
4550         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
4551                 ext = nft_set_elem_ext(set, catchall->elem);
4552                 if (!nft_set_elem_active(ext, genmask))
4553                         continue;
4554
4555                 elem.priv = catchall->elem;
4556                 ret = nft_setelem_data_validate(ctx, set, &elem);
4557                 if (ret < 0)
4558                         break;
4559         }
4560
4561         return ret;
4562 }
4563
4564 int nf_tables_bind_set(const struct nft_ctx *ctx, struct nft_set *set,
4565                        struct nft_set_binding *binding)
4566 {
4567         struct nft_set_binding *i;
4568         struct nft_set_iter iter;
4569
4570         if (set->use == UINT_MAX)
4571                 return -EOVERFLOW;
4572
4573         if (!list_empty(&set->bindings) && nft_set_is_anonymous(set))
4574                 return -EBUSY;
4575
4576         if (binding->flags & NFT_SET_MAP) {
4577                 /* If the set is already bound to the same chain all
4578                  * jumps are already validated for that chain.
4579                  */
4580                 list_for_each_entry(i, &set->bindings, list) {
4581                         if (i->flags & NFT_SET_MAP &&
4582                             i->chain == binding->chain)
4583                                 goto bind;
4584                 }
4585
4586                 iter.genmask    = nft_genmask_next(ctx->net);
4587                 iter.skip       = 0;
4588                 iter.count      = 0;
4589                 iter.err        = 0;
4590                 iter.fn         = nf_tables_bind_check_setelem;
4591
4592                 set->ops->walk(ctx, set, &iter);
4593                 if (!iter.err)
4594                         iter.err = nft_set_catchall_bind_check(ctx, set);
4595
4596                 if (iter.err < 0)
4597                         return iter.err;
4598         }
4599 bind:
4600         binding->chain = ctx->chain;
4601         list_add_tail_rcu(&binding->list, &set->bindings);
4602         nft_set_trans_bind(ctx, set);
4603         set->use++;
4604
4605         return 0;
4606 }
4607 EXPORT_SYMBOL_GPL(nf_tables_bind_set);
4608
4609 static void nf_tables_unbind_set(const struct nft_ctx *ctx, struct nft_set *set,
4610                                  struct nft_set_binding *binding, bool event)
4611 {
4612         list_del_rcu(&binding->list);
4613
4614         if (list_empty(&set->bindings) && nft_set_is_anonymous(set)) {
4615                 list_del_rcu(&set->list);
4616                 if (event)
4617                         nf_tables_set_notify(ctx, set, NFT_MSG_DELSET,
4618                                              GFP_KERNEL);
4619         }
4620 }
4621
4622 void nf_tables_deactivate_set(const struct nft_ctx *ctx, struct nft_set *set,
4623                               struct nft_set_binding *binding,
4624                               enum nft_trans_phase phase)
4625 {
4626         switch (phase) {
4627         case NFT_TRANS_PREPARE:
4628                 set->use--;
4629                 return;
4630         case NFT_TRANS_ABORT:
4631         case NFT_TRANS_RELEASE:
4632                 set->use--;
4633                 fallthrough;
4634         default:
4635                 nf_tables_unbind_set(ctx, set, binding,
4636                                      phase == NFT_TRANS_COMMIT);
4637         }
4638 }
4639 EXPORT_SYMBOL_GPL(nf_tables_deactivate_set);
4640
4641 void nf_tables_destroy_set(const struct nft_ctx *ctx, struct nft_set *set)
4642 {
4643         if (list_empty(&set->bindings) && nft_set_is_anonymous(set))
4644                 nft_set_destroy(ctx, set);
4645 }
4646 EXPORT_SYMBOL_GPL(nf_tables_destroy_set);
4647
4648 const struct nft_set_ext_type nft_set_ext_types[] = {
4649         [NFT_SET_EXT_KEY]               = {
4650                 .align  = __alignof__(u32),
4651         },
4652         [NFT_SET_EXT_DATA]              = {
4653                 .align  = __alignof__(u32),
4654         },
4655         [NFT_SET_EXT_EXPRESSIONS]       = {
4656                 .align  = __alignof__(struct nft_set_elem_expr),
4657         },
4658         [NFT_SET_EXT_OBJREF]            = {
4659                 .len    = sizeof(struct nft_object *),
4660                 .align  = __alignof__(struct nft_object *),
4661         },
4662         [NFT_SET_EXT_FLAGS]             = {
4663                 .len    = sizeof(u8),
4664                 .align  = __alignof__(u8),
4665         },
4666         [NFT_SET_EXT_TIMEOUT]           = {
4667                 .len    = sizeof(u64),
4668                 .align  = __alignof__(u64),
4669         },
4670         [NFT_SET_EXT_EXPIRATION]        = {
4671                 .len    = sizeof(u64),
4672                 .align  = __alignof__(u64),
4673         },
4674         [NFT_SET_EXT_USERDATA]          = {
4675                 .len    = sizeof(struct nft_userdata),
4676                 .align  = __alignof__(struct nft_userdata),
4677         },
4678         [NFT_SET_EXT_KEY_END]           = {
4679                 .align  = __alignof__(u32),
4680         },
4681 };
4682
4683 /*
4684  * Set elements
4685  */
4686
4687 static const struct nla_policy nft_set_elem_policy[NFTA_SET_ELEM_MAX + 1] = {
4688         [NFTA_SET_ELEM_KEY]             = { .type = NLA_NESTED },
4689         [NFTA_SET_ELEM_DATA]            = { .type = NLA_NESTED },
4690         [NFTA_SET_ELEM_FLAGS]           = { .type = NLA_U32 },
4691         [NFTA_SET_ELEM_TIMEOUT]         = { .type = NLA_U64 },
4692         [NFTA_SET_ELEM_EXPIRATION]      = { .type = NLA_U64 },
4693         [NFTA_SET_ELEM_USERDATA]        = { .type = NLA_BINARY,
4694                                             .len = NFT_USERDATA_MAXLEN },
4695         [NFTA_SET_ELEM_EXPR]            = { .type = NLA_NESTED },
4696         [NFTA_SET_ELEM_OBJREF]          = { .type = NLA_STRING,
4697                                             .len = NFT_OBJ_MAXNAMELEN - 1 },
4698         [NFTA_SET_ELEM_KEY_END]         = { .type = NLA_NESTED },
4699         [NFTA_SET_ELEM_EXPRESSIONS]     = { .type = NLA_NESTED },
4700 };
4701
4702 static const struct nla_policy nft_set_elem_list_policy[NFTA_SET_ELEM_LIST_MAX + 1] = {
4703         [NFTA_SET_ELEM_LIST_TABLE]      = { .type = NLA_STRING,
4704                                             .len = NFT_TABLE_MAXNAMELEN - 1 },
4705         [NFTA_SET_ELEM_LIST_SET]        = { .type = NLA_STRING,
4706                                             .len = NFT_SET_MAXNAMELEN - 1 },
4707         [NFTA_SET_ELEM_LIST_ELEMENTS]   = { .type = NLA_NESTED },
4708         [NFTA_SET_ELEM_LIST_SET_ID]     = { .type = NLA_U32 },
4709 };
4710
4711 static int nft_set_elem_expr_dump(struct sk_buff *skb,
4712                                   const struct nft_set *set,
4713                                   const struct nft_set_ext *ext)
4714 {
4715         struct nft_set_elem_expr *elem_expr;
4716         u32 size, num_exprs = 0;
4717         struct nft_expr *expr;
4718         struct nlattr *nest;
4719
4720         elem_expr = nft_set_ext_expr(ext);
4721         nft_setelem_expr_foreach(expr, elem_expr, size)
4722                 num_exprs++;
4723
4724         if (num_exprs == 1) {
4725                 expr = nft_setelem_expr_at(elem_expr, 0);
4726                 if (nft_expr_dump(skb, NFTA_SET_ELEM_EXPR, expr) < 0)
4727                         return -1;
4728
4729                 return 0;
4730         } else if (num_exprs > 1) {
4731                 nest = nla_nest_start_noflag(skb, NFTA_SET_ELEM_EXPRESSIONS);
4732                 if (nest == NULL)
4733                         goto nla_put_failure;
4734
4735                 nft_setelem_expr_foreach(expr, elem_expr, size) {
4736                         expr = nft_setelem_expr_at(elem_expr, size);
4737                         if (nft_expr_dump(skb, NFTA_LIST_ELEM, expr) < 0)
4738                                 goto nla_put_failure;
4739                 }
4740                 nla_nest_end(skb, nest);
4741         }
4742         return 0;
4743
4744 nla_put_failure:
4745         return -1;
4746 }
4747
4748 static int nf_tables_fill_setelem(struct sk_buff *skb,
4749                                   const struct nft_set *set,
4750                                   const struct nft_set_elem *elem)
4751 {
4752         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
4753         unsigned char *b = skb_tail_pointer(skb);
4754         struct nlattr *nest;
4755
4756         nest = nla_nest_start_noflag(skb, NFTA_LIST_ELEM);
4757         if (nest == NULL)
4758                 goto nla_put_failure;
4759
4760         if (nft_set_ext_exists(ext, NFT_SET_EXT_KEY) &&
4761             nft_data_dump(skb, NFTA_SET_ELEM_KEY, nft_set_ext_key(ext),
4762                           NFT_DATA_VALUE, set->klen) < 0)
4763                 goto nla_put_failure;
4764
4765         if (nft_set_ext_exists(ext, NFT_SET_EXT_KEY_END) &&
4766             nft_data_dump(skb, NFTA_SET_ELEM_KEY_END, nft_set_ext_key_end(ext),
4767                           NFT_DATA_VALUE, set->klen) < 0)
4768                 goto nla_put_failure;
4769
4770         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA) &&
4771             nft_data_dump(skb, NFTA_SET_ELEM_DATA, nft_set_ext_data(ext),
4772                           set->dtype == NFT_DATA_VERDICT ? NFT_DATA_VERDICT : NFT_DATA_VALUE,
4773                           set->dlen) < 0)
4774                 goto nla_put_failure;
4775
4776         if (nft_set_ext_exists(ext, NFT_SET_EXT_EXPRESSIONS) &&
4777             nft_set_elem_expr_dump(skb, set, ext))
4778                 goto nla_put_failure;
4779
4780         if (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF) &&
4781             nla_put_string(skb, NFTA_SET_ELEM_OBJREF,
4782                            (*nft_set_ext_obj(ext))->key.name) < 0)
4783                 goto nla_put_failure;
4784
4785         if (nft_set_ext_exists(ext, NFT_SET_EXT_FLAGS) &&
4786             nla_put_be32(skb, NFTA_SET_ELEM_FLAGS,
4787                          htonl(*nft_set_ext_flags(ext))))
4788                 goto nla_put_failure;
4789
4790         if (nft_set_ext_exists(ext, NFT_SET_EXT_TIMEOUT) &&
4791             nla_put_be64(skb, NFTA_SET_ELEM_TIMEOUT,
4792                          nf_jiffies64_to_msecs(*nft_set_ext_timeout(ext)),
4793                          NFTA_SET_ELEM_PAD))
4794                 goto nla_put_failure;
4795
4796         if (nft_set_ext_exists(ext, NFT_SET_EXT_EXPIRATION)) {
4797                 u64 expires, now = get_jiffies_64();
4798
4799                 expires = *nft_set_ext_expiration(ext);
4800                 if (time_before64(now, expires))
4801                         expires -= now;
4802                 else
4803                         expires = 0;
4804
4805                 if (nla_put_be64(skb, NFTA_SET_ELEM_EXPIRATION,
4806                                  nf_jiffies64_to_msecs(expires),
4807                                  NFTA_SET_ELEM_PAD))
4808                         goto nla_put_failure;
4809         }
4810
4811         if (nft_set_ext_exists(ext, NFT_SET_EXT_USERDATA)) {
4812                 struct nft_userdata *udata;
4813
4814                 udata = nft_set_ext_userdata(ext);
4815                 if (nla_put(skb, NFTA_SET_ELEM_USERDATA,
4816                             udata->len + 1, udata->data))
4817                         goto nla_put_failure;
4818         }
4819
4820         nla_nest_end(skb, nest);
4821         return 0;
4822
4823 nla_put_failure:
4824         nlmsg_trim(skb, b);
4825         return -EMSGSIZE;
4826 }
4827
4828 struct nft_set_dump_args {
4829         const struct netlink_callback   *cb;
4830         struct nft_set_iter             iter;
4831         struct sk_buff                  *skb;
4832 };
4833
4834 static int nf_tables_dump_setelem(const struct nft_ctx *ctx,
4835                                   struct nft_set *set,
4836                                   const struct nft_set_iter *iter,
4837                                   struct nft_set_elem *elem)
4838 {
4839         struct nft_set_dump_args *args;
4840
4841         args = container_of(iter, struct nft_set_dump_args, iter);
4842         return nf_tables_fill_setelem(args->skb, set, elem);
4843 }
4844
4845 struct nft_set_dump_ctx {
4846         const struct nft_set    *set;
4847         struct nft_ctx          ctx;
4848 };
4849
4850 static int nft_set_catchall_dump(struct net *net, struct sk_buff *skb,
4851                                  const struct nft_set *set)
4852 {
4853         struct nft_set_elem_catchall *catchall;
4854         u8 genmask = nft_genmask_cur(net);
4855         struct nft_set_elem elem;
4856         struct nft_set_ext *ext;
4857         int ret = 0;
4858
4859         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
4860                 ext = nft_set_elem_ext(set, catchall->elem);
4861                 if (!nft_set_elem_active(ext, genmask) ||
4862                     nft_set_elem_expired(ext))
4863                         continue;
4864
4865                 elem.priv = catchall->elem;
4866                 ret = nf_tables_fill_setelem(skb, set, &elem);
4867                 break;
4868         }
4869
4870         return ret;
4871 }
4872
4873 static int nf_tables_dump_set(struct sk_buff *skb, struct netlink_callback *cb)
4874 {
4875         struct nft_set_dump_ctx *dump_ctx = cb->data;
4876         struct net *net = sock_net(skb->sk);
4877         struct nftables_pernet *nft_net;
4878         struct nft_table *table;
4879         struct nft_set *set;
4880         struct nft_set_dump_args args;
4881         bool set_found = false;
4882         struct nlmsghdr *nlh;
4883         struct nlattr *nest;
4884         u32 portid, seq;
4885         int event;
4886
4887         rcu_read_lock();
4888         nft_net = nft_pernet(net);
4889         list_for_each_entry_rcu(table, &nft_net->tables, list) {
4890                 if (dump_ctx->ctx.family != NFPROTO_UNSPEC &&
4891                     dump_ctx->ctx.family != table->family)
4892                         continue;
4893
4894                 if (table != dump_ctx->ctx.table)
4895                         continue;
4896
4897                 list_for_each_entry_rcu(set, &table->sets, list) {
4898                         if (set == dump_ctx->set) {
4899                                 set_found = true;
4900                                 break;
4901                         }
4902                 }
4903                 break;
4904         }
4905
4906         if (!set_found) {
4907                 rcu_read_unlock();
4908                 return -ENOENT;
4909         }
4910
4911         event  = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, NFT_MSG_NEWSETELEM);
4912         portid = NETLINK_CB(cb->skb).portid;
4913         seq    = cb->nlh->nlmsg_seq;
4914
4915         nlh = nfnl_msg_put(skb, portid, seq, event, NLM_F_MULTI,
4916                            table->family, NFNETLINK_V0, nft_base_seq(net));
4917         if (!nlh)
4918                 goto nla_put_failure;
4919
4920         if (nla_put_string(skb, NFTA_SET_ELEM_LIST_TABLE, table->name))
4921                 goto nla_put_failure;
4922         if (nla_put_string(skb, NFTA_SET_ELEM_LIST_SET, set->name))
4923                 goto nla_put_failure;
4924
4925         nest = nla_nest_start_noflag(skb, NFTA_SET_ELEM_LIST_ELEMENTS);
4926         if (nest == NULL)
4927                 goto nla_put_failure;
4928
4929         args.cb                 = cb;
4930         args.skb                = skb;
4931         args.iter.genmask       = nft_genmask_cur(net);
4932         args.iter.skip          = cb->args[0];
4933         args.iter.count         = 0;
4934         args.iter.err           = 0;
4935         args.iter.fn            = nf_tables_dump_setelem;
4936         set->ops->walk(&dump_ctx->ctx, set, &args.iter);
4937
4938         if (!args.iter.err && args.iter.count == cb->args[0])
4939                 args.iter.err = nft_set_catchall_dump(net, skb, set);
4940         rcu_read_unlock();
4941
4942         nla_nest_end(skb, nest);
4943         nlmsg_end(skb, nlh);
4944
4945         if (args.iter.err && args.iter.err != -EMSGSIZE)
4946                 return args.iter.err;
4947         if (args.iter.count == cb->args[0])
4948                 return 0;
4949
4950         cb->args[0] = args.iter.count;
4951         return skb->len;
4952
4953 nla_put_failure:
4954         rcu_read_unlock();
4955         return -ENOSPC;
4956 }
4957
4958 static int nf_tables_dump_set_start(struct netlink_callback *cb)
4959 {
4960         struct nft_set_dump_ctx *dump_ctx = cb->data;
4961
4962         cb->data = kmemdup(dump_ctx, sizeof(*dump_ctx), GFP_ATOMIC);
4963
4964         return cb->data ? 0 : -ENOMEM;
4965 }
4966
4967 static int nf_tables_dump_set_done(struct netlink_callback *cb)
4968 {
4969         kfree(cb->data);
4970         return 0;
4971 }
4972
4973 static int nf_tables_fill_setelem_info(struct sk_buff *skb,
4974                                        const struct nft_ctx *ctx, u32 seq,
4975                                        u32 portid, int event, u16 flags,
4976                                        const struct nft_set *set,
4977                                        const struct nft_set_elem *elem)
4978 {
4979         struct nlmsghdr *nlh;
4980         struct nlattr *nest;
4981         int err;
4982
4983         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
4984         nlh = nfnl_msg_put(skb, portid, seq, event, flags, ctx->family,
4985                            NFNETLINK_V0, nft_base_seq(ctx->net));
4986         if (!nlh)
4987                 goto nla_put_failure;
4988
4989         if (nla_put_string(skb, NFTA_SET_TABLE, ctx->table->name))
4990                 goto nla_put_failure;
4991         if (nla_put_string(skb, NFTA_SET_NAME, set->name))
4992                 goto nla_put_failure;
4993
4994         nest = nla_nest_start_noflag(skb, NFTA_SET_ELEM_LIST_ELEMENTS);
4995         if (nest == NULL)
4996                 goto nla_put_failure;
4997
4998         err = nf_tables_fill_setelem(skb, set, elem);
4999         if (err < 0)
5000                 goto nla_put_failure;
5001
5002         nla_nest_end(skb, nest);
5003
5004         nlmsg_end(skb, nlh);
5005         return 0;
5006
5007 nla_put_failure:
5008         nlmsg_trim(skb, nlh);
5009         return -1;
5010 }
5011
5012 static int nft_setelem_parse_flags(const struct nft_set *set,
5013                                    const struct nlattr *attr, u32 *flags)
5014 {
5015         if (attr == NULL)
5016                 return 0;
5017
5018         *flags = ntohl(nla_get_be32(attr));
5019         if (*flags & ~(NFT_SET_ELEM_INTERVAL_END | NFT_SET_ELEM_CATCHALL))
5020                 return -EOPNOTSUPP;
5021         if (!(set->flags & NFT_SET_INTERVAL) &&
5022             *flags & NFT_SET_ELEM_INTERVAL_END)
5023                 return -EINVAL;
5024
5025         return 0;
5026 }
5027
5028 static int nft_setelem_parse_key(struct nft_ctx *ctx, struct nft_set *set,
5029                                  struct nft_data *key, struct nlattr *attr)
5030 {
5031         struct nft_data_desc desc;
5032         int err;
5033
5034         err = nft_data_init(ctx, key, NFT_DATA_VALUE_MAXLEN, &desc, attr);
5035         if (err < 0)
5036                 return err;
5037
5038         if (desc.type != NFT_DATA_VALUE || desc.len != set->klen) {
5039                 nft_data_release(key, desc.type);
5040                 return -EINVAL;
5041         }
5042
5043         return 0;
5044 }
5045
5046 static int nft_setelem_parse_data(struct nft_ctx *ctx, struct nft_set *set,
5047                                   struct nft_data_desc *desc,
5048                                   struct nft_data *data,
5049                                   struct nlattr *attr)
5050 {
5051         int err;
5052
5053         err = nft_data_init(ctx, data, NFT_DATA_VALUE_MAXLEN, desc, attr);
5054         if (err < 0)
5055                 return err;
5056
5057         if (desc->type != NFT_DATA_VERDICT && desc->len != set->dlen) {
5058                 nft_data_release(data, desc->type);
5059                 return -EINVAL;
5060         }
5061
5062         return 0;
5063 }
5064
5065 static void *nft_setelem_catchall_get(const struct net *net,
5066                                       const struct nft_set *set)
5067 {
5068         struct nft_set_elem_catchall *catchall;
5069         u8 genmask = nft_genmask_cur(net);
5070         struct nft_set_ext *ext;
5071         void *priv = NULL;
5072
5073         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
5074                 ext = nft_set_elem_ext(set, catchall->elem);
5075                 if (!nft_set_elem_active(ext, genmask) ||
5076                     nft_set_elem_expired(ext))
5077                         continue;
5078
5079                 priv = catchall->elem;
5080                 break;
5081         }
5082
5083         return priv;
5084 }
5085
5086 static int nft_setelem_get(struct nft_ctx *ctx, struct nft_set *set,
5087                            struct nft_set_elem *elem, u32 flags)
5088 {
5089         void *priv;
5090
5091         if (!(flags & NFT_SET_ELEM_CATCHALL)) {
5092                 priv = set->ops->get(ctx->net, set, elem, flags);
5093                 if (IS_ERR(priv))
5094                         return PTR_ERR(priv);
5095         } else {
5096                 priv = nft_setelem_catchall_get(ctx->net, set);
5097                 if (!priv)
5098                         return -ENOENT;
5099         }
5100         elem->priv = priv;
5101
5102         return 0;
5103 }
5104
5105 static int nft_get_set_elem(struct nft_ctx *ctx, struct nft_set *set,
5106                             const struct nlattr *attr)
5107 {
5108         struct nlattr *nla[NFTA_SET_ELEM_MAX + 1];
5109         struct nft_set_elem elem;
5110         struct sk_buff *skb;
5111         uint32_t flags = 0;
5112         int err;
5113
5114         err = nla_parse_nested_deprecated(nla, NFTA_SET_ELEM_MAX, attr,
5115                                           nft_set_elem_policy, NULL);
5116         if (err < 0)
5117                 return err;
5118
5119         err = nft_setelem_parse_flags(set, nla[NFTA_SET_ELEM_FLAGS], &flags);
5120         if (err < 0)
5121                 return err;
5122
5123         if (!nla[NFTA_SET_ELEM_KEY] && !(flags & NFT_SET_ELEM_CATCHALL))
5124                 return -EINVAL;
5125
5126         if (nla[NFTA_SET_ELEM_KEY]) {
5127                 err = nft_setelem_parse_key(ctx, set, &elem.key.val,
5128                                             nla[NFTA_SET_ELEM_KEY]);
5129                 if (err < 0)
5130                         return err;
5131         }
5132
5133         if (nla[NFTA_SET_ELEM_KEY_END]) {
5134                 err = nft_setelem_parse_key(ctx, set, &elem.key_end.val,
5135                                             nla[NFTA_SET_ELEM_KEY_END]);
5136                 if (err < 0)
5137                         return err;
5138         }
5139
5140         err = nft_setelem_get(ctx, set, &elem, flags);
5141         if (err < 0)
5142                 return err;
5143
5144         err = -ENOMEM;
5145         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_ATOMIC);
5146         if (skb == NULL)
5147                 return err;
5148
5149         err = nf_tables_fill_setelem_info(skb, ctx, ctx->seq, ctx->portid,
5150                                           NFT_MSG_NEWSETELEM, 0, set, &elem);
5151         if (err < 0)
5152                 goto err_fill_setelem;
5153
5154         return nfnetlink_unicast(skb, ctx->net, ctx->portid);
5155
5156 err_fill_setelem:
5157         kfree_skb(skb);
5158         return err;
5159 }
5160
5161 /* called with rcu_read_lock held */
5162 static int nf_tables_getsetelem(struct sk_buff *skb,
5163                                 const struct nfnl_info *info,
5164                                 const struct nlattr * const nla[])
5165 {
5166         struct netlink_ext_ack *extack = info->extack;
5167         u8 genmask = nft_genmask_cur(info->net);
5168         u8 family = info->nfmsg->nfgen_family;
5169         struct net *net = info->net;
5170         struct nft_table *table;
5171         struct nft_set *set;
5172         struct nlattr *attr;
5173         struct nft_ctx ctx;
5174         int rem, err = 0;
5175
5176         table = nft_table_lookup(net, nla[NFTA_SET_ELEM_LIST_TABLE], family,
5177                                  genmask, NETLINK_CB(skb).portid);
5178         if (IS_ERR(table)) {
5179                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_ELEM_LIST_TABLE]);
5180                 return PTR_ERR(table);
5181         }
5182
5183         set = nft_set_lookup(table, nla[NFTA_SET_ELEM_LIST_SET], genmask);
5184         if (IS_ERR(set))
5185                 return PTR_ERR(set);
5186
5187         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
5188
5189         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
5190                 struct netlink_dump_control c = {
5191                         .start = nf_tables_dump_set_start,
5192                         .dump = nf_tables_dump_set,
5193                         .done = nf_tables_dump_set_done,
5194                         .module = THIS_MODULE,
5195                 };
5196                 struct nft_set_dump_ctx dump_ctx = {
5197                         .set = set,
5198                         .ctx = ctx,
5199                 };
5200
5201                 c.data = &dump_ctx;
5202                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
5203         }
5204
5205         if (!nla[NFTA_SET_ELEM_LIST_ELEMENTS])
5206                 return -EINVAL;
5207
5208         nla_for_each_nested(attr, nla[NFTA_SET_ELEM_LIST_ELEMENTS], rem) {
5209                 err = nft_get_set_elem(&ctx, set, attr);
5210                 if (err < 0)
5211                         break;
5212         }
5213
5214         return err;
5215 }
5216
5217 static void nf_tables_setelem_notify(const struct nft_ctx *ctx,
5218                                      const struct nft_set *set,
5219                                      const struct nft_set_elem *elem,
5220                                      int event, u16 flags)
5221 {
5222         struct nftables_pernet *nft_net;
5223         struct net *net = ctx->net;
5224         u32 portid = ctx->portid;
5225         struct sk_buff *skb;
5226         int err;
5227
5228         if (!ctx->report && !nfnetlink_has_listeners(net, NFNLGRP_NFTABLES))
5229                 return;
5230
5231         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
5232         if (skb == NULL)
5233                 goto err;
5234
5235         err = nf_tables_fill_setelem_info(skb, ctx, 0, portid, event, flags,
5236                                           set, elem);
5237         if (err < 0) {
5238                 kfree_skb(skb);
5239                 goto err;
5240         }
5241
5242         nft_net = nft_pernet(net);
5243         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
5244         return;
5245 err:
5246         nfnetlink_set_err(net, portid, NFNLGRP_NFTABLES, -ENOBUFS);
5247 }
5248
5249 static struct nft_trans *nft_trans_elem_alloc(struct nft_ctx *ctx,
5250                                               int msg_type,
5251                                               struct nft_set *set)
5252 {
5253         struct nft_trans *trans;
5254
5255         trans = nft_trans_alloc(ctx, msg_type, sizeof(struct nft_trans_elem));
5256         if (trans == NULL)
5257                 return NULL;
5258
5259         nft_trans_elem_set(trans) = set;
5260         return trans;
5261 }
5262
5263 struct nft_expr *nft_set_elem_expr_alloc(const struct nft_ctx *ctx,
5264                                          const struct nft_set *set,
5265                                          const struct nlattr *attr)
5266 {
5267         struct nft_expr *expr;
5268         int err;
5269
5270         expr = nft_expr_init(ctx, attr);
5271         if (IS_ERR(expr))
5272                 return expr;
5273
5274         err = -EOPNOTSUPP;
5275         if (!(expr->ops->type->flags & NFT_EXPR_STATEFUL))
5276                 goto err_set_elem_expr;
5277
5278         if (expr->ops->type->flags & NFT_EXPR_GC) {
5279                 if (set->flags & NFT_SET_TIMEOUT)
5280                         goto err_set_elem_expr;
5281                 if (!set->ops->gc_init)
5282                         goto err_set_elem_expr;
5283                 set->ops->gc_init(set);
5284         }
5285
5286         return expr;
5287
5288 err_set_elem_expr:
5289         nft_expr_destroy(ctx, expr);
5290         return ERR_PTR(err);
5291 }
5292
5293 void *nft_set_elem_init(const struct nft_set *set,
5294                         const struct nft_set_ext_tmpl *tmpl,
5295                         const u32 *key, const u32 *key_end,
5296                         const u32 *data, u64 timeout, u64 expiration, gfp_t gfp)
5297 {
5298         struct nft_set_ext *ext;
5299         void *elem;
5300
5301         elem = kzalloc(set->ops->elemsize + tmpl->len, gfp);
5302         if (elem == NULL)
5303                 return NULL;
5304
5305         ext = nft_set_elem_ext(set, elem);
5306         nft_set_ext_init(ext, tmpl);
5307
5308         if (nft_set_ext_exists(ext, NFT_SET_EXT_KEY))
5309                 memcpy(nft_set_ext_key(ext), key, set->klen);
5310         if (nft_set_ext_exists(ext, NFT_SET_EXT_KEY_END))
5311                 memcpy(nft_set_ext_key_end(ext), key_end, set->klen);
5312         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA))
5313                 memcpy(nft_set_ext_data(ext), data, set->dlen);
5314         if (nft_set_ext_exists(ext, NFT_SET_EXT_EXPIRATION)) {
5315                 *nft_set_ext_expiration(ext) = get_jiffies_64() + expiration;
5316                 if (expiration == 0)
5317                         *nft_set_ext_expiration(ext) += timeout;
5318         }
5319         if (nft_set_ext_exists(ext, NFT_SET_EXT_TIMEOUT))
5320                 *nft_set_ext_timeout(ext) = timeout;
5321
5322         return elem;
5323 }
5324
5325 static void __nft_set_elem_expr_destroy(const struct nft_ctx *ctx,
5326                                         struct nft_expr *expr)
5327 {
5328         if (expr->ops->destroy_clone) {
5329                 expr->ops->destroy_clone(ctx, expr);
5330                 module_put(expr->ops->type->owner);
5331         } else {
5332                 nf_tables_expr_destroy(ctx, expr);
5333         }
5334 }
5335
5336 static void nft_set_elem_expr_destroy(const struct nft_ctx *ctx,
5337                                       struct nft_set_elem_expr *elem_expr)
5338 {
5339         struct nft_expr *expr;
5340         u32 size;
5341
5342         nft_setelem_expr_foreach(expr, elem_expr, size)
5343                 __nft_set_elem_expr_destroy(ctx, expr);
5344 }
5345
5346 void nft_set_elem_destroy(const struct nft_set *set, void *elem,
5347                           bool destroy_expr)
5348 {
5349         struct nft_set_ext *ext = nft_set_elem_ext(set, elem);
5350         struct nft_ctx ctx = {
5351                 .net    = read_pnet(&set->net),
5352                 .family = set->table->family,
5353         };
5354
5355         nft_data_release(nft_set_ext_key(ext), NFT_DATA_VALUE);
5356         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA))
5357                 nft_data_release(nft_set_ext_data(ext), set->dtype);
5358         if (destroy_expr && nft_set_ext_exists(ext, NFT_SET_EXT_EXPRESSIONS))
5359                 nft_set_elem_expr_destroy(&ctx, nft_set_ext_expr(ext));
5360
5361         if (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF))
5362                 (*nft_set_ext_obj(ext))->use--;
5363         kfree(elem);
5364 }
5365 EXPORT_SYMBOL_GPL(nft_set_elem_destroy);
5366
5367 /* Only called from commit path, nft_setelem_data_deactivate() already deals
5368  * with the refcounting from the preparation phase.
5369  */
5370 static void nf_tables_set_elem_destroy(const struct nft_ctx *ctx,
5371                                        const struct nft_set *set, void *elem)
5372 {
5373         struct nft_set_ext *ext = nft_set_elem_ext(set, elem);
5374
5375         if (nft_set_ext_exists(ext, NFT_SET_EXT_EXPRESSIONS))
5376                 nft_set_elem_expr_destroy(ctx, nft_set_ext_expr(ext));
5377
5378         kfree(elem);
5379 }
5380
5381 int nft_set_elem_expr_clone(const struct nft_ctx *ctx, struct nft_set *set,
5382                             struct nft_expr *expr_array[])
5383 {
5384         struct nft_expr *expr;
5385         int err, i, k;
5386
5387         for (i = 0; i < set->num_exprs; i++) {
5388                 expr = kzalloc(set->exprs[i]->ops->size, GFP_KERNEL);
5389                 if (!expr)
5390                         goto err_expr;
5391
5392                 err = nft_expr_clone(expr, set->exprs[i]);
5393                 if (err < 0) {
5394                         nft_expr_destroy(ctx, expr);
5395                         goto err_expr;
5396                 }
5397                 expr_array[i] = expr;
5398         }
5399
5400         return 0;
5401
5402 err_expr:
5403         for (k = i - 1; k >= 0; k--)
5404                 nft_expr_destroy(ctx, expr_array[k]);
5405
5406         return -ENOMEM;
5407 }
5408
5409 static int nft_set_elem_expr_setup(struct nft_ctx *ctx,
5410                                    const struct nft_set_ext *ext,
5411                                    struct nft_expr *expr_array[],
5412                                    u32 num_exprs)
5413 {
5414         struct nft_set_elem_expr *elem_expr = nft_set_ext_expr(ext);
5415         struct nft_expr *expr;
5416         int i, err;
5417
5418         for (i = 0; i < num_exprs; i++) {
5419                 expr = nft_setelem_expr_at(elem_expr, elem_expr->size);
5420                 err = nft_expr_clone(expr, expr_array[i]);
5421                 if (err < 0)
5422                         goto err_elem_expr_setup;
5423
5424                 elem_expr->size += expr_array[i]->ops->size;
5425                 nft_expr_destroy(ctx, expr_array[i]);
5426                 expr_array[i] = NULL;
5427         }
5428
5429         return 0;
5430
5431 err_elem_expr_setup:
5432         for (; i < num_exprs; i++) {
5433                 nft_expr_destroy(ctx, expr_array[i]);
5434                 expr_array[i] = NULL;
5435         }
5436
5437         return -ENOMEM;
5438 }
5439
5440 struct nft_set_ext *nft_set_catchall_lookup(const struct net *net,
5441                                             const struct nft_set *set)
5442 {
5443         struct nft_set_elem_catchall *catchall;
5444         u8 genmask = nft_genmask_cur(net);
5445         struct nft_set_ext *ext;
5446
5447         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
5448                 ext = nft_set_elem_ext(set, catchall->elem);
5449                 if (nft_set_elem_active(ext, genmask) &&
5450                     !nft_set_elem_expired(ext))
5451                         return ext;
5452         }
5453
5454         return NULL;
5455 }
5456 EXPORT_SYMBOL_GPL(nft_set_catchall_lookup);
5457
5458 void *nft_set_catchall_gc(const struct nft_set *set)
5459 {
5460         struct nft_set_elem_catchall *catchall, *next;
5461         struct nft_set_ext *ext;
5462         void *elem = NULL;
5463
5464         list_for_each_entry_safe(catchall, next, &set->catchall_list, list) {
5465                 ext = nft_set_elem_ext(set, catchall->elem);
5466
5467                 if (!nft_set_elem_expired(ext) ||
5468                     nft_set_elem_mark_busy(ext))
5469                         continue;
5470
5471                 elem = catchall->elem;
5472                 list_del_rcu(&catchall->list);
5473                 kfree_rcu(catchall, rcu);
5474                 break;
5475         }
5476
5477         return elem;
5478 }
5479 EXPORT_SYMBOL_GPL(nft_set_catchall_gc);
5480
5481 static int nft_setelem_catchall_insert(const struct net *net,
5482                                        struct nft_set *set,
5483                                        const struct nft_set_elem *elem,
5484                                        struct nft_set_ext **pext)
5485 {
5486         struct nft_set_elem_catchall *catchall;
5487         u8 genmask = nft_genmask_next(net);
5488         struct nft_set_ext *ext;
5489
5490         list_for_each_entry(catchall, &set->catchall_list, list) {
5491                 ext = nft_set_elem_ext(set, catchall->elem);
5492                 if (nft_set_elem_active(ext, genmask)) {
5493                         *pext = ext;
5494                         return -EEXIST;
5495                 }
5496         }
5497
5498         catchall = kmalloc(sizeof(*catchall), GFP_KERNEL);
5499         if (!catchall)
5500                 return -ENOMEM;
5501
5502         catchall->elem = elem->priv;
5503         list_add_tail_rcu(&catchall->list, &set->catchall_list);
5504
5505         return 0;
5506 }
5507
5508 static int nft_setelem_insert(const struct net *net,
5509                               struct nft_set *set,
5510                               const struct nft_set_elem *elem,
5511                               struct nft_set_ext **ext, unsigned int flags)
5512 {
5513         int ret;
5514
5515         if (flags & NFT_SET_ELEM_CATCHALL)
5516                 ret = nft_setelem_catchall_insert(net, set, elem, ext);
5517         else
5518                 ret = set->ops->insert(net, set, elem, ext);
5519
5520         return ret;
5521 }
5522
5523 static bool nft_setelem_is_catchall(const struct nft_set *set,
5524                                     const struct nft_set_elem *elem)
5525 {
5526         struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
5527
5528         if (nft_set_ext_exists(ext, NFT_SET_EXT_FLAGS) &&
5529             *nft_set_ext_flags(ext) & NFT_SET_ELEM_CATCHALL)
5530                 return true;
5531
5532         return false;
5533 }
5534
5535 static void nft_setelem_activate(struct net *net, struct nft_set *set,
5536                                  struct nft_set_elem *elem)
5537 {
5538         struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
5539
5540         if (nft_setelem_is_catchall(set, elem)) {
5541                 nft_set_elem_change_active(net, set, ext);
5542                 nft_set_elem_clear_busy(ext);
5543         } else {
5544                 set->ops->activate(net, set, elem);
5545         }
5546 }
5547
5548 static int nft_setelem_catchall_deactivate(const struct net *net,
5549                                            struct nft_set *set,
5550                                            struct nft_set_elem *elem)
5551 {
5552         struct nft_set_elem_catchall *catchall;
5553         struct nft_set_ext *ext;
5554
5555         list_for_each_entry(catchall, &set->catchall_list, list) {
5556                 ext = nft_set_elem_ext(set, catchall->elem);
5557                 if (!nft_is_active(net, ext) ||
5558                     nft_set_elem_mark_busy(ext))
5559                         continue;
5560
5561                 kfree(elem->priv);
5562                 elem->priv = catchall->elem;
5563                 nft_set_elem_change_active(net, set, ext);
5564                 return 0;
5565         }
5566
5567         return -ENOENT;
5568 }
5569
5570 static int __nft_setelem_deactivate(const struct net *net,
5571                                     struct nft_set *set,
5572                                     struct nft_set_elem *elem)
5573 {
5574         void *priv;
5575
5576         priv = set->ops->deactivate(net, set, elem);
5577         if (!priv)
5578                 return -ENOENT;
5579
5580         kfree(elem->priv);
5581         elem->priv = priv;
5582         set->ndeact++;
5583
5584         return 0;
5585 }
5586
5587 static int nft_setelem_deactivate(const struct net *net,
5588                                   struct nft_set *set,
5589                                   struct nft_set_elem *elem, u32 flags)
5590 {
5591         int ret;
5592
5593         if (flags & NFT_SET_ELEM_CATCHALL)
5594                 ret = nft_setelem_catchall_deactivate(net, set, elem);
5595         else
5596                 ret = __nft_setelem_deactivate(net, set, elem);
5597
5598         return ret;
5599 }
5600
5601 static void nft_setelem_catchall_remove(const struct net *net,
5602                                         const struct nft_set *set,
5603                                         const struct nft_set_elem *elem)
5604 {
5605         struct nft_set_elem_catchall *catchall, *next;
5606
5607         list_for_each_entry_safe(catchall, next, &set->catchall_list, list) {
5608                 if (catchall->elem == elem->priv) {
5609                         list_del_rcu(&catchall->list);
5610                         kfree_rcu(catchall);
5611                         break;
5612                 }
5613         }
5614 }
5615
5616 static void nft_setelem_remove(const struct net *net,
5617                                const struct nft_set *set,
5618                                const struct nft_set_elem *elem)
5619 {
5620         if (nft_setelem_is_catchall(set, elem))
5621                 nft_setelem_catchall_remove(net, set, elem);
5622         else
5623                 set->ops->remove(net, set, elem);
5624 }
5625
5626 static int nft_add_set_elem(struct nft_ctx *ctx, struct nft_set *set,
5627                             const struct nlattr *attr, u32 nlmsg_flags)
5628 {
5629         struct nft_expr *expr_array[NFT_SET_EXPR_MAX] = {};
5630         struct nlattr *nla[NFTA_SET_ELEM_MAX + 1];
5631         u8 genmask = nft_genmask_next(ctx->net);
5632         u32 flags = 0, size = 0, num_exprs = 0;
5633         struct nft_set_ext_tmpl tmpl;
5634         struct nft_set_ext *ext, *ext2;
5635         struct nft_set_elem elem;
5636         struct nft_set_binding *binding;
5637         struct nft_object *obj = NULL;
5638         struct nft_userdata *udata;
5639         struct nft_data_desc desc;
5640         enum nft_registers dreg;
5641         struct nft_trans *trans;
5642         u64 timeout;
5643         u64 expiration;
5644         int err, i;
5645         u8 ulen;
5646
5647         err = nla_parse_nested_deprecated(nla, NFTA_SET_ELEM_MAX, attr,
5648                                           nft_set_elem_policy, NULL);
5649         if (err < 0)
5650                 return err;
5651
5652         nft_set_ext_prepare(&tmpl);
5653
5654         err = nft_setelem_parse_flags(set, nla[NFTA_SET_ELEM_FLAGS], &flags);
5655         if (err < 0)
5656                 return err;
5657
5658         if (!nla[NFTA_SET_ELEM_KEY] && !(flags & NFT_SET_ELEM_CATCHALL))
5659                 return -EINVAL;
5660
5661         if (flags != 0)
5662                 nft_set_ext_add(&tmpl, NFT_SET_EXT_FLAGS);
5663
5664         if (set->flags & NFT_SET_MAP) {
5665                 if (nla[NFTA_SET_ELEM_DATA] == NULL &&
5666                     !(flags & NFT_SET_ELEM_INTERVAL_END))
5667                         return -EINVAL;
5668         } else {
5669                 if (nla[NFTA_SET_ELEM_DATA] != NULL)
5670                         return -EINVAL;
5671         }
5672
5673         if ((flags & NFT_SET_ELEM_INTERVAL_END) &&
5674              (nla[NFTA_SET_ELEM_DATA] ||
5675               nla[NFTA_SET_ELEM_OBJREF] ||
5676               nla[NFTA_SET_ELEM_TIMEOUT] ||
5677               nla[NFTA_SET_ELEM_EXPIRATION] ||
5678               nla[NFTA_SET_ELEM_USERDATA] ||
5679               nla[NFTA_SET_ELEM_EXPR] ||
5680               nla[NFTA_SET_ELEM_EXPRESSIONS]))
5681                 return -EINVAL;
5682
5683         timeout = 0;
5684         if (nla[NFTA_SET_ELEM_TIMEOUT] != NULL) {
5685                 if (!(set->flags & NFT_SET_TIMEOUT))
5686                         return -EINVAL;
5687                 err = nf_msecs_to_jiffies64(nla[NFTA_SET_ELEM_TIMEOUT],
5688                                             &timeout);
5689                 if (err)
5690                         return err;
5691         } else if (set->flags & NFT_SET_TIMEOUT) {
5692                 timeout = set->timeout;
5693         }
5694
5695         expiration = 0;
5696         if (nla[NFTA_SET_ELEM_EXPIRATION] != NULL) {
5697                 if (!(set->flags & NFT_SET_TIMEOUT))
5698                         return -EINVAL;
5699                 err = nf_msecs_to_jiffies64(nla[NFTA_SET_ELEM_EXPIRATION],
5700                                             &expiration);
5701                 if (err)
5702                         return err;
5703         }
5704
5705         if (nla[NFTA_SET_ELEM_EXPR]) {
5706                 struct nft_expr *expr;
5707
5708                 if (set->num_exprs && set->num_exprs != 1)
5709                         return -EOPNOTSUPP;
5710
5711                 expr = nft_set_elem_expr_alloc(ctx, set,
5712                                                nla[NFTA_SET_ELEM_EXPR]);
5713                 if (IS_ERR(expr))
5714                         return PTR_ERR(expr);
5715
5716                 expr_array[0] = expr;
5717                 num_exprs = 1;
5718
5719                 if (set->num_exprs && set->exprs[0]->ops != expr->ops) {
5720                         err = -EOPNOTSUPP;
5721                         goto err_set_elem_expr;
5722                 }
5723         } else if (nla[NFTA_SET_ELEM_EXPRESSIONS]) {
5724                 struct nft_expr *expr;
5725                 struct nlattr *tmp;
5726                 int left;
5727
5728                 i = 0;
5729                 nla_for_each_nested(tmp, nla[NFTA_SET_ELEM_EXPRESSIONS], left) {
5730                         if (i == NFT_SET_EXPR_MAX ||
5731                             (set->num_exprs && set->num_exprs == i)) {
5732                                 err = -E2BIG;
5733                                 goto err_set_elem_expr;
5734                         }
5735                         if (nla_type(tmp) != NFTA_LIST_ELEM) {
5736                                 err = -EINVAL;
5737                                 goto err_set_elem_expr;
5738                         }
5739                         expr = nft_set_elem_expr_alloc(ctx, set, tmp);
5740                         if (IS_ERR(expr)) {
5741                                 err = PTR_ERR(expr);
5742                                 goto err_set_elem_expr;
5743                         }
5744                         expr_array[i] = expr;
5745                         num_exprs++;
5746
5747                         if (set->num_exprs && expr->ops != set->exprs[i]->ops) {
5748                                 err = -EOPNOTSUPP;
5749                                 goto err_set_elem_expr;
5750                         }
5751                         i++;
5752                 }
5753                 if (set->num_exprs && set->num_exprs != i) {
5754                         err = -EOPNOTSUPP;
5755                         goto err_set_elem_expr;
5756                 }
5757         } else if (set->num_exprs > 0) {
5758                 err = nft_set_elem_expr_clone(ctx, set, expr_array);
5759                 if (err < 0)
5760                         goto err_set_elem_expr_clone;
5761
5762                 num_exprs = set->num_exprs;
5763         }
5764
5765         if (nla[NFTA_SET_ELEM_KEY]) {
5766                 err = nft_setelem_parse_key(ctx, set, &elem.key.val,
5767                                             nla[NFTA_SET_ELEM_KEY]);
5768                 if (err < 0)
5769                         goto err_set_elem_expr;
5770
5771                 nft_set_ext_add_length(&tmpl, NFT_SET_EXT_KEY, set->klen);
5772         }
5773
5774         if (nla[NFTA_SET_ELEM_KEY_END]) {
5775                 err = nft_setelem_parse_key(ctx, set, &elem.key_end.val,
5776                                             nla[NFTA_SET_ELEM_KEY_END]);
5777                 if (err < 0)
5778                         goto err_parse_key;
5779
5780                 nft_set_ext_add_length(&tmpl, NFT_SET_EXT_KEY_END, set->klen);
5781         }
5782
5783         if (timeout > 0) {
5784                 nft_set_ext_add(&tmpl, NFT_SET_EXT_EXPIRATION);
5785                 if (timeout != set->timeout)
5786                         nft_set_ext_add(&tmpl, NFT_SET_EXT_TIMEOUT);
5787         }
5788
5789         if (num_exprs) {
5790                 for (i = 0; i < num_exprs; i++)
5791                         size += expr_array[i]->ops->size;
5792
5793                 nft_set_ext_add_length(&tmpl, NFT_SET_EXT_EXPRESSIONS,
5794                                        sizeof(struct nft_set_elem_expr) +
5795                                        size);
5796         }
5797
5798         if (nla[NFTA_SET_ELEM_OBJREF] != NULL) {
5799                 if (!(set->flags & NFT_SET_OBJECT)) {
5800                         err = -EINVAL;
5801                         goto err_parse_key_end;
5802                 }
5803                 obj = nft_obj_lookup(ctx->net, ctx->table,
5804                                      nla[NFTA_SET_ELEM_OBJREF],
5805                                      set->objtype, genmask);
5806                 if (IS_ERR(obj)) {
5807                         err = PTR_ERR(obj);
5808                         goto err_parse_key_end;
5809                 }
5810                 nft_set_ext_add(&tmpl, NFT_SET_EXT_OBJREF);
5811         }
5812
5813         if (nla[NFTA_SET_ELEM_DATA] != NULL) {
5814                 err = nft_setelem_parse_data(ctx, set, &desc, &elem.data.val,
5815                                              nla[NFTA_SET_ELEM_DATA]);
5816                 if (err < 0)
5817                         goto err_parse_key_end;
5818
5819                 dreg = nft_type_to_reg(set->dtype);
5820                 list_for_each_entry(binding, &set->bindings, list) {
5821                         struct nft_ctx bind_ctx = {
5822                                 .net    = ctx->net,
5823                                 .family = ctx->family,
5824                                 .table  = ctx->table,
5825                                 .chain  = (struct nft_chain *)binding->chain,
5826                         };
5827
5828                         if (!(binding->flags & NFT_SET_MAP))
5829                                 continue;
5830
5831                         err = nft_validate_register_store(&bind_ctx, dreg,
5832                                                           &elem.data.val,
5833                                                           desc.type, desc.len);
5834                         if (err < 0)
5835                                 goto err_parse_data;
5836
5837                         if (desc.type == NFT_DATA_VERDICT &&
5838                             (elem.data.val.verdict.code == NFT_GOTO ||
5839                              elem.data.val.verdict.code == NFT_JUMP))
5840                                 nft_validate_state_update(ctx->net,
5841                                                           NFT_VALIDATE_NEED);
5842                 }
5843
5844                 nft_set_ext_add_length(&tmpl, NFT_SET_EXT_DATA, desc.len);
5845         }
5846
5847         /* The full maximum length of userdata can exceed the maximum
5848          * offset value (U8_MAX) for following extensions, therefor it
5849          * must be the last extension added.
5850          */
5851         ulen = 0;
5852         if (nla[NFTA_SET_ELEM_USERDATA] != NULL) {
5853                 ulen = nla_len(nla[NFTA_SET_ELEM_USERDATA]);
5854                 if (ulen > 0)
5855                         nft_set_ext_add_length(&tmpl, NFT_SET_EXT_USERDATA,
5856                                                ulen);
5857         }
5858
5859         err = -ENOMEM;
5860         elem.priv = nft_set_elem_init(set, &tmpl, elem.key.val.data,
5861                                       elem.key_end.val.data, elem.data.val.data,
5862                                       timeout, expiration, GFP_KERNEL);
5863         if (elem.priv == NULL)
5864                 goto err_parse_data;
5865
5866         ext = nft_set_elem_ext(set, elem.priv);
5867         if (flags)
5868                 *nft_set_ext_flags(ext) = flags;
5869         if (ulen > 0) {
5870                 udata = nft_set_ext_userdata(ext);
5871                 udata->len = ulen - 1;
5872                 nla_memcpy(&udata->data, nla[NFTA_SET_ELEM_USERDATA], ulen);
5873         }
5874         if (obj) {
5875                 *nft_set_ext_obj(ext) = obj;
5876                 obj->use++;
5877         }
5878         err = nft_set_elem_expr_setup(ctx, ext, expr_array, num_exprs);
5879         if (err < 0)
5880                 goto err_elem_expr;
5881
5882         trans = nft_trans_elem_alloc(ctx, NFT_MSG_NEWSETELEM, set);
5883         if (trans == NULL) {
5884                 err = -ENOMEM;
5885                 goto err_elem_expr;
5886         }
5887
5888         ext->genmask = nft_genmask_cur(ctx->net) | NFT_SET_ELEM_BUSY_MASK;
5889
5890         err = nft_setelem_insert(ctx->net, set, &elem, &ext2, flags);
5891         if (err) {
5892                 if (err == -EEXIST) {
5893                         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA) ^
5894                             nft_set_ext_exists(ext2, NFT_SET_EXT_DATA) ||
5895                             nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF) ^
5896                             nft_set_ext_exists(ext2, NFT_SET_EXT_OBJREF))
5897                                 goto err_element_clash;
5898                         if ((nft_set_ext_exists(ext, NFT_SET_EXT_DATA) &&
5899                              nft_set_ext_exists(ext2, NFT_SET_EXT_DATA) &&
5900                              memcmp(nft_set_ext_data(ext),
5901                                     nft_set_ext_data(ext2), set->dlen) != 0) ||
5902                             (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF) &&
5903                              nft_set_ext_exists(ext2, NFT_SET_EXT_OBJREF) &&
5904                              *nft_set_ext_obj(ext) != *nft_set_ext_obj(ext2)))
5905                                 goto err_element_clash;
5906                         else if (!(nlmsg_flags & NLM_F_EXCL))
5907                                 err = 0;
5908                 } else if (err == -ENOTEMPTY) {
5909                         /* ENOTEMPTY reports overlapping between this element
5910                          * and an existing one.
5911                          */
5912                         err = -EEXIST;
5913                 }
5914                 goto err_element_clash;
5915         }
5916
5917         if (!(flags & NFT_SET_ELEM_CATCHALL) && set->size &&
5918             !atomic_add_unless(&set->nelems, 1, set->size + set->ndeact)) {
5919                 err = -ENFILE;
5920                 goto err_set_full;
5921         }
5922
5923         nft_trans_elem(trans) = elem;
5924         nft_trans_commit_list_add_tail(ctx->net, trans);
5925         return 0;
5926
5927 err_set_full:
5928         nft_setelem_remove(ctx->net, set, &elem);
5929 err_element_clash:
5930         kfree(trans);
5931 err_elem_expr:
5932         if (obj)
5933                 obj->use--;
5934
5935         nf_tables_set_elem_destroy(ctx, set, elem.priv);
5936 err_parse_data:
5937         if (nla[NFTA_SET_ELEM_DATA] != NULL)
5938                 nft_data_release(&elem.data.val, desc.type);
5939 err_parse_key_end:
5940         nft_data_release(&elem.key_end.val, NFT_DATA_VALUE);
5941 err_parse_key:
5942         nft_data_release(&elem.key.val, NFT_DATA_VALUE);
5943 err_set_elem_expr:
5944         for (i = 0; i < num_exprs && expr_array[i]; i++)
5945                 nft_expr_destroy(ctx, expr_array[i]);
5946 err_set_elem_expr_clone:
5947         return err;
5948 }
5949
5950 static int nf_tables_newsetelem(struct sk_buff *skb,
5951                                 const struct nfnl_info *info,
5952                                 const struct nlattr * const nla[])
5953 {
5954         struct nftables_pernet *nft_net = nft_pernet(info->net);
5955         struct netlink_ext_ack *extack = info->extack;
5956         u8 genmask = nft_genmask_next(info->net);
5957         u8 family = info->nfmsg->nfgen_family;
5958         struct net *net = info->net;
5959         const struct nlattr *attr;
5960         struct nft_table *table;
5961         struct nft_set *set;
5962         struct nft_ctx ctx;
5963         int rem, err;
5964
5965         if (nla[NFTA_SET_ELEM_LIST_ELEMENTS] == NULL)
5966                 return -EINVAL;
5967
5968         table = nft_table_lookup(net, nla[NFTA_SET_ELEM_LIST_TABLE], family,
5969                                  genmask, NETLINK_CB(skb).portid);
5970         if (IS_ERR(table)) {
5971                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_ELEM_LIST_TABLE]);
5972                 return PTR_ERR(table);
5973         }
5974
5975         set = nft_set_lookup_global(net, table, nla[NFTA_SET_ELEM_LIST_SET],
5976                                     nla[NFTA_SET_ELEM_LIST_SET_ID], genmask);
5977         if (IS_ERR(set))
5978                 return PTR_ERR(set);
5979
5980         if (!list_empty(&set->bindings) && set->flags & NFT_SET_CONSTANT)
5981                 return -EBUSY;
5982
5983         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
5984
5985         nla_for_each_nested(attr, nla[NFTA_SET_ELEM_LIST_ELEMENTS], rem) {
5986                 err = nft_add_set_elem(&ctx, set, attr, info->nlh->nlmsg_flags);
5987                 if (err < 0)
5988                         return err;
5989         }
5990
5991         if (nft_net->validate_state == NFT_VALIDATE_DO)
5992                 return nft_table_validate(net, table);
5993
5994         return 0;
5995 }
5996
5997 /**
5998  *      nft_data_hold - hold a nft_data item
5999  *
6000  *      @data: struct nft_data to release
6001  *      @type: type of data
6002  *
6003  *      Hold a nft_data item. NFT_DATA_VALUE types can be silently discarded,
6004  *      NFT_DATA_VERDICT bumps the reference to chains in case of NFT_JUMP and
6005  *      NFT_GOTO verdicts. This function must be called on active data objects
6006  *      from the second phase of the commit protocol.
6007  */
6008 void nft_data_hold(const struct nft_data *data, enum nft_data_types type)
6009 {
6010         struct nft_chain *chain;
6011         struct nft_rule *rule;
6012
6013         if (type == NFT_DATA_VERDICT) {
6014                 switch (data->verdict.code) {
6015                 case NFT_JUMP:
6016                 case NFT_GOTO:
6017                         chain = data->verdict.chain;
6018                         chain->use++;
6019
6020                         if (!nft_chain_is_bound(chain))
6021                                 break;
6022
6023                         chain->table->use++;
6024                         list_for_each_entry(rule, &chain->rules, list)
6025                                 chain->use++;
6026
6027                         nft_chain_add(chain->table, chain);
6028                         break;
6029                 }
6030         }
6031 }
6032
6033 static void nft_setelem_data_activate(const struct net *net,
6034                                       const struct nft_set *set,
6035                                       struct nft_set_elem *elem)
6036 {
6037         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
6038
6039         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA))
6040                 nft_data_hold(nft_set_ext_data(ext), set->dtype);
6041         if (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF))
6042                 (*nft_set_ext_obj(ext))->use++;
6043 }
6044
6045 static void nft_setelem_data_deactivate(const struct net *net,
6046                                         const struct nft_set *set,
6047                                         struct nft_set_elem *elem)
6048 {
6049         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
6050
6051         if (nft_set_ext_exists(ext, NFT_SET_EXT_DATA))
6052                 nft_data_release(nft_set_ext_data(ext), set->dtype);
6053         if (nft_set_ext_exists(ext, NFT_SET_EXT_OBJREF))
6054                 (*nft_set_ext_obj(ext))->use--;
6055 }
6056
6057 static int nft_del_setelem(struct nft_ctx *ctx, struct nft_set *set,
6058                            const struct nlattr *attr)
6059 {
6060         struct nlattr *nla[NFTA_SET_ELEM_MAX + 1];
6061         struct nft_set_ext_tmpl tmpl;
6062         struct nft_set_elem elem;
6063         struct nft_set_ext *ext;
6064         struct nft_trans *trans;
6065         u32 flags = 0;
6066         int err;
6067
6068         err = nla_parse_nested_deprecated(nla, NFTA_SET_ELEM_MAX, attr,
6069                                           nft_set_elem_policy, NULL);
6070         if (err < 0)
6071                 return err;
6072
6073         err = nft_setelem_parse_flags(set, nla[NFTA_SET_ELEM_FLAGS], &flags);
6074         if (err < 0)
6075                 return err;
6076
6077         if (!nla[NFTA_SET_ELEM_KEY] && !(flags & NFT_SET_ELEM_CATCHALL))
6078                 return -EINVAL;
6079
6080         nft_set_ext_prepare(&tmpl);
6081
6082         if (flags != 0)
6083                 nft_set_ext_add(&tmpl, NFT_SET_EXT_FLAGS);
6084
6085         if (nla[NFTA_SET_ELEM_KEY]) {
6086                 err = nft_setelem_parse_key(ctx, set, &elem.key.val,
6087                                             nla[NFTA_SET_ELEM_KEY]);
6088                 if (err < 0)
6089                         return err;
6090
6091                 nft_set_ext_add_length(&tmpl, NFT_SET_EXT_KEY, set->klen);
6092         }
6093
6094         if (nla[NFTA_SET_ELEM_KEY_END]) {
6095                 err = nft_setelem_parse_key(ctx, set, &elem.key_end.val,
6096                                             nla[NFTA_SET_ELEM_KEY_END]);
6097                 if (err < 0)
6098                         return err;
6099
6100                 nft_set_ext_add_length(&tmpl, NFT_SET_EXT_KEY_END, set->klen);
6101         }
6102
6103         err = -ENOMEM;
6104         elem.priv = nft_set_elem_init(set, &tmpl, elem.key.val.data,
6105                                       elem.key_end.val.data, NULL, 0, 0,
6106                                       GFP_KERNEL);
6107         if (elem.priv == NULL)
6108                 goto fail_elem;
6109
6110         ext = nft_set_elem_ext(set, elem.priv);
6111         if (flags)
6112                 *nft_set_ext_flags(ext) = flags;
6113
6114         trans = nft_trans_elem_alloc(ctx, NFT_MSG_DELSETELEM, set);
6115         if (trans == NULL)
6116                 goto fail_trans;
6117
6118         err = nft_setelem_deactivate(ctx->net, set, &elem, flags);
6119         if (err < 0)
6120                 goto fail_ops;
6121
6122         nft_setelem_data_deactivate(ctx->net, set, &elem);
6123
6124         nft_trans_elem(trans) = elem;
6125         nft_trans_commit_list_add_tail(ctx->net, trans);
6126         return 0;
6127
6128 fail_ops:
6129         kfree(trans);
6130 fail_trans:
6131         kfree(elem.priv);
6132 fail_elem:
6133         nft_data_release(&elem.key.val, NFT_DATA_VALUE);
6134         return err;
6135 }
6136
6137 static int nft_setelem_flush(const struct nft_ctx *ctx,
6138                              struct nft_set *set,
6139                              const struct nft_set_iter *iter,
6140                              struct nft_set_elem *elem)
6141 {
6142         struct nft_trans *trans;
6143         int err;
6144
6145         trans = nft_trans_alloc_gfp(ctx, NFT_MSG_DELSETELEM,
6146                                     sizeof(struct nft_trans_elem), GFP_ATOMIC);
6147         if (!trans)
6148                 return -ENOMEM;
6149
6150         if (!set->ops->flush(ctx->net, set, elem->priv)) {
6151                 err = -ENOENT;
6152                 goto err1;
6153         }
6154         set->ndeact++;
6155
6156         nft_setelem_data_deactivate(ctx->net, set, elem);
6157         nft_trans_elem_set(trans) = set;
6158         nft_trans_elem(trans) = *elem;
6159         nft_trans_commit_list_add_tail(ctx->net, trans);
6160
6161         return 0;
6162 err1:
6163         kfree(trans);
6164         return err;
6165 }
6166
6167 static int __nft_set_catchall_flush(const struct nft_ctx *ctx,
6168                                     struct nft_set *set,
6169                                     struct nft_set_elem *elem)
6170 {
6171         struct nft_trans *trans;
6172
6173         trans = nft_trans_alloc_gfp(ctx, NFT_MSG_DELSETELEM,
6174                                     sizeof(struct nft_trans_elem), GFP_KERNEL);
6175         if (!trans)
6176                 return -ENOMEM;
6177
6178         nft_setelem_data_deactivate(ctx->net, set, elem);
6179         nft_trans_elem_set(trans) = set;
6180         nft_trans_elem(trans) = *elem;
6181         nft_trans_commit_list_add_tail(ctx->net, trans);
6182
6183         return 0;
6184 }
6185
6186 static int nft_set_catchall_flush(const struct nft_ctx *ctx,
6187                                   struct nft_set *set)
6188 {
6189         u8 genmask = nft_genmask_next(ctx->net);
6190         struct nft_set_elem_catchall *catchall;
6191         struct nft_set_elem elem;
6192         struct nft_set_ext *ext;
6193         int ret = 0;
6194
6195         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
6196                 ext = nft_set_elem_ext(set, catchall->elem);
6197                 if (!nft_set_elem_active(ext, genmask) ||
6198                     nft_set_elem_mark_busy(ext))
6199                         continue;
6200
6201                 elem.priv = catchall->elem;
6202                 ret = __nft_set_catchall_flush(ctx, set, &elem);
6203                 if (ret < 0)
6204                         break;
6205         }
6206
6207         return ret;
6208 }
6209
6210 static int nft_set_flush(struct nft_ctx *ctx, struct nft_set *set, u8 genmask)
6211 {
6212         struct nft_set_iter iter = {
6213                 .genmask        = genmask,
6214                 .fn             = nft_setelem_flush,
6215         };
6216
6217         set->ops->walk(ctx, set, &iter);
6218         if (!iter.err)
6219                 iter.err = nft_set_catchall_flush(ctx, set);
6220
6221         return iter.err;
6222 }
6223
6224 static int nf_tables_delsetelem(struct sk_buff *skb,
6225                                 const struct nfnl_info *info,
6226                                 const struct nlattr * const nla[])
6227 {
6228         struct netlink_ext_ack *extack = info->extack;
6229         u8 genmask = nft_genmask_next(info->net);
6230         u8 family = info->nfmsg->nfgen_family;
6231         struct net *net = info->net;
6232         const struct nlattr *attr;
6233         struct nft_table *table;
6234         struct nft_set *set;
6235         struct nft_ctx ctx;
6236         int rem, err = 0;
6237
6238         table = nft_table_lookup(net, nla[NFTA_SET_ELEM_LIST_TABLE], family,
6239                                  genmask, NETLINK_CB(skb).portid);
6240         if (IS_ERR(table)) {
6241                 NL_SET_BAD_ATTR(extack, nla[NFTA_SET_ELEM_LIST_TABLE]);
6242                 return PTR_ERR(table);
6243         }
6244
6245         set = nft_set_lookup(table, nla[NFTA_SET_ELEM_LIST_SET], genmask);
6246         if (IS_ERR(set))
6247                 return PTR_ERR(set);
6248         if (!list_empty(&set->bindings) && set->flags & NFT_SET_CONSTANT)
6249                 return -EBUSY;
6250
6251         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
6252
6253         if (!nla[NFTA_SET_ELEM_LIST_ELEMENTS])
6254                 return nft_set_flush(&ctx, set, genmask);
6255
6256         nla_for_each_nested(attr, nla[NFTA_SET_ELEM_LIST_ELEMENTS], rem) {
6257                 err = nft_del_setelem(&ctx, set, attr);
6258                 if (err < 0)
6259                         break;
6260         }
6261         return err;
6262 }
6263
6264 void nft_set_gc_batch_release(struct rcu_head *rcu)
6265 {
6266         struct nft_set_gc_batch *gcb;
6267         unsigned int i;
6268
6269         gcb = container_of(rcu, struct nft_set_gc_batch, head.rcu);
6270         for (i = 0; i < gcb->head.cnt; i++)
6271                 nft_set_elem_destroy(gcb->head.set, gcb->elems[i], true);
6272         kfree(gcb);
6273 }
6274
6275 struct nft_set_gc_batch *nft_set_gc_batch_alloc(const struct nft_set *set,
6276                                                 gfp_t gfp)
6277 {
6278         struct nft_set_gc_batch *gcb;
6279
6280         gcb = kzalloc(sizeof(*gcb), gfp);
6281         if (gcb == NULL)
6282                 return gcb;
6283         gcb->head.set = set;
6284         return gcb;
6285 }
6286
6287 /*
6288  * Stateful objects
6289  */
6290
6291 /**
6292  *      nft_register_obj- register nf_tables stateful object type
6293  *      @obj_type: object type
6294  *
6295  *      Registers the object type for use with nf_tables. Returns zero on
6296  *      success or a negative errno code otherwise.
6297  */
6298 int nft_register_obj(struct nft_object_type *obj_type)
6299 {
6300         if (obj_type->type == NFT_OBJECT_UNSPEC)
6301                 return -EINVAL;
6302
6303         nfnl_lock(NFNL_SUBSYS_NFTABLES);
6304         list_add_rcu(&obj_type->list, &nf_tables_objects);
6305         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
6306         return 0;
6307 }
6308 EXPORT_SYMBOL_GPL(nft_register_obj);
6309
6310 /**
6311  *      nft_unregister_obj - unregister nf_tables object type
6312  *      @obj_type: object type
6313  *
6314  *      Unregisters the object type for use with nf_tables.
6315  */
6316 void nft_unregister_obj(struct nft_object_type *obj_type)
6317 {
6318         nfnl_lock(NFNL_SUBSYS_NFTABLES);
6319         list_del_rcu(&obj_type->list);
6320         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
6321 }
6322 EXPORT_SYMBOL_GPL(nft_unregister_obj);
6323
6324 struct nft_object *nft_obj_lookup(const struct net *net,
6325                                   const struct nft_table *table,
6326                                   const struct nlattr *nla, u32 objtype,
6327                                   u8 genmask)
6328 {
6329         struct nft_object_hash_key k = { .table = table };
6330         char search[NFT_OBJ_MAXNAMELEN];
6331         struct rhlist_head *tmp, *list;
6332         struct nft_object *obj;
6333
6334         nla_strscpy(search, nla, sizeof(search));
6335         k.name = search;
6336
6337         WARN_ON_ONCE(!rcu_read_lock_held() &&
6338                      !lockdep_commit_lock_is_held(net));
6339
6340         rcu_read_lock();
6341         list = rhltable_lookup(&nft_objname_ht, &k, nft_objname_ht_params);
6342         if (!list)
6343                 goto out;
6344
6345         rhl_for_each_entry_rcu(obj, tmp, list, rhlhead) {
6346                 if (objtype == obj->ops->type->type &&
6347                     nft_active_genmask(obj, genmask)) {
6348                         rcu_read_unlock();
6349                         return obj;
6350                 }
6351         }
6352 out:
6353         rcu_read_unlock();
6354         return ERR_PTR(-ENOENT);
6355 }
6356 EXPORT_SYMBOL_GPL(nft_obj_lookup);
6357
6358 static struct nft_object *nft_obj_lookup_byhandle(const struct nft_table *table,
6359                                                   const struct nlattr *nla,
6360                                                   u32 objtype, u8 genmask)
6361 {
6362         struct nft_object *obj;
6363
6364         list_for_each_entry(obj, &table->objects, list) {
6365                 if (be64_to_cpu(nla_get_be64(nla)) == obj->handle &&
6366                     objtype == obj->ops->type->type &&
6367                     nft_active_genmask(obj, genmask))
6368                         return obj;
6369         }
6370         return ERR_PTR(-ENOENT);
6371 }
6372
6373 static const struct nla_policy nft_obj_policy[NFTA_OBJ_MAX + 1] = {
6374         [NFTA_OBJ_TABLE]        = { .type = NLA_STRING,
6375                                     .len = NFT_TABLE_MAXNAMELEN - 1 },
6376         [NFTA_OBJ_NAME]         = { .type = NLA_STRING,
6377                                     .len = NFT_OBJ_MAXNAMELEN - 1 },
6378         [NFTA_OBJ_TYPE]         = { .type = NLA_U32 },
6379         [NFTA_OBJ_DATA]         = { .type = NLA_NESTED },
6380         [NFTA_OBJ_HANDLE]       = { .type = NLA_U64},
6381         [NFTA_OBJ_USERDATA]     = { .type = NLA_BINARY,
6382                                     .len = NFT_USERDATA_MAXLEN },
6383 };
6384
6385 static struct nft_object *nft_obj_init(const struct nft_ctx *ctx,
6386                                        const struct nft_object_type *type,
6387                                        const struct nlattr *attr)
6388 {
6389         struct nlattr **tb;
6390         const struct nft_object_ops *ops;
6391         struct nft_object *obj;
6392         int err = -ENOMEM;
6393
6394         tb = kmalloc_array(type->maxattr + 1, sizeof(*tb), GFP_KERNEL);
6395         if (!tb)
6396                 goto err1;
6397
6398         if (attr) {
6399                 err = nla_parse_nested_deprecated(tb, type->maxattr, attr,
6400                                                   type->policy, NULL);
6401                 if (err < 0)
6402                         goto err2;
6403         } else {
6404                 memset(tb, 0, sizeof(tb[0]) * (type->maxattr + 1));
6405         }
6406
6407         if (type->select_ops) {
6408                 ops = type->select_ops(ctx, (const struct nlattr * const *)tb);
6409                 if (IS_ERR(ops)) {
6410                         err = PTR_ERR(ops);
6411                         goto err2;
6412                 }
6413         } else {
6414                 ops = type->ops;
6415         }
6416
6417         err = -ENOMEM;
6418         obj = kzalloc(sizeof(*obj) + ops->size, GFP_KERNEL);
6419         if (!obj)
6420                 goto err2;
6421
6422         err = ops->init(ctx, (const struct nlattr * const *)tb, obj);
6423         if (err < 0)
6424                 goto err3;
6425
6426         obj->ops = ops;
6427
6428         kfree(tb);
6429         return obj;
6430 err3:
6431         kfree(obj);
6432 err2:
6433         kfree(tb);
6434 err1:
6435         return ERR_PTR(err);
6436 }
6437
6438 static int nft_object_dump(struct sk_buff *skb, unsigned int attr,
6439                            struct nft_object *obj, bool reset)
6440 {
6441         struct nlattr *nest;
6442
6443         nest = nla_nest_start_noflag(skb, attr);
6444         if (!nest)
6445                 goto nla_put_failure;
6446         if (obj->ops->dump(skb, obj, reset) < 0)
6447                 goto nla_put_failure;
6448         nla_nest_end(skb, nest);
6449         return 0;
6450
6451 nla_put_failure:
6452         return -1;
6453 }
6454
6455 static const struct nft_object_type *__nft_obj_type_get(u32 objtype)
6456 {
6457         const struct nft_object_type *type;
6458
6459         list_for_each_entry(type, &nf_tables_objects, list) {
6460                 if (objtype == type->type)
6461                         return type;
6462         }
6463         return NULL;
6464 }
6465
6466 static const struct nft_object_type *
6467 nft_obj_type_get(struct net *net, u32 objtype)
6468 {
6469         const struct nft_object_type *type;
6470
6471         type = __nft_obj_type_get(objtype);
6472         if (type != NULL && try_module_get(type->owner))
6473                 return type;
6474
6475         lockdep_nfnl_nft_mutex_not_held();
6476 #ifdef CONFIG_MODULES
6477         if (type == NULL) {
6478                 if (nft_request_module(net, "nft-obj-%u", objtype) == -EAGAIN)
6479                         return ERR_PTR(-EAGAIN);
6480         }
6481 #endif
6482         return ERR_PTR(-ENOENT);
6483 }
6484
6485 static int nf_tables_updobj(const struct nft_ctx *ctx,
6486                             const struct nft_object_type *type,
6487                             const struct nlattr *attr,
6488                             struct nft_object *obj)
6489 {
6490         struct nft_object *newobj;
6491         struct nft_trans *trans;
6492         int err;
6493
6494         trans = nft_trans_alloc(ctx, NFT_MSG_NEWOBJ,
6495                                 sizeof(struct nft_trans_obj));
6496         if (!trans)
6497                 return -ENOMEM;
6498
6499         newobj = nft_obj_init(ctx, type, attr);
6500         if (IS_ERR(newobj)) {
6501                 err = PTR_ERR(newobj);
6502                 goto err_free_trans;
6503         }
6504
6505         nft_trans_obj(trans) = obj;
6506         nft_trans_obj_update(trans) = true;
6507         nft_trans_obj_newobj(trans) = newobj;
6508         nft_trans_commit_list_add_tail(ctx->net, trans);
6509
6510         return 0;
6511
6512 err_free_trans:
6513         kfree(trans);
6514         return err;
6515 }
6516
6517 static int nf_tables_newobj(struct sk_buff *skb, const struct nfnl_info *info,
6518                             const struct nlattr * const nla[])
6519 {
6520         struct netlink_ext_ack *extack = info->extack;
6521         u8 genmask = nft_genmask_next(info->net);
6522         u8 family = info->nfmsg->nfgen_family;
6523         const struct nft_object_type *type;
6524         struct net *net = info->net;
6525         struct nft_table *table;
6526         struct nft_object *obj;
6527         struct nft_ctx ctx;
6528         u32 objtype;
6529         int err;
6530
6531         if (!nla[NFTA_OBJ_TYPE] ||
6532             !nla[NFTA_OBJ_NAME] ||
6533             !nla[NFTA_OBJ_DATA])
6534                 return -EINVAL;
6535
6536         table = nft_table_lookup(net, nla[NFTA_OBJ_TABLE], family, genmask,
6537                                  NETLINK_CB(skb).portid);
6538         if (IS_ERR(table)) {
6539                 NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_TABLE]);
6540                 return PTR_ERR(table);
6541         }
6542
6543         objtype = ntohl(nla_get_be32(nla[NFTA_OBJ_TYPE]));
6544         obj = nft_obj_lookup(net, table, nla[NFTA_OBJ_NAME], objtype, genmask);
6545         if (IS_ERR(obj)) {
6546                 err = PTR_ERR(obj);
6547                 if (err != -ENOENT) {
6548                         NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_NAME]);
6549                         return err;
6550                 }
6551         } else {
6552                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
6553                         NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_NAME]);
6554                         return -EEXIST;
6555                 }
6556                 if (info->nlh->nlmsg_flags & NLM_F_REPLACE)
6557                         return -EOPNOTSUPP;
6558
6559                 type = __nft_obj_type_get(objtype);
6560                 nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
6561
6562                 return nf_tables_updobj(&ctx, type, nla[NFTA_OBJ_DATA], obj);
6563         }
6564
6565         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
6566
6567         type = nft_obj_type_get(net, objtype);
6568         if (IS_ERR(type))
6569                 return PTR_ERR(type);
6570
6571         obj = nft_obj_init(&ctx, type, nla[NFTA_OBJ_DATA]);
6572         if (IS_ERR(obj)) {
6573                 err = PTR_ERR(obj);
6574                 goto err_init;
6575         }
6576         obj->key.table = table;
6577         obj->handle = nf_tables_alloc_handle(table);
6578
6579         obj->key.name = nla_strdup(nla[NFTA_OBJ_NAME], GFP_KERNEL);
6580         if (!obj->key.name) {
6581                 err = -ENOMEM;
6582                 goto err_strdup;
6583         }
6584
6585         if (nla[NFTA_OBJ_USERDATA]) {
6586                 obj->udata = nla_memdup(nla[NFTA_OBJ_USERDATA], GFP_KERNEL);
6587                 if (obj->udata == NULL)
6588                         goto err_userdata;
6589
6590                 obj->udlen = nla_len(nla[NFTA_OBJ_USERDATA]);
6591         }
6592
6593         err = nft_trans_obj_add(&ctx, NFT_MSG_NEWOBJ, obj);
6594         if (err < 0)
6595                 goto err_trans;
6596
6597         err = rhltable_insert(&nft_objname_ht, &obj->rhlhead,
6598                               nft_objname_ht_params);
6599         if (err < 0)
6600                 goto err_obj_ht;
6601
6602         list_add_tail_rcu(&obj->list, &table->objects);
6603         table->use++;
6604         return 0;
6605 err_obj_ht:
6606         /* queued in transaction log */
6607         INIT_LIST_HEAD(&obj->list);
6608         return err;
6609 err_trans:
6610         kfree(obj->udata);
6611 err_userdata:
6612         kfree(obj->key.name);
6613 err_strdup:
6614         if (obj->ops->destroy)
6615                 obj->ops->destroy(&ctx, obj);
6616         kfree(obj);
6617 err_init:
6618         module_put(type->owner);
6619         return err;
6620 }
6621
6622 static int nf_tables_fill_obj_info(struct sk_buff *skb, struct net *net,
6623                                    u32 portid, u32 seq, int event, u32 flags,
6624                                    int family, const struct nft_table *table,
6625                                    struct nft_object *obj, bool reset)
6626 {
6627         struct nlmsghdr *nlh;
6628
6629         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
6630         nlh = nfnl_msg_put(skb, portid, seq, event, flags, family,
6631                            NFNETLINK_V0, nft_base_seq(net));
6632         if (!nlh)
6633                 goto nla_put_failure;
6634
6635         if (nla_put_string(skb, NFTA_OBJ_TABLE, table->name) ||
6636             nla_put_string(skb, NFTA_OBJ_NAME, obj->key.name) ||
6637             nla_put_be32(skb, NFTA_OBJ_TYPE, htonl(obj->ops->type->type)) ||
6638             nla_put_be32(skb, NFTA_OBJ_USE, htonl(obj->use)) ||
6639             nft_object_dump(skb, NFTA_OBJ_DATA, obj, reset) ||
6640             nla_put_be64(skb, NFTA_OBJ_HANDLE, cpu_to_be64(obj->handle),
6641                          NFTA_OBJ_PAD))
6642                 goto nla_put_failure;
6643
6644         if (obj->udata &&
6645             nla_put(skb, NFTA_OBJ_USERDATA, obj->udlen, obj->udata))
6646                 goto nla_put_failure;
6647
6648         nlmsg_end(skb, nlh);
6649         return 0;
6650
6651 nla_put_failure:
6652         nlmsg_trim(skb, nlh);
6653         return -1;
6654 }
6655
6656 struct nft_obj_filter {
6657         char            *table;
6658         u32             type;
6659 };
6660
6661 static int nf_tables_dump_obj(struct sk_buff *skb, struct netlink_callback *cb)
6662 {
6663         const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
6664         const struct nft_table *table;
6665         unsigned int idx = 0, s_idx = cb->args[0];
6666         struct nft_obj_filter *filter = cb->data;
6667         struct net *net = sock_net(skb->sk);
6668         int family = nfmsg->nfgen_family;
6669         struct nftables_pernet *nft_net;
6670         struct nft_object *obj;
6671         bool reset = false;
6672
6673         if (NFNL_MSG_TYPE(cb->nlh->nlmsg_type) == NFT_MSG_GETOBJ_RESET)
6674                 reset = true;
6675
6676         rcu_read_lock();
6677         nft_net = nft_pernet(net);
6678         cb->seq = nft_net->base_seq;
6679
6680         list_for_each_entry_rcu(table, &nft_net->tables, list) {
6681                 if (family != NFPROTO_UNSPEC && family != table->family)
6682                         continue;
6683
6684                 list_for_each_entry_rcu(obj, &table->objects, list) {
6685                         if (!nft_is_active(net, obj))
6686                                 goto cont;
6687                         if (idx < s_idx)
6688                                 goto cont;
6689                         if (idx > s_idx)
6690                                 memset(&cb->args[1], 0,
6691                                        sizeof(cb->args) - sizeof(cb->args[0]));
6692                         if (filter && filter->table &&
6693                             strcmp(filter->table, table->name))
6694                                 goto cont;
6695                         if (filter &&
6696                             filter->type != NFT_OBJECT_UNSPEC &&
6697                             obj->ops->type->type != filter->type)
6698                                 goto cont;
6699                         if (reset) {
6700                                 char *buf = kasprintf(GFP_ATOMIC,
6701                                                       "%s:%u",
6702                                                       table->name,
6703                                                       nft_net->base_seq);
6704
6705                                 audit_log_nfcfg(buf,
6706                                                 family,
6707                                                 obj->handle,
6708                                                 AUDIT_NFT_OP_OBJ_RESET,
6709                                                 GFP_ATOMIC);
6710                                 kfree(buf);
6711                         }
6712
6713                         if (nf_tables_fill_obj_info(skb, net, NETLINK_CB(cb->skb).portid,
6714                                                     cb->nlh->nlmsg_seq,
6715                                                     NFT_MSG_NEWOBJ,
6716                                                     NLM_F_MULTI | NLM_F_APPEND,
6717                                                     table->family, table,
6718                                                     obj, reset) < 0)
6719                                 goto done;
6720
6721                         nl_dump_check_consistent(cb, nlmsg_hdr(skb));
6722 cont:
6723                         idx++;
6724                 }
6725         }
6726 done:
6727         rcu_read_unlock();
6728
6729         cb->args[0] = idx;
6730         return skb->len;
6731 }
6732
6733 static int nf_tables_dump_obj_start(struct netlink_callback *cb)
6734 {
6735         const struct nlattr * const *nla = cb->data;
6736         struct nft_obj_filter *filter = NULL;
6737
6738         if (nla[NFTA_OBJ_TABLE] || nla[NFTA_OBJ_TYPE]) {
6739                 filter = kzalloc(sizeof(*filter), GFP_ATOMIC);
6740                 if (!filter)
6741                         return -ENOMEM;
6742
6743                 if (nla[NFTA_OBJ_TABLE]) {
6744                         filter->table = nla_strdup(nla[NFTA_OBJ_TABLE], GFP_ATOMIC);
6745                         if (!filter->table) {
6746                                 kfree(filter);
6747                                 return -ENOMEM;
6748                         }
6749                 }
6750
6751                 if (nla[NFTA_OBJ_TYPE])
6752                         filter->type = ntohl(nla_get_be32(nla[NFTA_OBJ_TYPE]));
6753         }
6754
6755         cb->data = filter;
6756         return 0;
6757 }
6758
6759 static int nf_tables_dump_obj_done(struct netlink_callback *cb)
6760 {
6761         struct nft_obj_filter *filter = cb->data;
6762
6763         if (filter) {
6764                 kfree(filter->table);
6765                 kfree(filter);
6766         }
6767
6768         return 0;
6769 }
6770
6771 /* called with rcu_read_lock held */
6772 static int nf_tables_getobj(struct sk_buff *skb, const struct nfnl_info *info,
6773                             const struct nlattr * const nla[])
6774 {
6775         struct netlink_ext_ack *extack = info->extack;
6776         u8 genmask = nft_genmask_cur(info->net);
6777         u8 family = info->nfmsg->nfgen_family;
6778         const struct nft_table *table;
6779         struct net *net = info->net;
6780         struct nft_object *obj;
6781         struct sk_buff *skb2;
6782         bool reset = false;
6783         u32 objtype;
6784         int err;
6785
6786         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
6787                 struct netlink_dump_control c = {
6788                         .start = nf_tables_dump_obj_start,
6789                         .dump = nf_tables_dump_obj,
6790                         .done = nf_tables_dump_obj_done,
6791                         .module = THIS_MODULE,
6792                         .data = (void *)nla,
6793                 };
6794
6795                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
6796         }
6797
6798         if (!nla[NFTA_OBJ_NAME] ||
6799             !nla[NFTA_OBJ_TYPE])
6800                 return -EINVAL;
6801
6802         table = nft_table_lookup(net, nla[NFTA_OBJ_TABLE], family, genmask, 0);
6803         if (IS_ERR(table)) {
6804                 NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_TABLE]);
6805                 return PTR_ERR(table);
6806         }
6807
6808         objtype = ntohl(nla_get_be32(nla[NFTA_OBJ_TYPE]));
6809         obj = nft_obj_lookup(net, table, nla[NFTA_OBJ_NAME], objtype, genmask);
6810         if (IS_ERR(obj)) {
6811                 NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_NAME]);
6812                 return PTR_ERR(obj);
6813         }
6814
6815         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
6816         if (!skb2)
6817                 return -ENOMEM;
6818
6819         if (NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_GETOBJ_RESET)
6820                 reset = true;
6821
6822         if (reset) {
6823                 const struct nftables_pernet *nft_net;
6824                 char *buf;
6825
6826                 nft_net = nft_pernet(net);
6827                 buf = kasprintf(GFP_ATOMIC, "%s:%u", table->name, nft_net->base_seq);
6828
6829                 audit_log_nfcfg(buf,
6830                                 family,
6831                                 obj->handle,
6832                                 AUDIT_NFT_OP_OBJ_RESET,
6833                                 GFP_ATOMIC);
6834                 kfree(buf);
6835         }
6836
6837         err = nf_tables_fill_obj_info(skb2, net, NETLINK_CB(skb).portid,
6838                                       info->nlh->nlmsg_seq, NFT_MSG_NEWOBJ, 0,
6839                                       family, table, obj, reset);
6840         if (err < 0)
6841                 goto err_fill_obj_info;
6842
6843         return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
6844
6845 err_fill_obj_info:
6846         kfree_skb(skb2);
6847         return err;
6848 }
6849
6850 static void nft_obj_destroy(const struct nft_ctx *ctx, struct nft_object *obj)
6851 {
6852         if (obj->ops->destroy)
6853                 obj->ops->destroy(ctx, obj);
6854
6855         module_put(obj->ops->type->owner);
6856         kfree(obj->key.name);
6857         kfree(obj->udata);
6858         kfree(obj);
6859 }
6860
6861 static int nf_tables_delobj(struct sk_buff *skb, const struct nfnl_info *info,
6862                             const struct nlattr * const nla[])
6863 {
6864         struct netlink_ext_ack *extack = info->extack;
6865         u8 genmask = nft_genmask_next(info->net);
6866         u8 family = info->nfmsg->nfgen_family;
6867         struct net *net = info->net;
6868         const struct nlattr *attr;
6869         struct nft_table *table;
6870         struct nft_object *obj;
6871         struct nft_ctx ctx;
6872         u32 objtype;
6873
6874         if (!nla[NFTA_OBJ_TYPE] ||
6875             (!nla[NFTA_OBJ_NAME] && !nla[NFTA_OBJ_HANDLE]))
6876                 return -EINVAL;
6877
6878         table = nft_table_lookup(net, nla[NFTA_OBJ_TABLE], family, genmask,
6879                                  NETLINK_CB(skb).portid);
6880         if (IS_ERR(table)) {
6881                 NL_SET_BAD_ATTR(extack, nla[NFTA_OBJ_TABLE]);
6882                 return PTR_ERR(table);
6883         }
6884
6885         objtype = ntohl(nla_get_be32(nla[NFTA_OBJ_TYPE]));
6886         if (nla[NFTA_OBJ_HANDLE]) {
6887                 attr = nla[NFTA_OBJ_HANDLE];
6888                 obj = nft_obj_lookup_byhandle(table, attr, objtype, genmask);
6889         } else {
6890                 attr = nla[NFTA_OBJ_NAME];
6891                 obj = nft_obj_lookup(net, table, attr, objtype, genmask);
6892         }
6893
6894         if (IS_ERR(obj)) {
6895                 NL_SET_BAD_ATTR(extack, attr);
6896                 return PTR_ERR(obj);
6897         }
6898         if (obj->use > 0) {
6899                 NL_SET_BAD_ATTR(extack, attr);
6900                 return -EBUSY;
6901         }
6902
6903         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
6904
6905         return nft_delobj(&ctx, obj);
6906 }
6907
6908 void nft_obj_notify(struct net *net, const struct nft_table *table,
6909                     struct nft_object *obj, u32 portid, u32 seq, int event,
6910                     int family, int report, gfp_t gfp)
6911 {
6912         struct nftables_pernet *nft_net = nft_pernet(net);
6913         struct sk_buff *skb;
6914         int err;
6915         char *buf = kasprintf(gfp, "%s:%u",
6916                               table->name, nft_net->base_seq);
6917
6918         audit_log_nfcfg(buf,
6919                         family,
6920                         obj->handle,
6921                         event == NFT_MSG_NEWOBJ ?
6922                                  AUDIT_NFT_OP_OBJ_REGISTER :
6923                                  AUDIT_NFT_OP_OBJ_UNREGISTER,
6924                         gfp);
6925         kfree(buf);
6926
6927         if (!report &&
6928             !nfnetlink_has_listeners(net, NFNLGRP_NFTABLES))
6929                 return;
6930
6931         skb = nlmsg_new(NLMSG_GOODSIZE, gfp);
6932         if (skb == NULL)
6933                 goto err;
6934
6935         err = nf_tables_fill_obj_info(skb, net, portid, seq, event, 0, family,
6936                                       table, obj, false);
6937         if (err < 0) {
6938                 kfree_skb(skb);
6939                 goto err;
6940         }
6941
6942         nft_notify_enqueue(skb, report, &nft_net->notify_list);
6943         return;
6944 err:
6945         nfnetlink_set_err(net, portid, NFNLGRP_NFTABLES, -ENOBUFS);
6946 }
6947 EXPORT_SYMBOL_GPL(nft_obj_notify);
6948
6949 static void nf_tables_obj_notify(const struct nft_ctx *ctx,
6950                                  struct nft_object *obj, int event)
6951 {
6952         nft_obj_notify(ctx->net, ctx->table, obj, ctx->portid, ctx->seq, event,
6953                        ctx->family, ctx->report, GFP_KERNEL);
6954 }
6955
6956 /*
6957  * Flow tables
6958  */
6959 void nft_register_flowtable_type(struct nf_flowtable_type *type)
6960 {
6961         nfnl_lock(NFNL_SUBSYS_NFTABLES);
6962         list_add_tail_rcu(&type->list, &nf_tables_flowtables);
6963         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
6964 }
6965 EXPORT_SYMBOL_GPL(nft_register_flowtable_type);
6966
6967 void nft_unregister_flowtable_type(struct nf_flowtable_type *type)
6968 {
6969         nfnl_lock(NFNL_SUBSYS_NFTABLES);
6970         list_del_rcu(&type->list);
6971         nfnl_unlock(NFNL_SUBSYS_NFTABLES);
6972 }
6973 EXPORT_SYMBOL_GPL(nft_unregister_flowtable_type);
6974
6975 static const struct nla_policy nft_flowtable_policy[NFTA_FLOWTABLE_MAX + 1] = {
6976         [NFTA_FLOWTABLE_TABLE]          = { .type = NLA_STRING,
6977                                             .len = NFT_NAME_MAXLEN - 1 },
6978         [NFTA_FLOWTABLE_NAME]           = { .type = NLA_STRING,
6979                                             .len = NFT_NAME_MAXLEN - 1 },
6980         [NFTA_FLOWTABLE_HOOK]           = { .type = NLA_NESTED },
6981         [NFTA_FLOWTABLE_HANDLE]         = { .type = NLA_U64 },
6982         [NFTA_FLOWTABLE_FLAGS]          = { .type = NLA_U32 },
6983 };
6984
6985 struct nft_flowtable *nft_flowtable_lookup(const struct nft_table *table,
6986                                            const struct nlattr *nla, u8 genmask)
6987 {
6988         struct nft_flowtable *flowtable;
6989
6990         list_for_each_entry_rcu(flowtable, &table->flowtables, list) {
6991                 if (!nla_strcmp(nla, flowtable->name) &&
6992                     nft_active_genmask(flowtable, genmask))
6993                         return flowtable;
6994         }
6995         return ERR_PTR(-ENOENT);
6996 }
6997 EXPORT_SYMBOL_GPL(nft_flowtable_lookup);
6998
6999 void nf_tables_deactivate_flowtable(const struct nft_ctx *ctx,
7000                                     struct nft_flowtable *flowtable,
7001                                     enum nft_trans_phase phase)
7002 {
7003         switch (phase) {
7004         case NFT_TRANS_PREPARE:
7005         case NFT_TRANS_ABORT:
7006         case NFT_TRANS_RELEASE:
7007                 flowtable->use--;
7008                 fallthrough;
7009         default:
7010                 return;
7011         }
7012 }
7013 EXPORT_SYMBOL_GPL(nf_tables_deactivate_flowtable);
7014
7015 static struct nft_flowtable *
7016 nft_flowtable_lookup_byhandle(const struct nft_table *table,
7017                               const struct nlattr *nla, u8 genmask)
7018 {
7019        struct nft_flowtable *flowtable;
7020
7021        list_for_each_entry(flowtable, &table->flowtables, list) {
7022                if (be64_to_cpu(nla_get_be64(nla)) == flowtable->handle &&
7023                    nft_active_genmask(flowtable, genmask))
7024                        return flowtable;
7025        }
7026        return ERR_PTR(-ENOENT);
7027 }
7028
7029 struct nft_flowtable_hook {
7030         u32                     num;
7031         int                     priority;
7032         struct list_head        list;
7033 };
7034
7035 static const struct nla_policy nft_flowtable_hook_policy[NFTA_FLOWTABLE_HOOK_MAX + 1] = {
7036         [NFTA_FLOWTABLE_HOOK_NUM]       = { .type = NLA_U32 },
7037         [NFTA_FLOWTABLE_HOOK_PRIORITY]  = { .type = NLA_U32 },
7038         [NFTA_FLOWTABLE_HOOK_DEVS]      = { .type = NLA_NESTED },
7039 };
7040
7041 static int nft_flowtable_parse_hook(const struct nft_ctx *ctx,
7042                                     const struct nlattr *attr,
7043                                     struct nft_flowtable_hook *flowtable_hook,
7044                                     struct nft_flowtable *flowtable, bool add)
7045 {
7046         struct nlattr *tb[NFTA_FLOWTABLE_HOOK_MAX + 1];
7047         struct nft_hook *hook;
7048         int hooknum, priority;
7049         int err;
7050
7051         INIT_LIST_HEAD(&flowtable_hook->list);
7052
7053         err = nla_parse_nested_deprecated(tb, NFTA_FLOWTABLE_HOOK_MAX, attr,
7054                                           nft_flowtable_hook_policy, NULL);
7055         if (err < 0)
7056                 return err;
7057
7058         if (add) {
7059                 if (!tb[NFTA_FLOWTABLE_HOOK_NUM] ||
7060                     !tb[NFTA_FLOWTABLE_HOOK_PRIORITY])
7061                         return -EINVAL;
7062
7063                 hooknum = ntohl(nla_get_be32(tb[NFTA_FLOWTABLE_HOOK_NUM]));
7064                 if (hooknum != NF_NETDEV_INGRESS)
7065                         return -EOPNOTSUPP;
7066
7067                 priority = ntohl(nla_get_be32(tb[NFTA_FLOWTABLE_HOOK_PRIORITY]));
7068
7069                 flowtable_hook->priority        = priority;
7070                 flowtable_hook->num             = hooknum;
7071         } else {
7072                 if (tb[NFTA_FLOWTABLE_HOOK_NUM]) {
7073                         hooknum = ntohl(nla_get_be32(tb[NFTA_FLOWTABLE_HOOK_NUM]));
7074                         if (hooknum != flowtable->hooknum)
7075                                 return -EOPNOTSUPP;
7076                 }
7077
7078                 if (tb[NFTA_FLOWTABLE_HOOK_PRIORITY]) {
7079                         priority = ntohl(nla_get_be32(tb[NFTA_FLOWTABLE_HOOK_PRIORITY]));
7080                         if (priority != flowtable->data.priority)
7081                                 return -EOPNOTSUPP;
7082                 }
7083
7084                 flowtable_hook->priority        = flowtable->data.priority;
7085                 flowtable_hook->num             = flowtable->hooknum;
7086         }
7087
7088         if (tb[NFTA_FLOWTABLE_HOOK_DEVS]) {
7089                 err = nf_tables_parse_netdev_hooks(ctx->net,
7090                                                    tb[NFTA_FLOWTABLE_HOOK_DEVS],
7091                                                    &flowtable_hook->list);
7092                 if (err < 0)
7093                         return err;
7094         }
7095
7096         list_for_each_entry(hook, &flowtable_hook->list, list) {
7097                 hook->ops.pf            = NFPROTO_NETDEV;
7098                 hook->ops.hooknum       = flowtable_hook->num;
7099                 hook->ops.priority      = flowtable_hook->priority;
7100                 hook->ops.priv          = &flowtable->data;
7101                 hook->ops.hook          = flowtable->data.type->hook;
7102         }
7103
7104         return err;
7105 }
7106
7107 static const struct nf_flowtable_type *__nft_flowtable_type_get(u8 family)
7108 {
7109         const struct nf_flowtable_type *type;
7110
7111         list_for_each_entry(type, &nf_tables_flowtables, list) {
7112                 if (family == type->family)
7113                         return type;
7114         }
7115         return NULL;
7116 }
7117
7118 static const struct nf_flowtable_type *
7119 nft_flowtable_type_get(struct net *net, u8 family)
7120 {
7121         const struct nf_flowtable_type *type;
7122
7123         type = __nft_flowtable_type_get(family);
7124         if (type != NULL && try_module_get(type->owner))
7125                 return type;
7126
7127         lockdep_nfnl_nft_mutex_not_held();
7128 #ifdef CONFIG_MODULES
7129         if (type == NULL) {
7130                 if (nft_request_module(net, "nf-flowtable-%u", family) == -EAGAIN)
7131                         return ERR_PTR(-EAGAIN);
7132         }
7133 #endif
7134         return ERR_PTR(-ENOENT);
7135 }
7136
7137 /* Only called from error and netdev event paths. */
7138 static void nft_unregister_flowtable_hook(struct net *net,
7139                                           struct nft_flowtable *flowtable,
7140                                           struct nft_hook *hook)
7141 {
7142         nf_unregister_net_hook(net, &hook->ops);
7143         flowtable->data.type->setup(&flowtable->data, hook->ops.dev,
7144                                     FLOW_BLOCK_UNBIND);
7145 }
7146
7147 static void nft_unregister_flowtable_net_hooks(struct net *net,
7148                                                struct list_head *hook_list)
7149 {
7150         struct nft_hook *hook;
7151
7152         list_for_each_entry(hook, hook_list, list)
7153                 nf_unregister_net_hook(net, &hook->ops);
7154 }
7155
7156 static int nft_register_flowtable_net_hooks(struct net *net,
7157                                             struct nft_table *table,
7158                                             struct list_head *hook_list,
7159                                             struct nft_flowtable *flowtable)
7160 {
7161         struct nft_hook *hook, *hook2, *next;
7162         struct nft_flowtable *ft;
7163         int err, i = 0;
7164
7165         list_for_each_entry(hook, hook_list, list) {
7166                 list_for_each_entry(ft, &table->flowtables, list) {
7167                         if (!nft_is_active_next(net, ft))
7168                                 continue;
7169
7170                         list_for_each_entry(hook2, &ft->hook_list, list) {
7171                                 if (hook->ops.dev == hook2->ops.dev &&
7172                                     hook->ops.pf == hook2->ops.pf) {
7173                                         err = -EEXIST;
7174                                         goto err_unregister_net_hooks;
7175                                 }
7176                         }
7177                 }
7178
7179                 err = flowtable->data.type->setup(&flowtable->data,
7180                                                   hook->ops.dev,
7181                                                   FLOW_BLOCK_BIND);
7182                 if (err < 0)
7183                         goto err_unregister_net_hooks;
7184
7185                 err = nf_register_net_hook(net, &hook->ops);
7186                 if (err < 0) {
7187                         flowtable->data.type->setup(&flowtable->data,
7188                                                     hook->ops.dev,
7189                                                     FLOW_BLOCK_UNBIND);
7190                         goto err_unregister_net_hooks;
7191                 }
7192
7193                 i++;
7194         }
7195
7196         return 0;
7197
7198 err_unregister_net_hooks:
7199         list_for_each_entry_safe(hook, next, hook_list, list) {
7200                 if (i-- <= 0)
7201                         break;
7202
7203                 nft_unregister_flowtable_hook(net, flowtable, hook);
7204                 list_del_rcu(&hook->list);
7205                 kfree_rcu(hook, rcu);
7206         }
7207
7208         return err;
7209 }
7210
7211 static void nft_flowtable_hooks_destroy(struct list_head *hook_list)
7212 {
7213         struct nft_hook *hook, *next;
7214
7215         list_for_each_entry_safe(hook, next, hook_list, list) {
7216                 list_del_rcu(&hook->list);
7217                 kfree_rcu(hook, rcu);
7218         }
7219 }
7220
7221 static int nft_flowtable_update(struct nft_ctx *ctx, const struct nlmsghdr *nlh,
7222                                 struct nft_flowtable *flowtable)
7223 {
7224         const struct nlattr * const *nla = ctx->nla;
7225         struct nft_flowtable_hook flowtable_hook;
7226         struct nft_hook *hook, *next;
7227         struct nft_trans *trans;
7228         bool unregister = false;
7229         u32 flags;
7230         int err;
7231
7232         err = nft_flowtable_parse_hook(ctx, nla[NFTA_FLOWTABLE_HOOK],
7233                                        &flowtable_hook, flowtable, false);
7234         if (err < 0)
7235                 return err;
7236
7237         list_for_each_entry_safe(hook, next, &flowtable_hook.list, list) {
7238                 if (nft_hook_list_find(&flowtable->hook_list, hook)) {
7239                         list_del(&hook->list);
7240                         kfree(hook);
7241                 }
7242         }
7243
7244         if (nla[NFTA_FLOWTABLE_FLAGS]) {
7245                 flags = ntohl(nla_get_be32(nla[NFTA_FLOWTABLE_FLAGS]));
7246                 if (flags & ~NFT_FLOWTABLE_MASK)
7247                         return -EOPNOTSUPP;
7248                 if ((flowtable->data.flags & NFT_FLOWTABLE_HW_OFFLOAD) ^
7249                     (flags & NFT_FLOWTABLE_HW_OFFLOAD))
7250                         return -EOPNOTSUPP;
7251         } else {
7252                 flags = flowtable->data.flags;
7253         }
7254
7255         err = nft_register_flowtable_net_hooks(ctx->net, ctx->table,
7256                                                &flowtable_hook.list, flowtable);
7257         if (err < 0)
7258                 goto err_flowtable_update_hook;
7259
7260         trans = nft_trans_alloc(ctx, NFT_MSG_NEWFLOWTABLE,
7261                                 sizeof(struct nft_trans_flowtable));
7262         if (!trans) {
7263                 unregister = true;
7264                 err = -ENOMEM;
7265                 goto err_flowtable_update_hook;
7266         }
7267
7268         nft_trans_flowtable_flags(trans) = flags;
7269         nft_trans_flowtable(trans) = flowtable;
7270         nft_trans_flowtable_update(trans) = true;
7271         INIT_LIST_HEAD(&nft_trans_flowtable_hooks(trans));
7272         list_splice(&flowtable_hook.list, &nft_trans_flowtable_hooks(trans));
7273
7274         nft_trans_commit_list_add_tail(ctx->net, trans);
7275
7276         return 0;
7277
7278 err_flowtable_update_hook:
7279         list_for_each_entry_safe(hook, next, &flowtable_hook.list, list) {
7280                 if (unregister)
7281                         nft_unregister_flowtable_hook(ctx->net, flowtable, hook);
7282                 list_del_rcu(&hook->list);
7283                 kfree_rcu(hook, rcu);
7284         }
7285
7286         return err;
7287
7288 }
7289
7290 static int nf_tables_newflowtable(struct sk_buff *skb,
7291                                   const struct nfnl_info *info,
7292                                   const struct nlattr * const nla[])
7293 {
7294         struct netlink_ext_ack *extack = info->extack;
7295         struct nft_flowtable_hook flowtable_hook;
7296         u8 genmask = nft_genmask_next(info->net);
7297         u8 family = info->nfmsg->nfgen_family;
7298         const struct nf_flowtable_type *type;
7299         struct nft_flowtable *flowtable;
7300         struct nft_hook *hook, *next;
7301         struct net *net = info->net;
7302         struct nft_table *table;
7303         struct nft_ctx ctx;
7304         int err;
7305
7306         if (!nla[NFTA_FLOWTABLE_TABLE] ||
7307             !nla[NFTA_FLOWTABLE_NAME] ||
7308             !nla[NFTA_FLOWTABLE_HOOK])
7309                 return -EINVAL;
7310
7311         table = nft_table_lookup(net, nla[NFTA_FLOWTABLE_TABLE], family,
7312                                  genmask, NETLINK_CB(skb).portid);
7313         if (IS_ERR(table)) {
7314                 NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_TABLE]);
7315                 return PTR_ERR(table);
7316         }
7317
7318         flowtable = nft_flowtable_lookup(table, nla[NFTA_FLOWTABLE_NAME],
7319                                          genmask);
7320         if (IS_ERR(flowtable)) {
7321                 err = PTR_ERR(flowtable);
7322                 if (err != -ENOENT) {
7323                         NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_NAME]);
7324                         return err;
7325                 }
7326         } else {
7327                 if (info->nlh->nlmsg_flags & NLM_F_EXCL) {
7328                         NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_NAME]);
7329                         return -EEXIST;
7330                 }
7331
7332                 nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
7333
7334                 return nft_flowtable_update(&ctx, info->nlh, flowtable);
7335         }
7336
7337         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
7338
7339         flowtable = kzalloc(sizeof(*flowtable), GFP_KERNEL);
7340         if (!flowtable)
7341                 return -ENOMEM;
7342
7343         flowtable->table = table;
7344         flowtable->handle = nf_tables_alloc_handle(table);
7345         INIT_LIST_HEAD(&flowtable->hook_list);
7346
7347         flowtable->name = nla_strdup(nla[NFTA_FLOWTABLE_NAME], GFP_KERNEL);
7348         if (!flowtable->name) {
7349                 err = -ENOMEM;
7350                 goto err1;
7351         }
7352
7353         type = nft_flowtable_type_get(net, family);
7354         if (IS_ERR(type)) {
7355                 err = PTR_ERR(type);
7356                 goto err2;
7357         }
7358
7359         if (nla[NFTA_FLOWTABLE_FLAGS]) {
7360                 flowtable->data.flags =
7361                         ntohl(nla_get_be32(nla[NFTA_FLOWTABLE_FLAGS]));
7362                 if (flowtable->data.flags & ~NFT_FLOWTABLE_MASK) {
7363                         err = -EOPNOTSUPP;
7364                         goto err3;
7365                 }
7366         }
7367
7368         write_pnet(&flowtable->data.net, net);
7369         flowtable->data.type = type;
7370         err = type->init(&flowtable->data);
7371         if (err < 0)
7372                 goto err3;
7373
7374         err = nft_flowtable_parse_hook(&ctx, nla[NFTA_FLOWTABLE_HOOK],
7375                                        &flowtable_hook, flowtable, true);
7376         if (err < 0)
7377                 goto err4;
7378
7379         list_splice(&flowtable_hook.list, &flowtable->hook_list);
7380         flowtable->data.priority = flowtable_hook.priority;
7381         flowtable->hooknum = flowtable_hook.num;
7382
7383         err = nft_register_flowtable_net_hooks(ctx.net, table,
7384                                                &flowtable->hook_list,
7385                                                flowtable);
7386         if (err < 0) {
7387                 nft_flowtable_hooks_destroy(&flowtable->hook_list);
7388                 goto err4;
7389         }
7390
7391         err = nft_trans_flowtable_add(&ctx, NFT_MSG_NEWFLOWTABLE, flowtable);
7392         if (err < 0)
7393                 goto err5;
7394
7395         list_add_tail_rcu(&flowtable->list, &table->flowtables);
7396         table->use++;
7397
7398         return 0;
7399 err5:
7400         list_for_each_entry_safe(hook, next, &flowtable->hook_list, list) {
7401                 nft_unregister_flowtable_hook(net, flowtable, hook);
7402                 list_del_rcu(&hook->list);
7403                 kfree_rcu(hook, rcu);
7404         }
7405 err4:
7406         flowtable->data.type->free(&flowtable->data);
7407 err3:
7408         module_put(type->owner);
7409 err2:
7410         kfree(flowtable->name);
7411 err1:
7412         kfree(flowtable);
7413         return err;
7414 }
7415
7416 static void nft_flowtable_hook_release(struct nft_flowtable_hook *flowtable_hook)
7417 {
7418         struct nft_hook *this, *next;
7419
7420         list_for_each_entry_safe(this, next, &flowtable_hook->list, list) {
7421                 list_del(&this->list);
7422                 kfree(this);
7423         }
7424 }
7425
7426 static int nft_delflowtable_hook(struct nft_ctx *ctx,
7427                                  struct nft_flowtable *flowtable)
7428 {
7429         const struct nlattr * const *nla = ctx->nla;
7430         struct nft_flowtable_hook flowtable_hook;
7431         struct nft_hook *this, *hook;
7432         struct nft_trans *trans;
7433         int err;
7434
7435         err = nft_flowtable_parse_hook(ctx, nla[NFTA_FLOWTABLE_HOOK],
7436                                        &flowtable_hook, flowtable, false);
7437         if (err < 0)
7438                 return err;
7439
7440         list_for_each_entry(this, &flowtable_hook.list, list) {
7441                 hook = nft_hook_list_find(&flowtable->hook_list, this);
7442                 if (!hook) {
7443                         err = -ENOENT;
7444                         goto err_flowtable_del_hook;
7445                 }
7446                 hook->inactive = true;
7447         }
7448
7449         trans = nft_trans_alloc(ctx, NFT_MSG_DELFLOWTABLE,
7450                                 sizeof(struct nft_trans_flowtable));
7451         if (!trans) {
7452                 err = -ENOMEM;
7453                 goto err_flowtable_del_hook;
7454         }
7455
7456         nft_trans_flowtable(trans) = flowtable;
7457         nft_trans_flowtable_update(trans) = true;
7458         INIT_LIST_HEAD(&nft_trans_flowtable_hooks(trans));
7459         nft_flowtable_hook_release(&flowtable_hook);
7460
7461         nft_trans_commit_list_add_tail(ctx->net, trans);
7462
7463         return 0;
7464
7465 err_flowtable_del_hook:
7466         list_for_each_entry(this, &flowtable_hook.list, list) {
7467                 hook = nft_hook_list_find(&flowtable->hook_list, this);
7468                 if (!hook)
7469                         break;
7470
7471                 hook->inactive = false;
7472         }
7473         nft_flowtable_hook_release(&flowtable_hook);
7474
7475         return err;
7476 }
7477
7478 static int nf_tables_delflowtable(struct sk_buff *skb,
7479                                   const struct nfnl_info *info,
7480                                   const struct nlattr * const nla[])
7481 {
7482         struct netlink_ext_ack *extack = info->extack;
7483         u8 genmask = nft_genmask_next(info->net);
7484         u8 family = info->nfmsg->nfgen_family;
7485         struct nft_flowtable *flowtable;
7486         struct net *net = info->net;
7487         const struct nlattr *attr;
7488         struct nft_table *table;
7489         struct nft_ctx ctx;
7490
7491         if (!nla[NFTA_FLOWTABLE_TABLE] ||
7492             (!nla[NFTA_FLOWTABLE_NAME] &&
7493              !nla[NFTA_FLOWTABLE_HANDLE]))
7494                 return -EINVAL;
7495
7496         table = nft_table_lookup(net, nla[NFTA_FLOWTABLE_TABLE], family,
7497                                  genmask, NETLINK_CB(skb).portid);
7498         if (IS_ERR(table)) {
7499                 NL_SET_BAD_ATTR(extack, nla[NFTA_FLOWTABLE_TABLE]);
7500                 return PTR_ERR(table);
7501         }
7502
7503         if (nla[NFTA_FLOWTABLE_HANDLE]) {
7504                 attr = nla[NFTA_FLOWTABLE_HANDLE];
7505                 flowtable = nft_flowtable_lookup_byhandle(table, attr, genmask);
7506         } else {
7507                 attr = nla[NFTA_FLOWTABLE_NAME];
7508                 flowtable = nft_flowtable_lookup(table, attr, genmask);
7509         }
7510
7511         if (IS_ERR(flowtable)) {
7512                 NL_SET_BAD_ATTR(extack, attr);
7513                 return PTR_ERR(flowtable);
7514         }
7515
7516         nft_ctx_init(&ctx, net, skb, info->nlh, family, table, NULL, nla);
7517
7518         if (nla[NFTA_FLOWTABLE_HOOK])
7519                 return nft_delflowtable_hook(&ctx, flowtable);
7520
7521         if (flowtable->use > 0) {
7522                 NL_SET_BAD_ATTR(extack, attr);
7523                 return -EBUSY;
7524         }
7525
7526         return nft_delflowtable(&ctx, flowtable);
7527 }
7528
7529 static int nf_tables_fill_flowtable_info(struct sk_buff *skb, struct net *net,
7530                                          u32 portid, u32 seq, int event,
7531                                          u32 flags, int family,
7532                                          struct nft_flowtable *flowtable,
7533                                          struct list_head *hook_list)
7534 {
7535         struct nlattr *nest, *nest_devs;
7536         struct nft_hook *hook;
7537         struct nlmsghdr *nlh;
7538
7539         event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, event);
7540         nlh = nfnl_msg_put(skb, portid, seq, event, flags, family,
7541                            NFNETLINK_V0, nft_base_seq(net));
7542         if (!nlh)
7543                 goto nla_put_failure;
7544
7545         if (nla_put_string(skb, NFTA_FLOWTABLE_TABLE, flowtable->table->name) ||
7546             nla_put_string(skb, NFTA_FLOWTABLE_NAME, flowtable->name) ||
7547             nla_put_be32(skb, NFTA_FLOWTABLE_USE, htonl(flowtable->use)) ||
7548             nla_put_be64(skb, NFTA_FLOWTABLE_HANDLE, cpu_to_be64(flowtable->handle),
7549                          NFTA_FLOWTABLE_PAD) ||
7550             nla_put_be32(skb, NFTA_FLOWTABLE_FLAGS, htonl(flowtable->data.flags)))
7551                 goto nla_put_failure;
7552
7553         nest = nla_nest_start_noflag(skb, NFTA_FLOWTABLE_HOOK);
7554         if (!nest)
7555                 goto nla_put_failure;
7556         if (nla_put_be32(skb, NFTA_FLOWTABLE_HOOK_NUM, htonl(flowtable->hooknum)) ||
7557             nla_put_be32(skb, NFTA_FLOWTABLE_HOOK_PRIORITY, htonl(flowtable->data.priority)))
7558                 goto nla_put_failure;
7559
7560         nest_devs = nla_nest_start_noflag(skb, NFTA_FLOWTABLE_HOOK_DEVS);
7561         if (!nest_devs)
7562                 goto nla_put_failure;
7563
7564         list_for_each_entry_rcu(hook, hook_list, list) {
7565                 if (nla_put_string(skb, NFTA_DEVICE_NAME, hook->ops.dev->name))
7566                         goto nla_put_failure;
7567         }
7568         nla_nest_end(skb, nest_devs);
7569         nla_nest_end(skb, nest);
7570
7571         nlmsg_end(skb, nlh);
7572         return 0;
7573
7574 nla_put_failure:
7575         nlmsg_trim(skb, nlh);
7576         return -1;
7577 }
7578
7579 struct nft_flowtable_filter {
7580         char            *table;
7581 };
7582
7583 static int nf_tables_dump_flowtable(struct sk_buff *skb,
7584                                     struct netlink_callback *cb)
7585 {
7586         const struct nfgenmsg *nfmsg = nlmsg_data(cb->nlh);
7587         struct nft_flowtable_filter *filter = cb->data;
7588         unsigned int idx = 0, s_idx = cb->args[0];
7589         struct net *net = sock_net(skb->sk);
7590         int family = nfmsg->nfgen_family;
7591         struct nft_flowtable *flowtable;
7592         struct nftables_pernet *nft_net;
7593         const struct nft_table *table;
7594
7595         rcu_read_lock();
7596         nft_net = nft_pernet(net);
7597         cb->seq = nft_net->base_seq;
7598
7599         list_for_each_entry_rcu(table, &nft_net->tables, list) {
7600                 if (family != NFPROTO_UNSPEC && family != table->family)
7601                         continue;
7602
7603                 list_for_each_entry_rcu(flowtable, &table->flowtables, list) {
7604                         if (!nft_is_active(net, flowtable))
7605                                 goto cont;
7606                         if (idx < s_idx)
7607                                 goto cont;
7608                         if (idx > s_idx)
7609                                 memset(&cb->args[1], 0,
7610                                        sizeof(cb->args) - sizeof(cb->args[0]));
7611                         if (filter && filter->table &&
7612                             strcmp(filter->table, table->name))
7613                                 goto cont;
7614
7615                         if (nf_tables_fill_flowtable_info(skb, net, NETLINK_CB(cb->skb).portid,
7616                                                           cb->nlh->nlmsg_seq,
7617                                                           NFT_MSG_NEWFLOWTABLE,
7618                                                           NLM_F_MULTI | NLM_F_APPEND,
7619                                                           table->family,
7620                                                           flowtable,
7621                                                           &flowtable->hook_list) < 0)
7622                                 goto done;
7623
7624                         nl_dump_check_consistent(cb, nlmsg_hdr(skb));
7625 cont:
7626                         idx++;
7627                 }
7628         }
7629 done:
7630         rcu_read_unlock();
7631
7632         cb->args[0] = idx;
7633         return skb->len;
7634 }
7635
7636 static int nf_tables_dump_flowtable_start(struct netlink_callback *cb)
7637 {
7638         const struct nlattr * const *nla = cb->data;
7639         struct nft_flowtable_filter *filter = NULL;
7640
7641         if (nla[NFTA_FLOWTABLE_TABLE]) {
7642                 filter = kzalloc(sizeof(*filter), GFP_ATOMIC);
7643                 if (!filter)
7644                         return -ENOMEM;
7645
7646                 filter->table = nla_strdup(nla[NFTA_FLOWTABLE_TABLE],
7647                                            GFP_ATOMIC);
7648                 if (!filter->table) {
7649                         kfree(filter);
7650                         return -ENOMEM;
7651                 }
7652         }
7653
7654         cb->data = filter;
7655         return 0;
7656 }
7657
7658 static int nf_tables_dump_flowtable_done(struct netlink_callback *cb)
7659 {
7660         struct nft_flowtable_filter *filter = cb->data;
7661
7662         if (!filter)
7663                 return 0;
7664
7665         kfree(filter->table);
7666         kfree(filter);
7667
7668         return 0;
7669 }
7670
7671 /* called with rcu_read_lock held */
7672 static int nf_tables_getflowtable(struct sk_buff *skb,
7673                                   const struct nfnl_info *info,
7674                                   const struct nlattr * const nla[])
7675 {
7676         u8 genmask = nft_genmask_cur(info->net);
7677         u8 family = info->nfmsg->nfgen_family;
7678         struct nft_flowtable *flowtable;
7679         const struct nft_table *table;
7680         struct net *net = info->net;
7681         struct sk_buff *skb2;
7682         int err;
7683
7684         if (info->nlh->nlmsg_flags & NLM_F_DUMP) {
7685                 struct netlink_dump_control c = {
7686                         .start = nf_tables_dump_flowtable_start,
7687                         .dump = nf_tables_dump_flowtable,
7688                         .done = nf_tables_dump_flowtable_done,
7689                         .module = THIS_MODULE,
7690                         .data = (void *)nla,
7691                 };
7692
7693                 return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c);
7694         }
7695
7696         if (!nla[NFTA_FLOWTABLE_NAME])
7697                 return -EINVAL;
7698
7699         table = nft_table_lookup(net, nla[NFTA_FLOWTABLE_TABLE], family,
7700                                  genmask, 0);
7701         if (IS_ERR(table))
7702                 return PTR_ERR(table);
7703
7704         flowtable = nft_flowtable_lookup(table, nla[NFTA_FLOWTABLE_NAME],
7705                                          genmask);
7706         if (IS_ERR(flowtable))
7707                 return PTR_ERR(flowtable);
7708
7709         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
7710         if (!skb2)
7711                 return -ENOMEM;
7712
7713         err = nf_tables_fill_flowtable_info(skb2, net, NETLINK_CB(skb).portid,
7714                                             info->nlh->nlmsg_seq,
7715                                             NFT_MSG_NEWFLOWTABLE, 0, family,
7716                                             flowtable, &flowtable->hook_list);
7717         if (err < 0)
7718                 goto err_fill_flowtable_info;
7719
7720         return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid);
7721
7722 err_fill_flowtable_info:
7723         kfree_skb(skb2);
7724         return err;
7725 }
7726
7727 static void nf_tables_flowtable_notify(struct nft_ctx *ctx,
7728                                        struct nft_flowtable *flowtable,
7729                                        struct list_head *hook_list,
7730                                        int event)
7731 {
7732         struct nftables_pernet *nft_net = nft_pernet(ctx->net);
7733         struct sk_buff *skb;
7734         int err;
7735
7736         if (!ctx->report &&
7737             !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
7738                 return;
7739
7740         skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
7741         if (skb == NULL)
7742                 goto err;
7743
7744         err = nf_tables_fill_flowtable_info(skb, ctx->net, ctx->portid,
7745                                             ctx->seq, event, 0,
7746                                             ctx->family, flowtable, hook_list);
7747         if (err < 0) {
7748                 kfree_skb(skb);
7749                 goto err;
7750         }
7751
7752         nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
7753         return;
7754 err:
7755         nfnetlink_set_err(ctx->net, ctx->portid, NFNLGRP_NFTABLES, -ENOBUFS);
7756 }
7757
7758 static void nf_tables_flowtable_destroy(struct nft_flowtable *flowtable)
7759 {
7760         struct nft_hook *hook, *next;
7761
7762         flowtable->data.type->free(&flowtable->data);
7763         list_for_each_entry_safe(hook, next, &flowtable->hook_list, list) {
7764                 flowtable->data.type->setup(&flowtable->data, hook->ops.dev,
7765                                             FLOW_BLOCK_UNBIND);
7766                 list_del_rcu(&hook->list);
7767                 kfree(hook);
7768         }
7769         kfree(flowtable->name);
7770         module_put(flowtable->data.type->owner);
7771         kfree(flowtable);
7772 }
7773
7774 static int nf_tables_fill_gen_info(struct sk_buff *skb, struct net *net,
7775                                    u32 portid, u32 seq)
7776 {
7777         struct nftables_pernet *nft_net = nft_pernet(net);
7778         struct nlmsghdr *nlh;
7779         char buf[TASK_COMM_LEN];
7780         int event = nfnl_msg_type(NFNL_SUBSYS_NFTABLES, NFT_MSG_NEWGEN);
7781
7782         nlh = nfnl_msg_put(skb, portid, seq, event, 0, AF_UNSPEC,
7783                            NFNETLINK_V0, nft_base_seq(net));
7784         if (!nlh)
7785                 goto nla_put_failure;
7786
7787         if (nla_put_be32(skb, NFTA_GEN_ID, htonl(nft_net->base_seq)) ||
7788             nla_put_be32(skb, NFTA_GEN_PROC_PID, htonl(task_pid_nr(current))) ||
7789             nla_put_string(skb, NFTA_GEN_PROC_NAME, get_task_comm(buf, current)))
7790                 goto nla_put_failure;
7791
7792         nlmsg_end(skb, nlh);
7793         return 0;
7794
7795 nla_put_failure:
7796         nlmsg_trim(skb, nlh);
7797         return -EMSGSIZE;
7798 }
7799
7800 static void nft_flowtable_event(unsigned long event, struct net_device *dev,
7801                                 struct nft_flowtable *flowtable)
7802 {
7803         struct nft_hook *hook;
7804
7805         list_for_each_entry(hook, &flowtable->hook_list, list) {
7806                 if (hook->ops.dev != dev)
7807                         continue;
7808
7809                 /* flow_offload_netdev_event() cleans up entries for us. */
7810                 nft_unregister_flowtable_hook(dev_net(dev), flowtable, hook);
7811                 list_del_rcu(&hook->list);
7812                 kfree_rcu(hook, rcu);
7813                 break;
7814         }
7815 }
7816
7817 static int nf_tables_flowtable_event(struct notifier_block *this,
7818                                      unsigned long event, void *ptr)
7819 {
7820         struct net_device *dev = netdev_notifier_info_to_dev(ptr);
7821         struct nft_flowtable *flowtable;
7822         struct nftables_pernet *nft_net;
7823         struct nft_table *table;
7824         struct net *net;
7825
7826         if (event != NETDEV_UNREGISTER)
7827                 return 0;
7828
7829         net = dev_net(dev);
7830         nft_net = nft_pernet(net);
7831         mutex_lock(&nft_net->commit_mutex);
7832         list_for_each_entry(table, &nft_net->tables, list) {
7833                 list_for_each_entry(flowtable, &table->flowtables, list) {
7834                         nft_flowtable_event(event, dev, flowtable);
7835                 }
7836         }
7837         mutex_unlock(&nft_net->commit_mutex);
7838
7839         return NOTIFY_DONE;
7840 }
7841
7842 static struct notifier_block nf_tables_flowtable_notifier = {
7843         .notifier_call  = nf_tables_flowtable_event,
7844 };
7845
7846 static void nf_tables_gen_notify(struct net *net, struct sk_buff *skb,
7847                                  int event)
7848 {
7849         struct nlmsghdr *nlh = nlmsg_hdr(skb);
7850         struct sk_buff *skb2;
7851         int err;
7852
7853         if (!nlmsg_report(nlh) &&
7854             !nfnetlink_has_listeners(net, NFNLGRP_NFTABLES))
7855                 return;
7856
7857         skb2 = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
7858         if (skb2 == NULL)
7859                 goto err;
7860
7861         err = nf_tables_fill_gen_info(skb2, net, NETLINK_CB(skb).portid,
7862                                       nlh->nlmsg_seq);
7863         if (err < 0) {
7864                 kfree_skb(skb2);
7865                 goto err;
7866         }
7867
7868         nfnetlink_send(skb2, net, NETLINK_CB(skb).portid, NFNLGRP_NFTABLES,
7869                        nlmsg_report(nlh), GFP_KERNEL);
7870         return;
7871 err:
7872         nfnetlink_set_err(net, NETLINK_CB(skb).portid, NFNLGRP_NFTABLES,
7873                           -ENOBUFS);
7874 }
7875
7876 static int nf_tables_getgen(struct sk_buff *skb, const struct nfnl_info *info,
7877                             const struct nlattr * const nla[])
7878 {
7879         struct sk_buff *skb2;
7880         int err;
7881
7882         skb2 = alloc_skb(NLMSG_GOODSIZE, GFP_ATOMIC);
7883         if (skb2 == NULL)
7884                 return -ENOMEM;
7885
7886         err = nf_tables_fill_gen_info(skb2, info->net, NETLINK_CB(skb).portid,
7887                                       info->nlh->nlmsg_seq);
7888         if (err < 0)
7889                 goto err_fill_gen_info;
7890
7891         return nfnetlink_unicast(skb2, info->net, NETLINK_CB(skb).portid);
7892
7893 err_fill_gen_info:
7894         kfree_skb(skb2);
7895         return err;
7896 }
7897
7898 static const struct nfnl_callback nf_tables_cb[NFT_MSG_MAX] = {
7899         [NFT_MSG_NEWTABLE] = {
7900                 .call           = nf_tables_newtable,
7901                 .type           = NFNL_CB_BATCH,
7902                 .attr_count     = NFTA_TABLE_MAX,
7903                 .policy         = nft_table_policy,
7904         },
7905         [NFT_MSG_GETTABLE] = {
7906                 .call           = nf_tables_gettable,
7907                 .type           = NFNL_CB_RCU,
7908                 .attr_count     = NFTA_TABLE_MAX,
7909                 .policy         = nft_table_policy,
7910         },
7911         [NFT_MSG_DELTABLE] = {
7912                 .call           = nf_tables_deltable,
7913                 .type           = NFNL_CB_BATCH,
7914                 .attr_count     = NFTA_TABLE_MAX,
7915                 .policy         = nft_table_policy,
7916         },
7917         [NFT_MSG_NEWCHAIN] = {
7918                 .call           = nf_tables_newchain,
7919                 .type           = NFNL_CB_BATCH,
7920                 .attr_count     = NFTA_CHAIN_MAX,
7921                 .policy         = nft_chain_policy,
7922         },
7923         [NFT_MSG_GETCHAIN] = {
7924                 .call           = nf_tables_getchain,
7925                 .type           = NFNL_CB_RCU,
7926                 .attr_count     = NFTA_CHAIN_MAX,
7927                 .policy         = nft_chain_policy,
7928         },
7929         [NFT_MSG_DELCHAIN] = {
7930                 .call           = nf_tables_delchain,
7931                 .type           = NFNL_CB_BATCH,
7932                 .attr_count     = NFTA_CHAIN_MAX,
7933                 .policy         = nft_chain_policy,
7934         },
7935         [NFT_MSG_NEWRULE] = {
7936                 .call           = nf_tables_newrule,
7937                 .type           = NFNL_CB_BATCH,
7938                 .attr_count     = NFTA_RULE_MAX,
7939                 .policy         = nft_rule_policy,
7940         },
7941         [NFT_MSG_GETRULE] = {
7942                 .call           = nf_tables_getrule,
7943                 .type           = NFNL_CB_RCU,
7944                 .attr_count     = NFTA_RULE_MAX,
7945                 .policy         = nft_rule_policy,
7946         },
7947         [NFT_MSG_DELRULE] = {
7948                 .call           = nf_tables_delrule,
7949                 .type           = NFNL_CB_BATCH,
7950                 .attr_count     = NFTA_RULE_MAX,
7951                 .policy         = nft_rule_policy,
7952         },
7953         [NFT_MSG_NEWSET] = {
7954                 .call           = nf_tables_newset,
7955                 .type           = NFNL_CB_BATCH,
7956                 .attr_count     = NFTA_SET_MAX,
7957                 .policy         = nft_set_policy,
7958         },
7959         [NFT_MSG_GETSET] = {
7960                 .call           = nf_tables_getset,
7961                 .type           = NFNL_CB_RCU,
7962                 .attr_count     = NFTA_SET_MAX,
7963                 .policy         = nft_set_policy,
7964         },
7965         [NFT_MSG_DELSET] = {
7966                 .call           = nf_tables_delset,
7967                 .type           = NFNL_CB_BATCH,
7968                 .attr_count     = NFTA_SET_MAX,
7969                 .policy         = nft_set_policy,
7970         },
7971         [NFT_MSG_NEWSETELEM] = {
7972                 .call           = nf_tables_newsetelem,
7973                 .type           = NFNL_CB_BATCH,
7974                 .attr_count     = NFTA_SET_ELEM_LIST_MAX,
7975                 .policy         = nft_set_elem_list_policy,
7976         },
7977         [NFT_MSG_GETSETELEM] = {
7978                 .call           = nf_tables_getsetelem,
7979                 .type           = NFNL_CB_RCU,
7980                 .attr_count     = NFTA_SET_ELEM_LIST_MAX,
7981                 .policy         = nft_set_elem_list_policy,
7982         },
7983         [NFT_MSG_DELSETELEM] = {
7984                 .call           = nf_tables_delsetelem,
7985                 .type           = NFNL_CB_BATCH,
7986                 .attr_count     = NFTA_SET_ELEM_LIST_MAX,
7987                 .policy         = nft_set_elem_list_policy,
7988         },
7989         [NFT_MSG_GETGEN] = {
7990                 .call           = nf_tables_getgen,
7991                 .type           = NFNL_CB_RCU,
7992         },
7993         [NFT_MSG_NEWOBJ] = {
7994                 .call           = nf_tables_newobj,
7995                 .type           = NFNL_CB_BATCH,
7996                 .attr_count     = NFTA_OBJ_MAX,
7997                 .policy         = nft_obj_policy,
7998         },
7999         [NFT_MSG_GETOBJ] = {
8000                 .call           = nf_tables_getobj,
8001                 .type           = NFNL_CB_RCU,
8002                 .attr_count     = NFTA_OBJ_MAX,
8003                 .policy         = nft_obj_policy,
8004         },
8005         [NFT_MSG_DELOBJ] = {
8006                 .call           = nf_tables_delobj,
8007                 .type           = NFNL_CB_BATCH,
8008                 .attr_count     = NFTA_OBJ_MAX,
8009                 .policy         = nft_obj_policy,
8010         },
8011         [NFT_MSG_GETOBJ_RESET] = {
8012                 .call           = nf_tables_getobj,
8013                 .type           = NFNL_CB_RCU,
8014                 .attr_count     = NFTA_OBJ_MAX,
8015                 .policy         = nft_obj_policy,
8016         },
8017         [NFT_MSG_NEWFLOWTABLE] = {
8018                 .call           = nf_tables_newflowtable,
8019                 .type           = NFNL_CB_BATCH,
8020                 .attr_count     = NFTA_FLOWTABLE_MAX,
8021                 .policy         = nft_flowtable_policy,
8022         },
8023         [NFT_MSG_GETFLOWTABLE] = {
8024                 .call           = nf_tables_getflowtable,
8025                 .type           = NFNL_CB_RCU,
8026                 .attr_count     = NFTA_FLOWTABLE_MAX,
8027                 .policy         = nft_flowtable_policy,
8028         },
8029         [NFT_MSG_DELFLOWTABLE] = {
8030                 .call           = nf_tables_delflowtable,
8031                 .type           = NFNL_CB_BATCH,
8032                 .attr_count     = NFTA_FLOWTABLE_MAX,
8033                 .policy         = nft_flowtable_policy,
8034         },
8035 };
8036
8037 static int nf_tables_validate(struct net *net)
8038 {
8039         struct nftables_pernet *nft_net = nft_pernet(net);
8040         struct nft_table *table;
8041
8042         switch (nft_net->validate_state) {
8043         case NFT_VALIDATE_SKIP:
8044                 break;
8045         case NFT_VALIDATE_NEED:
8046                 nft_validate_state_update(net, NFT_VALIDATE_DO);
8047                 fallthrough;
8048         case NFT_VALIDATE_DO:
8049                 list_for_each_entry(table, &nft_net->tables, list) {
8050                         if (nft_table_validate(net, table) < 0)
8051                                 return -EAGAIN;
8052                 }
8053                 break;
8054         }
8055
8056         return 0;
8057 }
8058
8059 /* a drop policy has to be deferred until all rules have been activated,
8060  * otherwise a large ruleset that contains a drop-policy base chain will
8061  * cause all packets to get dropped until the full transaction has been
8062  * processed.
8063  *
8064  * We defer the drop policy until the transaction has been finalized.
8065  */
8066 static void nft_chain_commit_drop_policy(struct nft_trans *trans)
8067 {
8068         struct nft_base_chain *basechain;
8069
8070         if (nft_trans_chain_policy(trans) != NF_DROP)
8071                 return;
8072
8073         if (!nft_is_base_chain(trans->ctx.chain))
8074                 return;
8075
8076         basechain = nft_base_chain(trans->ctx.chain);
8077         basechain->policy = NF_DROP;
8078 }
8079
8080 static void nft_chain_commit_update(struct nft_trans *trans)
8081 {
8082         struct nft_base_chain *basechain;
8083
8084         if (nft_trans_chain_name(trans)) {
8085                 rhltable_remove(&trans->ctx.table->chains_ht,
8086                                 &trans->ctx.chain->rhlhead,
8087                                 nft_chain_ht_params);
8088                 swap(trans->ctx.chain->name, nft_trans_chain_name(trans));
8089                 rhltable_insert_key(&trans->ctx.table->chains_ht,
8090                                     trans->ctx.chain->name,
8091                                     &trans->ctx.chain->rhlhead,
8092                                     nft_chain_ht_params);
8093         }
8094
8095         if (!nft_is_base_chain(trans->ctx.chain))
8096                 return;
8097
8098         nft_chain_stats_replace(trans);
8099
8100         basechain = nft_base_chain(trans->ctx.chain);
8101
8102         switch (nft_trans_chain_policy(trans)) {
8103         case NF_DROP:
8104         case NF_ACCEPT:
8105                 basechain->policy = nft_trans_chain_policy(trans);
8106                 break;
8107         }
8108 }
8109
8110 static void nft_obj_commit_update(struct nft_trans *trans)
8111 {
8112         struct nft_object *newobj;
8113         struct nft_object *obj;
8114
8115         obj = nft_trans_obj(trans);
8116         newobj = nft_trans_obj_newobj(trans);
8117
8118         if (obj->ops->update)
8119                 obj->ops->update(obj, newobj);
8120
8121         kfree(newobj);
8122 }
8123
8124 static void nft_commit_release(struct nft_trans *trans)
8125 {
8126         switch (trans->msg_type) {
8127         case NFT_MSG_DELTABLE:
8128                 nf_tables_table_destroy(&trans->ctx);
8129                 break;
8130         case NFT_MSG_NEWCHAIN:
8131                 free_percpu(nft_trans_chain_stats(trans));
8132                 kfree(nft_trans_chain_name(trans));
8133                 break;
8134         case NFT_MSG_DELCHAIN:
8135                 nf_tables_chain_destroy(&trans->ctx);
8136                 break;
8137         case NFT_MSG_DELRULE:
8138                 nf_tables_rule_destroy(&trans->ctx, nft_trans_rule(trans));
8139                 break;
8140         case NFT_MSG_DELSET:
8141                 nft_set_destroy(&trans->ctx, nft_trans_set(trans));
8142                 break;
8143         case NFT_MSG_DELSETELEM:
8144                 nf_tables_set_elem_destroy(&trans->ctx,
8145                                            nft_trans_elem_set(trans),
8146                                            nft_trans_elem(trans).priv);
8147                 break;
8148         case NFT_MSG_DELOBJ:
8149                 nft_obj_destroy(&trans->ctx, nft_trans_obj(trans));
8150                 break;
8151         case NFT_MSG_DELFLOWTABLE:
8152                 if (nft_trans_flowtable_update(trans))
8153                         nft_flowtable_hooks_destroy(&nft_trans_flowtable_hooks(trans));
8154                 else
8155                         nf_tables_flowtable_destroy(nft_trans_flowtable(trans));
8156                 break;
8157         }
8158
8159         if (trans->put_net)
8160                 put_net(trans->ctx.net);
8161
8162         kfree(trans);
8163 }
8164
8165 static void nf_tables_trans_destroy_work(struct work_struct *w)
8166 {
8167         struct nft_trans *trans, *next;
8168         LIST_HEAD(head);
8169
8170         spin_lock(&nf_tables_destroy_list_lock);
8171         list_splice_init(&nf_tables_destroy_list, &head);
8172         spin_unlock(&nf_tables_destroy_list_lock);
8173
8174         if (list_empty(&head))
8175                 return;
8176
8177         synchronize_rcu();
8178
8179         list_for_each_entry_safe(trans, next, &head, list) {
8180                 list_del(&trans->list);
8181                 nft_commit_release(trans);
8182         }
8183 }
8184
8185 void nf_tables_trans_destroy_flush_work(void)
8186 {
8187         flush_work(&trans_destroy_work);
8188 }
8189 EXPORT_SYMBOL_GPL(nf_tables_trans_destroy_flush_work);
8190
8191 static int nf_tables_commit_chain_prepare(struct net *net, struct nft_chain *chain)
8192 {
8193         struct nft_rule *rule;
8194         unsigned int alloc = 0;
8195         int i;
8196
8197         /* already handled or inactive chain? */
8198         if (chain->rules_next || !nft_is_active_next(net, chain))
8199                 return 0;
8200
8201         rule = list_entry(&chain->rules, struct nft_rule, list);
8202         i = 0;
8203
8204         list_for_each_entry_continue(rule, &chain->rules, list) {
8205                 if (nft_is_active_next(net, rule))
8206                         alloc++;
8207         }
8208
8209         chain->rules_next = nf_tables_chain_alloc_rules(chain, alloc);
8210         if (!chain->rules_next)
8211                 return -ENOMEM;
8212
8213         list_for_each_entry_continue(rule, &chain->rules, list) {
8214                 if (nft_is_active_next(net, rule))
8215                         chain->rules_next[i++] = rule;
8216         }
8217
8218         chain->rules_next[i] = NULL;
8219         return 0;
8220 }
8221
8222 static void nf_tables_commit_chain_prepare_cancel(struct net *net)
8223 {
8224         struct nftables_pernet *nft_net = nft_pernet(net);
8225         struct nft_trans *trans, *next;
8226
8227         list_for_each_entry_safe(trans, next, &nft_net->commit_list, list) {
8228                 struct nft_chain *chain = trans->ctx.chain;
8229
8230                 if (trans->msg_type == NFT_MSG_NEWRULE ||
8231                     trans->msg_type == NFT_MSG_DELRULE) {
8232                         kvfree(chain->rules_next);
8233                         chain->rules_next = NULL;
8234                 }
8235         }
8236 }
8237
8238 static void __nf_tables_commit_chain_free_rules_old(struct rcu_head *h)
8239 {
8240         struct nft_rules_old *o = container_of(h, struct nft_rules_old, h);
8241
8242         kvfree(o->start);
8243 }
8244
8245 static void nf_tables_commit_chain_free_rules_old(struct nft_rule **rules)
8246 {
8247         struct nft_rule **r = rules;
8248         struct nft_rules_old *old;
8249
8250         while (*r)
8251                 r++;
8252
8253         r++;    /* rcu_head is after end marker */
8254         old = (void *) r;
8255         old->start = rules;
8256
8257         call_rcu(&old->h, __nf_tables_commit_chain_free_rules_old);
8258 }
8259
8260 static void nf_tables_commit_chain(struct net *net, struct nft_chain *chain)
8261 {
8262         struct nft_rule **g0, **g1;
8263         bool next_genbit;
8264
8265         next_genbit = nft_gencursor_next(net);
8266
8267         g0 = rcu_dereference_protected(chain->rules_gen_0,
8268                                        lockdep_commit_lock_is_held(net));
8269         g1 = rcu_dereference_protected(chain->rules_gen_1,
8270                                        lockdep_commit_lock_is_held(net));
8271
8272         /* No changes to this chain? */
8273         if (chain->rules_next == NULL) {
8274                 /* chain had no change in last or next generation */
8275                 if (g0 == g1)
8276                         return;
8277                 /*
8278                  * chain had no change in this generation; make sure next
8279                  * one uses same rules as current generation.
8280                  */
8281                 if (next_genbit) {
8282                         rcu_assign_pointer(chain->rules_gen_1, g0);
8283                         nf_tables_commit_chain_free_rules_old(g1);
8284                 } else {
8285                         rcu_assign_pointer(chain->rules_gen_0, g1);
8286                         nf_tables_commit_chain_free_rules_old(g0);
8287                 }
8288
8289                 return;
8290         }
8291
8292         if (next_genbit)
8293                 rcu_assign_pointer(chain->rules_gen_1, chain->rules_next);
8294         else
8295                 rcu_assign_pointer(chain->rules_gen_0, chain->rules_next);
8296
8297         chain->rules_next = NULL;
8298
8299         if (g0 == g1)
8300                 return;
8301
8302         if (next_genbit)
8303                 nf_tables_commit_chain_free_rules_old(g1);
8304         else
8305                 nf_tables_commit_chain_free_rules_old(g0);
8306 }
8307
8308 static void nft_obj_del(struct nft_object *obj)
8309 {
8310         rhltable_remove(&nft_objname_ht, &obj->rhlhead, nft_objname_ht_params);
8311         list_del_rcu(&obj->list);
8312 }
8313
8314 void nft_chain_del(struct nft_chain *chain)
8315 {
8316         struct nft_table *table = chain->table;
8317
8318         WARN_ON_ONCE(rhltable_remove(&table->chains_ht, &chain->rhlhead,
8319                                      nft_chain_ht_params));
8320         list_del_rcu(&chain->list);
8321 }
8322
8323 static void nft_flowtable_hooks_del(struct nft_flowtable *flowtable,
8324                                     struct list_head *hook_list)
8325 {
8326         struct nft_hook *hook, *next;
8327
8328         list_for_each_entry_safe(hook, next, &flowtable->hook_list, list) {
8329                 if (hook->inactive)
8330                         list_move(&hook->list, hook_list);
8331         }
8332 }
8333
8334 static void nf_tables_module_autoload_cleanup(struct net *net)
8335 {
8336         struct nftables_pernet *nft_net = nft_pernet(net);
8337         struct nft_module_request *req, *next;
8338
8339         WARN_ON_ONCE(!list_empty(&nft_net->commit_list));
8340         list_for_each_entry_safe(req, next, &nft_net->module_list, list) {
8341                 WARN_ON_ONCE(!req->done);
8342                 list_del(&req->list);
8343                 kfree(req);
8344         }
8345 }
8346
8347 static void nf_tables_commit_release(struct net *net)
8348 {
8349         struct nftables_pernet *nft_net = nft_pernet(net);
8350         struct nft_trans *trans;
8351
8352         /* all side effects have to be made visible.
8353          * For example, if a chain named 'foo' has been deleted, a
8354          * new transaction must not find it anymore.
8355          *
8356          * Memory reclaim happens asynchronously from work queue
8357          * to prevent expensive synchronize_rcu() in commit phase.
8358          */
8359         if (list_empty(&nft_net->commit_list)) {
8360                 nf_tables_module_autoload_cleanup(net);
8361                 mutex_unlock(&nft_net->commit_mutex);
8362                 return;
8363         }
8364
8365         trans = list_last_entry(&nft_net->commit_list,
8366                                 struct nft_trans, list);
8367         get_net(trans->ctx.net);
8368         WARN_ON_ONCE(trans->put_net);
8369
8370         trans->put_net = true;
8371         spin_lock(&nf_tables_destroy_list_lock);
8372         list_splice_tail_init(&nft_net->commit_list, &nf_tables_destroy_list);
8373         spin_unlock(&nf_tables_destroy_list_lock);
8374
8375         nf_tables_module_autoload_cleanup(net);
8376         schedule_work(&trans_destroy_work);
8377
8378         mutex_unlock(&nft_net->commit_mutex);
8379 }
8380
8381 static void nft_commit_notify(struct net *net, u32 portid)
8382 {
8383         struct nftables_pernet *nft_net = nft_pernet(net);
8384         struct sk_buff *batch_skb = NULL, *nskb, *skb;
8385         unsigned char *data;
8386         int len;
8387
8388         list_for_each_entry_safe(skb, nskb, &nft_net->notify_list, list) {
8389                 if (!batch_skb) {
8390 new_batch:
8391                         batch_skb = skb;
8392                         len = NLMSG_GOODSIZE - skb->len;
8393                         list_del(&skb->list);
8394                         continue;
8395                 }
8396                 len -= skb->len;
8397                 if (len > 0 && NFT_CB(skb).report == NFT_CB(batch_skb).report) {
8398                         data = skb_put(batch_skb, skb->len);
8399                         memcpy(data, skb->data, skb->len);
8400                         list_del(&skb->list);
8401                         kfree_skb(skb);
8402                         continue;
8403                 }
8404                 nfnetlink_send(batch_skb, net, portid, NFNLGRP_NFTABLES,
8405                                NFT_CB(batch_skb).report, GFP_KERNEL);
8406                 goto new_batch;
8407         }
8408
8409         if (batch_skb) {
8410                 nfnetlink_send(batch_skb, net, portid, NFNLGRP_NFTABLES,
8411                                NFT_CB(batch_skb).report, GFP_KERNEL);
8412         }
8413
8414         WARN_ON_ONCE(!list_empty(&nft_net->notify_list));
8415 }
8416
8417 static int nf_tables_commit_audit_alloc(struct list_head *adl,
8418                                         struct nft_table *table)
8419 {
8420         struct nft_audit_data *adp;
8421
8422         list_for_each_entry(adp, adl, list) {
8423                 if (adp->table == table)
8424                         return 0;
8425         }
8426         adp = kzalloc(sizeof(*adp), GFP_KERNEL);
8427         if (!adp)
8428                 return -ENOMEM;
8429         adp->table = table;
8430         list_add(&adp->list, adl);
8431         return 0;
8432 }
8433
8434 static void nf_tables_commit_audit_collect(struct list_head *adl,
8435                                            struct nft_table *table, u32 op)
8436 {
8437         struct nft_audit_data *adp;
8438
8439         list_for_each_entry(adp, adl, list) {
8440                 if (adp->table == table)
8441                         goto found;
8442         }
8443         WARN_ONCE(1, "table=%s not expected in commit list", table->name);
8444         return;
8445 found:
8446         adp->entries++;
8447         if (!adp->op || adp->op > op)
8448                 adp->op = op;
8449 }
8450
8451 #define AUNFTABLENAMELEN (NFT_TABLE_MAXNAMELEN + 22)
8452
8453 static void nf_tables_commit_audit_log(struct list_head *adl, u32 generation)
8454 {
8455         struct nft_audit_data *adp, *adn;
8456         char aubuf[AUNFTABLENAMELEN];
8457
8458         list_for_each_entry_safe(adp, adn, adl, list) {
8459                 snprintf(aubuf, AUNFTABLENAMELEN, "%s:%u", adp->table->name,
8460                          generation);
8461                 audit_log_nfcfg(aubuf, adp->table->family, adp->entries,
8462                                 nft2audit_op[adp->op], GFP_KERNEL);
8463                 list_del(&adp->list);
8464                 kfree(adp);
8465         }
8466 }
8467
8468 static int nf_tables_commit(struct net *net, struct sk_buff *skb)
8469 {
8470         struct nftables_pernet *nft_net = nft_pernet(net);
8471         struct nft_trans *trans, *next;
8472         struct nft_trans_elem *te;
8473         struct nft_chain *chain;
8474         struct nft_table *table;
8475         LIST_HEAD(adl);
8476         int err;
8477
8478         if (list_empty(&nft_net->commit_list)) {
8479                 mutex_unlock(&nft_net->commit_mutex);
8480                 return 0;
8481         }
8482
8483         /* 0. Validate ruleset, otherwise roll back for error reporting. */
8484         if (nf_tables_validate(net) < 0)
8485                 return -EAGAIN;
8486
8487         err = nft_flow_rule_offload_commit(net);
8488         if (err < 0)
8489                 return err;
8490
8491         /* 1.  Allocate space for next generation rules_gen_X[] */
8492         list_for_each_entry_safe(trans, next, &nft_net->commit_list, list) {
8493                 int ret;
8494
8495                 ret = nf_tables_commit_audit_alloc(&adl, trans->ctx.table);
8496                 if (ret) {
8497                         nf_tables_commit_chain_prepare_cancel(net);
8498                         return ret;
8499                 }
8500                 if (trans->msg_type == NFT_MSG_NEWRULE ||
8501                     trans->msg_type == NFT_MSG_DELRULE) {
8502                         chain = trans->ctx.chain;
8503
8504                         ret = nf_tables_commit_chain_prepare(net, chain);
8505                         if (ret < 0) {
8506                                 nf_tables_commit_chain_prepare_cancel(net);
8507                                 return ret;
8508                         }
8509                 }
8510         }
8511
8512         /* step 2.  Make rules_gen_X visible to packet path */
8513         list_for_each_entry(table, &nft_net->tables, list) {
8514                 list_for_each_entry(chain, &table->chains, list)
8515                         nf_tables_commit_chain(net, chain);
8516         }
8517
8518         /*
8519          * Bump generation counter, invalidate any dump in progress.
8520          * Cannot fail after this point.
8521          */
8522         while (++nft_net->base_seq == 0)
8523                 ;
8524
8525         /* step 3. Start new generation, rules_gen_X now in use. */
8526         net->nft.gencursor = nft_gencursor_next(net);
8527
8528         list_for_each_entry_safe(trans, next, &nft_net->commit_list, list) {
8529                 nf_tables_commit_audit_collect(&adl, trans->ctx.table,
8530                                                trans->msg_type);
8531                 switch (trans->msg_type) {
8532                 case NFT_MSG_NEWTABLE:
8533                         if (nft_trans_table_update(trans)) {
8534                                 if (!(trans->ctx.table->flags & __NFT_TABLE_F_UPDATE)) {
8535                                         nft_trans_destroy(trans);
8536                                         break;
8537                                 }
8538                                 if (trans->ctx.table->flags & NFT_TABLE_F_DORMANT)
8539                                         nf_tables_table_disable(net, trans->ctx.table);
8540
8541                                 trans->ctx.table->flags &= ~__NFT_TABLE_F_UPDATE;
8542                         } else {
8543                                 nft_clear(net, trans->ctx.table);
8544                         }
8545                         nf_tables_table_notify(&trans->ctx, NFT_MSG_NEWTABLE);
8546                         nft_trans_destroy(trans);
8547                         break;
8548                 case NFT_MSG_DELTABLE:
8549                         list_del_rcu(&trans->ctx.table->list);
8550                         nf_tables_table_notify(&trans->ctx, NFT_MSG_DELTABLE);
8551                         break;
8552                 case NFT_MSG_NEWCHAIN:
8553                         if (nft_trans_chain_update(trans)) {
8554                                 nft_chain_commit_update(trans);
8555                                 nf_tables_chain_notify(&trans->ctx, NFT_MSG_NEWCHAIN);
8556                                 /* trans destroyed after rcu grace period */
8557                         } else {
8558                                 nft_chain_commit_drop_policy(trans);
8559                                 nft_clear(net, trans->ctx.chain);
8560                                 nf_tables_chain_notify(&trans->ctx, NFT_MSG_NEWCHAIN);
8561                                 nft_trans_destroy(trans);
8562                         }
8563                         break;
8564                 case NFT_MSG_DELCHAIN:
8565                         nft_chain_del(trans->ctx.chain);
8566                         nf_tables_chain_notify(&trans->ctx, NFT_MSG_DELCHAIN);
8567                         nf_tables_unregister_hook(trans->ctx.net,
8568                                                   trans->ctx.table,
8569                                                   trans->ctx.chain);
8570                         break;
8571                 case NFT_MSG_NEWRULE:
8572                         nft_clear(trans->ctx.net, nft_trans_rule(trans));
8573                         nf_tables_rule_notify(&trans->ctx,
8574                                               nft_trans_rule(trans),
8575                                               NFT_MSG_NEWRULE);
8576                         nft_trans_destroy(trans);
8577                         break;
8578                 case NFT_MSG_DELRULE:
8579                         list_del_rcu(&nft_trans_rule(trans)->list);
8580                         nf_tables_rule_notify(&trans->ctx,
8581                                               nft_trans_rule(trans),
8582                                               NFT_MSG_DELRULE);
8583                         nft_rule_expr_deactivate(&trans->ctx,
8584                                                  nft_trans_rule(trans),
8585                                                  NFT_TRANS_COMMIT);
8586                         break;
8587                 case NFT_MSG_NEWSET:
8588                         nft_clear(net, nft_trans_set(trans));
8589                         /* This avoids hitting -EBUSY when deleting the table
8590                          * from the transaction.
8591                          */
8592                         if (nft_set_is_anonymous(nft_trans_set(trans)) &&
8593                             !list_empty(&nft_trans_set(trans)->bindings))
8594                                 trans->ctx.table->use--;
8595
8596                         nf_tables_set_notify(&trans->ctx, nft_trans_set(trans),
8597                                              NFT_MSG_NEWSET, GFP_KERNEL);
8598                         nft_trans_destroy(trans);
8599                         break;
8600                 case NFT_MSG_DELSET:
8601                         list_del_rcu(&nft_trans_set(trans)->list);
8602                         nf_tables_set_notify(&trans->ctx, nft_trans_set(trans),
8603                                              NFT_MSG_DELSET, GFP_KERNEL);
8604                         break;
8605                 case NFT_MSG_NEWSETELEM:
8606                         te = (struct nft_trans_elem *)trans->data;
8607
8608                         nft_setelem_activate(net, te->set, &te->elem);
8609                         nf_tables_setelem_notify(&trans->ctx, te->set,
8610                                                  &te->elem,
8611                                                  NFT_MSG_NEWSETELEM, 0);
8612                         nft_trans_destroy(trans);
8613                         break;
8614                 case NFT_MSG_DELSETELEM:
8615                         te = (struct nft_trans_elem *)trans->data;
8616
8617                         nf_tables_setelem_notify(&trans->ctx, te->set,
8618                                                  &te->elem,
8619                                                  NFT_MSG_DELSETELEM, 0);
8620                         nft_setelem_remove(net, te->set, &te->elem);
8621                         if (!nft_setelem_is_catchall(te->set, &te->elem)) {
8622                                 atomic_dec(&te->set->nelems);
8623                                 te->set->ndeact--;
8624                         }
8625                         break;
8626                 case NFT_MSG_NEWOBJ:
8627                         if (nft_trans_obj_update(trans)) {
8628                                 nft_obj_commit_update(trans);
8629                                 nf_tables_obj_notify(&trans->ctx,
8630                                                      nft_trans_obj(trans),
8631                                                      NFT_MSG_NEWOBJ);
8632                         } else {
8633                                 nft_clear(net, nft_trans_obj(trans));
8634                                 nf_tables_obj_notify(&trans->ctx,
8635                                                      nft_trans_obj(trans),
8636                                                      NFT_MSG_NEWOBJ);
8637                                 nft_trans_destroy(trans);
8638                         }
8639                         break;
8640                 case NFT_MSG_DELOBJ:
8641                         nft_obj_del(nft_trans_obj(trans));
8642                         nf_tables_obj_notify(&trans->ctx, nft_trans_obj(trans),
8643                                              NFT_MSG_DELOBJ);
8644                         break;
8645                 case NFT_MSG_NEWFLOWTABLE:
8646                         if (nft_trans_flowtable_update(trans)) {
8647                                 nft_trans_flowtable(trans)->data.flags =
8648                                         nft_trans_flowtable_flags(trans);
8649                                 nf_tables_flowtable_notify(&trans->ctx,
8650                                                            nft_trans_flowtable(trans),
8651                                                            &nft_trans_flowtable_hooks(trans),
8652                                                            NFT_MSG_NEWFLOWTABLE);
8653                                 list_splice(&nft_trans_flowtable_hooks(trans),
8654                                             &nft_trans_flowtable(trans)->hook_list);
8655                         } else {
8656                                 nft_clear(net, nft_trans_flowtable(trans));
8657                                 nf_tables_flowtable_notify(&trans->ctx,
8658                                                            nft_trans_flowtable(trans),
8659                                                            &nft_trans_flowtable(trans)->hook_list,
8660                                                            NFT_MSG_NEWFLOWTABLE);
8661                         }
8662                         nft_trans_destroy(trans);
8663                         break;
8664                 case NFT_MSG_DELFLOWTABLE:
8665                         if (nft_trans_flowtable_update(trans)) {
8666                                 nft_flowtable_hooks_del(nft_trans_flowtable(trans),
8667                                                         &nft_trans_flowtable_hooks(trans));
8668                                 nf_tables_flowtable_notify(&trans->ctx,
8669                                                            nft_trans_flowtable(trans),
8670                                                            &nft_trans_flowtable_hooks(trans),
8671                                                            NFT_MSG_DELFLOWTABLE);
8672                                 nft_unregister_flowtable_net_hooks(net,
8673                                                                    &nft_trans_flowtable_hooks(trans));
8674                         } else {
8675                                 list_del_rcu(&nft_trans_flowtable(trans)->list);
8676                                 nf_tables_flowtable_notify(&trans->ctx,
8677                                                            nft_trans_flowtable(trans),
8678                                                            &nft_trans_flowtable(trans)->hook_list,
8679                                                            NFT_MSG_DELFLOWTABLE);
8680                                 nft_unregister_flowtable_net_hooks(net,
8681                                                 &nft_trans_flowtable(trans)->hook_list);
8682                         }
8683                         break;
8684                 }
8685         }
8686
8687         nft_commit_notify(net, NETLINK_CB(skb).portid);
8688         nf_tables_gen_notify(net, skb, NFT_MSG_NEWGEN);
8689         nf_tables_commit_audit_log(&adl, nft_net->base_seq);
8690         nf_tables_commit_release(net);
8691
8692         return 0;
8693 }
8694
8695 static void nf_tables_module_autoload(struct net *net)
8696 {
8697         struct nftables_pernet *nft_net = nft_pernet(net);
8698         struct nft_module_request *req, *next;
8699         LIST_HEAD(module_list);
8700
8701         list_splice_init(&nft_net->module_list, &module_list);
8702         mutex_unlock(&nft_net->commit_mutex);
8703         list_for_each_entry_safe(req, next, &module_list, list) {
8704                 request_module("%s", req->module);
8705                 req->done = true;
8706         }
8707         mutex_lock(&nft_net->commit_mutex);
8708         list_splice(&module_list, &nft_net->module_list);
8709 }
8710
8711 static void nf_tables_abort_release(struct nft_trans *trans)
8712 {
8713         switch (trans->msg_type) {
8714         case NFT_MSG_NEWTABLE:
8715                 nf_tables_table_destroy(&trans->ctx);
8716                 break;
8717         case NFT_MSG_NEWCHAIN:
8718                 nf_tables_chain_destroy(&trans->ctx);
8719                 break;
8720         case NFT_MSG_NEWRULE:
8721                 nf_tables_rule_destroy(&trans->ctx, nft_trans_rule(trans));
8722                 break;
8723         case NFT_MSG_NEWSET:
8724                 nft_set_destroy(&trans->ctx, nft_trans_set(trans));
8725                 break;
8726         case NFT_MSG_NEWSETELEM:
8727                 nft_set_elem_destroy(nft_trans_elem_set(trans),
8728                                      nft_trans_elem(trans).priv, true);
8729                 break;
8730         case NFT_MSG_NEWOBJ:
8731                 nft_obj_destroy(&trans->ctx, nft_trans_obj(trans));
8732                 break;
8733         case NFT_MSG_NEWFLOWTABLE:
8734                 if (nft_trans_flowtable_update(trans))
8735                         nft_flowtable_hooks_destroy(&nft_trans_flowtable_hooks(trans));
8736                 else
8737                         nf_tables_flowtable_destroy(nft_trans_flowtable(trans));
8738                 break;
8739         }
8740         kfree(trans);
8741 }
8742
8743 static int __nf_tables_abort(struct net *net, enum nfnl_abort_action action)
8744 {
8745         struct nftables_pernet *nft_net = nft_pernet(net);
8746         struct nft_trans *trans, *next;
8747         struct nft_trans_elem *te;
8748         struct nft_hook *hook;
8749
8750         if (action == NFNL_ABORT_VALIDATE &&
8751             nf_tables_validate(net) < 0)
8752                 return -EAGAIN;
8753
8754         list_for_each_entry_safe_reverse(trans, next, &nft_net->commit_list,
8755                                          list) {
8756                 switch (trans->msg_type) {
8757                 case NFT_MSG_NEWTABLE:
8758                         if (nft_trans_table_update(trans)) {
8759                                 if (!(trans->ctx.table->flags & __NFT_TABLE_F_UPDATE)) {
8760                                         nft_trans_destroy(trans);
8761                                         break;
8762                                 }
8763                                 if (trans->ctx.table->flags & __NFT_TABLE_F_WAS_DORMANT) {
8764                                         nf_tables_table_disable(net, trans->ctx.table);
8765                                         trans->ctx.table->flags |= NFT_TABLE_F_DORMANT;
8766                                 } else if (trans->ctx.table->flags & __NFT_TABLE_F_WAS_AWAKEN) {
8767                                         trans->ctx.table->flags &= ~NFT_TABLE_F_DORMANT;
8768                                 }
8769                                 trans->ctx.table->flags &= ~__NFT_TABLE_F_UPDATE;
8770                                 nft_trans_destroy(trans);
8771                         } else {
8772                                 list_del_rcu(&trans->ctx.table->list);
8773                         }
8774                         break;
8775                 case NFT_MSG_DELTABLE:
8776                         nft_clear(trans->ctx.net, trans->ctx.table);
8777                         nft_trans_destroy(trans);
8778                         break;
8779                 case NFT_MSG_NEWCHAIN:
8780                         if (nft_trans_chain_update(trans)) {
8781                                 free_percpu(nft_trans_chain_stats(trans));
8782                                 kfree(nft_trans_chain_name(trans));
8783                                 nft_trans_destroy(trans);
8784                         } else {
8785                                 if (nft_chain_is_bound(trans->ctx.chain)) {
8786                                         nft_trans_destroy(trans);
8787                                         break;
8788                                 }
8789                                 trans->ctx.table->use--;
8790                                 nft_chain_del(trans->ctx.chain);
8791                                 nf_tables_unregister_hook(trans->ctx.net,
8792                                                           trans->ctx.table,
8793                                                           trans->ctx.chain);
8794                         }
8795                         break;
8796                 case NFT_MSG_DELCHAIN:
8797                         trans->ctx.table->use++;
8798                         nft_clear(trans->ctx.net, trans->ctx.chain);
8799                         nft_trans_destroy(trans);
8800                         break;
8801                 case NFT_MSG_NEWRULE:
8802                         trans->ctx.chain->use--;
8803                         list_del_rcu(&nft_trans_rule(trans)->list);
8804                         nft_rule_expr_deactivate(&trans->ctx,
8805                                                  nft_trans_rule(trans),
8806                                                  NFT_TRANS_ABORT);
8807                         break;
8808                 case NFT_MSG_DELRULE:
8809                         trans->ctx.chain->use++;
8810                         nft_clear(trans->ctx.net, nft_trans_rule(trans));
8811                         nft_rule_expr_activate(&trans->ctx, nft_trans_rule(trans));
8812                         nft_trans_destroy(trans);
8813                         break;
8814                 case NFT_MSG_NEWSET:
8815                         trans->ctx.table->use--;
8816                         if (nft_trans_set_bound(trans)) {
8817                                 nft_trans_destroy(trans);
8818                                 break;
8819                         }
8820                         list_del_rcu(&nft_trans_set(trans)->list);
8821                         break;
8822                 case NFT_MSG_DELSET:
8823                         trans->ctx.table->use++;
8824                         nft_clear(trans->ctx.net, nft_trans_set(trans));
8825                         nft_trans_destroy(trans);
8826                         break;
8827                 case NFT_MSG_NEWSETELEM:
8828                         if (nft_trans_elem_set_bound(trans)) {
8829                                 nft_trans_destroy(trans);
8830                                 break;
8831                         }
8832                         te = (struct nft_trans_elem *)trans->data;
8833                         nft_setelem_remove(net, te->set, &te->elem);
8834                         if (!nft_setelem_is_catchall(te->set, &te->elem))
8835                                 atomic_dec(&te->set->nelems);
8836                         break;
8837                 case NFT_MSG_DELSETELEM:
8838                         te = (struct nft_trans_elem *)trans->data;
8839
8840                         nft_setelem_data_activate(net, te->set, &te->elem);
8841                         nft_setelem_activate(net, te->set, &te->elem);
8842                         if (!nft_setelem_is_catchall(te->set, &te->elem))
8843                                 te->set->ndeact--;
8844
8845                         nft_trans_destroy(trans);
8846                         break;
8847                 case NFT_MSG_NEWOBJ:
8848                         if (nft_trans_obj_update(trans)) {
8849                                 kfree(nft_trans_obj_newobj(trans));
8850                                 nft_trans_destroy(trans);
8851                         } else {
8852                                 trans->ctx.table->use--;
8853                                 nft_obj_del(nft_trans_obj(trans));
8854                         }
8855                         break;
8856                 case NFT_MSG_DELOBJ:
8857                         trans->ctx.table->use++;
8858                         nft_clear(trans->ctx.net, nft_trans_obj(trans));
8859                         nft_trans_destroy(trans);
8860                         break;
8861                 case NFT_MSG_NEWFLOWTABLE:
8862                         if (nft_trans_flowtable_update(trans)) {
8863                                 nft_unregister_flowtable_net_hooks(net,
8864                                                 &nft_trans_flowtable_hooks(trans));
8865                         } else {
8866                                 trans->ctx.table->use--;
8867                                 list_del_rcu(&nft_trans_flowtable(trans)->list);
8868                                 nft_unregister_flowtable_net_hooks(net,
8869                                                 &nft_trans_flowtable(trans)->hook_list);
8870                         }
8871                         break;
8872                 case NFT_MSG_DELFLOWTABLE:
8873                         if (nft_trans_flowtable_update(trans)) {
8874                                 list_for_each_entry(hook, &nft_trans_flowtable(trans)->hook_list, list)
8875                                         hook->inactive = false;
8876                         } else {
8877                                 trans->ctx.table->use++;
8878                                 nft_clear(trans->ctx.net, nft_trans_flowtable(trans));
8879                         }
8880                         nft_trans_destroy(trans);
8881                         break;
8882                 }
8883         }
8884
8885         synchronize_rcu();
8886
8887         list_for_each_entry_safe_reverse(trans, next,
8888                                          &nft_net->commit_list, list) {
8889                 list_del(&trans->list);
8890                 nf_tables_abort_release(trans);
8891         }
8892
8893         if (action == NFNL_ABORT_AUTOLOAD)
8894                 nf_tables_module_autoload(net);
8895         else
8896                 nf_tables_module_autoload_cleanup(net);
8897
8898         return 0;
8899 }
8900
8901 static void nf_tables_cleanup(struct net *net)
8902 {
8903         nft_validate_state_update(net, NFT_VALIDATE_SKIP);
8904 }
8905
8906 static int nf_tables_abort(struct net *net, struct sk_buff *skb,
8907                            enum nfnl_abort_action action)
8908 {
8909         struct nftables_pernet *nft_net = nft_pernet(net);
8910         int ret = __nf_tables_abort(net, action);
8911
8912         mutex_unlock(&nft_net->commit_mutex);
8913
8914         return ret;
8915 }
8916
8917 static bool nf_tables_valid_genid(struct net *net, u32 genid)
8918 {
8919         struct nftables_pernet *nft_net = nft_pernet(net);
8920         bool genid_ok;
8921
8922         mutex_lock(&nft_net->commit_mutex);
8923
8924         genid_ok = genid == 0 || nft_net->base_seq == genid;
8925         if (!genid_ok)
8926                 mutex_unlock(&nft_net->commit_mutex);
8927
8928         /* else, commit mutex has to be released by commit or abort function */
8929         return genid_ok;
8930 }
8931
8932 static const struct nfnetlink_subsystem nf_tables_subsys = {
8933         .name           = "nf_tables",
8934         .subsys_id      = NFNL_SUBSYS_NFTABLES,
8935         .cb_count       = NFT_MSG_MAX,
8936         .cb             = nf_tables_cb,
8937         .commit         = nf_tables_commit,
8938         .abort          = nf_tables_abort,
8939         .cleanup        = nf_tables_cleanup,
8940         .valid_genid    = nf_tables_valid_genid,
8941         .owner          = THIS_MODULE,
8942 };
8943
8944 int nft_chain_validate_dependency(const struct nft_chain *chain,
8945                                   enum nft_chain_types type)
8946 {
8947         const struct nft_base_chain *basechain;
8948
8949         if (nft_is_base_chain(chain)) {
8950                 basechain = nft_base_chain(chain);
8951                 if (basechain->type->type != type)
8952                         return -EOPNOTSUPP;
8953         }
8954         return 0;
8955 }
8956 EXPORT_SYMBOL_GPL(nft_chain_validate_dependency);
8957
8958 int nft_chain_validate_hooks(const struct nft_chain *chain,
8959                              unsigned int hook_flags)
8960 {
8961         struct nft_base_chain *basechain;
8962
8963         if (nft_is_base_chain(chain)) {
8964                 basechain = nft_base_chain(chain);
8965
8966                 if ((1 << basechain->ops.hooknum) & hook_flags)
8967                         return 0;
8968
8969                 return -EOPNOTSUPP;
8970         }
8971
8972         return 0;
8973 }
8974 EXPORT_SYMBOL_GPL(nft_chain_validate_hooks);
8975
8976 /*
8977  * Loop detection - walk through the ruleset beginning at the destination chain
8978  * of a new jump until either the source chain is reached (loop) or all
8979  * reachable chains have been traversed.
8980  *
8981  * The loop check is performed whenever a new jump verdict is added to an
8982  * expression or verdict map or a verdict map is bound to a new chain.
8983  */
8984
8985 static int nf_tables_check_loops(const struct nft_ctx *ctx,
8986                                  const struct nft_chain *chain);
8987
8988 static int nft_check_loops(const struct nft_ctx *ctx,
8989                            const struct nft_set_ext *ext)
8990 {
8991         const struct nft_data *data;
8992         int ret;
8993
8994         data = nft_set_ext_data(ext);
8995         switch (data->verdict.code) {
8996         case NFT_JUMP:
8997         case NFT_GOTO:
8998                 ret = nf_tables_check_loops(ctx, data->verdict.chain);
8999                 break;
9000         default:
9001                 ret = 0;
9002                 break;
9003         }
9004
9005         return ret;
9006 }
9007
9008 static int nf_tables_loop_check_setelem(const struct nft_ctx *ctx,
9009                                         struct nft_set *set,
9010                                         const struct nft_set_iter *iter,
9011                                         struct nft_set_elem *elem)
9012 {
9013         const struct nft_set_ext *ext = nft_set_elem_ext(set, elem->priv);
9014
9015         if (nft_set_ext_exists(ext, NFT_SET_EXT_FLAGS) &&
9016             *nft_set_ext_flags(ext) & NFT_SET_ELEM_INTERVAL_END)
9017                 return 0;
9018
9019         return nft_check_loops(ctx, ext);
9020 }
9021
9022 static int nft_set_catchall_loops(const struct nft_ctx *ctx,
9023                                   struct nft_set *set)
9024 {
9025         u8 genmask = nft_genmask_next(ctx->net);
9026         struct nft_set_elem_catchall *catchall;
9027         struct nft_set_ext *ext;
9028         int ret = 0;
9029
9030         list_for_each_entry_rcu(catchall, &set->catchall_list, list) {
9031                 ext = nft_set_elem_ext(set, catchall->elem);
9032                 if (!nft_set_elem_active(ext, genmask))
9033                         continue;
9034
9035                 ret = nft_check_loops(ctx, ext);
9036                 if (ret < 0)
9037                         return ret;
9038         }
9039
9040         return ret;
9041 }
9042
9043 static int nf_tables_check_loops(const struct nft_ctx *ctx,
9044                                  const struct nft_chain *chain)
9045 {
9046         const struct nft_rule *rule;
9047         const struct nft_expr *expr, *last;
9048         struct nft_set *set;
9049         struct nft_set_binding *binding;
9050         struct nft_set_iter iter;
9051
9052         if (ctx->chain == chain)
9053                 return -ELOOP;
9054
9055         list_for_each_entry(rule, &chain->rules, list) {
9056                 nft_rule_for_each_expr(expr, last, rule) {
9057                         struct nft_immediate_expr *priv;
9058                         const struct nft_data *data;
9059                         int err;
9060
9061                         if (strcmp(expr->ops->type->name, "immediate"))
9062                                 continue;
9063
9064                         priv = nft_expr_priv(expr);
9065                         if (priv->dreg != NFT_REG_VERDICT)
9066                                 continue;
9067
9068                         data = &priv->data;
9069                         switch (data->verdict.code) {
9070                         case NFT_JUMP:
9071                         case NFT_GOTO:
9072                                 err = nf_tables_check_loops(ctx,
9073                                                         data->verdict.chain);
9074                                 if (err < 0)
9075                                         return err;
9076                                 break;
9077                         default:
9078                                 break;
9079                         }
9080                 }
9081         }
9082
9083         list_for_each_entry(set, &ctx->table->sets, list) {
9084                 if (!nft_is_active_next(ctx->net, set))
9085                         continue;
9086                 if (!(set->flags & NFT_SET_MAP) ||
9087                     set->dtype != NFT_DATA_VERDICT)
9088                         continue;
9089
9090                 list_for_each_entry(binding, &set->bindings, list) {
9091                         if (!(binding->flags & NFT_SET_MAP) ||
9092                             binding->chain != chain)
9093                                 continue;
9094
9095                         iter.genmask    = nft_genmask_next(ctx->net);
9096                         iter.skip       = 0;
9097                         iter.count      = 0;
9098                         iter.err        = 0;
9099                         iter.fn         = nf_tables_loop_check_setelem;
9100
9101                         set->ops->walk(ctx, set, &iter);
9102                         if (!iter.err)
9103                                 iter.err = nft_set_catchall_loops(ctx, set);
9104
9105                         if (iter.err < 0)
9106                                 return iter.err;
9107                 }
9108         }
9109
9110         return 0;
9111 }
9112
9113 /**
9114  *      nft_parse_u32_check - fetch u32 attribute and check for maximum value
9115  *
9116  *      @attr: netlink attribute to fetch value from
9117  *      @max: maximum value to be stored in dest
9118  *      @dest: pointer to the variable
9119  *
9120  *      Parse, check and store a given u32 netlink attribute into variable.
9121  *      This function returns -ERANGE if the value goes over maximum value.
9122  *      Otherwise a 0 is returned and the attribute value is stored in the
9123  *      destination variable.
9124  */
9125 int nft_parse_u32_check(const struct nlattr *attr, int max, u32 *dest)
9126 {
9127         u32 val;
9128
9129         val = ntohl(nla_get_be32(attr));
9130         if (val > max)
9131                 return -ERANGE;
9132
9133         *dest = val;
9134         return 0;
9135 }
9136 EXPORT_SYMBOL_GPL(nft_parse_u32_check);
9137
9138 static unsigned int nft_parse_register(const struct nlattr *attr)
9139 {
9140         unsigned int reg;
9141
9142         reg = ntohl(nla_get_be32(attr));
9143         switch (reg) {
9144         case NFT_REG_VERDICT...NFT_REG_4:
9145                 return reg * NFT_REG_SIZE / NFT_REG32_SIZE;
9146         default:
9147                 return reg + NFT_REG_SIZE / NFT_REG32_SIZE - NFT_REG32_00;
9148         }
9149 }
9150
9151 /**
9152  *      nft_dump_register - dump a register value to a netlink attribute
9153  *
9154  *      @skb: socket buffer
9155  *      @attr: attribute number
9156  *      @reg: register number
9157  *
9158  *      Construct a netlink attribute containing the register number. For
9159  *      compatibility reasons, register numbers being a multiple of 4 are
9160  *      translated to the corresponding 128 bit register numbers.
9161  */
9162 int nft_dump_register(struct sk_buff *skb, unsigned int attr, unsigned int reg)
9163 {
9164         if (reg % (NFT_REG_SIZE / NFT_REG32_SIZE) == 0)
9165                 reg = reg / (NFT_REG_SIZE / NFT_REG32_SIZE);
9166         else
9167                 reg = reg - NFT_REG_SIZE / NFT_REG32_SIZE + NFT_REG32_00;
9168
9169         return nla_put_be32(skb, attr, htonl(reg));
9170 }
9171 EXPORT_SYMBOL_GPL(nft_dump_register);
9172
9173 static int nft_validate_register_load(enum nft_registers reg, unsigned int len)
9174 {
9175         if (reg < NFT_REG_1 * NFT_REG_SIZE / NFT_REG32_SIZE)
9176                 return -EINVAL;
9177         if (len == 0)
9178                 return -EINVAL;
9179         if (reg * NFT_REG32_SIZE + len > sizeof_field(struct nft_regs, data))
9180                 return -ERANGE;
9181
9182         return 0;
9183 }
9184
9185 int nft_parse_register_load(const struct nlattr *attr, u8 *sreg, u32 len)
9186 {
9187         u32 reg;
9188         int err;
9189
9190         reg = nft_parse_register(attr);
9191         err = nft_validate_register_load(reg, len);
9192         if (err < 0)
9193                 return err;
9194
9195         *sreg = reg;
9196         return 0;
9197 }
9198 EXPORT_SYMBOL_GPL(nft_parse_register_load);
9199
9200 static int nft_validate_register_store(const struct nft_ctx *ctx,
9201                                        enum nft_registers reg,
9202                                        const struct nft_data *data,
9203                                        enum nft_data_types type,
9204                                        unsigned int len)
9205 {
9206         int err;
9207
9208         switch (reg) {
9209         case NFT_REG_VERDICT:
9210                 if (type != NFT_DATA_VERDICT)
9211                         return -EINVAL;
9212
9213                 if (data != NULL &&
9214                     (data->verdict.code == NFT_GOTO ||
9215                      data->verdict.code == NFT_JUMP)) {
9216                         err = nf_tables_check_loops(ctx, data->verdict.chain);
9217                         if (err < 0)
9218                                 return err;
9219                 }
9220
9221                 return 0;
9222         default:
9223                 if (reg < NFT_REG_1 * NFT_REG_SIZE / NFT_REG32_SIZE)
9224                         return -EINVAL;
9225                 if (len == 0)
9226                         return -EINVAL;
9227                 if (reg * NFT_REG32_SIZE + len >
9228                     sizeof_field(struct nft_regs, data))
9229                         return -ERANGE;
9230
9231                 if (data != NULL && type != NFT_DATA_VALUE)
9232                         return -EINVAL;
9233                 return 0;
9234         }
9235 }
9236
9237 int nft_parse_register_store(const struct nft_ctx *ctx,
9238                              const struct nlattr *attr, u8 *dreg,
9239                              const struct nft_data *data,
9240                              enum nft_data_types type, unsigned int len)
9241 {
9242         int err;
9243         u32 reg;
9244
9245         reg = nft_parse_register(attr);
9246         err = nft_validate_register_store(ctx, reg, data, type, len);
9247         if (err < 0)
9248                 return err;
9249
9250         *dreg = reg;
9251         return 0;
9252 }
9253 EXPORT_SYMBOL_GPL(nft_parse_register_store);
9254
9255 static const struct nla_policy nft_verdict_policy[NFTA_VERDICT_MAX + 1] = {
9256         [NFTA_VERDICT_CODE]     = { .type = NLA_U32 },
9257         [NFTA_VERDICT_CHAIN]    = { .type = NLA_STRING,
9258                                     .len = NFT_CHAIN_MAXNAMELEN - 1 },
9259         [NFTA_VERDICT_CHAIN_ID] = { .type = NLA_U32 },
9260 };
9261
9262 static int nft_verdict_init(const struct nft_ctx *ctx, struct nft_data *data,
9263                             struct nft_data_desc *desc, const struct nlattr *nla)
9264 {
9265         u8 genmask = nft_genmask_next(ctx->net);
9266         struct nlattr *tb[NFTA_VERDICT_MAX + 1];
9267         struct nft_chain *chain;
9268         int err;
9269
9270         err = nla_parse_nested_deprecated(tb, NFTA_VERDICT_MAX, nla,
9271                                           nft_verdict_policy, NULL);
9272         if (err < 0)
9273                 return err;
9274
9275         if (!tb[NFTA_VERDICT_CODE])
9276                 return -EINVAL;
9277         data->verdict.code = ntohl(nla_get_be32(tb[NFTA_VERDICT_CODE]));
9278
9279         switch (data->verdict.code) {
9280         default:
9281                 switch (data->verdict.code & NF_VERDICT_MASK) {
9282                 case NF_ACCEPT:
9283                 case NF_DROP:
9284                 case NF_QUEUE:
9285                         break;
9286                 default:
9287                         return -EINVAL;
9288                 }
9289                 fallthrough;
9290         case NFT_CONTINUE:
9291         case NFT_BREAK:
9292         case NFT_RETURN:
9293                 break;
9294         case NFT_JUMP:
9295         case NFT_GOTO:
9296                 if (tb[NFTA_VERDICT_CHAIN]) {
9297                         chain = nft_chain_lookup(ctx->net, ctx->table,
9298                                                  tb[NFTA_VERDICT_CHAIN],
9299                                                  genmask);
9300                 } else if (tb[NFTA_VERDICT_CHAIN_ID]) {
9301                         chain = nft_chain_lookup_byid(ctx->net,
9302                                                       tb[NFTA_VERDICT_CHAIN_ID]);
9303                         if (IS_ERR(chain))
9304                                 return PTR_ERR(chain);
9305                 } else {
9306                         return -EINVAL;
9307                 }
9308
9309                 if (IS_ERR(chain))
9310                         return PTR_ERR(chain);
9311                 if (nft_is_base_chain(chain))
9312                         return -EOPNOTSUPP;
9313
9314                 chain->use++;
9315                 data->verdict.chain = chain;
9316                 break;
9317         }
9318
9319         desc->len = sizeof(data->verdict);
9320         desc->type = NFT_DATA_VERDICT;
9321         return 0;
9322 }
9323
9324 static void nft_verdict_uninit(const struct nft_data *data)
9325 {
9326         struct nft_chain *chain;
9327         struct nft_rule *rule;
9328
9329         switch (data->verdict.code) {
9330         case NFT_JUMP:
9331         case NFT_GOTO:
9332                 chain = data->verdict.chain;
9333                 chain->use--;
9334
9335                 if (!nft_chain_is_bound(chain))
9336                         break;
9337
9338                 chain->table->use--;
9339                 list_for_each_entry(rule, &chain->rules, list)
9340                         chain->use--;
9341
9342                 nft_chain_del(chain);
9343                 break;
9344         }
9345 }
9346
9347 int nft_verdict_dump(struct sk_buff *skb, int type, const struct nft_verdict *v)
9348 {
9349         struct nlattr *nest;
9350
9351         nest = nla_nest_start_noflag(skb, type);
9352         if (!nest)
9353                 goto nla_put_failure;
9354
9355         if (nla_put_be32(skb, NFTA_VERDICT_CODE, htonl(v->code)))
9356                 goto nla_put_failure;
9357
9358         switch (v->code) {
9359         case NFT_JUMP:
9360         case NFT_GOTO:
9361                 if (nla_put_string(skb, NFTA_VERDICT_CHAIN,
9362                                    v->chain->name))
9363                         goto nla_put_failure;
9364         }
9365         nla_nest_end(skb, nest);
9366         return 0;
9367
9368 nla_put_failure:
9369         return -1;
9370 }
9371
9372 static int nft_value_init(const struct nft_ctx *ctx,
9373                           struct nft_data *data, unsigned int size,
9374                           struct nft_data_desc *desc, const struct nlattr *nla)
9375 {
9376         unsigned int len;
9377
9378         len = nla_len(nla);
9379         if (len == 0)
9380                 return -EINVAL;
9381         if (len > size)
9382                 return -EOVERFLOW;
9383
9384         nla_memcpy(data->data, nla, len);
9385         desc->type = NFT_DATA_VALUE;
9386         desc->len  = len;
9387         return 0;
9388 }
9389
9390 static int nft_value_dump(struct sk_buff *skb, const struct nft_data *data,
9391                           unsigned int len)
9392 {
9393         return nla_put(skb, NFTA_DATA_VALUE, len, data->data);
9394 }
9395
9396 static const struct nla_policy nft_data_policy[NFTA_DATA_MAX + 1] = {
9397         [NFTA_DATA_VALUE]       = { .type = NLA_BINARY },
9398         [NFTA_DATA_VERDICT]     = { .type = NLA_NESTED },
9399 };
9400
9401 /**
9402  *      nft_data_init - parse nf_tables data netlink attributes
9403  *
9404  *      @ctx: context of the expression using the data
9405  *      @data: destination struct nft_data
9406  *      @size: maximum data length
9407  *      @desc: data description
9408  *      @nla: netlink attribute containing data
9409  *
9410  *      Parse the netlink data attributes and initialize a struct nft_data.
9411  *      The type and length of data are returned in the data description.
9412  *
9413  *      The caller can indicate that it only wants to accept data of type
9414  *      NFT_DATA_VALUE by passing NULL for the ctx argument.
9415  */
9416 int nft_data_init(const struct nft_ctx *ctx,
9417                   struct nft_data *data, unsigned int size,
9418                   struct nft_data_desc *desc, const struct nlattr *nla)
9419 {
9420         struct nlattr *tb[NFTA_DATA_MAX + 1];
9421         int err;
9422
9423         err = nla_parse_nested_deprecated(tb, NFTA_DATA_MAX, nla,
9424                                           nft_data_policy, NULL);
9425         if (err < 0)
9426                 return err;
9427
9428         if (tb[NFTA_DATA_VALUE])
9429                 return nft_value_init(ctx, data, size, desc,
9430                                       tb[NFTA_DATA_VALUE]);
9431         if (tb[NFTA_DATA_VERDICT] && ctx != NULL)
9432                 return nft_verdict_init(ctx, data, desc, tb[NFTA_DATA_VERDICT]);
9433         return -EINVAL;
9434 }
9435 EXPORT_SYMBOL_GPL(nft_data_init);
9436
9437 /**
9438  *      nft_data_release - release a nft_data item
9439  *
9440  *      @data: struct nft_data to release
9441  *      @type: type of data
9442  *
9443  *      Release a nft_data item. NFT_DATA_VALUE types can be silently discarded,
9444  *      all others need to be released by calling this function.
9445  */
9446 void nft_data_release(const struct nft_data *data, enum nft_data_types type)
9447 {
9448         if (type < NFT_DATA_VERDICT)
9449                 return;
9450         switch (type) {
9451         case NFT_DATA_VERDICT:
9452                 return nft_verdict_uninit(data);
9453         default:
9454                 WARN_ON(1);
9455         }
9456 }
9457 EXPORT_SYMBOL_GPL(nft_data_release);
9458
9459 int nft_data_dump(struct sk_buff *skb, int attr, const struct nft_data *data,
9460                   enum nft_data_types type, unsigned int len)
9461 {
9462         struct nlattr *nest;
9463         int err;
9464
9465         nest = nla_nest_start_noflag(skb, attr);
9466         if (nest == NULL)
9467                 return -1;
9468
9469         switch (type) {
9470         case NFT_DATA_VALUE:
9471                 err = nft_value_dump(skb, data, len);
9472                 break;
9473         case NFT_DATA_VERDICT:
9474                 err = nft_verdict_dump(skb, NFTA_DATA_VERDICT, &data->verdict);
9475                 break;
9476         default:
9477                 err = -EINVAL;
9478                 WARN_ON(1);
9479         }
9480
9481         nla_nest_end(skb, nest);
9482         return err;
9483 }
9484 EXPORT_SYMBOL_GPL(nft_data_dump);
9485
9486 int __nft_release_basechain(struct nft_ctx *ctx)
9487 {
9488         struct nft_rule *rule, *nr;
9489
9490         if (WARN_ON(!nft_is_base_chain(ctx->chain)))
9491                 return 0;
9492
9493         nf_tables_unregister_hook(ctx->net, ctx->chain->table, ctx->chain);
9494         list_for_each_entry_safe(rule, nr, &ctx->chain->rules, list) {
9495                 list_del(&rule->list);
9496                 ctx->chain->use--;
9497                 nf_tables_rule_release(ctx, rule);
9498         }
9499         nft_chain_del(ctx->chain);
9500         ctx->table->use--;
9501         nf_tables_chain_destroy(ctx);
9502
9503         return 0;
9504 }
9505 EXPORT_SYMBOL_GPL(__nft_release_basechain);
9506
9507 static void __nft_release_hook(struct net *net, struct nft_table *table)
9508 {
9509         struct nft_chain *chain;
9510
9511         list_for_each_entry(chain, &table->chains, list)
9512                 nf_tables_unregister_hook(net, table, chain);
9513 }
9514
9515 static void __nft_release_hooks(struct net *net)
9516 {
9517         struct nftables_pernet *nft_net = nft_pernet(net);
9518         struct nft_table *table;
9519
9520         list_for_each_entry(table, &nft_net->tables, list) {
9521                 if (nft_table_has_owner(table))
9522                         continue;
9523
9524                 __nft_release_hook(net, table);
9525         }
9526 }
9527
9528 static void __nft_release_table(struct net *net, struct nft_table *table)
9529 {
9530         struct nft_flowtable *flowtable, *nf;
9531         struct nft_chain *chain, *nc;
9532         struct nft_object *obj, *ne;
9533         struct nft_rule *rule, *nr;
9534         struct nft_set *set, *ns;
9535         struct nft_ctx ctx = {
9536                 .net    = net,
9537                 .family = NFPROTO_NETDEV,
9538         };
9539
9540         ctx.family = table->family;
9541         ctx.table = table;
9542         list_for_each_entry(chain, &table->chains, list) {
9543                 ctx.chain = chain;
9544                 list_for_each_entry_safe(rule, nr, &chain->rules, list) {
9545                         list_del(&rule->list);
9546                         chain->use--;
9547                         nf_tables_rule_release(&ctx, rule);
9548                 }
9549         }
9550         list_for_each_entry_safe(flowtable, nf, &table->flowtables, list) {
9551                 list_del(&flowtable->list);
9552                 table->use--;
9553                 nf_tables_flowtable_destroy(flowtable);
9554         }
9555         list_for_each_entry_safe(set, ns, &table->sets, list) {
9556                 list_del(&set->list);
9557                 table->use--;
9558                 nft_set_destroy(&ctx, set);
9559         }
9560         list_for_each_entry_safe(obj, ne, &table->objects, list) {
9561                 nft_obj_del(obj);
9562                 table->use--;
9563                 nft_obj_destroy(&ctx, obj);
9564         }
9565         list_for_each_entry_safe(chain, nc, &table->chains, list) {
9566                 ctx.chain = chain;
9567                 nft_chain_del(chain);
9568                 table->use--;
9569                 nf_tables_chain_destroy(&ctx);
9570         }
9571         list_del(&table->list);
9572         nf_tables_table_destroy(&ctx);
9573 }
9574
9575 static void __nft_release_tables(struct net *net)
9576 {
9577         struct nftables_pernet *nft_net = nft_pernet(net);
9578         struct nft_table *table, *nt;
9579
9580         list_for_each_entry_safe(table, nt, &nft_net->tables, list) {
9581                 if (nft_table_has_owner(table))
9582                         continue;
9583
9584                 __nft_release_table(net, table);
9585         }
9586 }
9587
9588 static int nft_rcv_nl_event(struct notifier_block *this, unsigned long event,
9589                             void *ptr)
9590 {
9591         struct nftables_pernet *nft_net;
9592         struct netlink_notify *n = ptr;
9593         struct nft_table *table, *nt;
9594         struct net *net = n->net;
9595         bool release = false;
9596
9597         if (event != NETLINK_URELEASE || n->protocol != NETLINK_NETFILTER)
9598                 return NOTIFY_DONE;
9599
9600         nft_net = nft_pernet(net);
9601         mutex_lock(&nft_net->commit_mutex);
9602         list_for_each_entry(table, &nft_net->tables, list) {
9603                 if (nft_table_has_owner(table) &&
9604                     n->portid == table->nlpid) {
9605                         __nft_release_hook(net, table);
9606                         release = true;
9607                 }
9608         }
9609         if (release) {
9610                 synchronize_rcu();
9611                 list_for_each_entry_safe(table, nt, &nft_net->tables, list) {
9612                         if (nft_table_has_owner(table) &&
9613                             n->portid == table->nlpid)
9614                                 __nft_release_table(net, table);
9615                 }
9616         }
9617         mutex_unlock(&nft_net->commit_mutex);
9618
9619         return NOTIFY_DONE;
9620 }
9621
9622 static struct notifier_block nft_nl_notifier = {
9623         .notifier_call  = nft_rcv_nl_event,
9624 };
9625
9626 static int __net_init nf_tables_init_net(struct net *net)
9627 {
9628         struct nftables_pernet *nft_net = nft_pernet(net);
9629
9630         INIT_LIST_HEAD(&nft_net->tables);
9631         INIT_LIST_HEAD(&nft_net->commit_list);
9632         INIT_LIST_HEAD(&nft_net->module_list);
9633         INIT_LIST_HEAD(&nft_net->notify_list);
9634         mutex_init(&nft_net->commit_mutex);
9635         nft_net->base_seq = 1;
9636         nft_net->validate_state = NFT_VALIDATE_SKIP;
9637
9638         return 0;
9639 }
9640
9641 static void __net_exit nf_tables_pre_exit_net(struct net *net)
9642 {
9643         __nft_release_hooks(net);
9644 }
9645
9646 static void __net_exit nf_tables_exit_net(struct net *net)
9647 {
9648         struct nftables_pernet *nft_net = nft_pernet(net);
9649
9650         mutex_lock(&nft_net->commit_mutex);
9651         if (!list_empty(&nft_net->commit_list))
9652                 __nf_tables_abort(net, NFNL_ABORT_NONE);
9653         __nft_release_tables(net);
9654         mutex_unlock(&nft_net->commit_mutex);
9655         WARN_ON_ONCE(!list_empty(&nft_net->tables));
9656         WARN_ON_ONCE(!list_empty(&nft_net->module_list));
9657         WARN_ON_ONCE(!list_empty(&nft_net->notify_list));
9658 }
9659
9660 static struct pernet_operations nf_tables_net_ops = {
9661         .init           = nf_tables_init_net,
9662         .pre_exit       = nf_tables_pre_exit_net,
9663         .exit           = nf_tables_exit_net,
9664         .id             = &nf_tables_net_id,
9665         .size           = sizeof(struct nftables_pernet),
9666 };
9667
9668 static int __init nf_tables_module_init(void)
9669 {
9670         int err;
9671
9672         err = register_pernet_subsys(&nf_tables_net_ops);
9673         if (err < 0)
9674                 return err;
9675
9676         err = nft_chain_filter_init();
9677         if (err < 0)
9678                 goto err_chain_filter;
9679
9680         err = nf_tables_core_module_init();
9681         if (err < 0)
9682                 goto err_core_module;
9683
9684         err = register_netdevice_notifier(&nf_tables_flowtable_notifier);
9685         if (err < 0)
9686                 goto err_netdev_notifier;
9687
9688         err = rhltable_init(&nft_objname_ht, &nft_objname_ht_params);
9689         if (err < 0)
9690                 goto err_rht_objname;
9691
9692         err = nft_offload_init();
9693         if (err < 0)
9694                 goto err_offload;
9695
9696         err = netlink_register_notifier(&nft_nl_notifier);
9697         if (err < 0)
9698                 goto err_netlink_notifier;
9699
9700         /* must be last */
9701         err = nfnetlink_subsys_register(&nf_tables_subsys);
9702         if (err < 0)
9703                 goto err_nfnl_subsys;
9704
9705         nft_chain_route_init();
9706
9707         return err;
9708
9709 err_nfnl_subsys:
9710         netlink_unregister_notifier(&nft_nl_notifier);
9711 err_netlink_notifier:
9712         nft_offload_exit();
9713 err_offload:
9714         rhltable_destroy(&nft_objname_ht);
9715 err_rht_objname:
9716         unregister_netdevice_notifier(&nf_tables_flowtable_notifier);
9717 err_netdev_notifier:
9718         nf_tables_core_module_exit();
9719 err_core_module:
9720         nft_chain_filter_fini();
9721 err_chain_filter:
9722         unregister_pernet_subsys(&nf_tables_net_ops);
9723         return err;
9724 }
9725
9726 static void __exit nf_tables_module_exit(void)
9727 {
9728         nfnetlink_subsys_unregister(&nf_tables_subsys);
9729         netlink_unregister_notifier(&nft_nl_notifier);
9730         nft_offload_exit();
9731         unregister_netdevice_notifier(&nf_tables_flowtable_notifier);
9732         nft_chain_filter_fini();
9733         nft_chain_route_fini();
9734         unregister_pernet_subsys(&nf_tables_net_ops);
9735         cancel_work_sync(&trans_destroy_work);
9736         rcu_barrier();
9737         rhltable_destroy(&nft_objname_ht);
9738         nf_tables_core_module_exit();
9739 }
9740
9741 module_init(nf_tables_module_init);
9742 module_exit(nf_tables_module_exit);
9743
9744 MODULE_LICENSE("GPL");
9745 MODULE_AUTHOR("Patrick McHardy <kaber@trash.net>");
9746 MODULE_ALIAS_NFNL_SUBSYS(NFNL_SUBSYS_NFTABLES);