net/sched/cls_api.c

   1 /*
   2  * net/sched/cls_api.c  Packet classifier API.
   3  *
   4  *              This program is free software; you can redistribute it and/or
   5  *              modify it under the terms of the GNU General Public License
   6  *              as published by the Free Software Foundation; either version
   7  *              2 of the License, or (at your option) any later version.
   8  *
   9  * Authors:     Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
  10  *
  11  * Changes:
  12  *
  13  * Eduardo J. Blanco <ejbs@netlabs.com.uy> :990222: kmod support
  14  *
  15  */
  16
  17 #include <linux/module.h>
  18 #include <linux/types.h>
  19 #include <linux/kernel.h>
  20 #include <linux/string.h>
  21 #include <linux/errno.h>
  22 #include <linux/err.h>
  23 #include <linux/skbuff.h>
  24 #include <linux/init.h>
  25 #include <linux/kmod.h>
  26 #include <linux/slab.h>
  27 #include <linux/idr.h>
  28 #include <linux/rhashtable.h>
  29 #include <net/net_namespace.h>
  30 #include <net/sock.h>
  31 #include <net/netlink.h>
  32 #include <net/pkt_sched.h>
  33 #include <net/pkt_cls.h>
  34 #include <net/tc_act/tc_pedit.h>
  35 #include <net/tc_act/tc_mirred.h>
  36 #include <net/tc_act/tc_vlan.h>
  37 #include <net/tc_act/tc_tunnel_key.h>
  38 #include <net/tc_act/tc_csum.h>
  39 #include <net/tc_act/tc_gact.h>
  40 #include <net/tc_act/tc_skbedit.h>
  41 #include <net/tc_act/tc_mirred.h>
  42
  43 extern const struct nla_policy rtm_tca_policy[TCA_MAX + 1];
  44
  45 /* The list of all installed classifier types */
  46 static LIST_HEAD(tcf_proto_base);
  47
  48 /* Protects list of registered TC modules. It is pure SMP lock. */
  49 static DEFINE_RWLOCK(cls_mod_lock);
  50
  51 /* Find classifier type by string name */
  52
  53 static const struct tcf_proto_ops *__tcf_proto_lookup_ops(const char *kind)
  54 {
  55         const struct tcf_proto_ops *t, *res = NULL;
  56
  57         if (kind) {
  58                 read_lock(&cls_mod_lock);
  59                 list_for_each_entry(t, &tcf_proto_base, head) {
  60                         if (strcmp(kind, t->kind) == 0) {
  61                                 if (try_module_get(t->owner))
  62                                         res = t;
  63                                 break;
  64                         }
  65                 }
  66                 read_unlock(&cls_mod_lock);
  67         }
  68         return res;
  69 }
  70
  71 static const struct tcf_proto_ops *
  72 tcf_proto_lookup_ops(const char *kind, struct netlink_ext_ack *extack)
  73 {
  74         const struct tcf_proto_ops *ops;
  75
  76         ops = __tcf_proto_lookup_ops(kind);
  77         if (ops)
  78                 return ops;
  79 #ifdef CONFIG_MODULES
  80         rtnl_unlock();
  81         request_module("cls_%s", kind);
  82         rtnl_lock();
  83         ops = __tcf_proto_lookup_ops(kind);
  84         /* We dropped the RTNL semaphore in order to perform
  85          * the module load. So, even if we succeeded in loading
  86          * the module we have to replay the request. We indicate
  87          * this using -EAGAIN.
  88          */
  89         if (ops) {
  90                 module_put(ops->owner);
  91                 return ERR_PTR(-EAGAIN);
  92         }
  93 #endif
  94         NL_SET_ERR_MSG(extack, "TC classifier not found");
  95         return ERR_PTR(-ENOENT);
  96 }
  97
  98 /* Register(unregister) new classifier type */
  99
 100 int register_tcf_proto_ops(struct tcf_proto_ops *ops)
 101 {
 102         struct tcf_proto_ops *t;
 103         int rc = -EEXIST;
 104
 105         write_lock(&cls_mod_lock);
 106         list_for_each_entry(t, &tcf_proto_base, head)
 107                 if (!strcmp(ops->kind, t->kind))
 108                         goto out;
 109
 110         list_add_tail(&ops->head, &tcf_proto_base);
 111         rc = 0;
 112 out:
 113         write_unlock(&cls_mod_lock);
 114         return rc;
 115 }
 116 EXPORT_SYMBOL(register_tcf_proto_ops);
 117
 118 static struct workqueue_struct *tc_filter_wq;
 119
 120 int unregister_tcf_proto_ops(struct tcf_proto_ops *ops)
 121 {
 122         struct tcf_proto_ops *t;
 123         int rc = -ENOENT;
 124
 125         /* Wait for outstanding call_rcu()s, if any, from a
 126          * tcf_proto_ops's destroy() handler.
 127          */
 128         rcu_barrier();
 129         flush_workqueue(tc_filter_wq);
 130
 131         write_lock(&cls_mod_lock);
 132         list_for_each_entry(t, &tcf_proto_base, head) {
 133                 if (t == ops) {
 134                         list_del(&t->head);
 135                         rc = 0;
 136                         break;
 137                 }
 138         }
 139         write_unlock(&cls_mod_lock);
 140         return rc;
 141 }
 142 EXPORT_SYMBOL(unregister_tcf_proto_ops);
 143
 144 bool tcf_queue_work(struct rcu_work *rwork, work_func_t func)
 145 {
 146         INIT_RCU_WORK(rwork, func);
 147         return queue_rcu_work(tc_filter_wq, rwork);
 148 }
 149 EXPORT_SYMBOL(tcf_queue_work);
 150
 151 /* Select new prio value from the range, managed by kernel. */
 152
 153 static inline u32 tcf_auto_prio(struct tcf_proto *tp)
 154 {
 155         u32 first = TC_H_MAKE(0xC0000000U, 0U);
 156
 157         if (tp)
 158                 first = tp->prio - 1;
 159
 160         return TC_H_MAJ(first);
 161 }
 162
 163 static struct tcf_proto *tcf_proto_create(const char *kind, u32 protocol,
 164                                           u32 prio, struct tcf_chain *chain,
 165                                           struct netlink_ext_ack *extack)
 166 {
 167         struct tcf_proto *tp;
 168         int err;
 169
 170         tp = kzalloc(sizeof(*tp), GFP_KERNEL);
 171         if (!tp)
 172                 return ERR_PTR(-ENOBUFS);
 173
 174         tp->ops = tcf_proto_lookup_ops(kind, extack);
 175         if (IS_ERR(tp->ops)) {
 176                 err = PTR_ERR(tp->ops);
 177                 goto errout;
 178         }
 179         tp->classify = tp->ops->classify;
 180         tp->protocol = protocol;
 181         tp->prio = prio;
 182         tp->chain = chain;
 183
 184         err = tp->ops->init(tp);
 185         if (err) {
 186                 module_put(tp->ops->owner);
 187                 goto errout;
 188         }
 189         return tp;
 190
 191 errout:
 192         kfree(tp);
 193         return ERR_PTR(err);
 194 }
 195
 196 static void tcf_proto_destroy(struct tcf_proto *tp,
 197                               struct netlink_ext_ack *extack)
 198 {
 199         tp->ops->destroy(tp, extack);
 200         module_put(tp->ops->owner);
 201         kfree_rcu(tp, rcu);
 202 }
 203
 204 #define ASSERT_BLOCK_LOCKED(block)                                      \
 205         lockdep_assert_held(&(block)->lock)
 206
 207 struct tcf_filter_chain_list_item {
 208         struct list_head list;
 209         tcf_chain_head_change_t *chain_head_change;
 210         void *chain_head_change_priv;
 211 };
 212
 213 static struct tcf_chain *tcf_chain_create(struct tcf_block *block,
 214                                           u32 chain_index)
 215 {
 216         struct tcf_chain *chain;
 217
 218         ASSERT_BLOCK_LOCKED(block);
 219
 220         chain = kzalloc(sizeof(*chain), GFP_KERNEL);
 221         if (!chain)
 222                 return NULL;
 223         list_add_tail(&chain->list, &block->chain_list);
 224         chain->block = block;
 225         chain->index = chain_index;
 226         chain->refcnt = 1;
 227         if (!chain->index)
 228                 block->chain0.chain = chain;
 229         return chain;
 230 }
 231
 232 static void tcf_chain_head_change_item(struct tcf_filter_chain_list_item *item,
 233                                        struct tcf_proto *tp_head)
 234 {
 235         if (item->chain_head_change)
 236                 item->chain_head_change(tp_head, item->chain_head_change_priv);
 237 }
 238
 239 static void tcf_chain0_head_change(struct tcf_chain *chain,
 240                                    struct tcf_proto *tp_head)
 241 {
 242         struct tcf_filter_chain_list_item *item;
 243         struct tcf_block *block = chain->block;
 244
 245         if (chain->index)
 246                 return;
 247         list_for_each_entry(item, &block->chain0.filter_chain_list, list)
 248                 tcf_chain_head_change_item(item, tp_head);
 249 }
 250
 251 /* Returns true if block can be safely freed. */
 252
 253 static bool tcf_chain_detach(struct tcf_chain *chain)
 254 {
 255         struct tcf_block *block = chain->block;
 256
 257         ASSERT_BLOCK_LOCKED(block);
 258
 259         list_del(&chain->list);
 260         if (!chain->index)
 261                 block->chain0.chain = NULL;
 262
 263         if (list_empty(&block->chain_list) &&
 264             refcount_read(&block->refcnt) == 0)
 265                 return true;
 266
 267         return false;
 268 }
 269
 270 static void tcf_block_destroy(struct tcf_block *block)
 271 {
 272         mutex_destroy(&block->lock);
 273         kfree_rcu(block, rcu);
 274 }
 275
 276 static void tcf_chain_destroy(struct tcf_chain *chain, bool free_block)
 277 {
 278         struct tcf_block *block = chain->block;
 279
 280         kfree(chain);
 281         if (free_block)
 282                 tcf_block_destroy(block);
 283 }
 284
 285 static void tcf_chain_hold(struct tcf_chain *chain)
 286 {
 287         ASSERT_BLOCK_LOCKED(chain->block);
 288
 289         ++chain->refcnt;
 290 }
 291
 292 static bool tcf_chain_held_by_acts_only(struct tcf_chain *chain)
 293 {
 294         ASSERT_BLOCK_LOCKED(chain->block);
 295
 296         /* In case all the references are action references, this
 297          * chain should not be shown to the user.
 298          */
 299         return chain->refcnt == chain->action_refcnt;
 300 }
 301
 302 static struct tcf_chain *tcf_chain_lookup(struct tcf_block *block,
 303                                           u32 chain_index)
 304 {
 305         struct tcf_chain *chain;
 306
 307         ASSERT_BLOCK_LOCKED(block);
 308
 309         list_for_each_entry(chain, &block->chain_list, list) {
 310                 if (chain->index == chain_index)
 311                         return chain;
 312         }
 313         return NULL;
 314 }
 315
 316 static int tc_chain_notify(struct tcf_chain *chain, struct sk_buff *oskb,
 317                            u32 seq, u16 flags, int event, bool unicast);
 318
 319 static struct tcf_chain *__tcf_chain_get(struct tcf_block *block,
 320                                          u32 chain_index, bool create,
 321                                          bool by_act)
 322 {
 323         struct tcf_chain *chain = NULL;
 324         bool is_first_reference;
 325
 326         mutex_lock(&block->lock);
 327         chain = tcf_chain_lookup(block, chain_index);
 328         if (chain) {
 329                 tcf_chain_hold(chain);
 330         } else {
 331                 if (!create)
 332                         goto errout;
 333                 chain = tcf_chain_create(block, chain_index);
 334                 if (!chain)
 335                         goto errout;
 336         }
 337
 338         if (by_act)
 339                 ++chain->action_refcnt;
 340         is_first_reference = chain->refcnt - chain->action_refcnt == 1;
 341         mutex_unlock(&block->lock);
 342
 343         /* Send notification only in case we got the first
 344          * non-action reference. Until then, the chain acts only as
 345          * a placeholder for actions pointing to it and user ought
 346          * not know about them.
 347          */
 348         if (is_first_reference && !by_act)
 349                 tc_chain_notify(chain, NULL, 0, NLM_F_CREATE | NLM_F_EXCL,
 350                                 RTM_NEWCHAIN, false);
 351
 352         return chain;
 353
 354 errout:
 355         mutex_unlock(&block->lock);
 356         return chain;
 357 }
 358
 359 static struct tcf_chain *tcf_chain_get(struct tcf_block *block, u32 chain_index,
 360                                        bool create)
 361 {
 362         return __tcf_chain_get(block, chain_index, create, false);
 363 }
 364
 365 struct tcf_chain *tcf_chain_get_by_act(struct tcf_block *block, u32 chain_index)
 366 {
 367         return __tcf_chain_get(block, chain_index, true, true);
 368 }
 369 EXPORT_SYMBOL(tcf_chain_get_by_act);
 370
 371 static void tc_chain_tmplt_del(struct tcf_chain *chain);
 372
 373 static void __tcf_chain_put(struct tcf_chain *chain, bool by_act)
 374 {
 375         struct tcf_block *block = chain->block;
 376         bool is_last, free_block = false;
 377         unsigned int refcnt;
 378
 379         mutex_lock(&block->lock);
 380         if (by_act)
 381                 chain->action_refcnt--;
 382
 383         /* tc_chain_notify_delete can't be called while holding block lock.
 384          * However, when block is unlocked chain can be changed concurrently, so
 385          * save these to temporary variables.
 386          */
 387         refcnt = --chain->refcnt;
 388         is_last = refcnt - chain->action_refcnt == 0;
 389         if (refcnt == 0)
 390                 free_block = tcf_chain_detach(chain);
 391         mutex_unlock(&block->lock);
 392
 393         /* The last dropped non-action reference will trigger notification. */
 394         if (is_last && !by_act)
 395                 tc_chain_notify(chain, NULL, 0, 0, RTM_DELCHAIN, false);
 396
 397         if (refcnt == 0) {
 398                 tc_chain_tmplt_del(chain);
 399                 tcf_chain_destroy(chain, free_block);
 400         }
 401 }
 402
 403 static void tcf_chain_put(struct tcf_chain *chain)
 404 {
 405         __tcf_chain_put(chain, false);
 406 }
 407
 408 void tcf_chain_put_by_act(struct tcf_chain *chain)
 409 {
 410         __tcf_chain_put(chain, true);
 411 }
 412 EXPORT_SYMBOL(tcf_chain_put_by_act);
 413
 414 static void tcf_chain_put_explicitly_created(struct tcf_chain *chain)
 415 {
 416         if (chain->explicitly_created)
 417                 tcf_chain_put(chain);
 418 }
 419
 420 static void tcf_chain_flush(struct tcf_chain *chain)
 421 {
 422         struct tcf_proto *tp = rtnl_dereference(chain->filter_chain);
 423
 424         tcf_chain0_head_change(chain, NULL);
 425         while (tp) {
 426                 RCU_INIT_POINTER(chain->filter_chain, tp->next);
 427                 tcf_proto_destroy(tp, NULL);
 428                 tp = rtnl_dereference(chain->filter_chain);
 429                 tcf_chain_put(chain);
 430         }
 431 }
 432
 433 static struct tcf_block *tc_dev_ingress_block(struct net_device *dev)
 434 {
 435         const struct Qdisc_class_ops *cops;
 436         struct Qdisc *qdisc;
 437
 438         if (!dev_ingress_queue(dev))
 439                 return NULL;
 440
 441         qdisc = dev_ingress_queue(dev)->qdisc_sleeping;
 442         if (!qdisc)
 443                 return NULL;
 444
 445         cops = qdisc->ops->cl_ops;
 446         if (!cops)
 447                 return NULL;
 448
 449         if (!cops->tcf_block)
 450                 return NULL;
 451
 452         return cops->tcf_block(qdisc, TC_H_MIN_INGRESS, NULL);
 453 }
 454
 455 static struct rhashtable indr_setup_block_ht;
 456
 457 struct tc_indr_block_dev {
 458         struct rhash_head ht_node;
 459         struct net_device *dev;
 460         unsigned int refcnt;
 461         struct list_head cb_list;
 462         struct tcf_block *block;
 463 };
 464
 465 struct tc_indr_block_cb {
 466         struct list_head list;
 467         void *cb_priv;
 468         tc_indr_block_bind_cb_t *cb;
 469         void *cb_ident;
 470 };
 471
 472 static const struct rhashtable_params tc_indr_setup_block_ht_params = {
 473         .key_offset     = offsetof(struct tc_indr_block_dev, dev),
 474         .head_offset    = offsetof(struct tc_indr_block_dev, ht_node),
 475         .key_len        = sizeof(struct net_device *),
 476 };
 477
 478 static struct tc_indr_block_dev *
 479 tc_indr_block_dev_lookup(struct net_device *dev)
 480 {
 481         return rhashtable_lookup_fast(&indr_setup_block_ht, &dev,
 482                                       tc_indr_setup_block_ht_params);
 483 }
 484
 485 static struct tc_indr_block_dev *tc_indr_block_dev_get(struct net_device *dev)
 486 {
 487         struct tc_indr_block_dev *indr_dev;
 488
 489         indr_dev = tc_indr_block_dev_lookup(dev);
 490         if (indr_dev)
 491                 goto inc_ref;
 492
 493         indr_dev = kzalloc(sizeof(*indr_dev), GFP_KERNEL);
 494         if (!indr_dev)
 495                 return NULL;
 496
 497         INIT_LIST_HEAD(&indr_dev->cb_list);
 498         indr_dev->dev = dev;
 499         indr_dev->block = tc_dev_ingress_block(dev);
 500         if (rhashtable_insert_fast(&indr_setup_block_ht, &indr_dev->ht_node,
 501                                    tc_indr_setup_block_ht_params)) {
 502                 kfree(indr_dev);
 503                 return NULL;
 504         }
 505
 506 inc_ref:
 507         indr_dev->refcnt++;
 508         return indr_dev;
 509 }
 510
 511 static void tc_indr_block_dev_put(struct tc_indr_block_dev *indr_dev)
 512 {
 513         if (--indr_dev->refcnt)
 514                 return;
 515
 516         rhashtable_remove_fast(&indr_setup_block_ht, &indr_dev->ht_node,
 517                                tc_indr_setup_block_ht_params);
 518         kfree(indr_dev);
 519 }
 520
 521 static struct tc_indr_block_cb *
 522 tc_indr_block_cb_lookup(struct tc_indr_block_dev *indr_dev,
 523                         tc_indr_block_bind_cb_t *cb, void *cb_ident)
 524 {
 525         struct tc_indr_block_cb *indr_block_cb;
 526
 527         list_for_each_entry(indr_block_cb, &indr_dev->cb_list, list)
 528                 if (indr_block_cb->cb == cb &&
 529                     indr_block_cb->cb_ident == cb_ident)
 530                         return indr_block_cb;
 531         return NULL;
 532 }
 533
 534 static struct tc_indr_block_cb *
 535 tc_indr_block_cb_add(struct tc_indr_block_dev *indr_dev, void *cb_priv,
 536                      tc_indr_block_bind_cb_t *cb, void *cb_ident)
 537 {
 538         struct tc_indr_block_cb *indr_block_cb;
 539
 540         indr_block_cb = tc_indr_block_cb_lookup(indr_dev, cb, cb_ident);
 541         if (indr_block_cb)
 542                 return ERR_PTR(-EEXIST);
 543
 544         indr_block_cb = kzalloc(sizeof(*indr_block_cb), GFP_KERNEL);
 545         if (!indr_block_cb)
 546                 return ERR_PTR(-ENOMEM);
 547
 548         indr_block_cb->cb_priv = cb_priv;
 549         indr_block_cb->cb = cb;
 550         indr_block_cb->cb_ident = cb_ident;
 551         list_add(&indr_block_cb->list, &indr_dev->cb_list);
 552
 553         return indr_block_cb;
 554 }
 555
 556 static void tc_indr_block_cb_del(struct tc_indr_block_cb *indr_block_cb)
 557 {
 558         list_del(&indr_block_cb->list);
 559         kfree(indr_block_cb);
 560 }
 561
 562 static void tc_indr_block_ing_cmd(struct tc_indr_block_dev *indr_dev,
 563                                   struct tc_indr_block_cb *indr_block_cb,
 564                                   enum tc_block_command command)
 565 {
 566         struct tc_block_offload bo = {
 567                 .command        = command,
 568                 .binder_type    = TCF_BLOCK_BINDER_TYPE_CLSACT_INGRESS,
 569                 .block          = indr_dev->block,
 570         };
 571
 572         if (!indr_dev->block)
 573                 return;
 574
 575         indr_block_cb->cb(indr_dev->dev, indr_block_cb->cb_priv, TC_SETUP_BLOCK,
 576                           &bo);
 577 }
 578
 579 int __tc_indr_block_cb_register(struct net_device *dev, void *cb_priv,
 580                                 tc_indr_block_bind_cb_t *cb, void *cb_ident)
 581 {
 582         struct tc_indr_block_cb *indr_block_cb;
 583         struct tc_indr_block_dev *indr_dev;
 584         int err;
 585
 586         indr_dev = tc_indr_block_dev_get(dev);
 587         if (!indr_dev)
 588                 return -ENOMEM;
 589
 590         indr_block_cb = tc_indr_block_cb_add(indr_dev, cb_priv, cb, cb_ident);
 591         err = PTR_ERR_OR_ZERO(indr_block_cb);
 592         if (err)
 593                 goto err_dev_put;
 594
 595         tc_indr_block_ing_cmd(indr_dev, indr_block_cb, TC_BLOCK_BIND);
 596         return 0;
 597
 598 err_dev_put:
 599         tc_indr_block_dev_put(indr_dev);
 600         return err;
 601 }
 602 EXPORT_SYMBOL_GPL(__tc_indr_block_cb_register);
 603
 604 int tc_indr_block_cb_register(struct net_device *dev, void *cb_priv,
 605                               tc_indr_block_bind_cb_t *cb, void *cb_ident)
 606 {
 607         int err;
 608
 609         rtnl_lock();
 610         err = __tc_indr_block_cb_register(dev, cb_priv, cb, cb_ident);
 611         rtnl_unlock();
 612
 613         return err;
 614 }
 615 EXPORT_SYMBOL_GPL(tc_indr_block_cb_register);
 616
 617 void __tc_indr_block_cb_unregister(struct net_device *dev,
 618                                    tc_indr_block_bind_cb_t *cb, void *cb_ident)
 619 {
 620         struct tc_indr_block_cb *indr_block_cb;
 621         struct tc_indr_block_dev *indr_dev;
 622
 623         indr_dev = tc_indr_block_dev_lookup(dev);
 624         if (!indr_dev)
 625                 return;
 626
 627         indr_block_cb = tc_indr_block_cb_lookup(indr_dev, cb, cb_ident);
 628         if (!indr_block_cb)
 629                 return;
 630
 631         /* Send unbind message if required to free any block cbs. */
 632         tc_indr_block_ing_cmd(indr_dev, indr_block_cb, TC_BLOCK_UNBIND);
 633         tc_indr_block_cb_del(indr_block_cb);
 634         tc_indr_block_dev_put(indr_dev);
 635 }
 636 EXPORT_SYMBOL_GPL(__tc_indr_block_cb_unregister);
 637
 638 void tc_indr_block_cb_unregister(struct net_device *dev,
 639                                  tc_indr_block_bind_cb_t *cb, void *cb_ident)
 640 {
 641         rtnl_lock();
 642         __tc_indr_block_cb_unregister(dev, cb, cb_ident);
 643         rtnl_unlock();
 644 }
 645 EXPORT_SYMBOL_GPL(tc_indr_block_cb_unregister);
 646
 647 static void tc_indr_block_call(struct tcf_block *block, struct net_device *dev,
 648                                struct tcf_block_ext_info *ei,
 649                                enum tc_block_command command,
 650                                struct netlink_ext_ack *extack)
 651 {
 652         struct tc_indr_block_cb *indr_block_cb;
 653         struct tc_indr_block_dev *indr_dev;
 654         struct tc_block_offload bo = {
 655                 .command        = command,
 656                 .binder_type    = ei->binder_type,
 657                 .block          = block,
 658                 .extack         = extack,
 659         };
 660
 661         indr_dev = tc_indr_block_dev_lookup(dev);
 662         if (!indr_dev)
 663                 return;
 664
 665         indr_dev->block = command == TC_BLOCK_BIND ? block : NULL;
 666
 667         list_for_each_entry(indr_block_cb, &indr_dev->cb_list, list)
 668                 indr_block_cb->cb(dev, indr_block_cb->cb_priv, TC_SETUP_BLOCK,
 669                                   &bo);
 670 }
 671
 672 static bool tcf_block_offload_in_use(struct tcf_block *block)
 673 {
 674         return block->offloadcnt;
 675 }
 676
 677 static int tcf_block_offload_cmd(struct tcf_block *block,
 678                                  struct net_device *dev,
 679                                  struct tcf_block_ext_info *ei,
 680                                  enum tc_block_command command,
 681                                  struct netlink_ext_ack *extack)
 682 {
 683         struct tc_block_offload bo = {};
 684
 685         bo.command = command;
 686         bo.binder_type = ei->binder_type;
 687         bo.block = block;
 688         bo.extack = extack;
 689         return dev->netdev_ops->ndo_setup_tc(dev, TC_SETUP_BLOCK, &bo);
 690 }
 691
 692 static int tcf_block_offload_bind(struct tcf_block *block, struct Qdisc *q,
 693                                   struct tcf_block_ext_info *ei,
 694                                   struct netlink_ext_ack *extack)
 695 {
 696         struct net_device *dev = q->dev_queue->dev;
 697         int err;
 698
 699         if (!dev->netdev_ops->ndo_setup_tc)
 700                 goto no_offload_dev_inc;
 701
 702         /* If tc offload feature is disabled and the block we try to bind
 703          * to already has some offloaded filters, forbid to bind.
 704          */
 705         if (!tc_can_offload(dev) && tcf_block_offload_in_use(block)) {
 706                 NL_SET_ERR_MSG(extack, "Bind to offloaded block failed as dev has offload disabled");
 707                 return -EOPNOTSUPP;
 708         }
 709
 710         err = tcf_block_offload_cmd(block, dev, ei, TC_BLOCK_BIND, extack);
 711         if (err == -EOPNOTSUPP)
 712                 goto no_offload_dev_inc;
 713         if (err)
 714                 return err;
 715
 716         tc_indr_block_call(block, dev, ei, TC_BLOCK_BIND, extack);
 717         return 0;
 718
 719 no_offload_dev_inc:
 720         if (tcf_block_offload_in_use(block))
 721                 return -EOPNOTSUPP;
 722         block->nooffloaddevcnt++;
 723         tc_indr_block_call(block, dev, ei, TC_BLOCK_BIND, extack);
 724         return 0;
 725 }
 726
 727 static void tcf_block_offload_unbind(struct tcf_block *block, struct Qdisc *q,
 728                                      struct tcf_block_ext_info *ei)
 729 {
 730         struct net_device *dev = q->dev_queue->dev;
 731         int err;
 732
 733         tc_indr_block_call(block, dev, ei, TC_BLOCK_UNBIND, NULL);
 734
 735         if (!dev->netdev_ops->ndo_setup_tc)
 736                 goto no_offload_dev_dec;
 737         err = tcf_block_offload_cmd(block, dev, ei, TC_BLOCK_UNBIND, NULL);
 738         if (err == -EOPNOTSUPP)
 739                 goto no_offload_dev_dec;
 740         return;
 741
 742 no_offload_dev_dec:
 743         WARN_ON(block->nooffloaddevcnt-- == 0);
 744 }
 745
 746 static int
 747 tcf_chain0_head_change_cb_add(struct tcf_block *block,
 748                               struct tcf_block_ext_info *ei,
 749                               struct netlink_ext_ack *extack)
 750 {
 751         struct tcf_chain *chain0 = block->chain0.chain;
 752         struct tcf_filter_chain_list_item *item;
 753
 754         item = kmalloc(sizeof(*item), GFP_KERNEL);
 755         if (!item) {
 756                 NL_SET_ERR_MSG(extack, "Memory allocation for head change callback item failed");
 757                 return -ENOMEM;
 758         }
 759         item->chain_head_change = ei->chain_head_change;
 760         item->chain_head_change_priv = ei->chain_head_change_priv;
 761         if (chain0 && chain0->filter_chain)
 762                 tcf_chain_head_change_item(item, chain0->filter_chain);
 763         list_add(&item->list, &block->chain0.filter_chain_list);
 764         return 0;
 765 }
 766
 767 static void
 768 tcf_chain0_head_change_cb_del(struct tcf_block *block,
 769                               struct tcf_block_ext_info *ei)
 770 {
 771         struct tcf_chain *chain0 = block->chain0.chain;
 772         struct tcf_filter_chain_list_item *item;
 773
 774         list_for_each_entry(item, &block->chain0.filter_chain_list, list) {
 775                 if ((!ei->chain_head_change && !ei->chain_head_change_priv) ||
 776                     (item->chain_head_change == ei->chain_head_change &&
 777                      item->chain_head_change_priv == ei->chain_head_change_priv)) {
 778                         if (chain0)
 779                                 tcf_chain_head_change_item(item, NULL);
 780                         list_del(&item->list);
 781                         kfree(item);
 782                         return;
 783                 }
 784         }
 785         WARN_ON(1);
 786 }
 787
 788 struct tcf_net {
 789         spinlock_t idr_lock; /* Protects idr */
 790         struct idr idr;
 791 };
 792
 793 static unsigned int tcf_net_id;
 794
 795 static int tcf_block_insert(struct tcf_block *block, struct net *net,
 796                             struct netlink_ext_ack *extack)
 797 {
 798         struct tcf_net *tn = net_generic(net, tcf_net_id);
 799         int err;
 800
 801         idr_preload(GFP_KERNEL);
 802         spin_lock(&tn->idr_lock);
 803         err = idr_alloc_u32(&tn->idr, block, &block->index, block->index,
 804                             GFP_NOWAIT);
 805         spin_unlock(&tn->idr_lock);
 806         idr_preload_end();
 807
 808         return err;
 809 }
 810
 811 static void tcf_block_remove(struct tcf_block *block, struct net *net)
 812 {
 813         struct tcf_net *tn = net_generic(net, tcf_net_id);
 814
 815         spin_lock(&tn->idr_lock);
 816         idr_remove(&tn->idr, block->index);
 817         spin_unlock(&tn->idr_lock);
 818 }
 819
 820 static struct tcf_block *tcf_block_create(struct net *net, struct Qdisc *q,
 821                                           u32 block_index,
 822                                           struct netlink_ext_ack *extack)
 823 {
 824         struct tcf_block *block;
 825
 826         block = kzalloc(sizeof(*block), GFP_KERNEL);
 827         if (!block) {
 828                 NL_SET_ERR_MSG(extack, "Memory allocation for block failed");
 829                 return ERR_PTR(-ENOMEM);
 830         }
 831         mutex_init(&block->lock);
 832         INIT_LIST_HEAD(&block->chain_list);
 833         INIT_LIST_HEAD(&block->cb_list);
 834         INIT_LIST_HEAD(&block->owner_list);
 835         INIT_LIST_HEAD(&block->chain0.filter_chain_list);
 836
 837         refcount_set(&block->refcnt, 1);
 838         block->net = net;
 839         block->index = block_index;
 840
 841         /* Don't store q pointer for blocks which are shared */
 842         if (!tcf_block_shared(block))
 843                 block->q = q;
 844         return block;
 845 }
 846
 847 static struct tcf_block *tcf_block_lookup(struct net *net, u32 block_index)
 848 {
 849         struct tcf_net *tn = net_generic(net, tcf_net_id);
 850
 851         return idr_find(&tn->idr, block_index);
 852 }
 853
 854 static struct tcf_block *tcf_block_refcnt_get(struct net *net, u32 block_index)
 855 {
 856         struct tcf_block *block;
 857
 858         rcu_read_lock();
 859         block = tcf_block_lookup(net, block_index);
 860         if (block && !refcount_inc_not_zero(&block->refcnt))
 861                 block = NULL;
 862         rcu_read_unlock();
 863
 864         return block;
 865 }
 866
 867 static void tcf_block_flush_all_chains(struct tcf_block *block)
 868 {
 869         struct tcf_chain *chain;
 870
 871         /* Hold a refcnt for all chains, so that they don't disappear
 872          * while we are iterating.
 873          */
 874         list_for_each_entry(chain, &block->chain_list, list)
 875                 tcf_chain_hold(chain);
 876
 877         list_for_each_entry(chain, &block->chain_list, list)
 878                 tcf_chain_flush(chain);
 879 }
 880
 881 static void tcf_block_put_all_chains(struct tcf_block *block)
 882 {
 883         struct tcf_chain *chain, *tmp;
 884
 885         /* At this point, all the chains should have refcnt >= 1. */
 886         list_for_each_entry_safe(chain, tmp, &block->chain_list, list) {
 887                 tcf_chain_put_explicitly_created(chain);
 888                 tcf_chain_put(chain);
 889         }
 890 }
 891
 892 static void __tcf_block_put(struct tcf_block *block, struct Qdisc *q,
 893                             struct tcf_block_ext_info *ei)
 894 {
 895         if (refcount_dec_and_mutex_lock(&block->refcnt, &block->lock)) {
 896                 /* Flushing/putting all chains will cause the block to be
 897                  * deallocated when last chain is freed. However, if chain_list
 898                  * is empty, block has to be manually deallocated. After block
 899                  * reference counter reached 0, it is no longer possible to
 900                  * increment it or add new chains to block.
 901                  */
 902                 bool free_block = list_empty(&block->chain_list);
 903
 904                 mutex_unlock(&block->lock);
 905                 if (tcf_block_shared(block))
 906                         tcf_block_remove(block, block->net);
 907                 if (!free_block)
 908                         tcf_block_flush_all_chains(block);
 909
 910                 if (q)
 911                         tcf_block_offload_unbind(block, q, ei);
 912
 913                 if (free_block)
 914                         tcf_block_destroy(block);
 915                 else
 916                         tcf_block_put_all_chains(block);
 917         } else if (q) {
 918                 tcf_block_offload_unbind(block, q, ei);
 919         }
 920 }
 921
 922 static void tcf_block_refcnt_put(struct tcf_block *block)
 923 {
 924         __tcf_block_put(block, NULL, NULL);
 925 }
 926
 927 /* Find tcf block.
 928  * Set q, parent, cl when appropriate.
 929  */
 930
 931 static struct tcf_block *tcf_block_find(struct net *net, struct Qdisc **q,
 932                                         u32 *parent, unsigned long *cl,
 933                                         int ifindex, u32 block_index,
 934                                         struct netlink_ext_ack *extack)
 935 {
 936         struct tcf_block *block;
 937         int err = 0;
 938
 939         if (ifindex == TCM_IFINDEX_MAGIC_BLOCK) {
 940                 block = tcf_block_refcnt_get(net, block_index);
 941                 if (!block) {
 942                         NL_SET_ERR_MSG(extack, "Block of given index was not found");
 943                         return ERR_PTR(-EINVAL);
 944                 }
 945         } else {
 946                 const struct Qdisc_class_ops *cops;
 947                 struct net_device *dev;
 948
 949                 rcu_read_lock();
 950
 951                 /* Find link */
 952                 dev = dev_get_by_index_rcu(net, ifindex);
 953                 if (!dev) {
 954                         rcu_read_unlock();
 955                         return ERR_PTR(-ENODEV);
 956                 }
 957
 958                 /* Find qdisc */
 959                 if (!*parent) {
 960                         *q = dev->qdisc;
 961                         *parent = (*q)->handle;
 962                 } else {
 963                         *q = qdisc_lookup_rcu(dev, TC_H_MAJ(*parent));
 964                         if (!*q) {
 965                                 NL_SET_ERR_MSG(extack, "Parent Qdisc doesn't exists");
 966                                 err = -EINVAL;
 967                                 goto errout_rcu;
 968                         }
 969                 }
 970
 971                 *q = qdisc_refcount_inc_nz(*q);
 972                 if (!*q) {
 973                         NL_SET_ERR_MSG(extack, "Parent Qdisc doesn't exists");
 974                         err = -EINVAL;
 975                         goto errout_rcu;
 976                 }
 977
 978                 /* Is it classful? */
 979                 cops = (*q)->ops->cl_ops;
 980                 if (!cops) {
 981                         NL_SET_ERR_MSG(extack, "Qdisc not classful");
 982                         err = -EINVAL;
 983                         goto errout_rcu;
 984                 }
 985
 986                 if (!cops->tcf_block) {
 987                         NL_SET_ERR_MSG(extack, "Class doesn't support blocks");
 988                         err = -EOPNOTSUPP;
 989                         goto errout_rcu;
 990                 }
 991
 992                 /* At this point we know that qdisc is not noop_qdisc,
 993                  * which means that qdisc holds a reference to net_device
 994                  * and we hold a reference to qdisc, so it is safe to release
 995                  * rcu read lock.
 996                  */
 997                 rcu_read_unlock();
 998
 999                 /* Do we search for filter, attached to class? */
1000                 if (TC_H_MIN(*parent)) {
1001                         *cl = cops->find(*q, *parent);
1002                         if (*cl == 0) {
1003                                 NL_SET_ERR_MSG(extack, "Specified class doesn't exist");
1004                                 err = -ENOENT;
1005                                 goto errout_qdisc;
1006                         }
1007                 }
1008
1009                 /* And the last stroke */
1010                 block = cops->tcf_block(*q, *cl, extack);
1011                 if (!block) {
1012                         err = -EINVAL;
1013                         goto errout_qdisc;
1014                 }
1015                 if (tcf_block_shared(block)) {
1016                         NL_SET_ERR_MSG(extack, "This filter block is shared. Please use the block index to manipulate the filters");
1017                         err = -EOPNOTSUPP;
1018                         goto errout_qdisc;
1019                 }
1020
1021                 /* Always take reference to block in order to support execution
1022                  * of rules update path of cls API without rtnl lock. Caller
1023                  * must release block when it is finished using it. 'if' block
1024                  * of this conditional obtain reference to block by calling
1025                  * tcf_block_refcnt_get().
1026                  */
1027                 refcount_inc(&block->refcnt);
1028         }
1029
1030         return block;
1031
1032 errout_rcu:
1033         rcu_read_unlock();
1034 errout_qdisc:
1035         if (*q) {
1036                 qdisc_put(*q);
1037                 *q = NULL;
1038         }
1039         return ERR_PTR(err);
1040 }
1041
1042 static void tcf_block_release(struct Qdisc *q, struct tcf_block *block)
1043 {
1044         if (!IS_ERR_OR_NULL(block))
1045                 tcf_block_refcnt_put(block);
1046
1047         if (q)
1048                 qdisc_put(q);
1049 }
1050
1051 struct tcf_block_owner_item {
1052         struct list_head list;
1053         struct Qdisc *q;
1054         enum tcf_block_binder_type binder_type;
1055 };
1056
1057 static void
1058 tcf_block_owner_netif_keep_dst(struct tcf_block *block,
1059                                struct Qdisc *q,
1060                                enum tcf_block_binder_type binder_type)
1061 {
1062         if (block->keep_dst &&
1063             binder_type != TCF_BLOCK_BINDER_TYPE_CLSACT_INGRESS &&
1064             binder_type != TCF_BLOCK_BINDER_TYPE_CLSACT_EGRESS)
1065                 netif_keep_dst(qdisc_dev(q));
1066 }
1067
1068 void tcf_block_netif_keep_dst(struct tcf_block *block)
1069 {
1070         struct tcf_block_owner_item *item;
1071
1072         block->keep_dst = true;
1073         list_for_each_entry(item, &block->owner_list, list)
1074                 tcf_block_owner_netif_keep_dst(block, item->q,
1075                                                item->binder_type);
1076 }
1077 EXPORT_SYMBOL(tcf_block_netif_keep_dst);
1078
1079 static int tcf_block_owner_add(struct tcf_block *block,
1080                                struct Qdisc *q,
1081                                enum tcf_block_binder_type binder_type)
1082 {
1083         struct tcf_block_owner_item *item;
1084
1085         item = kmalloc(sizeof(*item), GFP_KERNEL);
1086         if (!item)
1087                 return -ENOMEM;
1088         item->q = q;
1089         item->binder_type = binder_type;
1090         list_add(&item->list, &block->owner_list);
1091         return 0;
1092 }
1093
1094 static void tcf_block_owner_del(struct tcf_block *block,
1095                                 struct Qdisc *q,
1096                                 enum tcf_block_binder_type binder_type)
1097 {
1098         struct tcf_block_owner_item *item;
1099
1100         list_for_each_entry(item, &block->owner_list, list) {
1101                 if (item->q == q && item->binder_type == binder_type) {
1102                         list_del(&item->list);
1103                         kfree(item);
1104                         return;
1105                 }
1106         }
1107         WARN_ON(1);
1108 }
1109
1110 int tcf_block_get_ext(struct tcf_block **p_block, struct Qdisc *q,
1111                       struct tcf_block_ext_info *ei,
1112                       struct netlink_ext_ack *extack)
1113 {
1114         struct net *net = qdisc_net(q);
1115         struct tcf_block *block = NULL;
1116         int err;
1117
1118         if (ei->block_index)
1119                 /* block_index not 0 means the shared block is requested */
1120                 block = tcf_block_refcnt_get(net, ei->block_index);
1121
1122         if (!block) {
1123                 block = tcf_block_create(net, q, ei->block_index, extack);
1124                 if (IS_ERR(block))
1125                         return PTR_ERR(block);
1126                 if (tcf_block_shared(block)) {
1127                         err = tcf_block_insert(block, net, extack);
1128                         if (err)
1129                                 goto err_block_insert;
1130                 }
1131         }
1132
1133         err = tcf_block_owner_add(block, q, ei->binder_type);
1134         if (err)
1135                 goto err_block_owner_add;
1136
1137         tcf_block_owner_netif_keep_dst(block, q, ei->binder_type);
1138
1139         err = tcf_chain0_head_change_cb_add(block, ei, extack);
1140         if (err)
1141                 goto err_chain0_head_change_cb_add;
1142
1143         err = tcf_block_offload_bind(block, q, ei, extack);
1144         if (err)
1145                 goto err_block_offload_bind;
1146
1147         *p_block = block;
1148         return 0;
1149
1150 err_block_offload_bind:
1151         tcf_chain0_head_change_cb_del(block, ei);
1152 err_chain0_head_change_cb_add:
1153         tcf_block_owner_del(block, q, ei->binder_type);
1154 err_block_owner_add:
1155 err_block_insert:
1156         tcf_block_refcnt_put(block);
1157         return err;
1158 }
1159 EXPORT_SYMBOL(tcf_block_get_ext);
1160
1161 static void tcf_chain_head_change_dflt(struct tcf_proto *tp_head, void *priv)
1162 {
1163         struct tcf_proto __rcu **p_filter_chain = priv;
1164
1165         rcu_assign_pointer(*p_filter_chain, tp_head);
1166 }
1167
1168 int tcf_block_get(struct tcf_block **p_block,
1169                   struct tcf_proto __rcu **p_filter_chain, struct Qdisc *q,
1170                   struct netlink_ext_ack *extack)
1171 {
1172         struct tcf_block_ext_info ei = {
1173                 .chain_head_change = tcf_chain_head_change_dflt,
1174                 .chain_head_change_priv = p_filter_chain,
1175         };
1176
1177         WARN_ON(!p_filter_chain);
1178         return tcf_block_get_ext(p_block, q, &ei, extack);
1179 }
1180 EXPORT_SYMBOL(tcf_block_get);
1181
1182 /* XXX: Standalone actions are not allowed to jump to any chain, and bound
1183  * actions should be all removed after flushing.
1184  */
1185 void tcf_block_put_ext(struct tcf_block *block, struct Qdisc *q,
1186                        struct tcf_block_ext_info *ei)
1187 {
1188         if (!block)
1189                 return;
1190         tcf_chain0_head_change_cb_del(block, ei);
1191         tcf_block_owner_del(block, q, ei->binder_type);
1192
1193         __tcf_block_put(block, q, ei);
1194 }
1195 EXPORT_SYMBOL(tcf_block_put_ext);
1196
1197 void tcf_block_put(struct tcf_block *block)
1198 {
1199         struct tcf_block_ext_info ei = {0, };
1200
1201         if (!block)
1202                 return;
1203         tcf_block_put_ext(block, block->q, &ei);
1204 }
1205
1206 EXPORT_SYMBOL(tcf_block_put);
1207
1208 struct tcf_block_cb {
1209         struct list_head list;
1210         tc_setup_cb_t *cb;
1211         void *cb_ident;
1212         void *cb_priv;
1213         unsigned int refcnt;
1214 };
1215
1216 void *tcf_block_cb_priv(struct tcf_block_cb *block_cb)
1217 {
1218         return block_cb->cb_priv;
1219 }
1220 EXPORT_SYMBOL(tcf_block_cb_priv);
1221
1222 struct tcf_block_cb *tcf_block_cb_lookup(struct tcf_block *block,
1223                                          tc_setup_cb_t *cb, void *cb_ident)
1224 {       struct tcf_block_cb *block_cb;
1225
1226         list_for_each_entry(block_cb, &block->cb_list, list)
1227                 if (block_cb->cb == cb && block_cb->cb_ident == cb_ident)
1228                         return block_cb;
1229         return NULL;
1230 }
1231 EXPORT_SYMBOL(tcf_block_cb_lookup);
1232
1233 void tcf_block_cb_incref(struct tcf_block_cb *block_cb)
1234 {
1235         block_cb->refcnt++;
1236 }
1237 EXPORT_SYMBOL(tcf_block_cb_incref);
1238
1239 unsigned int tcf_block_cb_decref(struct tcf_block_cb *block_cb)
1240 {
1241         return --block_cb->refcnt;
1242 }
1243 EXPORT_SYMBOL(tcf_block_cb_decref);
1244
1245 static int
1246 tcf_block_playback_offloads(struct tcf_block *block, tc_setup_cb_t *cb,
1247                             void *cb_priv, bool add, bool offload_in_use,
1248                             struct netlink_ext_ack *extack)
1249 {
1250         struct tcf_chain *chain;
1251         struct tcf_proto *tp;
1252         int err;
1253
1254         list_for_each_entry(chain, &block->chain_list, list) {
1255                 for (tp = rtnl_dereference(chain->filter_chain); tp;
1256                      tp = rtnl_dereference(tp->next)) {
1257                         if (tp->ops->reoffload) {
1258                                 err = tp->ops->reoffload(tp, add, cb, cb_priv,
1259                                                          extack);
1260                                 if (err && add)
1261                                         goto err_playback_remove;
1262                         } else if (add && offload_in_use) {
1263                                 err = -EOPNOTSUPP;
1264                                 NL_SET_ERR_MSG(extack, "Filter HW offload failed - classifier without re-offloading support");
1265                                 goto err_playback_remove;
1266                         }
1267                 }
1268         }
1269
1270         return 0;
1271
1272 err_playback_remove:
1273         tcf_block_playback_offloads(block, cb, cb_priv, false, offload_in_use,
1274                                     extack);
1275         return err;
1276 }
1277
1278 struct tcf_block_cb *__tcf_block_cb_register(struct tcf_block *block,
1279                                              tc_setup_cb_t *cb, void *cb_ident,
1280                                              void *cb_priv,
1281                                              struct netlink_ext_ack *extack)
1282 {
1283         struct tcf_block_cb *block_cb;
1284         int err;
1285
1286         /* Replay any already present rules */
1287         err = tcf_block_playback_offloads(block, cb, cb_priv, true,
1288                                           tcf_block_offload_in_use(block),
1289                                           extack);
1290         if (err)
1291                 return ERR_PTR(err);
1292
1293         block_cb = kzalloc(sizeof(*block_cb), GFP_KERNEL);
1294         if (!block_cb)
1295                 return ERR_PTR(-ENOMEM);
1296         block_cb->cb = cb;
1297         block_cb->cb_ident = cb_ident;
1298         block_cb->cb_priv = cb_priv;
1299         list_add(&block_cb->list, &block->cb_list);
1300         return block_cb;
1301 }
1302 EXPORT_SYMBOL(__tcf_block_cb_register);
1303
1304 int tcf_block_cb_register(struct tcf_block *block,
1305                           tc_setup_cb_t *cb, void *cb_ident,
1306                           void *cb_priv, struct netlink_ext_ack *extack)
1307 {
1308         struct tcf_block_cb *block_cb;
1309
1310         block_cb = __tcf_block_cb_register(block, cb, cb_ident, cb_priv,
1311                                            extack);
1312         return PTR_ERR_OR_ZERO(block_cb);
1313 }
1314 EXPORT_SYMBOL(tcf_block_cb_register);
1315
1316 void __tcf_block_cb_unregister(struct tcf_block *block,
1317                                struct tcf_block_cb *block_cb)
1318 {
1319         tcf_block_playback_offloads(block, block_cb->cb, block_cb->cb_priv,
1320                                     false, tcf_block_offload_in_use(block),
1321                                     NULL);
1322         list_del(&block_cb->list);
1323         kfree(block_cb);
1324 }
1325 EXPORT_SYMBOL(__tcf_block_cb_unregister);
1326
1327 void tcf_block_cb_unregister(struct tcf_block *block,
1328                              tc_setup_cb_t *cb, void *cb_ident)
1329 {
1330         struct tcf_block_cb *block_cb;
1331
1332         block_cb = tcf_block_cb_lookup(block, cb, cb_ident);
1333         if (!block_cb)
1334                 return;
1335         __tcf_block_cb_unregister(block, block_cb);
1336 }
1337 EXPORT_SYMBOL(tcf_block_cb_unregister);
1338
1339 /* Main classifier routine: scans classifier chain attached
1340  * to this qdisc, (optionally) tests for protocol and asks
1341  * specific classifiers.
1342  */
1343 int tcf_classify(struct sk_buff *skb, const struct tcf_proto *tp,
1344                  struct tcf_result *res, bool compat_mode)
1345 {
1346 #ifdef CONFIG_NET_CLS_ACT
1347         const int max_reclassify_loop = 4;
1348         const struct tcf_proto *orig_tp = tp;
1349         const struct tcf_proto *first_tp;
1350         int limit = 0;
1351
1352 reclassify:
1353 #endif
1354         for (; tp; tp = rcu_dereference_bh(tp->next)) {
1355                 __be16 protocol = tc_skb_protocol(skb);
1356                 int err;
1357
1358                 if (tp->protocol != protocol &&
1359                     tp->protocol != htons(ETH_P_ALL))
1360                         continue;
1361
1362                 err = tp->classify(skb, tp, res);
1363 #ifdef CONFIG_NET_CLS_ACT
1364                 if (unlikely(err == TC_ACT_RECLASSIFY && !compat_mode)) {
1365                         first_tp = orig_tp;
1366                         goto reset;
1367                 } else if (unlikely(TC_ACT_EXT_CMP(err, TC_ACT_GOTO_CHAIN))) {
1368                         first_tp = res->goto_tp;
1369                         goto reset;
1370                 }
1371 #endif
1372                 if (err >= 0)
1373                         return err;
1374         }
1375
1376         return TC_ACT_UNSPEC; /* signal: continue lookup */
1377 #ifdef CONFIG_NET_CLS_ACT
1378 reset:
1379         if (unlikely(limit++ >= max_reclassify_loop)) {
1380                 net_notice_ratelimited("%u: reclassify loop, rule prio %u, protocol %02x\n",
1381                                        tp->chain->block->index,
1382                                        tp->prio & 0xffff,
1383                                        ntohs(tp->protocol));
1384                 return TC_ACT_SHOT;
1385         }
1386
1387         tp = first_tp;
1388         goto reclassify;
1389 #endif
1390 }
1391 EXPORT_SYMBOL(tcf_classify);
1392
1393 struct tcf_chain_info {
1394         struct tcf_proto __rcu **pprev;
1395         struct tcf_proto __rcu *next;
1396 };
1397
1398 static struct tcf_proto *tcf_chain_tp_prev(struct tcf_chain_info *chain_info)
1399 {
1400         return rtnl_dereference(*chain_info->pprev);
1401 }
1402
1403 static void tcf_chain_tp_insert(struct tcf_chain *chain,
1404                                 struct tcf_chain_info *chain_info,
1405                                 struct tcf_proto *tp)
1406 {
1407         if (*chain_info->pprev == chain->filter_chain)
1408                 tcf_chain0_head_change(chain, tp);
1409         RCU_INIT_POINTER(tp->next, tcf_chain_tp_prev(chain_info));
1410         rcu_assign_pointer(*chain_info->pprev, tp);
1411         tcf_chain_hold(chain);
1412 }
1413
1414 static void tcf_chain_tp_remove(struct tcf_chain *chain,
1415                                 struct tcf_chain_info *chain_info,
1416                                 struct tcf_proto *tp)
1417 {
1418         struct tcf_proto *next = rtnl_dereference(chain_info->next);
1419
1420         if (tp == chain->filter_chain)
1421                 tcf_chain0_head_change(chain, next);
1422         RCU_INIT_POINTER(*chain_info->pprev, next);
1423         tcf_chain_put(chain);
1424 }
1425
1426 static struct tcf_proto *tcf_chain_tp_find(struct tcf_chain *chain,
1427                                            struct tcf_chain_info *chain_info,
1428                                            u32 protocol, u32 prio,
1429                                            bool prio_allocate)
1430 {
1431         struct tcf_proto **pprev;
1432         struct tcf_proto *tp;
1433
1434         /* Check the chain for existence of proto-tcf with this priority */
1435         for (pprev = &chain->filter_chain;
1436              (tp = rtnl_dereference(*pprev)); pprev = &tp->next) {
1437                 if (tp->prio >= prio) {
1438                         if (tp->prio == prio) {
1439                                 if (prio_allocate ||
1440                                     (tp->protocol != protocol && protocol))
1441                                         return ERR_PTR(-EINVAL);
1442                         } else {
1443                                 tp = NULL;
1444                         }
1445                         break;
1446                 }
1447         }
1448         chain_info->pprev = pprev;
1449         chain_info->next = tp ? tp->next : NULL;
1450         return tp;
1451 }
1452
1453 static int tcf_fill_node(struct net *net, struct sk_buff *skb,
1454                          struct tcf_proto *tp, struct tcf_block *block,
1455                          struct Qdisc *q, u32 parent, void *fh,
1456                          u32 portid, u32 seq, u16 flags, int event)
1457 {
1458         struct tcmsg *tcm;
1459         struct nlmsghdr  *nlh;
1460         unsigned char *b = skb_tail_pointer(skb);
1461
1462         nlh = nlmsg_put(skb, portid, seq, event, sizeof(*tcm), flags);
1463         if (!nlh)
1464                 goto out_nlmsg_trim;
1465         tcm = nlmsg_data(nlh);
1466         tcm->tcm_family = AF_UNSPEC;
1467         tcm->tcm__pad1 = 0;
1468         tcm->tcm__pad2 = 0;
1469         if (q) {
1470                 tcm->tcm_ifindex = qdisc_dev(q)->ifindex;
1471                 tcm->tcm_parent = parent;
1472         } else {
1473                 tcm->tcm_ifindex = TCM_IFINDEX_MAGIC_BLOCK;
1474                 tcm->tcm_block_index = block->index;
1475         }
1476         tcm->tcm_info = TC_H_MAKE(tp->prio, tp->protocol);
1477         if (nla_put_string(skb, TCA_KIND, tp->ops->kind))
1478                 goto nla_put_failure;
1479         if (nla_put_u32(skb, TCA_CHAIN, tp->chain->index))
1480                 goto nla_put_failure;
1481         if (!fh) {
1482                 tcm->tcm_handle = 0;
1483         } else {
1484                 if (tp->ops->dump && tp->ops->dump(net, tp, fh, skb, tcm) < 0)
1485                         goto nla_put_failure;
1486         }
1487         nlh->nlmsg_len = skb_tail_pointer(skb) - b;
1488         return skb->len;
1489
1490 out_nlmsg_trim:
1491 nla_put_failure:
1492         nlmsg_trim(skb, b);
1493         return -1;
1494 }
1495
1496 static int tfilter_notify(struct net *net, struct sk_buff *oskb,
1497                           struct nlmsghdr *n, struct tcf_proto *tp,
1498                           struct tcf_block *block, struct Qdisc *q,
1499                           u32 parent, void *fh, int event, bool unicast)
1500 {
1501         struct sk_buff *skb;
1502         u32 portid = oskb ? NETLINK_CB(oskb).portid : 0;
1503
1504         skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
1505         if (!skb)
1506                 return -ENOBUFS;
1507
1508         if (tcf_fill_node(net, skb, tp, block, q, parent, fh, portid,
1509                           n->nlmsg_seq, n->nlmsg_flags, event) <= 0) {
1510                 kfree_skb(skb);
1511                 return -EINVAL;
1512         }
1513
1514         if (unicast)
1515                 return netlink_unicast(net->rtnl, skb, portid, MSG_DONTWAIT);
1516
1517         return rtnetlink_send(skb, net, portid, RTNLGRP_TC,
1518                               n->nlmsg_flags & NLM_F_ECHO);
1519 }
1520
1521 static int tfilter_del_notify(struct net *net, struct sk_buff *oskb,
1522                               struct nlmsghdr *n, struct tcf_proto *tp,
1523                               struct tcf_block *block, struct Qdisc *q,
1524                               u32 parent, void *fh, bool unicast, bool *last,
1525                               struct netlink_ext_ack *extack)
1526 {
1527         struct sk_buff *skb;
1528         u32 portid = oskb ? NETLINK_CB(oskb).portid : 0;
1529         int err;
1530
1531         skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
1532         if (!skb)
1533                 return -ENOBUFS;
1534
1535         if (tcf_fill_node(net, skb, tp, block, q, parent, fh, portid,
1536                           n->nlmsg_seq, n->nlmsg_flags, RTM_DELTFILTER) <= 0) {
1537                 NL_SET_ERR_MSG(extack, "Failed to build del event notification");
1538                 kfree_skb(skb);
1539                 return -EINVAL;
1540         }
1541
1542         err = tp->ops->delete(tp, fh, last, extack);
1543         if (err) {
1544                 kfree_skb(skb);
1545                 return err;
1546         }
1547
1548         if (unicast)
1549                 return netlink_unicast(net->rtnl, skb, portid, MSG_DONTWAIT);
1550
1551         err = rtnetlink_send(skb, net, portid, RTNLGRP_TC,
1552                              n->nlmsg_flags & NLM_F_ECHO);
1553         if (err < 0)
1554                 NL_SET_ERR_MSG(extack, "Failed to send filter delete notification");
1555         return err;
1556 }
1557
1558 static void tfilter_notify_chain(struct net *net, struct sk_buff *oskb,
1559                                  struct tcf_block *block, struct Qdisc *q,
1560                                  u32 parent, struct nlmsghdr *n,
1561                                  struct tcf_chain *chain, int event)
1562 {
1563         struct tcf_proto *tp;
1564
1565         for (tp = rtnl_dereference(chain->filter_chain);
1566              tp; tp = rtnl_dereference(tp->next))
1567                 tfilter_notify(net, oskb, n, tp, block,
1568                                q, parent, NULL, event, false);
1569 }
1570
1571 static int tc_new_tfilter(struct sk_buff *skb, struct nlmsghdr *n,
1572                           struct netlink_ext_ack *extack)
1573 {
1574         struct net *net = sock_net(skb->sk);
1575         struct nlattr *tca[TCA_MAX + 1];
1576         struct tcmsg *t;
1577         u32 protocol;
1578         u32 prio;
1579         bool prio_allocate;
1580         u32 parent;
1581         u32 chain_index;
1582         struct Qdisc *q = NULL;
1583         struct tcf_chain_info chain_info;
1584         struct tcf_chain *chain = NULL;
1585         struct tcf_block *block;
1586         struct tcf_proto *tp;
1587         unsigned long cl;
1588         void *fh;
1589         int err;
1590         int tp_created;
1591
1592         if (!netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN))
1593                 return -EPERM;
1594
1595 replay:
1596         tp_created = 0;
1597
1598         err = nlmsg_parse(n, sizeof(*t), tca, TCA_MAX, rtm_tca_policy, extack);
1599         if (err < 0)
1600                 return err;
1601
1602         t = nlmsg_data(n);
1603         protocol = TC_H_MIN(t->tcm_info);
1604         prio = TC_H_MAJ(t->tcm_info);
1605         prio_allocate = false;
1606         parent = t->tcm_parent;
1607         cl = 0;
1608
1609         if (prio == 0) {
1610                 /* If no priority is provided by the user,
1611                  * we allocate one.
1612                  */
1613                 if (n->nlmsg_flags & NLM_F_CREATE) {
1614                         prio = TC_H_MAKE(0x80000000U, 0U);
1615                         prio_allocate = true;
1616                 } else {
1617                         NL_SET_ERR_MSG(extack, "Invalid filter command with priority of zero");
1618                         return -ENOENT;
1619                 }
1620         }
1621
1622         /* Find head of filter chain. */
1623
1624         block = tcf_block_find(net, &q, &parent, &cl,
1625                                t->tcm_ifindex, t->tcm_block_index, extack);
1626         if (IS_ERR(block)) {
1627                 err = PTR_ERR(block);
1628                 goto errout;
1629         }
1630
1631         chain_index = tca[TCA_CHAIN] ? nla_get_u32(tca[TCA_CHAIN]) : 0;
1632         if (chain_index > TC_ACT_EXT_VAL_MASK) {
1633                 NL_SET_ERR_MSG(extack, "Specified chain index exceeds upper limit");
1634                 err = -EINVAL;
1635                 goto errout;
1636         }
1637         chain = tcf_chain_get(block, chain_index, true);
1638         if (!chain) {
1639                 NL_SET_ERR_MSG(extack, "Cannot create specified filter chain");
1640                 err = -ENOMEM;
1641                 goto errout;
1642         }
1643
1644         tp = tcf_chain_tp_find(chain, &chain_info, protocol,
1645                                prio, prio_allocate);
1646         if (IS_ERR(tp)) {
1647                 NL_SET_ERR_MSG(extack, "Filter with specified priority/protocol not found");
1648                 err = PTR_ERR(tp);
1649                 goto errout;
1650         }
1651
1652         if (tp == NULL) {
1653                 /* Proto-tcf does not exist, create new one */
1654
1655                 if (tca[TCA_KIND] == NULL || !protocol) {
1656                         NL_SET_ERR_MSG(extack, "Filter kind and protocol must be specified");
1657                         err = -EINVAL;
1658                         goto errout;
1659                 }
1660
1661                 if (!(n->nlmsg_flags & NLM_F_CREATE)) {
1662                         NL_SET_ERR_MSG(extack, "Need both RTM_NEWTFILTER and NLM_F_CREATE to create a new filter");
1663                         err = -ENOENT;
1664                         goto errout;
1665                 }
1666
1667                 if (prio_allocate)
1668                         prio = tcf_auto_prio(tcf_chain_tp_prev(&chain_info));
1669
1670                 tp = tcf_proto_create(nla_data(tca[TCA_KIND]),
1671                                       protocol, prio, chain, extack);
1672                 if (IS_ERR(tp)) {
1673                         err = PTR_ERR(tp);
1674                         goto errout;
1675                 }
1676                 tp_created = 1;
1677         } else if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], tp->ops->kind)) {
1678                 NL_SET_ERR_MSG(extack, "Specified filter kind does not match existing one");
1679                 err = -EINVAL;
1680                 goto errout;
1681         }
1682
1683         fh = tp->ops->get(tp, t->tcm_handle);
1684
1685         if (!fh) {
1686                 if (!(n->nlmsg_flags & NLM_F_CREATE)) {
1687                         NL_SET_ERR_MSG(extack, "Need both RTM_NEWTFILTER and NLM_F_CREATE to create a new filter");
1688                         err = -ENOENT;
1689                         goto errout;
1690                 }
1691         } else if (n->nlmsg_flags & NLM_F_EXCL) {
1692                 NL_SET_ERR_MSG(extack, "Filter already exists");
1693                 err = -EEXIST;
1694                 goto errout;
1695         }
1696
1697         if (chain->tmplt_ops && chain->tmplt_ops != tp->ops) {
1698                 NL_SET_ERR_MSG(extack, "Chain template is set to a different filter kind");
1699                 err = -EINVAL;
1700                 goto errout;
1701         }
1702
1703         err = tp->ops->change(net, skb, tp, cl, t->tcm_handle, tca, &fh,
1704                               n->nlmsg_flags & NLM_F_CREATE ? TCA_ACT_NOREPLACE : TCA_ACT_REPLACE,
1705                               extack);
1706         if (err == 0) {
1707                 if (tp_created)
1708                         tcf_chain_tp_insert(chain, &chain_info, tp);
1709                 tfilter_notify(net, skb, n, tp, block, q, parent, fh,
1710                                RTM_NEWTFILTER, false);
1711         } else {
1712                 if (tp_created)
1713                         tcf_proto_destroy(tp, NULL);
1714         }
1715
1716 errout:
1717         if (chain)
1718                 tcf_chain_put(chain);
1719         tcf_block_release(q, block);
1720         if (err == -EAGAIN)
1721                 /* Replay the request. */
1722                 goto replay;
1723         return err;
1724 }
1725
1726 static int tc_del_tfilter(struct sk_buff *skb, struct nlmsghdr *n,
1727                           struct netlink_ext_ack *extack)
1728 {
1729         struct net *net = sock_net(skb->sk);
1730         struct nlattr *tca[TCA_MAX + 1];
1731         struct tcmsg *t;
1732         u32 protocol;
1733         u32 prio;
1734         u32 parent;
1735         u32 chain_index;
1736         struct Qdisc *q = NULL;
1737         struct tcf_chain_info chain_info;
1738         struct tcf_chain *chain = NULL;
1739         struct tcf_block *block;
1740         struct tcf_proto *tp = NULL;
1741         unsigned long cl = 0;
1742         void *fh = NULL;
1743         int err;
1744
1745         if (!netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN))
1746                 return -EPERM;
1747
1748         err = nlmsg_parse(n, sizeof(*t), tca, TCA_MAX, rtm_tca_policy, extack);
1749         if (err < 0)
1750                 return err;
1751
1752         t = nlmsg_data(n);
1753         protocol = TC_H_MIN(t->tcm_info);
1754         prio = TC_H_MAJ(t->tcm_info);
1755         parent = t->tcm_parent;
1756
1757         if (prio == 0 && (protocol || t->tcm_handle || tca[TCA_KIND])) {
1758                 NL_SET_ERR_MSG(extack, "Cannot flush filters with protocol, handle or kind set");
1759                 return -ENOENT;
1760         }
1761
1762         /* Find head of filter chain. */
1763
1764         block = tcf_block_find(net, &q, &parent, &cl,
1765                                t->tcm_ifindex, t->tcm_block_index, extack);
1766         if (IS_ERR(block)) {
1767                 err = PTR_ERR(block);
1768                 goto errout;
1769         }
1770
1771         chain_index = tca[TCA_CHAIN] ? nla_get_u32(tca[TCA_CHAIN]) : 0;
1772         if (chain_index > TC_ACT_EXT_VAL_MASK) {
1773                 NL_SET_ERR_MSG(extack, "Specified chain index exceeds upper limit");
1774                 err = -EINVAL;
1775                 goto errout;
1776         }
1777         chain = tcf_chain_get(block, chain_index, false);
1778         if (!chain) {
1779                 /* User requested flush on non-existent chain. Nothing to do,
1780                  * so just return success.
1781                  */
1782                 if (prio == 0) {
1783                         err = 0;
1784                         goto errout;
1785                 }
1786                 NL_SET_ERR_MSG(extack, "Cannot find specified filter chain");
1787                 err = -ENOENT;
1788                 goto errout;
1789         }
1790
1791         if (prio == 0) {
1792                 tfilter_notify_chain(net, skb, block, q, parent, n,
1793                                      chain, RTM_DELTFILTER);
1794                 tcf_chain_flush(chain);
1795                 err = 0;
1796                 goto errout;
1797         }
1798
1799         tp = tcf_chain_tp_find(chain, &chain_info, protocol,
1800                                prio, false);
1801         if (!tp || IS_ERR(tp)) {
1802                 NL_SET_ERR_MSG(extack, "Filter with specified priority/protocol not found");
1803                 err = tp ? PTR_ERR(tp) : -ENOENT;
1804                 goto errout;
1805         } else if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], tp->ops->kind)) {
1806                 NL_SET_ERR_MSG(extack, "Specified filter kind does not match existing one");
1807                 err = -EINVAL;
1808                 goto errout;
1809         }
1810
1811         fh = tp->ops->get(tp, t->tcm_handle);
1812
1813         if (!fh) {
1814                 if (t->tcm_handle == 0) {
1815                         tcf_chain_tp_remove(chain, &chain_info, tp);
1816                         tfilter_notify(net, skb, n, tp, block, q, parent, fh,
1817                                        RTM_DELTFILTER, false);
1818                         tcf_proto_destroy(tp, extack);
1819                         err = 0;
1820                 } else {
1821                         NL_SET_ERR_MSG(extack, "Specified filter handle not found");
1822                         err = -ENOENT;
1823                 }
1824         } else {
1825                 bool last;
1826
1827                 err = tfilter_del_notify(net, skb, n, tp, block,
1828                                          q, parent, fh, false, &last,
1829                                          extack);
1830                 if (err)
1831                         goto errout;
1832                 if (last) {
1833                         tcf_chain_tp_remove(chain, &chain_info, tp);
1834                         tcf_proto_destroy(tp, extack);
1835                 }
1836         }
1837
1838 errout:
1839         if (chain)
1840                 tcf_chain_put(chain);
1841         tcf_block_release(q, block);
1842         return err;
1843 }
1844
1845 static int tc_get_tfilter(struct sk_buff *skb, struct nlmsghdr *n,
1846                           struct netlink_ext_ack *extack)
1847 {
1848         struct net *net = sock_net(skb->sk);
1849         struct nlattr *tca[TCA_MAX + 1];
1850         struct tcmsg *t;
1851         u32 protocol;
1852         u32 prio;
1853         u32 parent;
1854         u32 chain_index;
1855         struct Qdisc *q = NULL;
1856         struct tcf_chain_info chain_info;
1857         struct tcf_chain *chain = NULL;
1858         struct tcf_block *block;
1859         struct tcf_proto *tp = NULL;
1860         unsigned long cl = 0;
1861         void *fh = NULL;
1862         int err;
1863
1864         err = nlmsg_parse(n, sizeof(*t), tca, TCA_MAX, rtm_tca_policy, extack);
1865         if (err < 0)
1866                 return err;
1867
1868         t = nlmsg_data(n);
1869         protocol = TC_H_MIN(t->tcm_info);
1870         prio = TC_H_MAJ(t->tcm_info);
1871         parent = t->tcm_parent;
1872
1873         if (prio == 0) {
1874                 NL_SET_ERR_MSG(extack, "Invalid filter command with priority of zero");
1875                 return -ENOENT;
1876         }
1877
1878         /* Find head of filter chain. */
1879
1880         block = tcf_block_find(net, &q, &parent, &cl,
1881                                t->tcm_ifindex, t->tcm_block_index, extack);
1882         if (IS_ERR(block)) {
1883                 err = PTR_ERR(block);
1884                 goto errout;
1885         }
1886
1887         chain_index = tca[TCA_CHAIN] ? nla_get_u32(tca[TCA_CHAIN]) : 0;
1888         if (chain_index > TC_ACT_EXT_VAL_MASK) {
1889                 NL_SET_ERR_MSG(extack, "Specified chain index exceeds upper limit");
1890                 err = -EINVAL;
1891                 goto errout;
1892         }
1893         chain = tcf_chain_get(block, chain_index, false);
1894         if (!chain) {
1895                 NL_SET_ERR_MSG(extack, "Cannot find specified filter chain");
1896                 err = -EINVAL;
1897                 goto errout;
1898         }
1899
1900         tp = tcf_chain_tp_find(chain, &chain_info, protocol,
1901                                prio, false);
1902         if (!tp || IS_ERR(tp)) {
1903                 NL_SET_ERR_MSG(extack, "Filter with specified priority/protocol not found");
1904                 err = tp ? PTR_ERR(tp) : -ENOENT;
1905                 goto errout;
1906         } else if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], tp->ops->kind)) {
1907                 NL_SET_ERR_MSG(extack, "Specified filter kind does not match existing one");
1908                 err = -EINVAL;
1909                 goto errout;
1910         }
1911
1912         fh = tp->ops->get(tp, t->tcm_handle);
1913
1914         if (!fh) {
1915                 NL_SET_ERR_MSG(extack, "Specified filter handle not found");
1916                 err = -ENOENT;
1917         } else {
1918                 err = tfilter_notify(net, skb, n, tp, block, q, parent,
1919                                      fh, RTM_NEWTFILTER, true);
1920                 if (err < 0)
1921                         NL_SET_ERR_MSG(extack, "Failed to send filter notify message");
1922         }
1923
1924 errout:
1925         if (chain)
1926                 tcf_chain_put(chain);
1927         tcf_block_release(q, block);
1928         return err;
1929 }
1930
1931 struct tcf_dump_args {
1932         struct tcf_walker w;
1933         struct sk_buff *skb;
1934         struct netlink_callback *cb;
1935         struct tcf_block *block;
1936         struct Qdisc *q;
1937         u32 parent;
1938 };
1939
1940 static int tcf_node_dump(struct tcf_proto *tp, void *n, struct tcf_walker *arg)
1941 {
1942         struct tcf_dump_args *a = (void *)arg;
1943         struct net *net = sock_net(a->skb->sk);
1944
1945         return tcf_fill_node(net, a->skb, tp, a->block, a->q, a->parent,
1946                              n, NETLINK_CB(a->cb->skb).portid,
1947                              a->cb->nlh->nlmsg_seq, NLM_F_MULTI,
1948                              RTM_NEWTFILTER);
1949 }
1950
1951 static bool tcf_chain_dump(struct tcf_chain *chain, struct Qdisc *q, u32 parent,
1952                            struct sk_buff *skb, struct netlink_callback *cb,
1953                            long index_start, long *p_index)
1954 {
1955         struct net *net = sock_net(skb->sk);
1956         struct tcf_block *block = chain->block;
1957         struct tcmsg *tcm = nlmsg_data(cb->nlh);
1958         struct tcf_dump_args arg;
1959         struct tcf_proto *tp;
1960
1961         for (tp = rtnl_dereference(chain->filter_chain);
1962              tp; tp = rtnl_dereference(tp->next), (*p_index)++) {
1963                 if (*p_index < index_start)
1964                         continue;
1965                 if (TC_H_MAJ(tcm->tcm_info) &&
1966                     TC_H_MAJ(tcm->tcm_info) != tp->prio)
1967                         continue;
1968                 if (TC_H_MIN(tcm->tcm_info) &&
1969                     TC_H_MIN(tcm->tcm_info) != tp->protocol)
1970                         continue;
1971                 if (*p_index > index_start)
1972                         memset(&cb->args[1], 0,
1973                                sizeof(cb->args) - sizeof(cb->args[0]));
1974                 if (cb->args[1] == 0) {
1975                         if (tcf_fill_node(net, skb, tp, block, q, parent, NULL,
1976                                           NETLINK_CB(cb->skb).portid,
1977                                           cb->nlh->nlmsg_seq, NLM_F_MULTI,
1978                                           RTM_NEWTFILTER) <= 0)
1979                                 return false;
1980
1981                         cb->args[1] = 1;
1982                 }
1983                 if (!tp->ops->walk)
1984                         continue;
1985                 arg.w.fn = tcf_node_dump;
1986                 arg.skb = skb;
1987                 arg.cb = cb;
1988                 arg.block = block;
1989                 arg.q = q;
1990                 arg.parent = parent;
1991                 arg.w.stop = 0;
1992                 arg.w.skip = cb->args[1] - 1;
1993                 arg.w.count = 0;
1994                 arg.w.cookie = cb->args[2];
1995                 tp->ops->walk(tp, &arg.w);
1996                 cb->args[2] = arg.w.cookie;
1997                 cb->args[1] = arg.w.count + 1;
1998                 if (arg.w.stop)
1999                         return false;
2000         }
2001         return true;
2002 }
2003
2004 /* called with RTNL */
2005 static int tc_dump_tfilter(struct sk_buff *skb, struct netlink_callback *cb)
2006 {
2007         struct net *net = sock_net(skb->sk);
2008         struct nlattr *tca[TCA_MAX + 1];
2009         struct Qdisc *q = NULL;
2010         struct tcf_block *block;
2011         struct tcf_chain *chain;
2012         struct tcmsg *tcm = nlmsg_data(cb->nlh);
2013         long index_start;
2014         long index;
2015         u32 parent;
2016         int err;
2017
2018         if (nlmsg_len(cb->nlh) < sizeof(*tcm))
2019                 return skb->len;
2020
2021         err = nlmsg_parse(cb->nlh, sizeof(*tcm), tca, TCA_MAX, NULL,
2022                           cb->extack);
2023         if (err)
2024                 return err;
2025
2026         if (tcm->tcm_ifindex == TCM_IFINDEX_MAGIC_BLOCK) {
2027                 block = tcf_block_refcnt_get(net, tcm->tcm_block_index);
2028                 if (!block)
2029                         goto out;
2030                 /* If we work with block index, q is NULL and parent value
2031                  * will never be used in the following code. The check
2032                  * in tcf_fill_node prevents it. However, compiler does not
2033                  * see that far, so set parent to zero to silence the warning
2034                  * about parent being uninitialized.
2035                  */
2036                 parent = 0;
2037         } else {
2038                 const struct Qdisc_class_ops *cops;
2039                 struct net_device *dev;
2040                 unsigned long cl = 0;
2041
2042                 dev = __dev_get_by_index(net, tcm->tcm_ifindex);
2043                 if (!dev)
2044                         return skb->len;
2045
2046                 parent = tcm->tcm_parent;
2047                 if (!parent) {
2048                         q = dev->qdisc;
2049                         parent = q->handle;
2050                 } else {
2051                         q = qdisc_lookup(dev, TC_H_MAJ(tcm->tcm_parent));
2052                 }
2053                 if (!q)
2054                         goto out;
2055                 cops = q->ops->cl_ops;
2056                 if (!cops)
2057                         goto out;
2058                 if (!cops->tcf_block)
2059                         goto out;
2060                 if (TC_H_MIN(tcm->tcm_parent)) {
2061                         cl = cops->find(q, tcm->tcm_parent);
2062                         if (cl == 0)
2063                                 goto out;
2064                 }
2065                 block = cops->tcf_block(q, cl, NULL);
2066                 if (!block)
2067                         goto out;
2068                 if (tcf_block_shared(block))
2069                         q = NULL;
2070         }
2071
2072         index_start = cb->args[0];
2073         index = 0;
2074
2075         list_for_each_entry(chain, &block->chain_list, list) {
2076                 if (tca[TCA_CHAIN] &&
2077                     nla_get_u32(tca[TCA_CHAIN]) != chain->index)
2078                         continue;
2079                 if (!tcf_chain_dump(chain, q, parent, skb, cb,
2080                                     index_start, &index)) {
2081                         err = -EMSGSIZE;
2082                         break;
2083                 }
2084         }
2085
2086         if (tcm->tcm_ifindex == TCM_IFINDEX_MAGIC_BLOCK)
2087                 tcf_block_refcnt_put(block);
2088         cb->args[0] = index;
2089
2090 out:
2091         /* If we did no progress, the error (EMSGSIZE) is real */
2092         if (skb->len == 0 && err)
2093                 return err;
2094         return skb->len;
2095 }
2096
2097 static int tc_chain_fill_node(struct tcf_chain *chain, struct net *net,
2098                               struct sk_buff *skb, struct tcf_block *block,
2099                               u32 portid, u32 seq, u16 flags, int event)
2100 {
2101         unsigned char *b = skb_tail_pointer(skb);
2102         const struct tcf_proto_ops *ops;
2103         struct nlmsghdr *nlh;
2104         struct tcmsg *tcm;
2105         void *priv;
2106
2107         ops = chain->tmplt_ops;
2108         priv = chain->tmplt_priv;
2109
2110         nlh = nlmsg_put(skb, portid, seq, event, sizeof(*tcm), flags);
2111         if (!nlh)
2112                 goto out_nlmsg_trim;
2113         tcm = nlmsg_data(nlh);
2114         tcm->tcm_family = AF_UNSPEC;
2115         tcm->tcm__pad1 = 0;
2116         tcm->tcm__pad2 = 0;
2117         tcm->tcm_handle = 0;
2118         if (block->q) {
2119                 tcm->tcm_ifindex = qdisc_dev(block->q)->ifindex;
2120                 tcm->tcm_parent = block->q->handle;
2121         } else {
2122                 tcm->tcm_ifindex = TCM_IFINDEX_MAGIC_BLOCK;
2123                 tcm->tcm_block_index = block->index;
2124         }
2125
2126         if (nla_put_u32(skb, TCA_CHAIN, chain->index))
2127                 goto nla_put_failure;
2128
2129         if (ops) {
2130                 if (nla_put_string(skb, TCA_KIND, ops->kind))
2131                         goto nla_put_failure;
2132                 if (ops->tmplt_dump(skb, net, priv) < 0)
2133                         goto nla_put_failure;
2134         }
2135
2136         nlh->nlmsg_len = skb_tail_pointer(skb) - b;
2137         return skb->len;
2138
2139 out_nlmsg_trim:
2140 nla_put_failure:
2141         nlmsg_trim(skb, b);
2142         return -EMSGSIZE;
2143 }
2144
2145 static int tc_chain_notify(struct tcf_chain *chain, struct sk_buff *oskb,
2146                            u32 seq, u16 flags, int event, bool unicast)
2147 {
2148         u32 portid = oskb ? NETLINK_CB(oskb).portid : 0;
2149         struct tcf_block *block = chain->block;
2150         struct net *net = block->net;
2151         struct sk_buff *skb;
2152
2153         skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
2154         if (!skb)
2155                 return -ENOBUFS;
2156
2157         if (tc_chain_fill_node(chain, net, skb, block, portid,
2158                                seq, flags, event) <= 0) {
2159                 kfree_skb(skb);
2160                 return -EINVAL;
2161         }
2162
2163         if (unicast)
2164                 return netlink_unicast(net->rtnl, skb, portid, MSG_DONTWAIT);
2165
2166         return rtnetlink_send(skb, net, portid, RTNLGRP_TC, flags & NLM_F_ECHO);
2167 }
2168
2169 static int tc_chain_tmplt_add(struct tcf_chain *chain, struct net *net,
2170                               struct nlattr **tca,
2171                               struct netlink_ext_ack *extack)
2172 {
2173         const struct tcf_proto_ops *ops;
2174         void *tmplt_priv;
2175
2176         /* If kind is not set, user did not specify template. */
2177         if (!tca[TCA_KIND])
2178                 return 0;
2179
2180         ops = tcf_proto_lookup_ops(nla_data(tca[TCA_KIND]), extack);
2181         if (IS_ERR(ops))
2182                 return PTR_ERR(ops);
2183         if (!ops->tmplt_create || !ops->tmplt_destroy || !ops->tmplt_dump) {
2184                 NL_SET_ERR_MSG(extack, "Chain templates are not supported with specified classifier");
2185                 return -EOPNOTSUPP;
2186         }
2187
2188         tmplt_priv = ops->tmplt_create(net, chain, tca, extack);
2189         if (IS_ERR(tmplt_priv)) {
2190                 module_put(ops->owner);
2191                 return PTR_ERR(tmplt_priv);
2192         }
2193         chain->tmplt_ops = ops;
2194         chain->tmplt_priv = tmplt_priv;
2195         return 0;
2196 }
2197
2198 static void tc_chain_tmplt_del(struct tcf_chain *chain)
2199 {
2200         const struct tcf_proto_ops *ops = chain->tmplt_ops;
2201
2202         /* If template ops are set, no work to do for us. */
2203         if (!ops)
2204                 return;
2205
2206         ops->tmplt_destroy(chain->tmplt_priv);
2207         module_put(ops->owner);
2208 }
2209
2210 /* Add/delete/get a chain */
2211
2212 static int tc_ctl_chain(struct sk_buff *skb, struct nlmsghdr *n,
2213                         struct netlink_ext_ack *extack)
2214 {
2215         struct net *net = sock_net(skb->sk);
2216         struct nlattr *tca[TCA_MAX + 1];
2217         struct tcmsg *t;
2218         u32 parent;
2219         u32 chain_index;
2220         struct Qdisc *q = NULL;
2221         struct tcf_chain *chain = NULL;
2222         struct tcf_block *block;
2223         unsigned long cl;
2224         int err;
2225
2226         if (n->nlmsg_type != RTM_GETCHAIN &&
2227             !netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN))
2228                 return -EPERM;
2229
2230 replay:
2231         err = nlmsg_parse(n, sizeof(*t), tca, TCA_MAX, rtm_tca_policy, extack);
2232         if (err < 0)
2233                 return err;
2234
2235         t = nlmsg_data(n);
2236         parent = t->tcm_parent;
2237         cl = 0;
2238
2239         block = tcf_block_find(net, &q, &parent, &cl,
2240                                t->tcm_ifindex, t->tcm_block_index, extack);
2241         if (IS_ERR(block))
2242                 return PTR_ERR(block);
2243
2244         chain_index = tca[TCA_CHAIN] ? nla_get_u32(tca[TCA_CHAIN]) : 0;
2245         if (chain_index > TC_ACT_EXT_VAL_MASK) {
2246                 NL_SET_ERR_MSG(extack, "Specified chain index exceeds upper limit");
2247                 err = -EINVAL;
2248                 goto errout_block;
2249         }
2250         chain = tcf_chain_lookup(block, chain_index);
2251         if (n->nlmsg_type == RTM_NEWCHAIN) {
2252                 if (chain) {
2253                         if (tcf_chain_held_by_acts_only(chain)) {
2254                                 /* The chain exists only because there is
2255                                  * some action referencing it.
2256                                  */
2257                                 tcf_chain_hold(chain);
2258                         } else {
2259                                 NL_SET_ERR_MSG(extack, "Filter chain already exists");
2260                                 err = -EEXIST;
2261                                 goto errout_block;
2262                         }
2263                 } else {
2264                         if (!(n->nlmsg_flags & NLM_F_CREATE)) {
2265                                 NL_SET_ERR_MSG(extack, "Need both RTM_NEWCHAIN and NLM_F_CREATE to create a new chain");
2266                                 err = -ENOENT;
2267                                 goto errout_block;
2268                         }
2269                         chain = tcf_chain_create(block, chain_index);
2270                         if (!chain) {
2271                                 NL_SET_ERR_MSG(extack, "Failed to create filter chain");
2272                                 err = -ENOMEM;
2273                                 goto errout_block;
2274                         }
2275                 }
2276         } else {
2277                 if (!chain || tcf_chain_held_by_acts_only(chain)) {
2278                         NL_SET_ERR_MSG(extack, "Cannot find specified filter chain");
2279                         err = -EINVAL;
2280                         goto errout_block;
2281                 }
2282                 tcf_chain_hold(chain);
2283         }
2284
2285         switch (n->nlmsg_type) {
2286         case RTM_NEWCHAIN:
2287                 err = tc_chain_tmplt_add(chain, net, tca, extack);
2288                 if (err)
2289                         goto errout;
2290                 /* In case the chain was successfully added, take a reference
2291                  * to the chain. This ensures that an empty chain
2292                  * does not disappear at the end of this function.
2293                  */
2294                 tcf_chain_hold(chain);
2295                 chain->explicitly_created = true;
2296                 tc_chain_notify(chain, NULL, 0, NLM_F_CREATE | NLM_F_EXCL,
2297                                 RTM_NEWCHAIN, false);
2298                 break;
2299         case RTM_DELCHAIN:
2300                 tfilter_notify_chain(net, skb, block, q, parent, n,
2301                                      chain, RTM_DELTFILTER);
2302                 /* Flush the chain first as the user requested chain removal. */
2303                 tcf_chain_flush(chain);
2304                 /* In case the chain was successfully deleted, put a reference
2305                  * to the chain previously taken during addition.
2306                  */
2307                 tcf_chain_put_explicitly_created(chain);
2308                 chain->explicitly_created = false;
2309                 break;
2310         case RTM_GETCHAIN:
2311                 err = tc_chain_notify(chain, skb, n->nlmsg_seq,
2312                                       n->nlmsg_seq, n->nlmsg_type, true);
2313                 if (err < 0)
2314                         NL_SET_ERR_MSG(extack, "Failed to send chain notify message");
2315                 break;
2316         default:
2317                 err = -EOPNOTSUPP;
2318                 NL_SET_ERR_MSG(extack, "Unsupported message type");
2319                 goto errout;
2320         }
2321
2322 errout:
2323         tcf_chain_put(chain);
2324 errout_block:
2325         tcf_block_release(q, block);
2326         if (err == -EAGAIN)
2327                 /* Replay the request. */
2328                 goto replay;
2329         return err;
2330 }
2331
2332 /* called with RTNL */
2333 static int tc_dump_chain(struct sk_buff *skb, struct netlink_callback *cb)
2334 {
2335         struct net *net = sock_net(skb->sk);
2336         struct nlattr *tca[TCA_MAX + 1];
2337         struct Qdisc *q = NULL;
2338         struct tcf_block *block;
2339         struct tcf_chain *chain;
2340         struct tcmsg *tcm = nlmsg_data(cb->nlh);
2341         long index_start;
2342         long index;
2343         u32 parent;
2344         int err;
2345
2346         if (nlmsg_len(cb->nlh) < sizeof(*tcm))
2347                 return skb->len;
2348
2349         err = nlmsg_parse(cb->nlh, sizeof(*tcm), tca, TCA_MAX, rtm_tca_policy,
2350                           cb->extack);
2351         if (err)
2352                 return err;
2353
2354         if (tcm->tcm_ifindex == TCM_IFINDEX_MAGIC_BLOCK) {
2355                 block = tcf_block_refcnt_get(net, tcm->tcm_block_index);
2356                 if (!block)
2357                         goto out;
2358                 /* If we work with block index, q is NULL and parent value
2359                  * will never be used in the following code. The check
2360                  * in tcf_fill_node prevents it. However, compiler does not
2361                  * see that far, so set parent to zero to silence the warning
2362                  * about parent being uninitialized.
2363                  */
2364                 parent = 0;
2365         } else {
2366                 const struct Qdisc_class_ops *cops;
2367                 struct net_device *dev;
2368                 unsigned long cl = 0;
2369
2370                 dev = __dev_get_by_index(net, tcm->tcm_ifindex);
2371                 if (!dev)
2372                         return skb->len;
2373
2374                 parent = tcm->tcm_parent;
2375                 if (!parent) {
2376                         q = dev->qdisc;
2377                         parent = q->handle;
2378                 } else {
2379                         q = qdisc_lookup(dev, TC_H_MAJ(tcm->tcm_parent));
2380                 }
2381                 if (!q)
2382                         goto out;
2383                 cops = q->ops->cl_ops;
2384                 if (!cops)
2385                         goto out;
2386                 if (!cops->tcf_block)
2387                         goto out;
2388                 if (TC_H_MIN(tcm->tcm_parent)) {
2389                         cl = cops->find(q, tcm->tcm_parent);
2390                         if (cl == 0)
2391                                 goto out;
2392                 }
2393                 block = cops->tcf_block(q, cl, NULL);
2394                 if (!block)
2395                         goto out;
2396                 if (tcf_block_shared(block))
2397                         q = NULL;
2398         }
2399
2400         index_start = cb->args[0];
2401         index = 0;
2402
2403         list_for_each_entry(chain, &block->chain_list, list) {
2404                 if ((tca[TCA_CHAIN] &&
2405                      nla_get_u32(tca[TCA_CHAIN]) != chain->index))
2406                         continue;
2407                 if (index < index_start) {
2408                         index++;
2409                         continue;
2410                 }
2411                 if (tcf_chain_held_by_acts_only(chain))
2412                         continue;
2413                 err = tc_chain_fill_node(chain, net, skb, block,
2414                                          NETLINK_CB(cb->skb).portid,
2415                                          cb->nlh->nlmsg_seq, NLM_F_MULTI,
2416                                          RTM_NEWCHAIN);
2417                 if (err <= 0)
2418                         break;
2419                 index++;
2420         }
2421
2422         if (tcm->tcm_ifindex == TCM_IFINDEX_MAGIC_BLOCK)
2423                 tcf_block_refcnt_put(block);
2424         cb->args[0] = index;
2425
2426 out:
2427         /* If we did no progress, the error (EMSGSIZE) is real */
2428         if (skb->len == 0 && err)
2429                 return err;
2430         return skb->len;
2431 }
2432
2433 void tcf_exts_destroy(struct tcf_exts *exts)
2434 {
2435 #ifdef CONFIG_NET_CLS_ACT
2436         tcf_action_destroy(exts->actions, TCA_ACT_UNBIND);
2437         kfree(exts->actions);
2438         exts->nr_actions = 0;
2439 #endif
2440 }
2441 EXPORT_SYMBOL(tcf_exts_destroy);
2442
2443 int tcf_exts_validate(struct net *net, struct tcf_proto *tp, struct nlattr **tb,
2444                       struct nlattr *rate_tlv, struct tcf_exts *exts, bool ovr,
2445                       struct netlink_ext_ack *extack)
2446 {
2447 #ifdef CONFIG_NET_CLS_ACT
2448         {
2449                 struct tc_action *act;
2450                 size_t attr_size = 0;
2451
2452                 if (exts->police && tb[exts->police]) {
2453                         act = tcf_action_init_1(net, tp, tb[exts->police],
2454                                                 rate_tlv, "police", ovr,
2455                                                 TCA_ACT_BIND, true, extack);
2456                         if (IS_ERR(act))
2457                                 return PTR_ERR(act);
2458
2459                         act->type = exts->type = TCA_OLD_COMPAT;
2460                         exts->actions[0] = act;
2461                         exts->nr_actions = 1;
2462                 } else if (exts->action && tb[exts->action]) {
2463                         int err;
2464
2465                         err = tcf_action_init(net, tp, tb[exts->action],
2466                                               rate_tlv, NULL, ovr, TCA_ACT_BIND,
2467                                               exts->actions, &attr_size, true,
2468                                               extack);
2469                         if (err < 0)
2470                                 return err;
2471                         exts->nr_actions = err;
2472                 }
2473                 exts->net = net;
2474         }
2475 #else
2476         if ((exts->action && tb[exts->action]) ||
2477             (exts->police && tb[exts->police])) {
2478                 NL_SET_ERR_MSG(extack, "Classifier actions are not supported per compile options (CONFIG_NET_CLS_ACT)");
2479                 return -EOPNOTSUPP;
2480         }
2481 #endif
2482
2483         return 0;
2484 }
2485 EXPORT_SYMBOL(tcf_exts_validate);
2486
2487 void tcf_exts_change(struct tcf_exts *dst, struct tcf_exts *src)
2488 {
2489 #ifdef CONFIG_NET_CLS_ACT
2490         struct tcf_exts old = *dst;
2491
2492         *dst = *src;
2493         tcf_exts_destroy(&old);
2494 #endif
2495 }
2496 EXPORT_SYMBOL(tcf_exts_change);
2497
2498 #ifdef CONFIG_NET_CLS_ACT
2499 static struct tc_action *tcf_exts_first_act(struct tcf_exts *exts)
2500 {
2501         if (exts->nr_actions == 0)
2502                 return NULL;
2503         else
2504                 return exts->actions[0];
2505 }
2506 #endif
2507
2508 int tcf_exts_dump(struct sk_buff *skb, struct tcf_exts *exts)
2509 {
2510 #ifdef CONFIG_NET_CLS_ACT
2511         struct nlattr *nest;
2512
2513         if (exts->action && tcf_exts_has_actions(exts)) {
2514                 /*
2515                  * again for backward compatible mode - we want
2516                  * to work with both old and new modes of entering
2517                  * tc data even if iproute2  was newer - jhs
2518                  */
2519                 if (exts->type != TCA_OLD_COMPAT) {
2520                         nest = nla_nest_start(skb, exts->action);
2521                         if (nest == NULL)
2522                                 goto nla_put_failure;
2523
2524                         if (tcf_action_dump(skb, exts->actions, 0, 0) < 0)
2525                                 goto nla_put_failure;
2526                         nla_nest_end(skb, nest);
2527                 } else if (exts->police) {
2528                         struct tc_action *act = tcf_exts_first_act(exts);
2529                         nest = nla_nest_start(skb, exts->police);
2530                         if (nest == NULL || !act)
2531                                 goto nla_put_failure;
2532                         if (tcf_action_dump_old(skb, act, 0, 0) < 0)
2533                                 goto nla_put_failure;
2534                         nla_nest_end(skb, nest);
2535                 }
2536         }
2537         return 0;
2538
2539 nla_put_failure:
2540         nla_nest_cancel(skb, nest);
2541         return -1;
2542 #else
2543         return 0;
2544 #endif
2545 }
2546 EXPORT_SYMBOL(tcf_exts_dump);
2547
2548
2549 int tcf_exts_dump_stats(struct sk_buff *skb, struct tcf_exts *exts)
2550 {
2551 #ifdef CONFIG_NET_CLS_ACT
2552         struct tc_action *a = tcf_exts_first_act(exts);
2553         if (a != NULL && tcf_action_copy_stats(skb, a, 1) < 0)
2554                 return -1;
2555 #endif
2556         return 0;
2557 }
2558 EXPORT_SYMBOL(tcf_exts_dump_stats);
2559
2560 int tc_setup_cb_call(struct tcf_block *block, enum tc_setup_type type,
2561                      void *type_data, bool err_stop)
2562 {
2563         struct tcf_block_cb *block_cb;
2564         int ok_count = 0;
2565         int err;
2566
2567         /* Make sure all netdevs sharing this block are offload-capable. */
2568         if (block->nooffloaddevcnt && err_stop)
2569                 return -EOPNOTSUPP;
2570
2571         list_for_each_entry(block_cb, &block->cb_list, list) {
2572                 err = block_cb->cb(type, type_data, block_cb->cb_priv);
2573                 if (err) {
2574                         if (err_stop)
2575                                 return err;
2576                 } else {
2577                         ok_count++;
2578                 }
2579         }
2580         return ok_count;
2581 }
2582 EXPORT_SYMBOL(tc_setup_cb_call);
2583
2584 int tc_setup_flow_action(struct flow_action *flow_action,
2585                          const struct tcf_exts *exts)
2586 {
2587         const struct tc_action *act;
2588         int i, j, k;
2589
2590         if (!exts)
2591                 return 0;
2592
2593         j = 0;
2594         tcf_exts_for_each_action(i, act, exts) {
2595                 struct flow_action_entry *entry;
2596
2597                 entry = &flow_action->entries[j];
2598                 if (is_tcf_gact_ok(act)) {
2599                         entry->id = FLOW_ACTION_ACCEPT;
2600                 } else if (is_tcf_gact_shot(act)) {
2601                         entry->id = FLOW_ACTION_DROP;
2602                 } else if (is_tcf_gact_trap(act)) {
2603                         entry->id = FLOW_ACTION_TRAP;
2604                 } else if (is_tcf_gact_goto_chain(act)) {
2605                         entry->id = FLOW_ACTION_GOTO;
2606                         entry->chain_index = tcf_gact_goto_chain_index(act);
2607                 } else if (is_tcf_mirred_egress_redirect(act)) {
2608                         entry->id = FLOW_ACTION_REDIRECT;
2609                         entry->dev = tcf_mirred_dev(act);
2610                 } else if (is_tcf_mirred_egress_mirror(act)) {
2611                         entry->id = FLOW_ACTION_MIRRED;
2612                         entry->dev = tcf_mirred_dev(act);
2613                 } else if (is_tcf_vlan(act)) {
2614                         switch (tcf_vlan_action(act)) {
2615                         case TCA_VLAN_ACT_PUSH:
2616                                 entry->id = FLOW_ACTION_VLAN_PUSH;
2617                                 entry->vlan.vid = tcf_vlan_push_vid(act);
2618                                 entry->vlan.proto = tcf_vlan_push_proto(act);
2619                                 entry->vlan.prio = tcf_vlan_push_prio(act);
2620                                 break;
2621                         case TCA_VLAN_ACT_POP:
2622                                 entry->id = FLOW_ACTION_VLAN_POP;
2623                                 break;
2624                         case TCA_VLAN_ACT_MODIFY:
2625                                 entry->id = FLOW_ACTION_VLAN_MANGLE;
2626                                 entry->vlan.vid = tcf_vlan_push_vid(act);
2627                                 entry->vlan.proto = tcf_vlan_push_proto(act);
2628                                 entry->vlan.prio = tcf_vlan_push_prio(act);
2629                                 break;
2630                         default:
2631                                 goto err_out;
2632                         }
2633                 } else if (is_tcf_tunnel_set(act)) {
2634                         entry->id = FLOW_ACTION_TUNNEL_ENCAP;
2635                         entry->tunnel = tcf_tunnel_info(act);
2636                 } else if (is_tcf_tunnel_release(act)) {
2637                         entry->id = FLOW_ACTION_TUNNEL_DECAP;
2638                         entry->tunnel = tcf_tunnel_info(act);
2639                 } else if (is_tcf_pedit(act)) {
2640                         for (k = 0; k < tcf_pedit_nkeys(act); k++) {
2641                                 switch (tcf_pedit_cmd(act, k)) {
2642                                 case TCA_PEDIT_KEY_EX_CMD_SET:
2643                                         entry->id = FLOW_ACTION_MANGLE;
2644                                         break;
2645                                 case TCA_PEDIT_KEY_EX_CMD_ADD:
2646                                         entry->id = FLOW_ACTION_ADD;
2647                                         break;
2648                                 default:
2649                                         goto err_out;
2650                                 }
2651                                 entry->mangle.htype = tcf_pedit_htype(act, k);
2652                                 entry->mangle.mask = tcf_pedit_mask(act, k);
2653                                 entry->mangle.val = tcf_pedit_val(act, k);
2654                                 entry->mangle.offset = tcf_pedit_offset(act, k);
2655                                 entry = &flow_action->entries[++j];
2656                         }
2657                 } else if (is_tcf_csum(act)) {
2658                         entry->id = FLOW_ACTION_CSUM;
2659                         entry->csum_flags = tcf_csum_update_flags(act);
2660                 } else if (is_tcf_skbedit_mark(act)) {
2661                         entry->id = FLOW_ACTION_MARK;
2662                         entry->mark = tcf_skbedit_mark(act);
2663                 } else {
2664                         goto err_out;
2665                 }
2666
2667                 if (!is_tcf_pedit(act))
2668                         j++;
2669         }
2670         return 0;
2671 err_out:
2672         return -EOPNOTSUPP;
2673 }
2674 EXPORT_SYMBOL(tc_setup_flow_action);
2675
2676 unsigned int tcf_exts_num_actions(struct tcf_exts *exts)
2677 {
2678         unsigned int num_acts = 0;
2679         struct tc_action *act;
2680         int i;
2681
2682         tcf_exts_for_each_action(i, act, exts) {
2683                 if (is_tcf_pedit(act))
2684                         num_acts += tcf_pedit_nkeys(act);
2685                 else
2686                         num_acts++;
2687         }
2688         return num_acts;
2689 }
2690 EXPORT_SYMBOL(tcf_exts_num_actions);
2691
2692 static __net_init int tcf_net_init(struct net *net)
2693 {
2694         struct tcf_net *tn = net_generic(net, tcf_net_id);
2695
2696         spin_lock_init(&tn->idr_lock);
2697         idr_init(&tn->idr);
2698         return 0;
2699 }
2700
2701 static void __net_exit tcf_net_exit(struct net *net)
2702 {
2703         struct tcf_net *tn = net_generic(net, tcf_net_id);
2704
2705         idr_destroy(&tn->idr);
2706 }
2707
2708 static struct pernet_operations tcf_net_ops = {
2709         .init = tcf_net_init,
2710         .exit = tcf_net_exit,
2711         .id   = &tcf_net_id,
2712         .size = sizeof(struct tcf_net),
2713 };
2714
2715 static int __init tc_filter_init(void)
2716 {
2717         int err;
2718
2719         tc_filter_wq = alloc_ordered_workqueue("tc_filter_workqueue", 0);
2720         if (!tc_filter_wq)
2721                 return -ENOMEM;
2722
2723         err = register_pernet_subsys(&tcf_net_ops);
2724         if (err)
2725                 goto err_register_pernet_subsys;
2726
2727         err = rhashtable_init(&indr_setup_block_ht,
2728                               &tc_indr_setup_block_ht_params);
2729         if (err)
2730                 goto err_rhash_setup_block_ht;
2731
2732         rtnl_register(PF_UNSPEC, RTM_NEWTFILTER, tc_new_tfilter, NULL, 0);
2733         rtnl_register(PF_UNSPEC, RTM_DELTFILTER, tc_del_tfilter, NULL, 0);
2734         rtnl_register(PF_UNSPEC, RTM_GETTFILTER, tc_get_tfilter,
2735                       tc_dump_tfilter, 0);
2736         rtnl_register(PF_UNSPEC, RTM_NEWCHAIN, tc_ctl_chain, NULL, 0);
2737         rtnl_register(PF_UNSPEC, RTM_DELCHAIN, tc_ctl_chain, NULL, 0);
2738         rtnl_register(PF_UNSPEC, RTM_GETCHAIN, tc_ctl_chain,
2739                       tc_dump_chain, 0);
2740
2741         return 0;
2742
2743 err_rhash_setup_block_ht:
2744         unregister_pernet_subsys(&tcf_net_ops);
2745 err_register_pernet_subsys:
2746         destroy_workqueue(tc_filter_wq);
2747         return err;
2748 }
2749
2750 subsys_initcall(tc_filter_init);