net/xfrm/xfrm_policy.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * xfrm_policy.c
   4  *
   5  * Changes:
   6  *      Mitsuru KANDA @USAGI
   7  *      Kazunori MIYAZAWA @USAGI
   8  *      Kunihiro Ishiguro <kunihiro@ipinfusion.com>
   9  *              IPv6 support
  10  *      Kazunori MIYAZAWA @USAGI
  11  *      YOSHIFUJI Hideaki
  12  *              Split up af-specific portion
  13  *      Derek Atkins <derek@ihtfp.com>          Add the post_input processor
  14  *
  15  */
  16
  17 #include <linux/err.h>
  18 #include <linux/slab.h>
  19 #include <linux/kmod.h>
  20 #include <linux/list.h>
  21 #include <linux/spinlock.h>
  22 #include <linux/workqueue.h>
  23 #include <linux/notifier.h>
  24 #include <linux/netdevice.h>
  25 #include <linux/netfilter.h>
  26 #include <linux/module.h>
  27 #include <linux/cache.h>
  28 #include <linux/cpu.h>
  29 #include <linux/audit.h>
  30 #include <linux/rhashtable.h>
  31 #include <linux/if_tunnel.h>
  32 #include <linux/icmp.h>
  33 #include <net/dst.h>
  34 #include <net/flow.h>
  35 #include <net/inet_ecn.h>
  36 #include <net/xfrm.h>
  37 #include <net/ip.h>
  38 #include <net/gre.h>
  39 #if IS_ENABLED(CONFIG_IPV6_MIP6)
  40 #include <net/mip6.h>
  41 #endif
  42 #ifdef CONFIG_XFRM_STATISTICS
  43 #include <net/snmp.h>
  44 #endif
  45 #ifdef CONFIG_XFRM_ESPINTCP
  46 #include <net/espintcp.h>
  47 #endif
  48
  49 #include "xfrm_hash.h"
  50
  51 #define XFRM_QUEUE_TMO_MIN ((unsigned)(HZ/10))
  52 #define XFRM_QUEUE_TMO_MAX ((unsigned)(60*HZ))
  53 #define XFRM_MAX_QUEUE_LEN      100
  54
  55 struct xfrm_flo {
  56         struct dst_entry *dst_orig;
  57         u8 flags;
  58 };
  59
  60 /* prefixes smaller than this are stored in lists, not trees. */
  61 #define INEXACT_PREFIXLEN_IPV4  16
  62 #define INEXACT_PREFIXLEN_IPV6  48
  63
  64 struct xfrm_pol_inexact_node {
  65         struct rb_node node;
  66         union {
  67                 xfrm_address_t addr;
  68                 struct rcu_head rcu;
  69         };
  70         u8 prefixlen;
  71
  72         struct rb_root root;
  73
  74         /* the policies matching this node, can be empty list */
  75         struct hlist_head hhead;
  76 };
  77
  78 /* xfrm inexact policy search tree:
  79  * xfrm_pol_inexact_bin = hash(dir,type,family,if_id);
  80  *  |
  81  * +---- root_d: sorted by daddr:prefix
  82  * |                 |
  83  * |        xfrm_pol_inexact_node
  84  * |                 |
  85  * |                 +- root: sorted by saddr/prefix
  86  * |                 |              |
  87  * |                 |         xfrm_pol_inexact_node
  88  * |                 |              |
  89  * |                 |              + root: unused
  90  * |                 |              |
  91  * |                 |              + hhead: saddr:daddr policies
  92  * |                 |
  93  * |                 +- coarse policies and all any:daddr policies
  94  * |
  95  * +---- root_s: sorted by saddr:prefix
  96  * |                 |
  97  * |        xfrm_pol_inexact_node
  98  * |                 |
  99  * |                 + root: unused
 100  * |                 |
 101  * |                 + hhead: saddr:any policies
 102  * |
 103  * +---- coarse policies and all any:any policies
 104  *
 105  * Lookups return four candidate lists:
 106  * 1. any:any list from top-level xfrm_pol_inexact_bin
 107  * 2. any:daddr list from daddr tree
 108  * 3. saddr:daddr list from 2nd level daddr tree
 109  * 4. saddr:any list from saddr tree
 110  *
 111  * This result set then needs to be searched for the policy with
 112  * the lowest priority.  If two results have same prio, youngest one wins.
 113  */
 114
 115 struct xfrm_pol_inexact_key {
 116         possible_net_t net;
 117         u32 if_id;
 118         u16 family;
 119         u8 dir, type;
 120 };
 121
 122 struct xfrm_pol_inexact_bin {
 123         struct xfrm_pol_inexact_key k;
 124         struct rhash_head head;
 125         /* list containing '*:*' policies */
 126         struct hlist_head hhead;
 127
 128         seqcount_spinlock_t count;
 129         /* tree sorted by daddr/prefix */
 130         struct rb_root root_d;
 131
 132         /* tree sorted by saddr/prefix */
 133         struct rb_root root_s;
 134
 135         /* slow path below */
 136         struct list_head inexact_bins;
 137         struct rcu_head rcu;
 138 };
 139
 140 enum xfrm_pol_inexact_candidate_type {
 141         XFRM_POL_CAND_BOTH,
 142         XFRM_POL_CAND_SADDR,
 143         XFRM_POL_CAND_DADDR,
 144         XFRM_POL_CAND_ANY,
 145
 146         XFRM_POL_CAND_MAX,
 147 };
 148
 149 struct xfrm_pol_inexact_candidates {
 150         struct hlist_head *res[XFRM_POL_CAND_MAX];
 151 };
 152
 153 struct xfrm_flow_keys {
 154         struct flow_dissector_key_basic basic;
 155         struct flow_dissector_key_control control;
 156         union {
 157                 struct flow_dissector_key_ipv4_addrs ipv4;
 158                 struct flow_dissector_key_ipv6_addrs ipv6;
 159         } addrs;
 160         struct flow_dissector_key_ip ip;
 161         struct flow_dissector_key_icmp icmp;
 162         struct flow_dissector_key_ports ports;
 163         struct flow_dissector_key_keyid gre;
 164 };
 165
 166 static struct flow_dissector xfrm_session_dissector __ro_after_init;
 167
 168 static DEFINE_SPINLOCK(xfrm_if_cb_lock);
 169 static struct xfrm_if_cb const __rcu *xfrm_if_cb __read_mostly;
 170
 171 static DEFINE_SPINLOCK(xfrm_policy_afinfo_lock);
 172 static struct xfrm_policy_afinfo const __rcu *xfrm_policy_afinfo[AF_INET6 + 1]
 173                                                 __read_mostly;
 174
 175 static struct kmem_cache *xfrm_dst_cache __ro_after_init;
 176
 177 static struct rhashtable xfrm_policy_inexact_table;
 178 static const struct rhashtable_params xfrm_pol_inexact_params;
 179
 180 static void xfrm_init_pmtu(struct xfrm_dst **bundle, int nr);
 181 static int stale_bundle(struct dst_entry *dst);
 182 static int xfrm_bundle_ok(struct xfrm_dst *xdst);
 183 static void xfrm_policy_queue_process(struct timer_list *t);
 184
 185 static void __xfrm_policy_link(struct xfrm_policy *pol, int dir);
 186 static struct xfrm_policy *__xfrm_policy_unlink(struct xfrm_policy *pol,
 187                                                 int dir);
 188
 189 static struct xfrm_pol_inexact_bin *
 190 xfrm_policy_inexact_lookup(struct net *net, u8 type, u16 family, u8 dir,
 191                            u32 if_id);
 192
 193 static struct xfrm_pol_inexact_bin *
 194 xfrm_policy_inexact_lookup_rcu(struct net *net,
 195                                u8 type, u16 family, u8 dir, u32 if_id);
 196 static struct xfrm_policy *
 197 xfrm_policy_insert_list(struct hlist_head *chain, struct xfrm_policy *policy,
 198                         bool excl);
 199 static void xfrm_policy_insert_inexact_list(struct hlist_head *chain,
 200                                             struct xfrm_policy *policy);
 201
 202 static bool
 203 xfrm_policy_find_inexact_candidates(struct xfrm_pol_inexact_candidates *cand,
 204                                     struct xfrm_pol_inexact_bin *b,
 205                                     const xfrm_address_t *saddr,
 206                                     const xfrm_address_t *daddr);
 207
 208 static inline bool xfrm_pol_hold_rcu(struct xfrm_policy *policy)
 209 {
 210         return refcount_inc_not_zero(&policy->refcnt);
 211 }
 212
 213 static inline bool
 214 __xfrm4_selector_match(const struct xfrm_selector *sel, const struct flowi *fl)
 215 {
 216         const struct flowi4 *fl4 = &fl->u.ip4;
 217
 218         return  addr4_match(fl4->daddr, sel->daddr.a4, sel->prefixlen_d) &&
 219                 addr4_match(fl4->saddr, sel->saddr.a4, sel->prefixlen_s) &&
 220                 !((xfrm_flowi_dport(fl, &fl4->uli) ^ sel->dport) & sel->dport_mask) &&
 221                 !((xfrm_flowi_sport(fl, &fl4->uli) ^ sel->sport) & sel->sport_mask) &&
 222                 (fl4->flowi4_proto == sel->proto || !sel->proto) &&
 223                 (fl4->flowi4_oif == sel->ifindex || !sel->ifindex);
 224 }
 225
 226 static inline bool
 227 __xfrm6_selector_match(const struct xfrm_selector *sel, const struct flowi *fl)
 228 {
 229         const struct flowi6 *fl6 = &fl->u.ip6;
 230
 231         return  addr_match(&fl6->daddr, &sel->daddr, sel->prefixlen_d) &&
 232                 addr_match(&fl6->saddr, &sel->saddr, sel->prefixlen_s) &&
 233                 !((xfrm_flowi_dport(fl, &fl6->uli) ^ sel->dport) & sel->dport_mask) &&
 234                 !((xfrm_flowi_sport(fl, &fl6->uli) ^ sel->sport) & sel->sport_mask) &&
 235                 (fl6->flowi6_proto == sel->proto || !sel->proto) &&
 236                 (fl6->flowi6_oif == sel->ifindex || !sel->ifindex);
 237 }
 238
 239 bool xfrm_selector_match(const struct xfrm_selector *sel, const struct flowi *fl,
 240                          unsigned short family)
 241 {
 242         switch (family) {
 243         case AF_INET:
 244                 return __xfrm4_selector_match(sel, fl);
 245         case AF_INET6:
 246                 return __xfrm6_selector_match(sel, fl);
 247         }
 248         return false;
 249 }
 250
 251 static const struct xfrm_policy_afinfo *xfrm_policy_get_afinfo(unsigned short family)
 252 {
 253         const struct xfrm_policy_afinfo *afinfo;
 254
 255         if (unlikely(family >= ARRAY_SIZE(xfrm_policy_afinfo)))
 256                 return NULL;
 257         rcu_read_lock();
 258         afinfo = rcu_dereference(xfrm_policy_afinfo[family]);
 259         if (unlikely(!afinfo))
 260                 rcu_read_unlock();
 261         return afinfo;
 262 }
 263
 264 /* Called with rcu_read_lock(). */
 265 static const struct xfrm_if_cb *xfrm_if_get_cb(void)
 266 {
 267         return rcu_dereference(xfrm_if_cb);
 268 }
 269
 270 struct dst_entry *__xfrm_dst_lookup(struct net *net, int tos, int oif,
 271                                     const xfrm_address_t *saddr,
 272                                     const xfrm_address_t *daddr,
 273                                     int family, u32 mark)
 274 {
 275         const struct xfrm_policy_afinfo *afinfo;
 276         struct dst_entry *dst;
 277
 278         afinfo = xfrm_policy_get_afinfo(family);
 279         if (unlikely(afinfo == NULL))
 280                 return ERR_PTR(-EAFNOSUPPORT);
 281
 282         dst = afinfo->dst_lookup(net, tos, oif, saddr, daddr, mark);
 283
 284         rcu_read_unlock();
 285
 286         return dst;
 287 }
 288 EXPORT_SYMBOL(__xfrm_dst_lookup);
 289
 290 static inline struct dst_entry *xfrm_dst_lookup(struct xfrm_state *x,
 291                                                 int tos, int oif,
 292                                                 xfrm_address_t *prev_saddr,
 293                                                 xfrm_address_t *prev_daddr,
 294                                                 int family, u32 mark)
 295 {
 296         struct net *net = xs_net(x);
 297         xfrm_address_t *saddr = &x->props.saddr;
 298         xfrm_address_t *daddr = &x->id.daddr;
 299         struct dst_entry *dst;
 300
 301         if (x->type->flags & XFRM_TYPE_LOCAL_COADDR) {
 302                 saddr = x->coaddr;
 303                 daddr = prev_daddr;
 304         }
 305         if (x->type->flags & XFRM_TYPE_REMOTE_COADDR) {
 306                 saddr = prev_saddr;
 307                 daddr = x->coaddr;
 308         }
 309
 310         dst = __xfrm_dst_lookup(net, tos, oif, saddr, daddr, family, mark);
 311
 312         if (!IS_ERR(dst)) {
 313                 if (prev_saddr != saddr)
 314                         memcpy(prev_saddr, saddr,  sizeof(*prev_saddr));
 315                 if (prev_daddr != daddr)
 316                         memcpy(prev_daddr, daddr,  sizeof(*prev_daddr));
 317         }
 318
 319         return dst;
 320 }
 321
 322 static inline unsigned long make_jiffies(long secs)
 323 {
 324         if (secs >= (MAX_SCHEDULE_TIMEOUT-1)/HZ)
 325                 return MAX_SCHEDULE_TIMEOUT-1;
 326         else
 327                 return secs*HZ;
 328 }
 329
 330 static void xfrm_policy_timer(struct timer_list *t)
 331 {
 332         struct xfrm_policy *xp = from_timer(xp, t, timer);
 333         time64_t now = ktime_get_real_seconds();
 334         time64_t next = TIME64_MAX;
 335         int warn = 0;
 336         int dir;
 337
 338         read_lock(&xp->lock);
 339
 340         if (unlikely(xp->walk.dead))
 341                 goto out;
 342
 343         dir = xfrm_policy_id2dir(xp->index);
 344
 345         if (xp->lft.hard_add_expires_seconds) {
 346                 time64_t tmo = xp->lft.hard_add_expires_seconds +
 347                         xp->curlft.add_time - now;
 348                 if (tmo <= 0)
 349                         goto expired;
 350                 if (tmo < next)
 351                         next = tmo;
 352         }
 353         if (xp->lft.hard_use_expires_seconds) {
 354                 time64_t tmo = xp->lft.hard_use_expires_seconds +
 355                         (READ_ONCE(xp->curlft.use_time) ? : xp->curlft.add_time) - now;
 356                 if (tmo <= 0)
 357                         goto expired;
 358                 if (tmo < next)
 359                         next = tmo;
 360         }
 361         if (xp->lft.soft_add_expires_seconds) {
 362                 time64_t tmo = xp->lft.soft_add_expires_seconds +
 363                         xp->curlft.add_time - now;
 364                 if (tmo <= 0) {
 365                         warn = 1;
 366                         tmo = XFRM_KM_TIMEOUT;
 367                 }
 368                 if (tmo < next)
 369                         next = tmo;
 370         }
 371         if (xp->lft.soft_use_expires_seconds) {
 372                 time64_t tmo = xp->lft.soft_use_expires_seconds +
 373                         (READ_ONCE(xp->curlft.use_time) ? : xp->curlft.add_time) - now;
 374                 if (tmo <= 0) {
 375                         warn = 1;
 376                         tmo = XFRM_KM_TIMEOUT;
 377                 }
 378                 if (tmo < next)
 379                         next = tmo;
 380         }
 381
 382         if (warn)
 383                 km_policy_expired(xp, dir, 0, 0);
 384         if (next != TIME64_MAX &&
 385             !mod_timer(&xp->timer, jiffies + make_jiffies(next)))
 386                 xfrm_pol_hold(xp);
 387
 388 out:
 389         read_unlock(&xp->lock);
 390         xfrm_pol_put(xp);
 391         return;
 392
 393 expired:
 394         read_unlock(&xp->lock);
 395         if (!xfrm_policy_delete(xp, dir))
 396                 km_policy_expired(xp, dir, 1, 0);
 397         xfrm_pol_put(xp);
 398 }
 399
 400 /* Allocate xfrm_policy. Not used here, it is supposed to be used by pfkeyv2
 401  * SPD calls.
 402  */
 403
 404 struct xfrm_policy *xfrm_policy_alloc(struct net *net, gfp_t gfp)
 405 {
 406         struct xfrm_policy *policy;
 407
 408         policy = kzalloc(sizeof(struct xfrm_policy), gfp);
 409
 410         if (policy) {
 411                 write_pnet(&policy->xp_net, net);
 412                 INIT_LIST_HEAD(&policy->walk.all);
 413                 INIT_HLIST_NODE(&policy->bydst_inexact_list);
 414                 INIT_HLIST_NODE(&policy->bydst);
 415                 INIT_HLIST_NODE(&policy->byidx);
 416                 rwlock_init(&policy->lock);
 417                 refcount_set(&policy->refcnt, 1);
 418                 skb_queue_head_init(&policy->polq.hold_queue);
 419                 timer_setup(&policy->timer, xfrm_policy_timer, 0);
 420                 timer_setup(&policy->polq.hold_timer,
 421                             xfrm_policy_queue_process, 0);
 422         }
 423         return policy;
 424 }
 425 EXPORT_SYMBOL(xfrm_policy_alloc);
 426
 427 static void xfrm_policy_destroy_rcu(struct rcu_head *head)
 428 {
 429         struct xfrm_policy *policy = container_of(head, struct xfrm_policy, rcu);
 430
 431         security_xfrm_policy_free(policy->security);
 432         kfree(policy);
 433 }
 434
 435 /* Destroy xfrm_policy: descendant resources must be released to this moment. */
 436
 437 void xfrm_policy_destroy(struct xfrm_policy *policy)
 438 {
 439         BUG_ON(!policy->walk.dead);
 440
 441         if (del_timer(&policy->timer) || del_timer(&policy->polq.hold_timer))
 442                 BUG();
 443
 444         xfrm_dev_policy_free(policy);
 445         call_rcu(&policy->rcu, xfrm_policy_destroy_rcu);
 446 }
 447 EXPORT_SYMBOL(xfrm_policy_destroy);
 448
 449 /* Rule must be locked. Release descendant resources, announce
 450  * entry dead. The rule must be unlinked from lists to the moment.
 451  */
 452
 453 static void xfrm_policy_kill(struct xfrm_policy *policy)
 454 {
 455         write_lock_bh(&policy->lock);
 456         policy->walk.dead = 1;
 457         write_unlock_bh(&policy->lock);
 458
 459         atomic_inc(&policy->genid);
 460
 461         if (del_timer(&policy->polq.hold_timer))
 462                 xfrm_pol_put(policy);
 463         skb_queue_purge(&policy->polq.hold_queue);
 464
 465         if (del_timer(&policy->timer))
 466                 xfrm_pol_put(policy);
 467
 468         xfrm_pol_put(policy);
 469 }
 470
 471 static unsigned int xfrm_policy_hashmax __read_mostly = 1 * 1024 * 1024;
 472
 473 static inline unsigned int idx_hash(struct net *net, u32 index)
 474 {
 475         return __idx_hash(index, net->xfrm.policy_idx_hmask);
 476 }
 477
 478 /* calculate policy hash thresholds */
 479 static void __get_hash_thresh(struct net *net,
 480                               unsigned short family, int dir,
 481                               u8 *dbits, u8 *sbits)
 482 {
 483         switch (family) {
 484         case AF_INET:
 485                 *dbits = net->xfrm.policy_bydst[dir].dbits4;
 486                 *sbits = net->xfrm.policy_bydst[dir].sbits4;
 487                 break;
 488
 489         case AF_INET6:
 490                 *dbits = net->xfrm.policy_bydst[dir].dbits6;
 491                 *sbits = net->xfrm.policy_bydst[dir].sbits6;
 492                 break;
 493
 494         default:
 495                 *dbits = 0;
 496                 *sbits = 0;
 497         }
 498 }
 499
 500 static struct hlist_head *policy_hash_bysel(struct net *net,
 501                                             const struct xfrm_selector *sel,
 502                                             unsigned short family, int dir)
 503 {
 504         unsigned int hmask = net->xfrm.policy_bydst[dir].hmask;
 505         unsigned int hash;
 506         u8 dbits;
 507         u8 sbits;
 508
 509         __get_hash_thresh(net, family, dir, &dbits, &sbits);
 510         hash = __sel_hash(sel, family, hmask, dbits, sbits);
 511
 512         if (hash == hmask + 1)
 513                 return NULL;
 514
 515         return rcu_dereference_check(net->xfrm.policy_bydst[dir].table,
 516                      lockdep_is_held(&net->xfrm.xfrm_policy_lock)) + hash;
 517 }
 518
 519 static struct hlist_head *policy_hash_direct(struct net *net,
 520                                              const xfrm_address_t *daddr,
 521                                              const xfrm_address_t *saddr,
 522                                              unsigned short family, int dir)
 523 {
 524         unsigned int hmask = net->xfrm.policy_bydst[dir].hmask;
 525         unsigned int hash;
 526         u8 dbits;
 527         u8 sbits;
 528
 529         __get_hash_thresh(net, family, dir, &dbits, &sbits);
 530         hash = __addr_hash(daddr, saddr, family, hmask, dbits, sbits);
 531
 532         return rcu_dereference_check(net->xfrm.policy_bydst[dir].table,
 533                      lockdep_is_held(&net->xfrm.xfrm_policy_lock)) + hash;
 534 }
 535
 536 static void xfrm_dst_hash_transfer(struct net *net,
 537                                    struct hlist_head *list,
 538                                    struct hlist_head *ndsttable,
 539                                    unsigned int nhashmask,
 540                                    int dir)
 541 {
 542         struct hlist_node *tmp, *entry0 = NULL;
 543         struct xfrm_policy *pol;
 544         unsigned int h0 = 0;
 545         u8 dbits;
 546         u8 sbits;
 547
 548 redo:
 549         hlist_for_each_entry_safe(pol, tmp, list, bydst) {
 550                 unsigned int h;
 551
 552                 __get_hash_thresh(net, pol->family, dir, &dbits, &sbits);
 553                 h = __addr_hash(&pol->selector.daddr, &pol->selector.saddr,
 554                                 pol->family, nhashmask, dbits, sbits);
 555                 if (!entry0 || pol->xdo.type == XFRM_DEV_OFFLOAD_PACKET) {
 556                         hlist_del_rcu(&pol->bydst);
 557                         hlist_add_head_rcu(&pol->bydst, ndsttable + h);
 558                         h0 = h;
 559                 } else {
 560                         if (h != h0)
 561                                 continue;
 562                         hlist_del_rcu(&pol->bydst);
 563                         hlist_add_behind_rcu(&pol->bydst, entry0);
 564                 }
 565                 entry0 = &pol->bydst;
 566         }
 567         if (!hlist_empty(list)) {
 568                 entry0 = NULL;
 569                 goto redo;
 570         }
 571 }
 572
 573 static void xfrm_idx_hash_transfer(struct hlist_head *list,
 574                                    struct hlist_head *nidxtable,
 575                                    unsigned int nhashmask)
 576 {
 577         struct hlist_node *tmp;
 578         struct xfrm_policy *pol;
 579
 580         hlist_for_each_entry_safe(pol, tmp, list, byidx) {
 581                 unsigned int h;
 582
 583                 h = __idx_hash(pol->index, nhashmask);
 584                 hlist_add_head(&pol->byidx, nidxtable+h);
 585         }
 586 }
 587
 588 static unsigned long xfrm_new_hash_mask(unsigned int old_hmask)
 589 {
 590         return ((old_hmask + 1) << 1) - 1;
 591 }
 592
 593 static void xfrm_bydst_resize(struct net *net, int dir)
 594 {
 595         unsigned int hmask = net->xfrm.policy_bydst[dir].hmask;
 596         unsigned int nhashmask = xfrm_new_hash_mask(hmask);
 597         unsigned int nsize = (nhashmask + 1) * sizeof(struct hlist_head);
 598         struct hlist_head *ndst = xfrm_hash_alloc(nsize);
 599         struct hlist_head *odst;
 600         int i;
 601
 602         if (!ndst)
 603                 return;
 604
 605         spin_lock_bh(&net->xfrm.xfrm_policy_lock);
 606         write_seqcount_begin(&net->xfrm.xfrm_policy_hash_generation);
 607
 608         odst = rcu_dereference_protected(net->xfrm.policy_bydst[dir].table,
 609                                 lockdep_is_held(&net->xfrm.xfrm_policy_lock));
 610
 611         for (i = hmask; i >= 0; i--)
 612                 xfrm_dst_hash_transfer(net, odst + i, ndst, nhashmask, dir);
 613
 614         rcu_assign_pointer(net->xfrm.policy_bydst[dir].table, ndst);
 615         net->xfrm.policy_bydst[dir].hmask = nhashmask;
 616
 617         write_seqcount_end(&net->xfrm.xfrm_policy_hash_generation);
 618         spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
 619
 620         synchronize_rcu();
 621
 622         xfrm_hash_free(odst, (hmask + 1) * sizeof(struct hlist_head));
 623 }
 624
 625 static void xfrm_byidx_resize(struct net *net)
 626 {
 627         unsigned int hmask = net->xfrm.policy_idx_hmask;
 628         unsigned int nhashmask = xfrm_new_hash_mask(hmask);
 629         unsigned int nsize = (nhashmask + 1) * sizeof(struct hlist_head);
 630         struct hlist_head *oidx = net->xfrm.policy_byidx;
 631         struct hlist_head *nidx = xfrm_hash_alloc(nsize);
 632         int i;
 633
 634         if (!nidx)
 635                 return;
 636
 637         spin_lock_bh(&net->xfrm.xfrm_policy_lock);
 638
 639         for (i = hmask; i >= 0; i--)
 640                 xfrm_idx_hash_transfer(oidx + i, nidx, nhashmask);
 641
 642         net->xfrm.policy_byidx = nidx;
 643         net->xfrm.policy_idx_hmask = nhashmask;
 644
 645         spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
 646
 647         xfrm_hash_free(oidx, (hmask + 1) * sizeof(struct hlist_head));
 648 }
 649
 650 static inline int xfrm_bydst_should_resize(struct net *net, int dir, int *total)
 651 {
 652         unsigned int cnt = net->xfrm.policy_count[dir];
 653         unsigned int hmask = net->xfrm.policy_bydst[dir].hmask;
 654
 655         if (total)
 656                 *total += cnt;
 657
 658         if ((hmask + 1) < xfrm_policy_hashmax &&
 659             cnt > hmask)
 660                 return 1;
 661
 662         return 0;
 663 }
 664
 665 static inline int xfrm_byidx_should_resize(struct net *net, int total)
 666 {
 667         unsigned int hmask = net->xfrm.policy_idx_hmask;
 668
 669         if ((hmask + 1) < xfrm_policy_hashmax &&
 670             total > hmask)
 671                 return 1;
 672
 673         return 0;
 674 }
 675
 676 void xfrm_spd_getinfo(struct net *net, struct xfrmk_spdinfo *si)
 677 {
 678         si->incnt = net->xfrm.policy_count[XFRM_POLICY_IN];
 679         si->outcnt = net->xfrm.policy_count[XFRM_POLICY_OUT];
 680         si->fwdcnt = net->xfrm.policy_count[XFRM_POLICY_FWD];
 681         si->inscnt = net->xfrm.policy_count[XFRM_POLICY_IN+XFRM_POLICY_MAX];
 682         si->outscnt = net->xfrm.policy_count[XFRM_POLICY_OUT+XFRM_POLICY_MAX];
 683         si->fwdscnt = net->xfrm.policy_count[XFRM_POLICY_FWD+XFRM_POLICY_MAX];
 684         si->spdhcnt = net->xfrm.policy_idx_hmask;
 685         si->spdhmcnt = xfrm_policy_hashmax;
 686 }
 687 EXPORT_SYMBOL(xfrm_spd_getinfo);
 688
 689 static DEFINE_MUTEX(hash_resize_mutex);
 690 static void xfrm_hash_resize(struct work_struct *work)
 691 {
 692         struct net *net = container_of(work, struct net, xfrm.policy_hash_work);
 693         int dir, total;
 694
 695         mutex_lock(&hash_resize_mutex);
 696
 697         total = 0;
 698         for (dir = 0; dir < XFRM_POLICY_MAX; dir++) {
 699                 if (xfrm_bydst_should_resize(net, dir, &total))
 700                         xfrm_bydst_resize(net, dir);
 701         }
 702         if (xfrm_byidx_should_resize(net, total))
 703                 xfrm_byidx_resize(net);
 704
 705         mutex_unlock(&hash_resize_mutex);
 706 }
 707
 708 /* Make sure *pol can be inserted into fastbin.
 709  * Useful to check that later insert requests will be successful
 710  * (provided xfrm_policy_lock is held throughout).
 711  */
 712 static struct xfrm_pol_inexact_bin *
 713 xfrm_policy_inexact_alloc_bin(const struct xfrm_policy *pol, u8 dir)
 714 {
 715         struct xfrm_pol_inexact_bin *bin, *prev;
 716         struct xfrm_pol_inexact_key k = {
 717                 .family = pol->family,
 718                 .type = pol->type,
 719                 .dir = dir,
 720                 .if_id = pol->if_id,
 721         };
 722         struct net *net = xp_net(pol);
 723
 724         lockdep_assert_held(&net->xfrm.xfrm_policy_lock);
 725
 726         write_pnet(&k.net, net);
 727         bin = rhashtable_lookup_fast(&xfrm_policy_inexact_table, &k,
 728                                      xfrm_pol_inexact_params);
 729         if (bin)
 730                 return bin;
 731
 732         bin = kzalloc(sizeof(*bin), GFP_ATOMIC);
 733         if (!bin)
 734                 return NULL;
 735
 736         bin->k = k;
 737         INIT_HLIST_HEAD(&bin->hhead);
 738         bin->root_d = RB_ROOT;
 739         bin->root_s = RB_ROOT;
 740         seqcount_spinlock_init(&bin->count, &net->xfrm.xfrm_policy_lock);
 741
 742         prev = rhashtable_lookup_get_insert_key(&xfrm_policy_inexact_table,
 743                                                 &bin->k, &bin->head,
 744                                                 xfrm_pol_inexact_params);
 745         if (!prev) {
 746                 list_add(&bin->inexact_bins, &net->xfrm.inexact_bins);
 747                 return bin;
 748         }
 749
 750         kfree(bin);
 751
 752         return IS_ERR(prev) ? NULL : prev;
 753 }
 754
 755 static bool xfrm_pol_inexact_addr_use_any_list(const xfrm_address_t *addr,
 756                                                int family, u8 prefixlen)
 757 {
 758         if (xfrm_addr_any(addr, family))
 759                 return true;
 760
 761         if (family == AF_INET6 && prefixlen < INEXACT_PREFIXLEN_IPV6)
 762                 return true;
 763
 764         if (family == AF_INET && prefixlen < INEXACT_PREFIXLEN_IPV4)
 765                 return true;
 766
 767         return false;
 768 }
 769
 770 static bool
 771 xfrm_policy_inexact_insert_use_any_list(const struct xfrm_policy *policy)
 772 {
 773         const xfrm_address_t *addr;
 774         bool saddr_any, daddr_any;
 775         u8 prefixlen;
 776
 777         addr = &policy->selector.saddr;
 778         prefixlen = policy->selector.prefixlen_s;
 779
 780         saddr_any = xfrm_pol_inexact_addr_use_any_list(addr,
 781                                                        policy->family,
 782                                                        prefixlen);
 783         addr = &policy->selector.daddr;
 784         prefixlen = policy->selector.prefixlen_d;
 785         daddr_any = xfrm_pol_inexact_addr_use_any_list(addr,
 786                                                        policy->family,
 787                                                        prefixlen);
 788         return saddr_any && daddr_any;
 789 }
 790
 791 static void xfrm_pol_inexact_node_init(struct xfrm_pol_inexact_node *node,
 792                                        const xfrm_address_t *addr, u8 prefixlen)
 793 {
 794         node->addr = *addr;
 795         node->prefixlen = prefixlen;
 796 }
 797
 798 static struct xfrm_pol_inexact_node *
 799 xfrm_pol_inexact_node_alloc(const xfrm_address_t *addr, u8 prefixlen)
 800 {
 801         struct xfrm_pol_inexact_node *node;
 802
 803         node = kzalloc(sizeof(*node), GFP_ATOMIC);
 804         if (node)
 805                 xfrm_pol_inexact_node_init(node, addr, prefixlen);
 806
 807         return node;
 808 }
 809
 810 static int xfrm_policy_addr_delta(const xfrm_address_t *a,
 811                                   const xfrm_address_t *b,
 812                                   u8 prefixlen, u16 family)
 813 {
 814         u32 ma, mb, mask;
 815         unsigned int pdw, pbi;
 816         int delta = 0;
 817
 818         switch (family) {
 819         case AF_INET:
 820                 if (prefixlen == 0)
 821                         return 0;
 822                 mask = ~0U << (32 - prefixlen);
 823                 ma = ntohl(a->a4) & mask;
 824                 mb = ntohl(b->a4) & mask;
 825                 if (ma < mb)
 826                         delta = -1;
 827                 else if (ma > mb)
 828                         delta = 1;
 829                 break;
 830         case AF_INET6:
 831                 pdw = prefixlen >> 5;
 832                 pbi = prefixlen & 0x1f;
 833
 834                 if (pdw) {
 835                         delta = memcmp(a->a6, b->a6, pdw << 2);
 836                         if (delta)
 837                                 return delta;
 838                 }
 839                 if (pbi) {
 840                         mask = ~0U << (32 - pbi);
 841                         ma = ntohl(a->a6[pdw]) & mask;
 842                         mb = ntohl(b->a6[pdw]) & mask;
 843                         if (ma < mb)
 844                                 delta = -1;
 845                         else if (ma > mb)
 846                                 delta = 1;
 847                 }
 848                 break;
 849         default:
 850                 break;
 851         }
 852
 853         return delta;
 854 }
 855
 856 static void xfrm_policy_inexact_list_reinsert(struct net *net,
 857                                               struct xfrm_pol_inexact_node *n,
 858                                               u16 family)
 859 {
 860         unsigned int matched_s, matched_d;
 861         struct xfrm_policy *policy, *p;
 862
 863         matched_s = 0;
 864         matched_d = 0;
 865
 866         list_for_each_entry_reverse(policy, &net->xfrm.policy_all, walk.all) {
 867                 struct hlist_node *newpos = NULL;
 868                 bool matches_s, matches_d;
 869
 870                 if (policy->walk.dead || !policy->bydst_reinsert)
 871                         continue;
 872
 873                 WARN_ON_ONCE(policy->family != family);
 874
 875                 policy->bydst_reinsert = false;
 876                 hlist_for_each_entry(p, &n->hhead, bydst) {
 877                         if (policy->priority > p->priority)
 878                                 newpos = &p->bydst;
 879                         else if (policy->priority == p->priority &&
 880                                  policy->pos > p->pos)
 881                                 newpos = &p->bydst;
 882                         else
 883                                 break;
 884                 }
 885
 886                 if (newpos && policy->xdo.type != XFRM_DEV_OFFLOAD_PACKET)
 887                         hlist_add_behind_rcu(&policy->bydst, newpos);
 888                 else
 889                         hlist_add_head_rcu(&policy->bydst, &n->hhead);
 890
 891                 /* paranoia checks follow.
 892                  * Check that the reinserted policy matches at least
 893                  * saddr or daddr for current node prefix.
 894                  *
 895                  * Matching both is fine, matching saddr in one policy
 896                  * (but not daddr) and then matching only daddr in another
 897                  * is a bug.
 898                  */
 899                 matches_s = xfrm_policy_addr_delta(&policy->selector.saddr,
 900                                                    &n->addr,
 901                                                    n->prefixlen,
 902                                                    family) == 0;
 903                 matches_d = xfrm_policy_addr_delta(&policy->selector.daddr,
 904                                                    &n->addr,
 905                                                    n->prefixlen,
 906                                                    family) == 0;
 907                 if (matches_s && matches_d)
 908                         continue;
 909
 910                 WARN_ON_ONCE(!matches_s && !matches_d);
 911                 if (matches_s)
 912                         matched_s++;
 913                 if (matches_d)
 914                         matched_d++;
 915                 WARN_ON_ONCE(matched_s && matched_d);
 916         }
 917 }
 918
 919 static void xfrm_policy_inexact_node_reinsert(struct net *net,
 920                                               struct xfrm_pol_inexact_node *n,
 921                                               struct rb_root *new,
 922                                               u16 family)
 923 {
 924         struct xfrm_pol_inexact_node *node;
 925         struct rb_node **p, *parent;
 926
 927         /* we should not have another subtree here */
 928         WARN_ON_ONCE(!RB_EMPTY_ROOT(&n->root));
 929 restart:
 930         parent = NULL;
 931         p = &new->rb_node;
 932         while (*p) {
 933                 u8 prefixlen;
 934                 int delta;
 935
 936                 parent = *p;
 937                 node = rb_entry(*p, struct xfrm_pol_inexact_node, node);
 938
 939                 prefixlen = min(node->prefixlen, n->prefixlen);
 940
 941                 delta = xfrm_policy_addr_delta(&n->addr, &node->addr,
 942                                                prefixlen, family);
 943                 if (delta < 0) {
 944                         p = &parent->rb_left;
 945                 } else if (delta > 0) {
 946                         p = &parent->rb_right;
 947                 } else {
 948                         bool same_prefixlen = node->prefixlen == n->prefixlen;
 949                         struct xfrm_policy *tmp;
 950
 951                         hlist_for_each_entry(tmp, &n->hhead, bydst) {
 952                                 tmp->bydst_reinsert = true;
 953                                 hlist_del_rcu(&tmp->bydst);
 954                         }
 955
 956                         node->prefixlen = prefixlen;
 957
 958                         xfrm_policy_inexact_list_reinsert(net, node, family);
 959
 960                         if (same_prefixlen) {
 961                                 kfree_rcu(n, rcu);
 962                                 return;
 963                         }
 964
 965                         rb_erase(*p, new);
 966                         kfree_rcu(n, rcu);
 967                         n = node;
 968                         goto restart;
 969                 }
 970         }
 971
 972         rb_link_node_rcu(&n->node, parent, p);
 973         rb_insert_color(&n->node, new);
 974 }
 975
 976 /* merge nodes v and n */
 977 static void xfrm_policy_inexact_node_merge(struct net *net,
 978                                            struct xfrm_pol_inexact_node *v,
 979                                            struct xfrm_pol_inexact_node *n,
 980                                            u16 family)
 981 {
 982         struct xfrm_pol_inexact_node *node;
 983         struct xfrm_policy *tmp;
 984         struct rb_node *rnode;
 985
 986         /* To-be-merged node v has a subtree.
 987          *
 988          * Dismantle it and insert its nodes to n->root.
 989          */
 990         while ((rnode = rb_first(&v->root)) != NULL) {
 991                 node = rb_entry(rnode, struct xfrm_pol_inexact_node, node);
 992                 rb_erase(&node->node, &v->root);
 993                 xfrm_policy_inexact_node_reinsert(net, node, &n->root,
 994                                                   family);
 995         }
 996
 997         hlist_for_each_entry(tmp, &v->hhead, bydst) {
 998                 tmp->bydst_reinsert = true;
 999                 hlist_del_rcu(&tmp->bydst);
1000         }
1001
1002         xfrm_policy_inexact_list_reinsert(net, n, family);
1003 }
1004
1005 static struct xfrm_pol_inexact_node *
1006 xfrm_policy_inexact_insert_node(struct net *net,
1007                                 struct rb_root *root,
1008                                 xfrm_address_t *addr,
1009                                 u16 family, u8 prefixlen, u8 dir)
1010 {
1011         struct xfrm_pol_inexact_node *cached = NULL;
1012         struct rb_node **p, *parent = NULL;
1013         struct xfrm_pol_inexact_node *node;
1014
1015         p = &root->rb_node;
1016         while (*p) {
1017                 int delta;
1018
1019                 parent = *p;
1020                 node = rb_entry(*p, struct xfrm_pol_inexact_node, node);
1021
1022                 delta = xfrm_policy_addr_delta(addr, &node->addr,
1023                                                node->prefixlen,
1024                                                family);
1025                 if (delta == 0 && prefixlen >= node->prefixlen) {
1026                         WARN_ON_ONCE(cached); /* ipsec policies got lost */
1027                         return node;
1028                 }
1029
1030                 if (delta < 0)
1031                         p = &parent->rb_left;
1032                 else
1033                         p = &parent->rb_right;
1034
1035                 if (prefixlen < node->prefixlen) {
1036                         delta = xfrm_policy_addr_delta(addr, &node->addr,
1037                                                        prefixlen,
1038                                                        family);
1039                         if (delta)
1040                                 continue;
1041
1042                         /* This node is a subnet of the new prefix. It needs
1043                          * to be removed and re-inserted with the smaller
1044                          * prefix and all nodes that are now also covered
1045                          * by the reduced prefixlen.
1046                          */
1047                         rb_erase(&node->node, root);
1048
1049                         if (!cached) {
1050                                 xfrm_pol_inexact_node_init(node, addr,
1051                                                            prefixlen);
1052                                 cached = node;
1053                         } else {
1054                                 /* This node also falls within the new
1055                                  * prefixlen. Merge the to-be-reinserted
1056                                  * node and this one.
1057                                  */
1058                                 xfrm_policy_inexact_node_merge(net, node,
1059                                                                cached, family);
1060                                 kfree_rcu(node, rcu);
1061                         }
1062
1063                         /* restart */
1064                         p = &root->rb_node;
1065                         parent = NULL;
1066                 }
1067         }
1068
1069         node = cached;
1070         if (!node) {
1071                 node = xfrm_pol_inexact_node_alloc(addr, prefixlen);
1072                 if (!node)
1073                         return NULL;
1074         }
1075
1076         rb_link_node_rcu(&node->node, parent, p);
1077         rb_insert_color(&node->node, root);
1078
1079         return node;
1080 }
1081
1082 static void xfrm_policy_inexact_gc_tree(struct rb_root *r, bool rm)
1083 {
1084         struct xfrm_pol_inexact_node *node;
1085         struct rb_node *rn = rb_first(r);
1086
1087         while (rn) {
1088                 node = rb_entry(rn, struct xfrm_pol_inexact_node, node);
1089
1090                 xfrm_policy_inexact_gc_tree(&node->root, rm);
1091                 rn = rb_next(rn);
1092
1093                 if (!hlist_empty(&node->hhead) || !RB_EMPTY_ROOT(&node->root)) {
1094                         WARN_ON_ONCE(rm);
1095                         continue;
1096                 }
1097
1098                 rb_erase(&node->node, r);
1099                 kfree_rcu(node, rcu);
1100         }
1101 }
1102
1103 static void __xfrm_policy_inexact_prune_bin(struct xfrm_pol_inexact_bin *b, bool net_exit)
1104 {
1105         write_seqcount_begin(&b->count);
1106         xfrm_policy_inexact_gc_tree(&b->root_d, net_exit);
1107         xfrm_policy_inexact_gc_tree(&b->root_s, net_exit);
1108         write_seqcount_end(&b->count);
1109
1110         if (!RB_EMPTY_ROOT(&b->root_d) || !RB_EMPTY_ROOT(&b->root_s) ||
1111             !hlist_empty(&b->hhead)) {
1112                 WARN_ON_ONCE(net_exit);
1113                 return;
1114         }
1115
1116         if (rhashtable_remove_fast(&xfrm_policy_inexact_table, &b->head,
1117                                    xfrm_pol_inexact_params) == 0) {
1118                 list_del(&b->inexact_bins);
1119                 kfree_rcu(b, rcu);
1120         }
1121 }
1122
1123 static void xfrm_policy_inexact_prune_bin(struct xfrm_pol_inexact_bin *b)
1124 {
1125         struct net *net = read_pnet(&b->k.net);
1126
1127         spin_lock_bh(&net->xfrm.xfrm_policy_lock);
1128         __xfrm_policy_inexact_prune_bin(b, false);
1129         spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1130 }
1131
1132 static void __xfrm_policy_inexact_flush(struct net *net)
1133 {
1134         struct xfrm_pol_inexact_bin *bin, *t;
1135
1136         lockdep_assert_held(&net->xfrm.xfrm_policy_lock);
1137
1138         list_for_each_entry_safe(bin, t, &net->xfrm.inexact_bins, inexact_bins)
1139                 __xfrm_policy_inexact_prune_bin(bin, false);
1140 }
1141
1142 static struct hlist_head *
1143 xfrm_policy_inexact_alloc_chain(struct xfrm_pol_inexact_bin *bin,
1144                                 struct xfrm_policy *policy, u8 dir)
1145 {
1146         struct xfrm_pol_inexact_node *n;
1147         struct net *net;
1148
1149         net = xp_net(policy);
1150         lockdep_assert_held(&net->xfrm.xfrm_policy_lock);
1151
1152         if (xfrm_policy_inexact_insert_use_any_list(policy))
1153                 return &bin->hhead;
1154
1155         if (xfrm_pol_inexact_addr_use_any_list(&policy->selector.daddr,
1156                                                policy->family,
1157                                                policy->selector.prefixlen_d)) {
1158                 write_seqcount_begin(&bin->count);
1159                 n = xfrm_policy_inexact_insert_node(net,
1160                                                     &bin->root_s,
1161                                                     &policy->selector.saddr,
1162                                                     policy->family,
1163                                                     policy->selector.prefixlen_s,
1164                                                     dir);
1165                 write_seqcount_end(&bin->count);
1166                 if (!n)
1167                         return NULL;
1168
1169                 return &n->hhead;
1170         }
1171
1172         /* daddr is fixed */
1173         write_seqcount_begin(&bin->count);
1174         n = xfrm_policy_inexact_insert_node(net,
1175                                             &bin->root_d,
1176                                             &policy->selector.daddr,
1177                                             policy->family,
1178                                             policy->selector.prefixlen_d, dir);
1179         write_seqcount_end(&bin->count);
1180         if (!n)
1181                 return NULL;
1182
1183         /* saddr is wildcard */
1184         if (xfrm_pol_inexact_addr_use_any_list(&policy->selector.saddr,
1185                                                policy->family,
1186                                                policy->selector.prefixlen_s))
1187                 return &n->hhead;
1188
1189         write_seqcount_begin(&bin->count);
1190         n = xfrm_policy_inexact_insert_node(net,
1191                                             &n->root,
1192                                             &policy->selector.saddr,
1193                                             policy->family,
1194                                             policy->selector.prefixlen_s, dir);
1195         write_seqcount_end(&bin->count);
1196         if (!n)
1197                 return NULL;
1198
1199         return &n->hhead;
1200 }
1201
1202 static struct xfrm_policy *
1203 xfrm_policy_inexact_insert(struct xfrm_policy *policy, u8 dir, int excl)
1204 {
1205         struct xfrm_pol_inexact_bin *bin;
1206         struct xfrm_policy *delpol;
1207         struct hlist_head *chain;
1208         struct net *net;
1209
1210         bin = xfrm_policy_inexact_alloc_bin(policy, dir);
1211         if (!bin)
1212                 return ERR_PTR(-ENOMEM);
1213
1214         net = xp_net(policy);
1215         lockdep_assert_held(&net->xfrm.xfrm_policy_lock);
1216
1217         chain = xfrm_policy_inexact_alloc_chain(bin, policy, dir);
1218         if (!chain) {
1219                 __xfrm_policy_inexact_prune_bin(bin, false);
1220                 return ERR_PTR(-ENOMEM);
1221         }
1222
1223         delpol = xfrm_policy_insert_list(chain, policy, excl);
1224         if (delpol && excl) {
1225                 __xfrm_policy_inexact_prune_bin(bin, false);
1226                 return ERR_PTR(-EEXIST);
1227         }
1228
1229         chain = &net->xfrm.policy_inexact[dir];
1230         xfrm_policy_insert_inexact_list(chain, policy);
1231
1232         if (delpol)
1233                 __xfrm_policy_inexact_prune_bin(bin, false);
1234
1235         return delpol;
1236 }
1237
1238 static void xfrm_hash_rebuild(struct work_struct *work)
1239 {
1240         struct net *net = container_of(work, struct net,
1241                                        xfrm.policy_hthresh.work);
1242         unsigned int hmask;
1243         struct xfrm_policy *pol;
1244         struct xfrm_policy *policy;
1245         struct hlist_head *chain;
1246         struct hlist_head *odst;
1247         struct hlist_node *newpos;
1248         int i;
1249         int dir;
1250         unsigned seq;
1251         u8 lbits4, rbits4, lbits6, rbits6;
1252
1253         mutex_lock(&hash_resize_mutex);
1254
1255         /* read selector prefixlen thresholds */
1256         do {
1257                 seq = read_seqbegin(&net->xfrm.policy_hthresh.lock);
1258
1259                 lbits4 = net->xfrm.policy_hthresh.lbits4;
1260                 rbits4 = net->xfrm.policy_hthresh.rbits4;
1261                 lbits6 = net->xfrm.policy_hthresh.lbits6;
1262                 rbits6 = net->xfrm.policy_hthresh.rbits6;
1263         } while (read_seqretry(&net->xfrm.policy_hthresh.lock, seq));
1264
1265         spin_lock_bh(&net->xfrm.xfrm_policy_lock);
1266         write_seqcount_begin(&net->xfrm.xfrm_policy_hash_generation);
1267
1268         /* make sure that we can insert the indirect policies again before
1269          * we start with destructive action.
1270          */
1271         list_for_each_entry(policy, &net->xfrm.policy_all, walk.all) {
1272                 struct xfrm_pol_inexact_bin *bin;
1273                 u8 dbits, sbits;
1274
1275                 if (policy->walk.dead)
1276                         continue;
1277
1278                 dir = xfrm_policy_id2dir(policy->index);
1279                 if (dir >= XFRM_POLICY_MAX)
1280                         continue;
1281
1282                 if ((dir & XFRM_POLICY_MASK) == XFRM_POLICY_OUT) {
1283                         if (policy->family == AF_INET) {
1284                                 dbits = rbits4;
1285                                 sbits = lbits4;
1286                         } else {
1287                                 dbits = rbits6;
1288                                 sbits = lbits6;
1289                         }
1290                 } else {
1291                         if (policy->family == AF_INET) {
1292                                 dbits = lbits4;
1293                                 sbits = rbits4;
1294                         } else {
1295                                 dbits = lbits6;
1296                                 sbits = rbits6;
1297                         }
1298                 }
1299
1300                 if (policy->selector.prefixlen_d < dbits ||
1301                     policy->selector.prefixlen_s < sbits)
1302                         continue;
1303
1304                 bin = xfrm_policy_inexact_alloc_bin(policy, dir);
1305                 if (!bin)
1306                         goto out_unlock;
1307
1308                 if (!xfrm_policy_inexact_alloc_chain(bin, policy, dir))
1309                         goto out_unlock;
1310         }
1311
1312         /* reset the bydst and inexact table in all directions */
1313         for (dir = 0; dir < XFRM_POLICY_MAX; dir++) {
1314                 struct hlist_node *n;
1315
1316                 hlist_for_each_entry_safe(policy, n,
1317                                           &net->xfrm.policy_inexact[dir],
1318                                           bydst_inexact_list) {
1319                         hlist_del_rcu(&policy->bydst);
1320                         hlist_del_init(&policy->bydst_inexact_list);
1321                 }
1322
1323                 hmask = net->xfrm.policy_bydst[dir].hmask;
1324                 odst = net->xfrm.policy_bydst[dir].table;
1325                 for (i = hmask; i >= 0; i--) {
1326                         hlist_for_each_entry_safe(policy, n, odst + i, bydst)
1327                                 hlist_del_rcu(&policy->bydst);
1328                 }
1329                 if ((dir & XFRM_POLICY_MASK) == XFRM_POLICY_OUT) {
1330                         /* dir out => dst = remote, src = local */
1331                         net->xfrm.policy_bydst[dir].dbits4 = rbits4;
1332                         net->xfrm.policy_bydst[dir].sbits4 = lbits4;
1333                         net->xfrm.policy_bydst[dir].dbits6 = rbits6;
1334                         net->xfrm.policy_bydst[dir].sbits6 = lbits6;
1335                 } else {
1336                         /* dir in/fwd => dst = local, src = remote */
1337                         net->xfrm.policy_bydst[dir].dbits4 = lbits4;
1338                         net->xfrm.policy_bydst[dir].sbits4 = rbits4;
1339                         net->xfrm.policy_bydst[dir].dbits6 = lbits6;
1340                         net->xfrm.policy_bydst[dir].sbits6 = rbits6;
1341                 }
1342         }
1343
1344         /* re-insert all policies by order of creation */
1345         list_for_each_entry_reverse(policy, &net->xfrm.policy_all, walk.all) {
1346                 if (policy->walk.dead)
1347                         continue;
1348                 dir = xfrm_policy_id2dir(policy->index);
1349                 if (dir >= XFRM_POLICY_MAX) {
1350                         /* skip socket policies */
1351                         continue;
1352                 }
1353                 newpos = NULL;
1354                 chain = policy_hash_bysel(net, &policy->selector,
1355                                           policy->family, dir);
1356
1357                 if (!chain) {
1358                         void *p = xfrm_policy_inexact_insert(policy, dir, 0);
1359
1360                         WARN_ONCE(IS_ERR(p), "reinsert: %ld\n", PTR_ERR(p));
1361                         continue;
1362                 }
1363
1364                 hlist_for_each_entry(pol, chain, bydst) {
1365                         if (policy->priority >= pol->priority)
1366                                 newpos = &pol->bydst;
1367                         else
1368                                 break;
1369                 }
1370                 if (newpos && policy->xdo.type != XFRM_DEV_OFFLOAD_PACKET)
1371                         hlist_add_behind_rcu(&policy->bydst, newpos);
1372                 else
1373                         hlist_add_head_rcu(&policy->bydst, chain);
1374         }
1375
1376 out_unlock:
1377         __xfrm_policy_inexact_flush(net);
1378         write_seqcount_end(&net->xfrm.xfrm_policy_hash_generation);
1379         spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1380
1381         mutex_unlock(&hash_resize_mutex);
1382 }
1383
1384 void xfrm_policy_hash_rebuild(struct net *net)
1385 {
1386         schedule_work(&net->xfrm.policy_hthresh.work);
1387 }
1388 EXPORT_SYMBOL(xfrm_policy_hash_rebuild);
1389
1390 /* Generate new index... KAME seems to generate them ordered by cost
1391  * of an absolute inpredictability of ordering of rules. This will not pass. */
1392 static u32 xfrm_gen_index(struct net *net, int dir, u32 index)
1393 {
1394         for (;;) {
1395                 struct hlist_head *list;
1396                 struct xfrm_policy *p;
1397                 u32 idx;
1398                 int found;
1399
1400                 if (!index) {
1401                         idx = (net->xfrm.idx_generator | dir);
1402                         net->xfrm.idx_generator += 8;
1403                 } else {
1404                         idx = index;
1405                         index = 0;
1406                 }
1407
1408                 if (idx == 0)
1409                         idx = 8;
1410                 list = net->xfrm.policy_byidx + idx_hash(net, idx);
1411                 found = 0;
1412                 hlist_for_each_entry(p, list, byidx) {
1413                         if (p->index == idx) {
1414                                 found = 1;
1415                                 break;
1416                         }
1417                 }
1418                 if (!found)
1419                         return idx;
1420         }
1421 }
1422
1423 static inline int selector_cmp(struct xfrm_selector *s1, struct xfrm_selector *s2)
1424 {
1425         u32 *p1 = (u32 *) s1;
1426         u32 *p2 = (u32 *) s2;
1427         int len = sizeof(struct xfrm_selector) / sizeof(u32);
1428         int i;
1429
1430         for (i = 0; i < len; i++) {
1431                 if (p1[i] != p2[i])
1432                         return 1;
1433         }
1434
1435         return 0;
1436 }
1437
1438 static void xfrm_policy_requeue(struct xfrm_policy *old,
1439                                 struct xfrm_policy *new)
1440 {
1441         struct xfrm_policy_queue *pq = &old->polq;
1442         struct sk_buff_head list;
1443
1444         if (skb_queue_empty(&pq->hold_queue))
1445                 return;
1446
1447         __skb_queue_head_init(&list);
1448
1449         spin_lock_bh(&pq->hold_queue.lock);
1450         skb_queue_splice_init(&pq->hold_queue, &list);
1451         if (del_timer(&pq->hold_timer))
1452                 xfrm_pol_put(old);
1453         spin_unlock_bh(&pq->hold_queue.lock);
1454
1455         pq = &new->polq;
1456
1457         spin_lock_bh(&pq->hold_queue.lock);
1458         skb_queue_splice(&list, &pq->hold_queue);
1459         pq->timeout = XFRM_QUEUE_TMO_MIN;
1460         if (!mod_timer(&pq->hold_timer, jiffies))
1461                 xfrm_pol_hold(new);
1462         spin_unlock_bh(&pq->hold_queue.lock);
1463 }
1464
1465 static inline bool xfrm_policy_mark_match(const struct xfrm_mark *mark,
1466                                           struct xfrm_policy *pol)
1467 {
1468         return mark->v == pol->mark.v && mark->m == pol->mark.m;
1469 }
1470
1471 static u32 xfrm_pol_bin_key(const void *data, u32 len, u32 seed)
1472 {
1473         const struct xfrm_pol_inexact_key *k = data;
1474         u32 a = k->type << 24 | k->dir << 16 | k->family;
1475
1476         return jhash_3words(a, k->if_id, net_hash_mix(read_pnet(&k->net)),
1477                             seed);
1478 }
1479
1480 static u32 xfrm_pol_bin_obj(const void *data, u32 len, u32 seed)
1481 {
1482         const struct xfrm_pol_inexact_bin *b = data;
1483
1484         return xfrm_pol_bin_key(&b->k, 0, seed);
1485 }
1486
1487 static int xfrm_pol_bin_cmp(struct rhashtable_compare_arg *arg,
1488                             const void *ptr)
1489 {
1490         const struct xfrm_pol_inexact_key *key = arg->key;
1491         const struct xfrm_pol_inexact_bin *b = ptr;
1492         int ret;
1493
1494         if (!net_eq(read_pnet(&b->k.net), read_pnet(&key->net)))
1495                 return -1;
1496
1497         ret = b->k.dir ^ key->dir;
1498         if (ret)
1499                 return ret;
1500
1501         ret = b->k.type ^ key->type;
1502         if (ret)
1503                 return ret;
1504
1505         ret = b->k.family ^ key->family;
1506         if (ret)
1507                 return ret;
1508
1509         return b->k.if_id ^ key->if_id;
1510 }
1511
1512 static const struct rhashtable_params xfrm_pol_inexact_params = {
1513         .head_offset            = offsetof(struct xfrm_pol_inexact_bin, head),
1514         .hashfn                 = xfrm_pol_bin_key,
1515         .obj_hashfn             = xfrm_pol_bin_obj,
1516         .obj_cmpfn              = xfrm_pol_bin_cmp,
1517         .automatic_shrinking    = true,
1518 };
1519
1520 static void xfrm_policy_insert_inexact_list(struct hlist_head *chain,
1521                                             struct xfrm_policy *policy)
1522 {
1523         struct xfrm_policy *pol, *delpol = NULL;
1524         struct hlist_node *newpos = NULL;
1525         int i = 0;
1526
1527         hlist_for_each_entry(pol, chain, bydst_inexact_list) {
1528                 if (pol->type == policy->type &&
1529                     pol->if_id == policy->if_id &&
1530                     !selector_cmp(&pol->selector, &policy->selector) &&
1531                     xfrm_policy_mark_match(&policy->mark, pol) &&
1532                     xfrm_sec_ctx_match(pol->security, policy->security) &&
1533                     !WARN_ON(delpol)) {
1534                         delpol = pol;
1535                         if (policy->priority > pol->priority)
1536                                 continue;
1537                 } else if (policy->priority >= pol->priority) {
1538                         newpos = &pol->bydst_inexact_list;
1539                         continue;
1540                 }
1541                 if (delpol)
1542                         break;
1543         }
1544
1545         if (newpos && policy->xdo.type != XFRM_DEV_OFFLOAD_PACKET)
1546                 hlist_add_behind_rcu(&policy->bydst_inexact_list, newpos);
1547         else
1548                 hlist_add_head_rcu(&policy->bydst_inexact_list, chain);
1549
1550         hlist_for_each_entry(pol, chain, bydst_inexact_list) {
1551                 pol->pos = i;
1552                 i++;
1553         }
1554 }
1555
1556 static struct xfrm_policy *xfrm_policy_insert_list(struct hlist_head *chain,
1557                                                    struct xfrm_policy *policy,
1558                                                    bool excl)
1559 {
1560         struct xfrm_policy *pol, *newpos = NULL, *delpol = NULL;
1561
1562         hlist_for_each_entry(pol, chain, bydst) {
1563                 if (pol->type == policy->type &&
1564                     pol->if_id == policy->if_id &&
1565                     !selector_cmp(&pol->selector, &policy->selector) &&
1566                     xfrm_policy_mark_match(&policy->mark, pol) &&
1567                     xfrm_sec_ctx_match(pol->security, policy->security) &&
1568                     !WARN_ON(delpol)) {
1569                         if (excl)
1570                                 return ERR_PTR(-EEXIST);
1571                         delpol = pol;
1572                         if (policy->priority > pol->priority)
1573                                 continue;
1574                 } else if (policy->priority >= pol->priority) {
1575                         newpos = pol;
1576                         continue;
1577                 }
1578                 if (delpol)
1579                         break;
1580         }
1581
1582         if (newpos && policy->xdo.type != XFRM_DEV_OFFLOAD_PACKET)
1583                 hlist_add_behind_rcu(&policy->bydst, &newpos->bydst);
1584         else
1585                 /* Packet offload policies enter to the head
1586                  * to speed-up lookups.
1587                  */
1588                 hlist_add_head_rcu(&policy->bydst, chain);
1589
1590         return delpol;
1591 }
1592
1593 int xfrm_policy_insert(int dir, struct xfrm_policy *policy, int excl)
1594 {
1595         struct net *net = xp_net(policy);
1596         struct xfrm_policy *delpol;
1597         struct hlist_head *chain;
1598
1599         spin_lock_bh(&net->xfrm.xfrm_policy_lock);
1600         chain = policy_hash_bysel(net, &policy->selector, policy->family, dir);
1601         if (chain)
1602                 delpol = xfrm_policy_insert_list(chain, policy, excl);
1603         else
1604                 delpol = xfrm_policy_inexact_insert(policy, dir, excl);
1605
1606         if (IS_ERR(delpol)) {
1607                 spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1608                 return PTR_ERR(delpol);
1609         }
1610
1611         __xfrm_policy_link(policy, dir);
1612
1613         /* After previous checking, family can either be AF_INET or AF_INET6 */
1614         if (policy->family == AF_INET)
1615                 rt_genid_bump_ipv4(net);
1616         else
1617                 rt_genid_bump_ipv6(net);
1618
1619         if (delpol) {
1620                 xfrm_policy_requeue(delpol, policy);
1621                 __xfrm_policy_unlink(delpol, dir);
1622         }
1623         policy->index = delpol ? delpol->index : xfrm_gen_index(net, dir, policy->index);
1624         hlist_add_head(&policy->byidx, net->xfrm.policy_byidx+idx_hash(net, policy->index));
1625         policy->curlft.add_time = ktime_get_real_seconds();
1626         policy->curlft.use_time = 0;
1627         if (!mod_timer(&policy->timer, jiffies + HZ))
1628                 xfrm_pol_hold(policy);
1629         spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1630
1631         if (delpol)
1632                 xfrm_policy_kill(delpol);
1633         else if (xfrm_bydst_should_resize(net, dir, NULL))
1634                 schedule_work(&net->xfrm.policy_hash_work);
1635
1636         return 0;
1637 }
1638 EXPORT_SYMBOL(xfrm_policy_insert);
1639
1640 static struct xfrm_policy *
1641 __xfrm_policy_bysel_ctx(struct hlist_head *chain, const struct xfrm_mark *mark,
1642                         u32 if_id, u8 type, int dir, struct xfrm_selector *sel,
1643                         struct xfrm_sec_ctx *ctx)
1644 {
1645         struct xfrm_policy *pol;
1646
1647         if (!chain)
1648                 return NULL;
1649
1650         hlist_for_each_entry(pol, chain, bydst) {
1651                 if (pol->type == type &&
1652                     pol->if_id == if_id &&
1653                     xfrm_policy_mark_match(mark, pol) &&
1654                     !selector_cmp(sel, &pol->selector) &&
1655                     xfrm_sec_ctx_match(ctx, pol->security))
1656                         return pol;
1657         }
1658
1659         return NULL;
1660 }
1661
1662 struct xfrm_policy *
1663 xfrm_policy_bysel_ctx(struct net *net, const struct xfrm_mark *mark, u32 if_id,
1664                       u8 type, int dir, struct xfrm_selector *sel,
1665                       struct xfrm_sec_ctx *ctx, int delete, int *err)
1666 {
1667         struct xfrm_pol_inexact_bin *bin = NULL;
1668         struct xfrm_policy *pol, *ret = NULL;
1669         struct hlist_head *chain;
1670
1671         *err = 0;
1672         spin_lock_bh(&net->xfrm.xfrm_policy_lock);
1673         chain = policy_hash_bysel(net, sel, sel->family, dir);
1674         if (!chain) {
1675                 struct xfrm_pol_inexact_candidates cand;
1676                 int i;
1677
1678                 bin = xfrm_policy_inexact_lookup(net, type,
1679                                                  sel->family, dir, if_id);
1680                 if (!bin) {
1681                         spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1682                         return NULL;
1683                 }
1684
1685                 if (!xfrm_policy_find_inexact_candidates(&cand, bin,
1686                                                          &sel->saddr,
1687                                                          &sel->daddr)) {
1688                         spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1689                         return NULL;
1690                 }
1691
1692                 pol = NULL;
1693                 for (i = 0; i < ARRAY_SIZE(cand.res); i++) {
1694                         struct xfrm_policy *tmp;
1695
1696                         tmp = __xfrm_policy_bysel_ctx(cand.res[i], mark,
1697                                                       if_id, type, dir,
1698                                                       sel, ctx);
1699                         if (!tmp)
1700                                 continue;
1701
1702                         if (!pol || tmp->pos < pol->pos)
1703                                 pol = tmp;
1704                 }
1705         } else {
1706                 pol = __xfrm_policy_bysel_ctx(chain, mark, if_id, type, dir,
1707                                               sel, ctx);
1708         }
1709
1710         if (pol) {
1711                 xfrm_pol_hold(pol);
1712                 if (delete) {
1713                         *err = security_xfrm_policy_delete(pol->security);
1714                         if (*err) {
1715                                 spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1716                                 return pol;
1717                         }
1718                         __xfrm_policy_unlink(pol, dir);
1719                 }
1720                 ret = pol;
1721         }
1722         spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1723
1724         if (ret && delete)
1725                 xfrm_policy_kill(ret);
1726         if (bin && delete)
1727                 xfrm_policy_inexact_prune_bin(bin);
1728         return ret;
1729 }
1730 EXPORT_SYMBOL(xfrm_policy_bysel_ctx);
1731
1732 struct xfrm_policy *
1733 xfrm_policy_byid(struct net *net, const struct xfrm_mark *mark, u32 if_id,
1734                  u8 type, int dir, u32 id, int delete, int *err)
1735 {
1736         struct xfrm_policy *pol, *ret;
1737         struct hlist_head *chain;
1738
1739         *err = -ENOENT;
1740         if (xfrm_policy_id2dir(id) != dir)
1741                 return NULL;
1742
1743         *err = 0;
1744         spin_lock_bh(&net->xfrm.xfrm_policy_lock);
1745         chain = net->xfrm.policy_byidx + idx_hash(net, id);
1746         ret = NULL;
1747         hlist_for_each_entry(pol, chain, byidx) {
1748                 if (pol->type == type && pol->index == id &&
1749                     pol->if_id == if_id && xfrm_policy_mark_match(mark, pol)) {
1750                         xfrm_pol_hold(pol);
1751                         if (delete) {
1752                                 *err = security_xfrm_policy_delete(
1753                                                                 pol->security);
1754                                 if (*err) {
1755                                         spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1756                                         return pol;
1757                                 }
1758                                 __xfrm_policy_unlink(pol, dir);
1759                         }
1760                         ret = pol;
1761                         break;
1762                 }
1763         }
1764         spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1765
1766         if (ret && delete)
1767                 xfrm_policy_kill(ret);
1768         return ret;
1769 }
1770 EXPORT_SYMBOL(xfrm_policy_byid);
1771
1772 #ifdef CONFIG_SECURITY_NETWORK_XFRM
1773 static inline int
1774 xfrm_policy_flush_secctx_check(struct net *net, u8 type, bool task_valid)
1775 {
1776         struct xfrm_policy *pol;
1777         int err = 0;
1778
1779         list_for_each_entry(pol, &net->xfrm.policy_all, walk.all) {
1780                 if (pol->walk.dead ||
1781                     xfrm_policy_id2dir(pol->index) >= XFRM_POLICY_MAX ||
1782                     pol->type != type)
1783                         continue;
1784
1785                 err = security_xfrm_policy_delete(pol->security);
1786                 if (err) {
1787                         xfrm_audit_policy_delete(pol, 0, task_valid);
1788                         return err;
1789                 }
1790         }
1791         return err;
1792 }
1793
1794 static inline int xfrm_dev_policy_flush_secctx_check(struct net *net,
1795                                                      struct net_device *dev,
1796                                                      bool task_valid)
1797 {
1798         struct xfrm_policy *pol;
1799         int err = 0;
1800
1801         list_for_each_entry(pol, &net->xfrm.policy_all, walk.all) {
1802                 if (pol->walk.dead ||
1803                     xfrm_policy_id2dir(pol->index) >= XFRM_POLICY_MAX ||
1804                     pol->xdo.dev != dev)
1805                         continue;
1806
1807                 err = security_xfrm_policy_delete(pol->security);
1808                 if (err) {
1809                         xfrm_audit_policy_delete(pol, 0, task_valid);
1810                         return err;
1811                 }
1812         }
1813         return err;
1814 }
1815 #else
1816 static inline int
1817 xfrm_policy_flush_secctx_check(struct net *net, u8 type, bool task_valid)
1818 {
1819         return 0;
1820 }
1821
1822 static inline int xfrm_dev_policy_flush_secctx_check(struct net *net,
1823                                                      struct net_device *dev,
1824                                                      bool task_valid)
1825 {
1826         return 0;
1827 }
1828 #endif
1829
1830 int xfrm_policy_flush(struct net *net, u8 type, bool task_valid)
1831 {
1832         int dir, err = 0, cnt = 0;
1833         struct xfrm_policy *pol;
1834
1835         spin_lock_bh(&net->xfrm.xfrm_policy_lock);
1836
1837         err = xfrm_policy_flush_secctx_check(net, type, task_valid);
1838         if (err)
1839                 goto out;
1840
1841 again:
1842         list_for_each_entry(pol, &net->xfrm.policy_all, walk.all) {
1843                 if (pol->walk.dead)
1844                         continue;
1845
1846                 dir = xfrm_policy_id2dir(pol->index);
1847                 if (dir >= XFRM_POLICY_MAX ||
1848                     pol->type != type)
1849                         continue;
1850
1851                 __xfrm_policy_unlink(pol, dir);
1852                 spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1853                 xfrm_dev_policy_delete(pol);
1854                 cnt++;
1855                 xfrm_audit_policy_delete(pol, 1, task_valid);
1856                 xfrm_policy_kill(pol);
1857                 spin_lock_bh(&net->xfrm.xfrm_policy_lock);
1858                 goto again;
1859         }
1860         if (cnt)
1861                 __xfrm_policy_inexact_flush(net);
1862         else
1863                 err = -ESRCH;
1864 out:
1865         spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1866         return err;
1867 }
1868 EXPORT_SYMBOL(xfrm_policy_flush);
1869
1870 int xfrm_dev_policy_flush(struct net *net, struct net_device *dev,
1871                           bool task_valid)
1872 {
1873         int dir, err = 0, cnt = 0;
1874         struct xfrm_policy *pol;
1875
1876         spin_lock_bh(&net->xfrm.xfrm_policy_lock);
1877
1878         err = xfrm_dev_policy_flush_secctx_check(net, dev, task_valid);
1879         if (err)
1880                 goto out;
1881
1882 again:
1883         list_for_each_entry(pol, &net->xfrm.policy_all, walk.all) {
1884                 if (pol->walk.dead)
1885                         continue;
1886
1887                 dir = xfrm_policy_id2dir(pol->index);
1888                 if (dir >= XFRM_POLICY_MAX ||
1889                     pol->xdo.dev != dev)
1890                         continue;
1891
1892                 __xfrm_policy_unlink(pol, dir);
1893                 spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1894                 xfrm_dev_policy_delete(pol);
1895                 cnt++;
1896                 xfrm_audit_policy_delete(pol, 1, task_valid);
1897                 xfrm_policy_kill(pol);
1898                 spin_lock_bh(&net->xfrm.xfrm_policy_lock);
1899                 goto again;
1900         }
1901         if (cnt)
1902                 __xfrm_policy_inexact_flush(net);
1903         else
1904                 err = -ESRCH;
1905 out:
1906         spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1907         return err;
1908 }
1909 EXPORT_SYMBOL(xfrm_dev_policy_flush);
1910
1911 int xfrm_policy_walk(struct net *net, struct xfrm_policy_walk *walk,
1912                      int (*func)(struct xfrm_policy *, int, int, void*),
1913                      void *data)
1914 {
1915         struct xfrm_policy *pol;
1916         struct xfrm_policy_walk_entry *x;
1917         int error = 0;
1918
1919         if (walk->type >= XFRM_POLICY_TYPE_MAX &&
1920             walk->type != XFRM_POLICY_TYPE_ANY)
1921                 return -EINVAL;
1922
1923         if (list_empty(&walk->walk.all) && walk->seq != 0)
1924                 return 0;
1925
1926         spin_lock_bh(&net->xfrm.xfrm_policy_lock);
1927         if (list_empty(&walk->walk.all))
1928                 x = list_first_entry(&net->xfrm.policy_all, struct xfrm_policy_walk_entry, all);
1929         else
1930                 x = list_first_entry(&walk->walk.all,
1931                                      struct xfrm_policy_walk_entry, all);
1932
1933         list_for_each_entry_from(x, &net->xfrm.policy_all, all) {
1934                 if (x->dead)
1935                         continue;
1936                 pol = container_of(x, struct xfrm_policy, walk);
1937                 if (walk->type != XFRM_POLICY_TYPE_ANY &&
1938                     walk->type != pol->type)
1939                         continue;
1940                 error = func(pol, xfrm_policy_id2dir(pol->index),
1941                              walk->seq, data);
1942                 if (error) {
1943                         list_move_tail(&walk->walk.all, &x->all);
1944                         goto out;
1945                 }
1946                 walk->seq++;
1947         }
1948         if (walk->seq == 0) {
1949                 error = -ENOENT;
1950                 goto out;
1951         }
1952         list_del_init(&walk->walk.all);
1953 out:
1954         spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1955         return error;
1956 }
1957 EXPORT_SYMBOL(xfrm_policy_walk);
1958
1959 void xfrm_policy_walk_init(struct xfrm_policy_walk *walk, u8 type)
1960 {
1961         INIT_LIST_HEAD(&walk->walk.all);
1962         walk->walk.dead = 1;
1963         walk->type = type;
1964         walk->seq = 0;
1965 }
1966 EXPORT_SYMBOL(xfrm_policy_walk_init);
1967
1968 void xfrm_policy_walk_done(struct xfrm_policy_walk *walk, struct net *net)
1969 {
1970         if (list_empty(&walk->walk.all))
1971                 return;
1972
1973         spin_lock_bh(&net->xfrm.xfrm_policy_lock); /*FIXME where is net? */
1974         list_del(&walk->walk.all);
1975         spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1976 }
1977 EXPORT_SYMBOL(xfrm_policy_walk_done);
1978
1979 /*
1980  * Find policy to apply to this flow.
1981  *
1982  * Returns 0 if policy found, else an -errno.
1983  */
1984 static int xfrm_policy_match(const struct xfrm_policy *pol,
1985                              const struct flowi *fl,
1986                              u8 type, u16 family, u32 if_id)
1987 {
1988         const struct xfrm_selector *sel = &pol->selector;
1989         int ret = -ESRCH;
1990         bool match;
1991
1992         if (pol->family != family ||
1993             pol->if_id != if_id ||
1994             (fl->flowi_mark & pol->mark.m) != pol->mark.v ||
1995             pol->type != type)
1996                 return ret;
1997
1998         match = xfrm_selector_match(sel, fl, family);
1999         if (match)
2000                 ret = security_xfrm_policy_lookup(pol->security, fl->flowi_secid);
2001         return ret;
2002 }
2003
2004 static struct xfrm_pol_inexact_node *
2005 xfrm_policy_lookup_inexact_addr(const struct rb_root *r,
2006                                 seqcount_spinlock_t *count,
2007                                 const xfrm_address_t *addr, u16 family)
2008 {
2009         const struct rb_node *parent;
2010         int seq;
2011
2012 again:
2013         seq = read_seqcount_begin(count);
2014
2015         parent = rcu_dereference_raw(r->rb_node);
2016         while (parent) {
2017                 struct xfrm_pol_inexact_node *node;
2018                 int delta;
2019
2020                 node = rb_entry(parent, struct xfrm_pol_inexact_node, node);
2021
2022                 delta = xfrm_policy_addr_delta(addr, &node->addr,
2023                                                node->prefixlen, family);
2024                 if (delta < 0) {
2025                         parent = rcu_dereference_raw(parent->rb_left);
2026                         continue;
2027                 } else if (delta > 0) {
2028                         parent = rcu_dereference_raw(parent->rb_right);
2029                         continue;
2030                 }
2031
2032                 return node;
2033         }
2034
2035         if (read_seqcount_retry(count, seq))
2036                 goto again;
2037
2038         return NULL;
2039 }
2040
2041 static bool
2042 xfrm_policy_find_inexact_candidates(struct xfrm_pol_inexact_candidates *cand,
2043                                     struct xfrm_pol_inexact_bin *b,
2044                                     const xfrm_address_t *saddr,
2045                                     const xfrm_address_t *daddr)
2046 {
2047         struct xfrm_pol_inexact_node *n;
2048         u16 family;
2049
2050         if (!b)
2051                 return false;
2052
2053         family = b->k.family;
2054         memset(cand, 0, sizeof(*cand));
2055         cand->res[XFRM_POL_CAND_ANY] = &b->hhead;
2056
2057         n = xfrm_policy_lookup_inexact_addr(&b->root_d, &b->count, daddr,
2058                                             family);
2059         if (n) {
2060                 cand->res[XFRM_POL_CAND_DADDR] = &n->hhead;
2061                 n = xfrm_policy_lookup_inexact_addr(&n->root, &b->count, saddr,
2062                                                     family);
2063                 if (n)
2064                         cand->res[XFRM_POL_CAND_BOTH] = &n->hhead;
2065         }
2066
2067         n = xfrm_policy_lookup_inexact_addr(&b->root_s, &b->count, saddr,
2068                                             family);
2069         if (n)
2070                 cand->res[XFRM_POL_CAND_SADDR] = &n->hhead;
2071
2072         return true;
2073 }
2074
2075 static struct xfrm_pol_inexact_bin *
2076 xfrm_policy_inexact_lookup_rcu(struct net *net, u8 type, u16 family,
2077                                u8 dir, u32 if_id)
2078 {
2079         struct xfrm_pol_inexact_key k = {
2080                 .family = family,
2081                 .type = type,
2082                 .dir = dir,
2083                 .if_id = if_id,
2084         };
2085
2086         write_pnet(&k.net, net);
2087
2088         return rhashtable_lookup(&xfrm_policy_inexact_table, &k,
2089                                  xfrm_pol_inexact_params);
2090 }
2091
2092 static struct xfrm_pol_inexact_bin *
2093 xfrm_policy_inexact_lookup(struct net *net, u8 type, u16 family,
2094                            u8 dir, u32 if_id)
2095 {
2096         struct xfrm_pol_inexact_bin *bin;
2097
2098         lockdep_assert_held(&net->xfrm.xfrm_policy_lock);
2099
2100         rcu_read_lock();
2101         bin = xfrm_policy_inexact_lookup_rcu(net, type, family, dir, if_id);
2102         rcu_read_unlock();
2103
2104         return bin;
2105 }
2106
2107 static struct xfrm_policy *
2108 __xfrm_policy_eval_candidates(struct hlist_head *chain,
2109                               struct xfrm_policy *prefer,
2110                               const struct flowi *fl,
2111                               u8 type, u16 family, u32 if_id)
2112 {
2113         u32 priority = prefer ? prefer->priority : ~0u;
2114         struct xfrm_policy *pol;
2115
2116         if (!chain)
2117                 return NULL;
2118
2119         hlist_for_each_entry_rcu(pol, chain, bydst) {
2120                 int err;
2121
2122                 if (pol->priority > priority)
2123                         break;
2124
2125                 err = xfrm_policy_match(pol, fl, type, family, if_id);
2126                 if (err) {
2127                         if (err != -ESRCH)
2128                                 return ERR_PTR(err);
2129
2130                         continue;
2131                 }
2132
2133                 if (prefer) {
2134                         /* matches.  Is it older than *prefer? */
2135                         if (pol->priority == priority &&
2136                             prefer->pos < pol->pos)
2137                                 return prefer;
2138                 }
2139
2140                 return pol;
2141         }
2142
2143         return NULL;
2144 }
2145
2146 static struct xfrm_policy *
2147 xfrm_policy_eval_candidates(struct xfrm_pol_inexact_candidates *cand,
2148                             struct xfrm_policy *prefer,
2149                             const struct flowi *fl,
2150                             u8 type, u16 family, u32 if_id)
2151 {
2152         struct xfrm_policy *tmp;
2153         int i;
2154
2155         for (i = 0; i < ARRAY_SIZE(cand->res); i++) {
2156                 tmp = __xfrm_policy_eval_candidates(cand->res[i],
2157                                                     prefer,
2158                                                     fl, type, family, if_id);
2159                 if (!tmp)
2160                         continue;
2161
2162                 if (IS_ERR(tmp))
2163                         return tmp;
2164                 prefer = tmp;
2165         }
2166
2167         return prefer;
2168 }
2169
2170 static struct xfrm_policy *xfrm_policy_lookup_bytype(struct net *net, u8 type,
2171                                                      const struct flowi *fl,
2172                                                      u16 family, u8 dir,
2173                                                      u32 if_id)
2174 {
2175         struct xfrm_pol_inexact_candidates cand;
2176         const xfrm_address_t *daddr, *saddr;
2177         struct xfrm_pol_inexact_bin *bin;
2178         struct xfrm_policy *pol, *ret;
2179         struct hlist_head *chain;
2180         unsigned int sequence;
2181         int err;
2182
2183         daddr = xfrm_flowi_daddr(fl, family);
2184         saddr = xfrm_flowi_saddr(fl, family);
2185         if (unlikely(!daddr || !saddr))
2186                 return NULL;
2187
2188         rcu_read_lock();
2189  retry:
2190         do {
2191                 sequence = read_seqcount_begin(&net->xfrm.xfrm_policy_hash_generation);
2192                 chain = policy_hash_direct(net, daddr, saddr, family, dir);
2193         } while (read_seqcount_retry(&net->xfrm.xfrm_policy_hash_generation, sequence));
2194
2195         ret = NULL;
2196         hlist_for_each_entry_rcu(pol, chain, bydst) {
2197                 err = xfrm_policy_match(pol, fl, type, family, if_id);
2198                 if (err) {
2199                         if (err == -ESRCH)
2200                                 continue;
2201                         else {
2202                                 ret = ERR_PTR(err);
2203                                 goto fail;
2204                         }
2205                 } else {
2206                         ret = pol;
2207                         break;
2208                 }
2209         }
2210         if (ret && ret->xdo.type == XFRM_DEV_OFFLOAD_PACKET)
2211                 goto skip_inexact;
2212
2213         bin = xfrm_policy_inexact_lookup_rcu(net, type, family, dir, if_id);
2214         if (!bin || !xfrm_policy_find_inexact_candidates(&cand, bin, saddr,
2215                                                          daddr))
2216                 goto skip_inexact;
2217
2218         pol = xfrm_policy_eval_candidates(&cand, ret, fl, type,
2219                                           family, if_id);
2220         if (pol) {
2221                 ret = pol;
2222                 if (IS_ERR(pol))
2223                         goto fail;
2224         }
2225
2226 skip_inexact:
2227         if (read_seqcount_retry(&net->xfrm.xfrm_policy_hash_generation, sequence))
2228                 goto retry;
2229
2230         if (ret && !xfrm_pol_hold_rcu(ret))
2231                 goto retry;
2232 fail:
2233         rcu_read_unlock();
2234
2235         return ret;
2236 }
2237
2238 static struct xfrm_policy *xfrm_policy_lookup(struct net *net,
2239                                               const struct flowi *fl,
2240                                               u16 family, u8 dir, u32 if_id)
2241 {
2242 #ifdef CONFIG_XFRM_SUB_POLICY
2243         struct xfrm_policy *pol;
2244
2245         pol = xfrm_policy_lookup_bytype(net, XFRM_POLICY_TYPE_SUB, fl, family,
2246                                         dir, if_id);
2247         if (pol != NULL)
2248                 return pol;
2249 #endif
2250         return xfrm_policy_lookup_bytype(net, XFRM_POLICY_TYPE_MAIN, fl, family,
2251                                          dir, if_id);
2252 }
2253
2254 static struct xfrm_policy *xfrm_sk_policy_lookup(const struct sock *sk, int dir,
2255                                                  const struct flowi *fl,
2256                                                  u16 family, u32 if_id)
2257 {
2258         struct xfrm_policy *pol;
2259
2260         rcu_read_lock();
2261  again:
2262         pol = rcu_dereference(sk->sk_policy[dir]);
2263         if (pol != NULL) {
2264                 bool match;
2265                 int err = 0;
2266
2267                 if (pol->family != family) {
2268                         pol = NULL;
2269                         goto out;
2270                 }
2271
2272                 match = xfrm_selector_match(&pol->selector, fl, family);
2273                 if (match) {
2274                         if ((READ_ONCE(sk->sk_mark) & pol->mark.m) != pol->mark.v ||
2275                             pol->if_id != if_id) {
2276                                 pol = NULL;
2277                                 goto out;
2278                         }
2279                         err = security_xfrm_policy_lookup(pol->security,
2280                                                       fl->flowi_secid);
2281                         if (!err) {
2282                                 if (!xfrm_pol_hold_rcu(pol))
2283                                         goto again;
2284                         } else if (err == -ESRCH) {
2285                                 pol = NULL;
2286                         } else {
2287                                 pol = ERR_PTR(err);
2288                         }
2289                 } else
2290                         pol = NULL;
2291         }
2292 out:
2293         rcu_read_unlock();
2294         return pol;
2295 }
2296
2297 static void __xfrm_policy_link(struct xfrm_policy *pol, int dir)
2298 {
2299         struct net *net = xp_net(pol);
2300
2301         list_add(&pol->walk.all, &net->xfrm.policy_all);
2302         net->xfrm.policy_count[dir]++;
2303         xfrm_pol_hold(pol);
2304 }
2305
2306 static struct xfrm_policy *__xfrm_policy_unlink(struct xfrm_policy *pol,
2307                                                 int dir)
2308 {
2309         struct net *net = xp_net(pol);
2310
2311         if (list_empty(&pol->walk.all))
2312                 return NULL;
2313
2314         /* Socket policies are not hashed. */
2315         if (!hlist_unhashed(&pol->bydst)) {
2316                 hlist_del_rcu(&pol->bydst);
2317                 hlist_del_init(&pol->bydst_inexact_list);
2318                 hlist_del(&pol->byidx);
2319         }
2320
2321         list_del_init(&pol->walk.all);
2322         net->xfrm.policy_count[dir]--;
2323
2324         return pol;
2325 }
2326
2327 static void xfrm_sk_policy_link(struct xfrm_policy *pol, int dir)
2328 {
2329         __xfrm_policy_link(pol, XFRM_POLICY_MAX + dir);
2330 }
2331
2332 static void xfrm_sk_policy_unlink(struct xfrm_policy *pol, int dir)
2333 {
2334         __xfrm_policy_unlink(pol, XFRM_POLICY_MAX + dir);
2335 }
2336
2337 int xfrm_policy_delete(struct xfrm_policy *pol, int dir)
2338 {
2339         struct net *net = xp_net(pol);
2340
2341         spin_lock_bh(&net->xfrm.xfrm_policy_lock);
2342         pol = __xfrm_policy_unlink(pol, dir);
2343         spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
2344         if (pol) {
2345                 xfrm_dev_policy_delete(pol);
2346                 xfrm_policy_kill(pol);
2347                 return 0;
2348         }
2349         return -ENOENT;
2350 }
2351 EXPORT_SYMBOL(xfrm_policy_delete);
2352
2353 int xfrm_sk_policy_insert(struct sock *sk, int dir, struct xfrm_policy *pol)
2354 {
2355         struct net *net = sock_net(sk);
2356         struct xfrm_policy *old_pol;
2357
2358 #ifdef CONFIG_XFRM_SUB_POLICY
2359         if (pol && pol->type != XFRM_POLICY_TYPE_MAIN)
2360                 return -EINVAL;
2361 #endif
2362
2363         spin_lock_bh(&net->xfrm.xfrm_policy_lock);
2364         old_pol = rcu_dereference_protected(sk->sk_policy[dir],
2365                                 lockdep_is_held(&net->xfrm.xfrm_policy_lock));
2366         if (pol) {
2367                 pol->curlft.add_time = ktime_get_real_seconds();
2368                 pol->index = xfrm_gen_index(net, XFRM_POLICY_MAX+dir, 0);
2369                 xfrm_sk_policy_link(pol, dir);
2370         }
2371         rcu_assign_pointer(sk->sk_policy[dir], pol);
2372         if (old_pol) {
2373                 if (pol)
2374                         xfrm_policy_requeue(old_pol, pol);
2375
2376                 /* Unlinking succeeds always. This is the only function
2377                  * allowed to delete or replace socket policy.
2378                  */
2379                 xfrm_sk_policy_unlink(old_pol, dir);
2380         }
2381         spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
2382
2383         if (old_pol) {
2384                 xfrm_policy_kill(old_pol);
2385         }
2386         return 0;
2387 }
2388
2389 static struct xfrm_policy *clone_policy(const struct xfrm_policy *old, int dir)
2390 {
2391         struct xfrm_policy *newp = xfrm_policy_alloc(xp_net(old), GFP_ATOMIC);
2392         struct net *net = xp_net(old);
2393
2394         if (newp) {
2395                 newp->selector = old->selector;
2396                 if (security_xfrm_policy_clone(old->security,
2397                                                &newp->security)) {
2398                         kfree(newp);
2399                         return NULL;  /* ENOMEM */
2400                 }
2401                 newp->lft = old->lft;
2402                 newp->curlft = old->curlft;
2403                 newp->mark = old->mark;
2404                 newp->if_id = old->if_id;
2405                 newp->action = old->action;
2406                 newp->flags = old->flags;
2407                 newp->xfrm_nr = old->xfrm_nr;
2408                 newp->index = old->index;
2409                 newp->type = old->type;
2410                 newp->family = old->family;
2411                 memcpy(newp->xfrm_vec, old->xfrm_vec,
2412                        newp->xfrm_nr*sizeof(struct xfrm_tmpl));
2413                 spin_lock_bh(&net->xfrm.xfrm_policy_lock);
2414                 xfrm_sk_policy_link(newp, dir);
2415                 spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
2416                 xfrm_pol_put(newp);
2417         }
2418         return newp;
2419 }
2420
2421 int __xfrm_sk_clone_policy(struct sock *sk, const struct sock *osk)
2422 {
2423         const struct xfrm_policy *p;
2424         struct xfrm_policy *np;
2425         int i, ret = 0;
2426
2427         rcu_read_lock();
2428         for (i = 0; i < 2; i++) {
2429                 p = rcu_dereference(osk->sk_policy[i]);
2430                 if (p) {
2431                         np = clone_policy(p, i);
2432                         if (unlikely(!np)) {
2433                                 ret = -ENOMEM;
2434                                 break;
2435                         }
2436                         rcu_assign_pointer(sk->sk_policy[i], np);
2437                 }
2438         }
2439         rcu_read_unlock();
2440         return ret;
2441 }
2442
2443 static int
2444 xfrm_get_saddr(struct net *net, int oif, xfrm_address_t *local,
2445                xfrm_address_t *remote, unsigned short family, u32 mark)
2446 {
2447         int err;
2448         const struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family);
2449
2450         if (unlikely(afinfo == NULL))
2451                 return -EINVAL;
2452         err = afinfo->get_saddr(net, oif, local, remote, mark);
2453         rcu_read_unlock();
2454         return err;
2455 }
2456
2457 /* Resolve list of templates for the flow, given policy. */
2458
2459 static int
2460 xfrm_tmpl_resolve_one(struct xfrm_policy *policy, const struct flowi *fl,
2461                       struct xfrm_state **xfrm, unsigned short family)
2462 {
2463         struct net *net = xp_net(policy);
2464         int nx;
2465         int i, error;
2466         xfrm_address_t *daddr = xfrm_flowi_daddr(fl, family);
2467         xfrm_address_t *saddr = xfrm_flowi_saddr(fl, family);
2468         xfrm_address_t tmp;
2469
2470         for (nx = 0, i = 0; i < policy->xfrm_nr; i++) {
2471                 struct xfrm_state *x;
2472                 xfrm_address_t *remote = daddr;
2473                 xfrm_address_t *local  = saddr;
2474                 struct xfrm_tmpl *tmpl = &policy->xfrm_vec[i];
2475
2476                 if (tmpl->mode == XFRM_MODE_TUNNEL ||
2477                     tmpl->mode == XFRM_MODE_BEET) {
2478                         remote = &tmpl->id.daddr;
2479                         local = &tmpl->saddr;
2480                         if (xfrm_addr_any(local, tmpl->encap_family)) {
2481                                 error = xfrm_get_saddr(net, fl->flowi_oif,
2482                                                        &tmp, remote,
2483                                                        tmpl->encap_family, 0);
2484                                 if (error)
2485                                         goto fail;
2486                                 local = &tmp;
2487                         }
2488                 }
2489
2490                 x = xfrm_state_find(remote, local, fl, tmpl, policy, &error,
2491                                     family, policy->if_id);
2492
2493                 if (x && x->km.state == XFRM_STATE_VALID) {
2494                         xfrm[nx++] = x;
2495                         daddr = remote;
2496                         saddr = local;
2497                         continue;
2498                 }
2499                 if (x) {
2500                         error = (x->km.state == XFRM_STATE_ERROR ?
2501                                  -EINVAL : -EAGAIN);
2502                         xfrm_state_put(x);
2503                 } else if (error == -ESRCH) {
2504                         error = -EAGAIN;
2505                 }
2506
2507                 if (!tmpl->optional)
2508                         goto fail;
2509         }
2510         return nx;
2511
2512 fail:
2513         for (nx--; nx >= 0; nx--)
2514                 xfrm_state_put(xfrm[nx]);
2515         return error;
2516 }
2517
2518 static int
2519 xfrm_tmpl_resolve(struct xfrm_policy **pols, int npols, const struct flowi *fl,
2520                   struct xfrm_state **xfrm, unsigned short family)
2521 {
2522         struct xfrm_state *tp[XFRM_MAX_DEPTH];
2523         struct xfrm_state **tpp = (npols > 1) ? tp : xfrm;
2524         int cnx = 0;
2525         int error;
2526         int ret;
2527         int i;
2528
2529         for (i = 0; i < npols; i++) {
2530                 if (cnx + pols[i]->xfrm_nr >= XFRM_MAX_DEPTH) {
2531                         error = -ENOBUFS;
2532                         goto fail;
2533                 }
2534
2535                 ret = xfrm_tmpl_resolve_one(pols[i], fl, &tpp[cnx], family);
2536                 if (ret < 0) {
2537                         error = ret;
2538                         goto fail;
2539                 } else
2540                         cnx += ret;
2541         }
2542
2543         /* found states are sorted for outbound processing */
2544         if (npols > 1)
2545                 xfrm_state_sort(xfrm, tpp, cnx, family);
2546
2547         return cnx;
2548
2549  fail:
2550         for (cnx--; cnx >= 0; cnx--)
2551                 xfrm_state_put(tpp[cnx]);
2552         return error;
2553
2554 }
2555
2556 static int xfrm_get_tos(const struct flowi *fl, int family)
2557 {
2558         if (family == AF_INET)
2559                 return IPTOS_RT_MASK & fl->u.ip4.flowi4_tos;
2560
2561         return 0;
2562 }
2563
2564 static inline struct xfrm_dst *xfrm_alloc_dst(struct net *net, int family)
2565 {
2566         const struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family);
2567         struct dst_ops *dst_ops;
2568         struct xfrm_dst *xdst;
2569
2570         if (!afinfo)
2571                 return ERR_PTR(-EINVAL);
2572
2573         switch (family) {
2574         case AF_INET:
2575                 dst_ops = &net->xfrm.xfrm4_dst_ops;
2576                 break;
2577 #if IS_ENABLED(CONFIG_IPV6)
2578         case AF_INET6:
2579                 dst_ops = &net->xfrm.xfrm6_dst_ops;
2580                 break;
2581 #endif
2582         default:
2583                 BUG();
2584         }
2585         xdst = dst_alloc(dst_ops, NULL, DST_OBSOLETE_NONE, 0);
2586
2587         if (likely(xdst)) {
2588                 memset_after(xdst, 0, u.dst);
2589         } else
2590                 xdst = ERR_PTR(-ENOBUFS);
2591
2592         rcu_read_unlock();
2593
2594         return xdst;
2595 }
2596
2597 static void xfrm_init_path(struct xfrm_dst *path, struct dst_entry *dst,
2598                            int nfheader_len)
2599 {
2600         if (dst->ops->family == AF_INET6) {
2601                 struct rt6_info *rt = (struct rt6_info *)dst;
2602                 path->path_cookie = rt6_get_cookie(rt);
2603                 path->u.rt6.rt6i_nfheader_len = nfheader_len;
2604         }
2605 }
2606
2607 static inline int xfrm_fill_dst(struct xfrm_dst *xdst, struct net_device *dev,
2608                                 const struct flowi *fl)
2609 {
2610         const struct xfrm_policy_afinfo *afinfo =
2611                 xfrm_policy_get_afinfo(xdst->u.dst.ops->family);
2612         int err;
2613
2614         if (!afinfo)
2615                 return -EINVAL;
2616
2617         err = afinfo->fill_dst(xdst, dev, fl);
2618
2619         rcu_read_unlock();
2620
2621         return err;
2622 }
2623
2624
2625 /* Allocate chain of dst_entry's, attach known xfrm's, calculate
2626  * all the metrics... Shortly, bundle a bundle.
2627  */
2628
2629 static struct dst_entry *xfrm_bundle_create(struct xfrm_policy *policy,
2630                                             struct xfrm_state **xfrm,
2631                                             struct xfrm_dst **bundle,
2632                                             int nx,
2633                                             const struct flowi *fl,
2634                                             struct dst_entry *dst)
2635 {
2636         const struct xfrm_state_afinfo *afinfo;
2637         const struct xfrm_mode *inner_mode;
2638         struct net *net = xp_net(policy);
2639         unsigned long now = jiffies;
2640         struct net_device *dev;
2641         struct xfrm_dst *xdst_prev = NULL;
2642         struct xfrm_dst *xdst0 = NULL;
2643         int i = 0;
2644         int err;
2645         int header_len = 0;
2646         int nfheader_len = 0;
2647         int trailer_len = 0;
2648         int tos;
2649         int family = policy->selector.family;
2650         xfrm_address_t saddr, daddr;
2651
2652         xfrm_flowi_addr_get(fl, &saddr, &daddr, family);
2653
2654         tos = xfrm_get_tos(fl, family);
2655
2656         dst_hold(dst);
2657
2658         for (; i < nx; i++) {
2659                 struct xfrm_dst *xdst = xfrm_alloc_dst(net, family);
2660                 struct dst_entry *dst1 = &xdst->u.dst;
2661
2662                 err = PTR_ERR(xdst);
2663                 if (IS_ERR(xdst)) {
2664                         dst_release(dst);
2665                         goto put_states;
2666                 }
2667
2668                 bundle[i] = xdst;
2669                 if (!xdst_prev)
2670                         xdst0 = xdst;
2671                 else
2672                         /* Ref count is taken during xfrm_alloc_dst()
2673                          * No need to do dst_clone() on dst1
2674                          */
2675                         xfrm_dst_set_child(xdst_prev, &xdst->u.dst);
2676
2677                 if (xfrm[i]->sel.family == AF_UNSPEC) {
2678                         inner_mode = xfrm_ip2inner_mode(xfrm[i],
2679                                                         xfrm_af2proto(family));
2680                         if (!inner_mode) {
2681                                 err = -EAFNOSUPPORT;
2682                                 dst_release(dst);
2683                                 goto put_states;
2684                         }
2685                 } else
2686                         inner_mode = &xfrm[i]->inner_mode;
2687
2688                 xdst->route = dst;
2689                 dst_copy_metrics(dst1, dst);
2690
2691                 if (xfrm[i]->props.mode != XFRM_MODE_TRANSPORT) {
2692                         __u32 mark = 0;
2693                         int oif;
2694
2695                         if (xfrm[i]->props.smark.v || xfrm[i]->props.smark.m)
2696                                 mark = xfrm_smark_get(fl->flowi_mark, xfrm[i]);
2697
2698                         if (xfrm[i]->xso.type != XFRM_DEV_OFFLOAD_PACKET)
2699                                 family = xfrm[i]->props.family;
2700
2701                         oif = fl->flowi_oif ? : fl->flowi_l3mdev;
2702                         dst = xfrm_dst_lookup(xfrm[i], tos, oif,
2703                                               &saddr, &daddr, family, mark);
2704                         err = PTR_ERR(dst);
2705                         if (IS_ERR(dst))
2706                                 goto put_states;
2707                 } else
2708                         dst_hold(dst);
2709
2710                 dst1->xfrm = xfrm[i];
2711                 xdst->xfrm_genid = xfrm[i]->genid;
2712
2713                 dst1->obsolete = DST_OBSOLETE_FORCE_CHK;
2714                 dst1->lastuse = now;
2715
2716                 dst1->input = dst_discard;
2717
2718                 rcu_read_lock();
2719                 afinfo = xfrm_state_afinfo_get_rcu(inner_mode->family);
2720                 if (likely(afinfo))
2721                         dst1->output = afinfo->output;
2722                 else
2723                         dst1->output = dst_discard_out;
2724                 rcu_read_unlock();
2725
2726                 xdst_prev = xdst;
2727
2728                 header_len += xfrm[i]->props.header_len;
2729                 if (xfrm[i]->type->flags & XFRM_TYPE_NON_FRAGMENT)
2730                         nfheader_len += xfrm[i]->props.header_len;
2731                 trailer_len += xfrm[i]->props.trailer_len;
2732         }
2733
2734         xfrm_dst_set_child(xdst_prev, dst);
2735         xdst0->path = dst;
2736
2737         err = -ENODEV;
2738         dev = dst->dev;
2739         if (!dev)
2740                 goto free_dst;
2741
2742         xfrm_init_path(xdst0, dst, nfheader_len);
2743         xfrm_init_pmtu(bundle, nx);
2744
2745         for (xdst_prev = xdst0; xdst_prev != (struct xfrm_dst *)dst;
2746              xdst_prev = (struct xfrm_dst *) xfrm_dst_child(&xdst_prev->u.dst)) {
2747                 err = xfrm_fill_dst(xdst_prev, dev, fl);
2748                 if (err)
2749                         goto free_dst;
2750
2751                 xdst_prev->u.dst.header_len = header_len;
2752                 xdst_prev->u.dst.trailer_len = trailer_len;
2753                 header_len -= xdst_prev->u.dst.xfrm->props.header_len;
2754                 trailer_len -= xdst_prev->u.dst.xfrm->props.trailer_len;
2755         }
2756
2757         return &xdst0->u.dst;
2758
2759 put_states:
2760         for (; i < nx; i++)
2761                 xfrm_state_put(xfrm[i]);
2762 free_dst:
2763         if (xdst0)
2764                 dst_release_immediate(&xdst0->u.dst);
2765
2766         return ERR_PTR(err);
2767 }
2768
2769 static int xfrm_expand_policies(const struct flowi *fl, u16 family,
2770                                 struct xfrm_policy **pols,
2771                                 int *num_pols, int *num_xfrms)
2772 {
2773         int i;
2774
2775         if (*num_pols == 0 || !pols[0]) {
2776                 *num_pols = 0;
2777                 *num_xfrms = 0;
2778                 return 0;
2779         }
2780         if (IS_ERR(pols[0])) {
2781                 *num_pols = 0;
2782                 return PTR_ERR(pols[0]);
2783         }
2784
2785         *num_xfrms = pols[0]->xfrm_nr;
2786
2787 #ifdef CONFIG_XFRM_SUB_POLICY
2788         if (pols[0]->action == XFRM_POLICY_ALLOW &&
2789             pols[0]->type != XFRM_POLICY_TYPE_MAIN) {
2790                 pols[1] = xfrm_policy_lookup_bytype(xp_net(pols[0]),
2791                                                     XFRM_POLICY_TYPE_MAIN,
2792                                                     fl, family,
2793                                                     XFRM_POLICY_OUT,
2794                                                     pols[0]->if_id);
2795                 if (pols[1]) {
2796                         if (IS_ERR(pols[1])) {
2797                                 xfrm_pols_put(pols, *num_pols);
2798                                 *num_pols = 0;
2799                                 return PTR_ERR(pols[1]);
2800                         }
2801                         (*num_pols)++;
2802                         (*num_xfrms) += pols[1]->xfrm_nr;
2803                 }
2804         }
2805 #endif
2806         for (i = 0; i < *num_pols; i++) {
2807                 if (pols[i]->action != XFRM_POLICY_ALLOW) {
2808                         *num_xfrms = -1;
2809                         break;
2810                 }
2811         }
2812
2813         return 0;
2814
2815 }
2816
2817 static struct xfrm_dst *
2818 xfrm_resolve_and_create_bundle(struct xfrm_policy **pols, int num_pols,
2819                                const struct flowi *fl, u16 family,
2820                                struct dst_entry *dst_orig)
2821 {
2822         struct net *net = xp_net(pols[0]);
2823         struct xfrm_state *xfrm[XFRM_MAX_DEPTH];
2824         struct xfrm_dst *bundle[XFRM_MAX_DEPTH];
2825         struct xfrm_dst *xdst;
2826         struct dst_entry *dst;
2827         int err;
2828
2829         /* Try to instantiate a bundle */
2830         err = xfrm_tmpl_resolve(pols, num_pols, fl, xfrm, family);
2831         if (err <= 0) {
2832                 if (err == 0)
2833                         return NULL;
2834
2835                 if (err != -EAGAIN)
2836                         XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTPOLERROR);
2837                 return ERR_PTR(err);
2838         }
2839
2840         dst = xfrm_bundle_create(pols[0], xfrm, bundle, err, fl, dst_orig);
2841         if (IS_ERR(dst)) {
2842                 XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTBUNDLEGENERROR);
2843                 return ERR_CAST(dst);
2844         }
2845
2846         xdst = (struct xfrm_dst *)dst;
2847         xdst->num_xfrms = err;
2848         xdst->num_pols = num_pols;
2849         memcpy(xdst->pols, pols, sizeof(struct xfrm_policy *) * num_pols);
2850         xdst->policy_genid = atomic_read(&pols[0]->genid);
2851
2852         return xdst;
2853 }
2854
2855 static void xfrm_policy_queue_process(struct timer_list *t)
2856 {
2857         struct sk_buff *skb;
2858         struct sock *sk;
2859         struct dst_entry *dst;
2860         struct xfrm_policy *pol = from_timer(pol, t, polq.hold_timer);
2861         struct net *net = xp_net(pol);
2862         struct xfrm_policy_queue *pq = &pol->polq;
2863         struct flowi fl;
2864         struct sk_buff_head list;
2865         __u32 skb_mark;
2866
2867         spin_lock(&pq->hold_queue.lock);
2868         skb = skb_peek(&pq->hold_queue);
2869         if (!skb) {
2870                 spin_unlock(&pq->hold_queue.lock);
2871                 goto out;
2872         }
2873         dst = skb_dst(skb);
2874         sk = skb->sk;
2875
2876         /* Fixup the mark to support VTI. */
2877         skb_mark = skb->mark;
2878         skb->mark = pol->mark.v;
2879         xfrm_decode_session(net, skb, &fl, dst->ops->family);
2880         skb->mark = skb_mark;
2881         spin_unlock(&pq->hold_queue.lock);
2882
2883         dst_hold(xfrm_dst_path(dst));
2884         dst = xfrm_lookup(net, xfrm_dst_path(dst), &fl, sk, XFRM_LOOKUP_QUEUE);
2885         if (IS_ERR(dst))
2886                 goto purge_queue;
2887
2888         if (dst->flags & DST_XFRM_QUEUE) {
2889                 dst_release(dst);
2890
2891                 if (pq->timeout >= XFRM_QUEUE_TMO_MAX)
2892                         goto purge_queue;
2893
2894                 pq->timeout = pq->timeout << 1;
2895                 if (!mod_timer(&pq->hold_timer, jiffies + pq->timeout))
2896                         xfrm_pol_hold(pol);
2897                 goto out;
2898         }
2899
2900         dst_release(dst);
2901
2902         __skb_queue_head_init(&list);
2903
2904         spin_lock(&pq->hold_queue.lock);
2905         pq->timeout = 0;
2906         skb_queue_splice_init(&pq->hold_queue, &list);
2907         spin_unlock(&pq->hold_queue.lock);
2908
2909         while (!skb_queue_empty(&list)) {
2910                 skb = __skb_dequeue(&list);
2911
2912                 /* Fixup the mark to support VTI. */
2913                 skb_mark = skb->mark;
2914                 skb->mark = pol->mark.v;
2915                 xfrm_decode_session(net, skb, &fl, skb_dst(skb)->ops->family);
2916                 skb->mark = skb_mark;
2917
2918                 dst_hold(xfrm_dst_path(skb_dst(skb)));
2919                 dst = xfrm_lookup(net, xfrm_dst_path(skb_dst(skb)), &fl, skb->sk, 0);
2920                 if (IS_ERR(dst)) {
2921                         kfree_skb(skb);
2922                         continue;
2923                 }
2924
2925                 nf_reset_ct(skb);
2926                 skb_dst_drop(skb);
2927                 skb_dst_set(skb, dst);
2928
2929                 dst_output(net, skb->sk, skb);
2930         }
2931
2932 out:
2933         xfrm_pol_put(pol);
2934         return;
2935
2936 purge_queue:
2937         pq->timeout = 0;
2938         skb_queue_purge(&pq->hold_queue);
2939         xfrm_pol_put(pol);
2940 }
2941
2942 static int xdst_queue_output(struct net *net, struct sock *sk, struct sk_buff *skb)
2943 {
2944         unsigned long sched_next;
2945         struct dst_entry *dst = skb_dst(skb);
2946         struct xfrm_dst *xdst = (struct xfrm_dst *) dst;
2947         struct xfrm_policy *pol = xdst->pols[0];
2948         struct xfrm_policy_queue *pq = &pol->polq;
2949
2950         if (unlikely(skb_fclone_busy(sk, skb))) {
2951                 kfree_skb(skb);
2952                 return 0;
2953         }
2954
2955         if (pq->hold_queue.qlen > XFRM_MAX_QUEUE_LEN) {
2956                 kfree_skb(skb);
2957                 return -EAGAIN;
2958         }
2959
2960         skb_dst_force(skb);
2961
2962         spin_lock_bh(&pq->hold_queue.lock);
2963
2964         if (!pq->timeout)
2965                 pq->timeout = XFRM_QUEUE_TMO_MIN;
2966
2967         sched_next = jiffies + pq->timeout;
2968
2969         if (del_timer(&pq->hold_timer)) {
2970                 if (time_before(pq->hold_timer.expires, sched_next))
2971                         sched_next = pq->hold_timer.expires;
2972                 xfrm_pol_put(pol);
2973         }
2974
2975         __skb_queue_tail(&pq->hold_queue, skb);
2976         if (!mod_timer(&pq->hold_timer, sched_next))
2977                 xfrm_pol_hold(pol);
2978
2979         spin_unlock_bh(&pq->hold_queue.lock);
2980
2981         return 0;
2982 }
2983
2984 static struct xfrm_dst *xfrm_create_dummy_bundle(struct net *net,
2985                                                  struct xfrm_flo *xflo,
2986                                                  const struct flowi *fl,
2987                                                  int num_xfrms,
2988                                                  u16 family)
2989 {
2990         int err;
2991         struct net_device *dev;
2992         struct dst_entry *dst;
2993         struct dst_entry *dst1;
2994         struct xfrm_dst *xdst;
2995
2996         xdst = xfrm_alloc_dst(net, family);
2997         if (IS_ERR(xdst))
2998                 return xdst;
2999
3000         if (!(xflo->flags & XFRM_LOOKUP_QUEUE) ||
3001             net->xfrm.sysctl_larval_drop ||
3002             num_xfrms <= 0)
3003                 return xdst;
3004
3005         dst = xflo->dst_orig;
3006         dst1 = &xdst->u.dst;
3007         dst_hold(dst);
3008         xdst->route = dst;
3009
3010         dst_copy_metrics(dst1, dst);
3011
3012         dst1->obsolete = DST_OBSOLETE_FORCE_CHK;
3013         dst1->flags |= DST_XFRM_QUEUE;
3014         dst1->lastuse = jiffies;
3015
3016         dst1->input = dst_discard;
3017         dst1->output = xdst_queue_output;
3018
3019         dst_hold(dst);
3020         xfrm_dst_set_child(xdst, dst);
3021         xdst->path = dst;
3022
3023         xfrm_init_path((struct xfrm_dst *)dst1, dst, 0);
3024
3025         err = -ENODEV;
3026         dev = dst->dev;
3027         if (!dev)
3028                 goto free_dst;
3029
3030         err = xfrm_fill_dst(xdst, dev, fl);
3031         if (err)
3032                 goto free_dst;
3033
3034 out:
3035         return xdst;
3036
3037 free_dst:
3038         dst_release(dst1);
3039         xdst = ERR_PTR(err);
3040         goto out;
3041 }
3042
3043 static struct xfrm_dst *xfrm_bundle_lookup(struct net *net,
3044                                            const struct flowi *fl,
3045                                            u16 family, u8 dir,
3046                                            struct xfrm_flo *xflo, u32 if_id)
3047 {
3048         struct xfrm_policy *pols[XFRM_POLICY_TYPE_MAX];
3049         int num_pols = 0, num_xfrms = 0, err;
3050         struct xfrm_dst *xdst;
3051
3052         /* Resolve policies to use if we couldn't get them from
3053          * previous cache entry */
3054         num_pols = 1;
3055         pols[0] = xfrm_policy_lookup(net, fl, family, dir, if_id);
3056         err = xfrm_expand_policies(fl, family, pols,
3057                                            &num_pols, &num_xfrms);
3058         if (err < 0)
3059                 goto inc_error;
3060         if (num_pols == 0)
3061                 return NULL;
3062         if (num_xfrms <= 0)
3063                 goto make_dummy_bundle;
3064
3065         xdst = xfrm_resolve_and_create_bundle(pols, num_pols, fl, family,
3066                                               xflo->dst_orig);
3067         if (IS_ERR(xdst)) {
3068                 err = PTR_ERR(xdst);
3069                 if (err == -EREMOTE) {
3070                         xfrm_pols_put(pols, num_pols);
3071                         return NULL;
3072                 }
3073
3074                 if (err != -EAGAIN)
3075                         goto error;
3076                 goto make_dummy_bundle;
3077         } else if (xdst == NULL) {
3078                 num_xfrms = 0;
3079                 goto make_dummy_bundle;
3080         }
3081
3082         return xdst;
3083
3084 make_dummy_bundle:
3085         /* We found policies, but there's no bundles to instantiate:
3086          * either because the policy blocks, has no transformations or
3087          * we could not build template (no xfrm_states).*/
3088         xdst = xfrm_create_dummy_bundle(net, xflo, fl, num_xfrms, family);
3089         if (IS_ERR(xdst)) {
3090                 xfrm_pols_put(pols, num_pols);
3091                 return ERR_CAST(xdst);
3092         }
3093         xdst->num_pols = num_pols;
3094         xdst->num_xfrms = num_xfrms;
3095         memcpy(xdst->pols, pols, sizeof(struct xfrm_policy *) * num_pols);
3096
3097         return xdst;
3098
3099 inc_error:
3100         XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTPOLERROR);
3101 error:
3102         xfrm_pols_put(pols, num_pols);
3103         return ERR_PTR(err);
3104 }
3105
3106 static struct dst_entry *make_blackhole(struct net *net, u16 family,
3107                                         struct dst_entry *dst_orig)
3108 {
3109         const struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family);
3110         struct dst_entry *ret;
3111
3112         if (!afinfo) {
3113                 dst_release(dst_orig);
3114                 return ERR_PTR(-EINVAL);
3115         } else {
3116                 ret = afinfo->blackhole_route(net, dst_orig);
3117         }
3118         rcu_read_unlock();
3119
3120         return ret;
3121 }
3122
3123 /* Finds/creates a bundle for given flow and if_id
3124  *
3125  * At the moment we eat a raw IP route. Mostly to speed up lookups
3126  * on interfaces with disabled IPsec.
3127  *
3128  * xfrm_lookup uses an if_id of 0 by default, and is provided for
3129  * compatibility
3130  */
3131 struct dst_entry *xfrm_lookup_with_ifid(struct net *net,
3132                                         struct dst_entry *dst_orig,
3133                                         const struct flowi *fl,
3134                                         const struct sock *sk,
3135                                         int flags, u32 if_id)
3136 {
3137         struct xfrm_policy *pols[XFRM_POLICY_TYPE_MAX];
3138         struct xfrm_dst *xdst;
3139         struct dst_entry *dst, *route;
3140         u16 family = dst_orig->ops->family;
3141         u8 dir = XFRM_POLICY_OUT;
3142         int i, err, num_pols, num_xfrms = 0, drop_pols = 0;
3143
3144         dst = NULL;
3145         xdst = NULL;
3146         route = NULL;
3147
3148         sk = sk_const_to_full_sk(sk);
3149         if (sk && sk->sk_policy[XFRM_POLICY_OUT]) {
3150                 num_pols = 1;
3151                 pols[0] = xfrm_sk_policy_lookup(sk, XFRM_POLICY_OUT, fl, family,
3152                                                 if_id);
3153                 err = xfrm_expand_policies(fl, family, pols,
3154                                            &num_pols, &num_xfrms);
3155                 if (err < 0)
3156                         goto dropdst;
3157
3158                 if (num_pols) {
3159                         if (num_xfrms <= 0) {
3160                                 drop_pols = num_pols;
3161                                 goto no_transform;
3162                         }
3163
3164                         xdst = xfrm_resolve_and_create_bundle(
3165                                         pols, num_pols, fl,
3166                                         family, dst_orig);
3167
3168                         if (IS_ERR(xdst)) {
3169                                 xfrm_pols_put(pols, num_pols);
3170                                 err = PTR_ERR(xdst);
3171                                 if (err == -EREMOTE)
3172                                         goto nopol;
3173
3174                                 goto dropdst;
3175                         } else if (xdst == NULL) {
3176                                 num_xfrms = 0;
3177                                 drop_pols = num_pols;
3178                                 goto no_transform;
3179                         }
3180
3181                         route = xdst->route;
3182                 }
3183         }
3184
3185         if (xdst == NULL) {
3186                 struct xfrm_flo xflo;
3187
3188                 xflo.dst_orig = dst_orig;
3189                 xflo.flags = flags;
3190
3191                 /* To accelerate a bit...  */
3192                 if (!if_id && ((dst_orig->flags & DST_NOXFRM) ||
3193                                !net->xfrm.policy_count[XFRM_POLICY_OUT]))
3194                         goto nopol;
3195
3196                 xdst = xfrm_bundle_lookup(net, fl, family, dir, &xflo, if_id);
3197                 if (xdst == NULL)
3198                         goto nopol;
3199                 if (IS_ERR(xdst)) {
3200                         err = PTR_ERR(xdst);
3201                         goto dropdst;
3202                 }
3203
3204                 num_pols = xdst->num_pols;
3205                 num_xfrms = xdst->num_xfrms;
3206                 memcpy(pols, xdst->pols, sizeof(struct xfrm_policy *) * num_pols);
3207                 route = xdst->route;
3208         }
3209
3210         dst = &xdst->u.dst;
3211         if (route == NULL && num_xfrms > 0) {
3212                 /* The only case when xfrm_bundle_lookup() returns a
3213                  * bundle with null route, is when the template could
3214                  * not be resolved. It means policies are there, but
3215                  * bundle could not be created, since we don't yet
3216                  * have the xfrm_state's. We need to wait for KM to
3217                  * negotiate new SA's or bail out with error.*/
3218                 if (net->xfrm.sysctl_larval_drop) {
3219                         XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTNOSTATES);
3220                         err = -EREMOTE;
3221                         goto error;
3222                 }
3223
3224                 err = -EAGAIN;
3225
3226                 XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTNOSTATES);
3227                 goto error;
3228         }
3229
3230 no_transform:
3231         if (num_pols == 0)
3232                 goto nopol;
3233
3234         if ((flags & XFRM_LOOKUP_ICMP) &&
3235             !(pols[0]->flags & XFRM_POLICY_ICMP)) {
3236                 err = -ENOENT;
3237                 goto error;
3238         }
3239
3240         for (i = 0; i < num_pols; i++)
3241                 WRITE_ONCE(pols[i]->curlft.use_time, ktime_get_real_seconds());
3242
3243         if (num_xfrms < 0) {
3244                 /* Prohibit the flow */
3245                 XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTPOLBLOCK);
3246                 err = -EPERM;
3247                 goto error;
3248         } else if (num_xfrms > 0) {
3249                 /* Flow transformed */
3250                 dst_release(dst_orig);
3251         } else {
3252                 /* Flow passes untransformed */
3253                 dst_release(dst);
3254                 dst = dst_orig;
3255         }
3256 ok:
3257         xfrm_pols_put(pols, drop_pols);
3258         if (dst && dst->xfrm &&
3259             dst->xfrm->props.mode == XFRM_MODE_TUNNEL)
3260                 dst->flags |= DST_XFRM_TUNNEL;
3261         return dst;
3262
3263 nopol:
3264         if ((!dst_orig->dev || !(dst_orig->dev->flags & IFF_LOOPBACK)) &&
3265             net->xfrm.policy_default[dir] == XFRM_USERPOLICY_BLOCK) {
3266                 err = -EPERM;
3267                 goto error;
3268         }
3269         if (!(flags & XFRM_LOOKUP_ICMP)) {
3270                 dst = dst_orig;
3271                 goto ok;
3272         }
3273         err = -ENOENT;
3274 error:
3275         dst_release(dst);
3276 dropdst:
3277         if (!(flags & XFRM_LOOKUP_KEEP_DST_REF))
3278                 dst_release(dst_orig);
3279         xfrm_pols_put(pols, drop_pols);
3280         return ERR_PTR(err);
3281 }
3282 EXPORT_SYMBOL(xfrm_lookup_with_ifid);
3283
3284 /* Main function: finds/creates a bundle for given flow.
3285  *
3286  * At the moment we eat a raw IP route. Mostly to speed up lookups
3287  * on interfaces with disabled IPsec.
3288  */
3289 struct dst_entry *xfrm_lookup(struct net *net, struct dst_entry *dst_orig,
3290                               const struct flowi *fl, const struct sock *sk,
3291                               int flags)
3292 {
3293         return xfrm_lookup_with_ifid(net, dst_orig, fl, sk, flags, 0);
3294 }
3295 EXPORT_SYMBOL(xfrm_lookup);
3296
3297 /* Callers of xfrm_lookup_route() must ensure a call to dst_output().
3298  * Otherwise we may send out blackholed packets.
3299  */
3300 struct dst_entry *xfrm_lookup_route(struct net *net, struct dst_entry *dst_orig,
3301                                     const struct flowi *fl,
3302                                     const struct sock *sk, int flags)
3303 {
3304         struct dst_entry *dst = xfrm_lookup(net, dst_orig, fl, sk,
3305                                             flags | XFRM_LOOKUP_QUEUE |
3306                                             XFRM_LOOKUP_KEEP_DST_REF);
3307
3308         if (PTR_ERR(dst) == -EREMOTE)
3309                 return make_blackhole(net, dst_orig->ops->family, dst_orig);
3310
3311         if (IS_ERR(dst))
3312                 dst_release(dst_orig);
3313
3314         return dst;
3315 }
3316 EXPORT_SYMBOL(xfrm_lookup_route);
3317
3318 static inline int
3319 xfrm_secpath_reject(int idx, struct sk_buff *skb, const struct flowi *fl)
3320 {
3321         struct sec_path *sp = skb_sec_path(skb);
3322         struct xfrm_state *x;
3323
3324         if (!sp || idx < 0 || idx >= sp->len)
3325                 return 0;
3326         x = sp->xvec[idx];
3327         if (!x->type->reject)
3328                 return 0;
3329         return x->type->reject(x, skb, fl);
3330 }
3331
3332 /* When skb is transformed back to its "native" form, we have to
3333  * check policy restrictions. At the moment we make this in maximally
3334  * stupid way. Shame on me. :-) Of course, connected sockets must
3335  * have policy cached at them.
3336  */
3337
3338 static inline int
3339 xfrm_state_ok(const struct xfrm_tmpl *tmpl, const struct xfrm_state *x,
3340               unsigned short family, u32 if_id)
3341 {
3342         if (xfrm_state_kern(x))
3343                 return tmpl->optional && !xfrm_state_addr_cmp(tmpl, x, tmpl->encap_family);
3344         return  x->id.proto == tmpl->id.proto &&
3345                 (x->id.spi == tmpl->id.spi || !tmpl->id.spi) &&
3346                 (x->props.reqid == tmpl->reqid || !tmpl->reqid) &&
3347                 x->props.mode == tmpl->mode &&
3348                 (tmpl->allalgs || (tmpl->aalgos & (1<<x->props.aalgo)) ||
3349                  !(xfrm_id_proto_match(tmpl->id.proto, IPSEC_PROTO_ANY))) &&
3350                 !(x->props.mode != XFRM_MODE_TRANSPORT &&
3351                   xfrm_state_addr_cmp(tmpl, x, family)) &&
3352                 (if_id == 0 || if_id == x->if_id);
3353 }
3354
3355 /*
3356  * 0 or more than 0 is returned when validation is succeeded (either bypass
3357  * because of optional transport mode, or next index of the matched secpath
3358  * state with the template.
3359  * -1 is returned when no matching template is found.
3360  * Otherwise "-2 - errored_index" is returned.
3361  */
3362 static inline int
3363 xfrm_policy_ok(const struct xfrm_tmpl *tmpl, const struct sec_path *sp, int start,
3364                unsigned short family, u32 if_id)
3365 {
3366         int idx = start;
3367
3368         if (tmpl->optional) {
3369                 if (tmpl->mode == XFRM_MODE_TRANSPORT)
3370                         return start;
3371         } else
3372                 start = -1;
3373         for (; idx < sp->len; idx++) {
3374                 if (xfrm_state_ok(tmpl, sp->xvec[idx], family, if_id))
3375                         return ++idx;
3376                 if (sp->xvec[idx]->props.mode != XFRM_MODE_TRANSPORT) {
3377                         if (idx < sp->verified_cnt) {
3378                                 /* Secpath entry previously verified, consider optional and
3379                                  * continue searching
3380                                  */
3381                                 continue;
3382                         }
3383
3384                         if (start == -1)
3385                                 start = -2-idx;
3386                         break;
3387                 }
3388         }
3389         return start;
3390 }
3391
3392 static void
3393 decode_session4(const struct xfrm_flow_keys *flkeys, struct flowi *fl, bool reverse)
3394 {
3395         struct flowi4 *fl4 = &fl->u.ip4;
3396
3397         memset(fl4, 0, sizeof(struct flowi4));
3398
3399         if (reverse) {
3400                 fl4->saddr = flkeys->addrs.ipv4.dst;
3401                 fl4->daddr = flkeys->addrs.ipv4.src;
3402                 fl4->fl4_sport = flkeys->ports.dst;
3403                 fl4->fl4_dport = flkeys->ports.src;
3404         } else {
3405                 fl4->saddr = flkeys->addrs.ipv4.src;
3406                 fl4->daddr = flkeys->addrs.ipv4.dst;
3407                 fl4->fl4_sport = flkeys->ports.src;
3408                 fl4->fl4_dport = flkeys->ports.dst;
3409         }
3410
3411         switch (flkeys->basic.ip_proto) {
3412         case IPPROTO_GRE:
3413                 fl4->fl4_gre_key = flkeys->gre.keyid;
3414                 break;
3415         case IPPROTO_ICMP:
3416                 fl4->fl4_icmp_type = flkeys->icmp.type;
3417                 fl4->fl4_icmp_code = flkeys->icmp.code;
3418                 break;
3419         }
3420
3421         fl4->flowi4_proto = flkeys->basic.ip_proto;
3422         fl4->flowi4_tos = flkeys->ip.tos & ~INET_ECN_MASK;
3423 }
3424
3425 #if IS_ENABLED(CONFIG_IPV6)
3426 static void
3427 decode_session6(const struct xfrm_flow_keys *flkeys, struct flowi *fl, bool reverse)
3428 {
3429         struct flowi6 *fl6 = &fl->u.ip6;
3430
3431         memset(fl6, 0, sizeof(struct flowi6));
3432
3433         if (reverse) {
3434                 fl6->saddr = flkeys->addrs.ipv6.dst;
3435                 fl6->daddr = flkeys->addrs.ipv6.src;
3436                 fl6->fl6_sport = flkeys->ports.dst;
3437                 fl6->fl6_dport = flkeys->ports.src;
3438         } else {
3439                 fl6->saddr = flkeys->addrs.ipv6.src;
3440                 fl6->daddr = flkeys->addrs.ipv6.dst;
3441                 fl6->fl6_sport = flkeys->ports.src;
3442                 fl6->fl6_dport = flkeys->ports.dst;
3443         }
3444
3445         switch (flkeys->basic.ip_proto) {
3446         case IPPROTO_GRE:
3447                 fl6->fl6_gre_key = flkeys->gre.keyid;
3448                 break;
3449         case IPPROTO_ICMPV6:
3450                 fl6->fl6_icmp_type = flkeys->icmp.type;
3451                 fl6->fl6_icmp_code = flkeys->icmp.code;
3452                 break;
3453         }
3454
3455         fl6->flowi6_proto = flkeys->basic.ip_proto;
3456 }
3457 #endif
3458
3459 int __xfrm_decode_session(struct net *net, struct sk_buff *skb, struct flowi *fl,
3460                           unsigned int family, int reverse)
3461 {
3462         struct xfrm_flow_keys flkeys;
3463
3464         memset(&flkeys, 0, sizeof(flkeys));
3465         __skb_flow_dissect(net, skb, &xfrm_session_dissector, &flkeys,
3466                            NULL, 0, 0, 0, FLOW_DISSECTOR_F_STOP_AT_ENCAP);
3467
3468         switch (family) {
3469         case AF_INET:
3470                 decode_session4(&flkeys, fl, reverse);
3471                 break;
3472 #if IS_ENABLED(CONFIG_IPV6)
3473         case AF_INET6:
3474                 decode_session6(&flkeys, fl, reverse);
3475                 break;
3476 #endif
3477         default:
3478                 return -EAFNOSUPPORT;
3479         }
3480
3481         fl->flowi_mark = skb->mark;
3482         if (reverse) {
3483                 fl->flowi_oif = skb->skb_iif;
3484         } else {
3485                 int oif = 0;
3486
3487                 if (skb_dst(skb) && skb_dst(skb)->dev)
3488                         oif = skb_dst(skb)->dev->ifindex;
3489
3490                 fl->flowi_oif = oif;
3491         }
3492
3493         return security_xfrm_decode_session(skb, &fl->flowi_secid);
3494 }
3495 EXPORT_SYMBOL(__xfrm_decode_session);
3496
3497 static inline int secpath_has_nontransport(const struct sec_path *sp, int k, int *idxp)
3498 {
3499         for (; k < sp->len; k++) {
3500                 if (sp->xvec[k]->props.mode != XFRM_MODE_TRANSPORT) {
3501                         *idxp = k;
3502                         return 1;
3503                 }
3504         }
3505
3506         return 0;
3507 }
3508
3509 static bool icmp_err_packet(const struct flowi *fl, unsigned short family)
3510 {
3511         const struct flowi4 *fl4 = &fl->u.ip4;
3512
3513         if (family == AF_INET &&
3514             fl4->flowi4_proto == IPPROTO_ICMP &&
3515             (fl4->fl4_icmp_type == ICMP_DEST_UNREACH ||
3516              fl4->fl4_icmp_type == ICMP_TIME_EXCEEDED))
3517                 return true;
3518
3519 #if IS_ENABLED(CONFIG_IPV6)
3520         if (family == AF_INET6) {
3521                 const struct flowi6 *fl6 = &fl->u.ip6;
3522
3523                 if (fl6->flowi6_proto == IPPROTO_ICMPV6 &&
3524                     (fl6->fl6_icmp_type == ICMPV6_DEST_UNREACH ||
3525                     fl6->fl6_icmp_type == ICMPV6_PKT_TOOBIG ||
3526                     fl6->fl6_icmp_type == ICMPV6_TIME_EXCEED))
3527                         return true;
3528         }
3529 #endif
3530         return false;
3531 }
3532
3533 static bool xfrm_icmp_flow_decode(struct sk_buff *skb, unsigned short family,
3534                                   const struct flowi *fl, struct flowi *fl1)
3535 {
3536         bool ret = true;
3537         struct sk_buff *newskb = skb_clone(skb, GFP_ATOMIC);
3538         int hl = family == AF_INET ? (sizeof(struct iphdr) +  sizeof(struct icmphdr)) :
3539                  (sizeof(struct ipv6hdr) + sizeof(struct icmp6hdr));
3540
3541         if (!newskb)
3542                 return true;
3543
3544         if (!pskb_pull(newskb, hl))
3545                 goto out;
3546
3547         skb_reset_network_header(newskb);
3548
3549         if (xfrm_decode_session_reverse(dev_net(skb->dev), newskb, fl1, family) < 0)
3550                 goto out;
3551
3552         fl1->flowi_oif = fl->flowi_oif;
3553         fl1->flowi_mark = fl->flowi_mark;
3554         fl1->flowi_tos = fl->flowi_tos;
3555         nf_nat_decode_session(newskb, fl1, family);
3556         ret = false;
3557
3558 out:
3559         consume_skb(newskb);
3560         return ret;
3561 }
3562
3563 static bool xfrm_selector_inner_icmp_match(struct sk_buff *skb, unsigned short family,
3564                                            const struct xfrm_selector *sel,
3565                                            const struct flowi *fl)
3566 {
3567         bool ret = false;
3568
3569         if (icmp_err_packet(fl, family)) {
3570                 struct flowi fl1;
3571
3572                 if (xfrm_icmp_flow_decode(skb, family, fl, &fl1))
3573                         return ret;
3574
3575                 ret = xfrm_selector_match(sel, &fl1, family);
3576         }
3577
3578         return ret;
3579 }
3580
3581 static inline struct
3582 xfrm_policy *xfrm_in_fwd_icmp(struct sk_buff *skb,
3583                               const struct flowi *fl, unsigned short family,
3584                               u32 if_id)
3585 {
3586         struct xfrm_policy *pol = NULL;
3587
3588         if (icmp_err_packet(fl, family)) {
3589                 struct flowi fl1;
3590                 struct net *net = dev_net(skb->dev);
3591
3592                 if (xfrm_icmp_flow_decode(skb, family, fl, &fl1))
3593                         return pol;
3594
3595                 pol = xfrm_policy_lookup(net, &fl1, family, XFRM_POLICY_FWD, if_id);
3596         }
3597
3598         return pol;
3599 }
3600
3601 static inline struct
3602 dst_entry *xfrm_out_fwd_icmp(struct sk_buff *skb, struct flowi *fl,
3603                              unsigned short family, struct dst_entry *dst)
3604 {
3605         if (icmp_err_packet(fl, family)) {
3606                 struct net *net = dev_net(skb->dev);
3607                 struct dst_entry *dst2;
3608                 struct flowi fl1;
3609
3610                 if (xfrm_icmp_flow_decode(skb, family, fl, &fl1))
3611                         return dst;
3612
3613                 dst_hold(dst);
3614
3615                 dst2 = xfrm_lookup(net, dst, &fl1, NULL, (XFRM_LOOKUP_QUEUE | XFRM_LOOKUP_ICMP));
3616
3617                 if (IS_ERR(dst2))
3618                         return dst;
3619
3620                 if (dst2->xfrm) {
3621                         dst_release(dst);
3622                         dst = dst2;
3623                 } else {
3624                         dst_release(dst2);
3625                 }
3626         }
3627
3628         return dst;
3629 }
3630
3631 int __xfrm_policy_check(struct sock *sk, int dir, struct sk_buff *skb,
3632                         unsigned short family)
3633 {
3634         struct net *net = dev_net(skb->dev);
3635         struct xfrm_policy *pol;
3636         struct xfrm_policy *pols[XFRM_POLICY_TYPE_MAX];
3637         int npols = 0;
3638         int xfrm_nr;
3639         int pi;
3640         int reverse;
3641         struct flowi fl;
3642         int xerr_idx = -1;
3643         const struct xfrm_if_cb *ifcb;
3644         struct sec_path *sp;
3645         u32 if_id = 0;
3646
3647         rcu_read_lock();
3648         ifcb = xfrm_if_get_cb();
3649
3650         if (ifcb) {
3651                 struct xfrm_if_decode_session_result r;
3652
3653                 if (ifcb->decode_session(skb, family, &r)) {
3654                         if_id = r.if_id;
3655                         net = r.net;
3656                 }
3657         }
3658         rcu_read_unlock();
3659
3660         reverse = dir & ~XFRM_POLICY_MASK;
3661         dir &= XFRM_POLICY_MASK;
3662
3663         if (__xfrm_decode_session(net, skb, &fl, family, reverse) < 0) {
3664                 XFRM_INC_STATS(net, LINUX_MIB_XFRMINHDRERROR);
3665                 return 0;
3666         }
3667
3668         nf_nat_decode_session(skb, &fl, family);
3669
3670         /* First, check used SA against their selectors. */
3671         sp = skb_sec_path(skb);
3672         if (sp) {
3673                 int i;
3674
3675                 for (i = sp->len - 1; i >= 0; i--) {
3676                         struct xfrm_state *x = sp->xvec[i];
3677                         int ret = 0;
3678
3679                         if (!xfrm_selector_match(&x->sel, &fl, family)) {
3680                                 ret = 1;
3681                                 if (x->props.flags & XFRM_STATE_ICMP &&
3682                                     xfrm_selector_inner_icmp_match(skb, family, &x->sel, &fl))
3683                                         ret = 0;
3684                                 if (ret) {
3685                                         XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMISMATCH);
3686                                         return 0;
3687                                 }
3688                         }
3689                 }
3690         }
3691
3692         pol = NULL;
3693         sk = sk_to_full_sk(sk);
3694         if (sk && sk->sk_policy[dir]) {
3695                 pol = xfrm_sk_policy_lookup(sk, dir, &fl, family, if_id);
3696                 if (IS_ERR(pol)) {
3697                         XFRM_INC_STATS(net, LINUX_MIB_XFRMINPOLERROR);
3698                         return 0;
3699                 }
3700         }
3701
3702         if (!pol)
3703                 pol = xfrm_policy_lookup(net, &fl, family, dir, if_id);
3704
3705         if (IS_ERR(pol)) {
3706                 XFRM_INC_STATS(net, LINUX_MIB_XFRMINPOLERROR);
3707                 return 0;
3708         }
3709
3710         if (!pol && dir == XFRM_POLICY_FWD)
3711                 pol = xfrm_in_fwd_icmp(skb, &fl, family, if_id);
3712
3713         if (!pol) {
3714                 if (net->xfrm.policy_default[dir] == XFRM_USERPOLICY_BLOCK) {
3715                         XFRM_INC_STATS(net, LINUX_MIB_XFRMINNOPOLS);
3716                         return 0;
3717                 }
3718
3719                 if (sp && secpath_has_nontransport(sp, 0, &xerr_idx)) {
3720                         xfrm_secpath_reject(xerr_idx, skb, &fl);
3721                         XFRM_INC_STATS(net, LINUX_MIB_XFRMINNOPOLS);
3722                         return 0;
3723                 }
3724                 return 1;
3725         }
3726
3727         /* This lockless write can happen from different cpus. */
3728         WRITE_ONCE(pol->curlft.use_time, ktime_get_real_seconds());
3729
3730         pols[0] = pol;
3731         npols++;
3732 #ifdef CONFIG_XFRM_SUB_POLICY
3733         if (pols[0]->type != XFRM_POLICY_TYPE_MAIN) {
3734                 pols[1] = xfrm_policy_lookup_bytype(net, XFRM_POLICY_TYPE_MAIN,
3735                                                     &fl, family,
3736                                                     XFRM_POLICY_IN, if_id);
3737                 if (pols[1]) {
3738                         if (IS_ERR(pols[1])) {
3739                                 XFRM_INC_STATS(net, LINUX_MIB_XFRMINPOLERROR);
3740                                 xfrm_pol_put(pols[0]);
3741                                 return 0;
3742                         }
3743                         /* This write can happen from different cpus. */
3744                         WRITE_ONCE(pols[1]->curlft.use_time,
3745                                    ktime_get_real_seconds());
3746                         npols++;
3747                 }
3748         }
3749 #endif
3750
3751         if (pol->action == XFRM_POLICY_ALLOW) {
3752                 static struct sec_path dummy;
3753                 struct xfrm_tmpl *tp[XFRM_MAX_DEPTH];
3754                 struct xfrm_tmpl *stp[XFRM_MAX_DEPTH];
3755                 struct xfrm_tmpl **tpp = tp;
3756                 int ti = 0;
3757                 int i, k;
3758
3759                 sp = skb_sec_path(skb);
3760                 if (!sp)
3761                         sp = &dummy;
3762
3763                 for (pi = 0; pi < npols; pi++) {
3764                         if (pols[pi] != pol &&
3765                             pols[pi]->action != XFRM_POLICY_ALLOW) {
3766                                 XFRM_INC_STATS(net, LINUX_MIB_XFRMINPOLBLOCK);
3767                                 goto reject;
3768                         }
3769                         if (ti + pols[pi]->xfrm_nr >= XFRM_MAX_DEPTH) {
3770                                 XFRM_INC_STATS(net, LINUX_MIB_XFRMINBUFFERERROR);
3771                                 goto reject_error;
3772                         }
3773                         for (i = 0; i < pols[pi]->xfrm_nr; i++)
3774                                 tpp[ti++] = &pols[pi]->xfrm_vec[i];
3775                 }
3776                 xfrm_nr = ti;
3777
3778                 if (npols > 1) {
3779                         xfrm_tmpl_sort(stp, tpp, xfrm_nr, family);
3780                         tpp = stp;
3781                 }
3782
3783                 /* For each tunnel xfrm, find the first matching tmpl.
3784                  * For each tmpl before that, find corresponding xfrm.
3785                  * Order is _important_. Later we will implement
3786                  * some barriers, but at the moment barriers
3787                  * are implied between each two transformations.
3788                  * Upon success, marks secpath entries as having been
3789                  * verified to allow them to be skipped in future policy
3790                  * checks (e.g. nested tunnels).
3791                  */
3792                 for (i = xfrm_nr-1, k = 0; i >= 0; i--) {
3793                         k = xfrm_policy_ok(tpp[i], sp, k, family, if_id);
3794                         if (k < 0) {
3795                                 if (k < -1)
3796                                         /* "-2 - errored_index" returned */
3797                                         xerr_idx = -(2+k);
3798                                 XFRM_INC_STATS(net, LINUX_MIB_XFRMINTMPLMISMATCH);
3799                                 goto reject;
3800                         }
3801                 }
3802
3803                 if (secpath_has_nontransport(sp, k, &xerr_idx)) {
3804                         XFRM_INC_STATS(net, LINUX_MIB_XFRMINTMPLMISMATCH);
3805                         goto reject;
3806                 }
3807
3808                 xfrm_pols_put(pols, npols);
3809                 sp->verified_cnt = k;
3810
3811                 return 1;
3812         }
3813         XFRM_INC_STATS(net, LINUX_MIB_XFRMINPOLBLOCK);
3814
3815 reject:
3816         xfrm_secpath_reject(xerr_idx, skb, &fl);
3817 reject_error:
3818         xfrm_pols_put(pols, npols);
3819         return 0;
3820 }
3821 EXPORT_SYMBOL(__xfrm_policy_check);
3822
3823 int __xfrm_route_forward(struct sk_buff *skb, unsigned short family)
3824 {
3825         struct net *net = dev_net(skb->dev);
3826         struct flowi fl;
3827         struct dst_entry *dst;
3828         int res = 1;
3829
3830         if (xfrm_decode_session(net, skb, &fl, family) < 0) {
3831                 XFRM_INC_STATS(net, LINUX_MIB_XFRMFWDHDRERROR);
3832                 return 0;
3833         }
3834
3835         skb_dst_force(skb);
3836         if (!skb_dst(skb)) {
3837                 XFRM_INC_STATS(net, LINUX_MIB_XFRMFWDHDRERROR);
3838                 return 0;
3839         }
3840
3841         dst = xfrm_lookup(net, skb_dst(skb), &fl, NULL, XFRM_LOOKUP_QUEUE);
3842         if (IS_ERR(dst)) {
3843                 res = 0;
3844                 dst = NULL;
3845         }
3846
3847         if (dst && !dst->xfrm)
3848                 dst = xfrm_out_fwd_icmp(skb, &fl, family, dst);
3849
3850         skb_dst_set(skb, dst);
3851         return res;
3852 }
3853 EXPORT_SYMBOL(__xfrm_route_forward);
3854
3855 /* Optimize later using cookies and generation ids. */
3856
3857 static struct dst_entry *xfrm_dst_check(struct dst_entry *dst, u32 cookie)
3858 {
3859         /* Code (such as __xfrm4_bundle_create()) sets dst->obsolete
3860          * to DST_OBSOLETE_FORCE_CHK to force all XFRM destinations to
3861          * get validated by dst_ops->check on every use.  We do this
3862          * because when a normal route referenced by an XFRM dst is
3863          * obsoleted we do not go looking around for all parent
3864          * referencing XFRM dsts so that we can invalidate them.  It
3865          * is just too much work.  Instead we make the checks here on
3866          * every use.  For example:
3867          *
3868          *      XFRM dst A --> IPv4 dst X
3869          *
3870          * X is the "xdst->route" of A (X is also the "dst->path" of A
3871          * in this example).  If X is marked obsolete, "A" will not
3872          * notice.  That's what we are validating here via the
3873          * stale_bundle() check.
3874          *
3875          * When a dst is removed from the fib tree, DST_OBSOLETE_DEAD will
3876          * be marked on it.
3877          * This will force stale_bundle() to fail on any xdst bundle with
3878          * this dst linked in it.
3879          */
3880         if (dst->obsolete < 0 && !stale_bundle(dst))
3881                 return dst;
3882
3883         return NULL;
3884 }
3885
3886 static int stale_bundle(struct dst_entry *dst)
3887 {
3888         return !xfrm_bundle_ok((struct xfrm_dst *)dst);
3889 }
3890
3891 void xfrm_dst_ifdown(struct dst_entry *dst, struct net_device *dev)
3892 {
3893         while ((dst = xfrm_dst_child(dst)) && dst->xfrm && dst->dev == dev) {
3894                 dst->dev = blackhole_netdev;
3895                 dev_hold(dst->dev);
3896                 dev_put(dev);
3897         }
3898 }
3899 EXPORT_SYMBOL(xfrm_dst_ifdown);
3900
3901 static void xfrm_link_failure(struct sk_buff *skb)
3902 {
3903         /* Impossible. Such dst must be popped before reaches point of failure. */
3904 }
3905
3906 static struct dst_entry *xfrm_negative_advice(struct dst_entry *dst)
3907 {
3908         if (dst) {
3909                 if (dst->obsolete) {
3910                         dst_release(dst);
3911                         dst = NULL;
3912                 }
3913         }
3914         return dst;
3915 }
3916
3917 static void xfrm_init_pmtu(struct xfrm_dst **bundle, int nr)
3918 {
3919         while (nr--) {
3920                 struct xfrm_dst *xdst = bundle[nr];
3921                 u32 pmtu, route_mtu_cached;
3922                 struct dst_entry *dst;
3923
3924                 dst = &xdst->u.dst;
3925                 pmtu = dst_mtu(xfrm_dst_child(dst));
3926                 xdst->child_mtu_cached = pmtu;
3927
3928                 pmtu = xfrm_state_mtu(dst->xfrm, pmtu);
3929
3930                 route_mtu_cached = dst_mtu(xdst->route);
3931                 xdst->route_mtu_cached = route_mtu_cached;
3932
3933                 if (pmtu > route_mtu_cached)
3934                         pmtu = route_mtu_cached;
3935
3936                 dst_metric_set(dst, RTAX_MTU, pmtu);
3937         }
3938 }
3939
3940 /* Check that the bundle accepts the flow and its components are
3941  * still valid.
3942  */
3943
3944 static int xfrm_bundle_ok(struct xfrm_dst *first)
3945 {
3946         struct xfrm_dst *bundle[XFRM_MAX_DEPTH];
3947         struct dst_entry *dst = &first->u.dst;
3948         struct xfrm_dst *xdst;
3949         int start_from, nr;
3950         u32 mtu;
3951
3952         if (!dst_check(xfrm_dst_path(dst), ((struct xfrm_dst *)dst)->path_cookie) ||
3953             (dst->dev && !netif_running(dst->dev)))
3954                 return 0;
3955
3956         if (dst->flags & DST_XFRM_QUEUE)
3957                 return 1;
3958
3959         start_from = nr = 0;
3960         do {
3961                 struct xfrm_dst *xdst = (struct xfrm_dst *)dst;
3962
3963                 if (dst->xfrm->km.state != XFRM_STATE_VALID)
3964                         return 0;
3965                 if (xdst->xfrm_genid != dst->xfrm->genid)
3966                         return 0;
3967                 if (xdst->num_pols > 0 &&
3968                     xdst->policy_genid != atomic_read(&xdst->pols[0]->genid))
3969                         return 0;
3970
3971                 bundle[nr++] = xdst;
3972
3973                 mtu = dst_mtu(xfrm_dst_child(dst));
3974                 if (xdst->child_mtu_cached != mtu) {
3975                         start_from = nr;
3976                         xdst->child_mtu_cached = mtu;
3977                 }
3978
3979                 if (!dst_check(xdst->route, xdst->route_cookie))
3980                         return 0;
3981                 mtu = dst_mtu(xdst->route);
3982                 if (xdst->route_mtu_cached != mtu) {
3983                         start_from = nr;
3984                         xdst->route_mtu_cached = mtu;
3985                 }
3986
3987                 dst = xfrm_dst_child(dst);
3988         } while (dst->xfrm);
3989
3990         if (likely(!start_from))
3991                 return 1;
3992
3993         xdst = bundle[start_from - 1];
3994         mtu = xdst->child_mtu_cached;
3995         while (start_from--) {
3996                 dst = &xdst->u.dst;
3997
3998                 mtu = xfrm_state_mtu(dst->xfrm, mtu);
3999                 if (mtu > xdst->route_mtu_cached)
4000                         mtu = xdst->route_mtu_cached;
4001                 dst_metric_set(dst, RTAX_MTU, mtu);
4002                 if (!start_from)
4003                         break;
4004
4005                 xdst = bundle[start_from - 1];
4006                 xdst->child_mtu_cached = mtu;
4007         }
4008
4009         return 1;
4010 }
4011
4012 static unsigned int xfrm_default_advmss(const struct dst_entry *dst)
4013 {
4014         return dst_metric_advmss(xfrm_dst_path(dst));
4015 }
4016
4017 static unsigned int xfrm_mtu(const struct dst_entry *dst)
4018 {
4019         unsigned int mtu = dst_metric_raw(dst, RTAX_MTU);
4020
4021         return mtu ? : dst_mtu(xfrm_dst_path(dst));
4022 }
4023
4024 static const void *xfrm_get_dst_nexthop(const struct dst_entry *dst,
4025                                         const void *daddr)
4026 {
4027         while (dst->xfrm) {
4028                 const struct xfrm_state *xfrm = dst->xfrm;
4029
4030                 dst = xfrm_dst_child(dst);
4031
4032                 if (xfrm->props.mode == XFRM_MODE_TRANSPORT)
4033                         continue;
4034                 if (xfrm->type->flags & XFRM_TYPE_REMOTE_COADDR)
4035                         daddr = xfrm->coaddr;
4036                 else if (!(xfrm->type->flags & XFRM_TYPE_LOCAL_COADDR))
4037                         daddr = &xfrm->id.daddr;
4038         }
4039         return daddr;
4040 }
4041
4042 static struct neighbour *xfrm_neigh_lookup(const struct dst_entry *dst,
4043                                            struct sk_buff *skb,
4044                                            const void *daddr)
4045 {
4046         const struct dst_entry *path = xfrm_dst_path(dst);
4047
4048         if (!skb)
4049                 daddr = xfrm_get_dst_nexthop(dst, daddr);
4050         return path->ops->neigh_lookup(path, skb, daddr);
4051 }
4052
4053 static void xfrm_confirm_neigh(const struct dst_entry *dst, const void *daddr)
4054 {
4055         const struct dst_entry *path = xfrm_dst_path(dst);
4056
4057         daddr = xfrm_get_dst_nexthop(dst, daddr);
4058         path->ops->confirm_neigh(path, daddr);
4059 }
4060
4061 int xfrm_policy_register_afinfo(const struct xfrm_policy_afinfo *afinfo, int family)
4062 {
4063         int err = 0;
4064
4065         if (WARN_ON(family >= ARRAY_SIZE(xfrm_policy_afinfo)))
4066                 return -EAFNOSUPPORT;
4067
4068         spin_lock(&xfrm_policy_afinfo_lock);
4069         if (unlikely(xfrm_policy_afinfo[family] != NULL))
4070                 err = -EEXIST;
4071         else {
4072                 struct dst_ops *dst_ops = afinfo->dst_ops;
4073                 if (likely(dst_ops->kmem_cachep == NULL))
4074                         dst_ops->kmem_cachep = xfrm_dst_cache;
4075                 if (likely(dst_ops->check == NULL))
4076                         dst_ops->check = xfrm_dst_check;
4077                 if (likely(dst_ops->default_advmss == NULL))
4078                         dst_ops->default_advmss = xfrm_default_advmss;
4079                 if (likely(dst_ops->mtu == NULL))
4080                         dst_ops->mtu = xfrm_mtu;
4081                 if (likely(dst_ops->negative_advice == NULL))
4082                         dst_ops->negative_advice = xfrm_negative_advice;
4083                 if (likely(dst_ops->link_failure == NULL))
4084                         dst_ops->link_failure = xfrm_link_failure;
4085                 if (likely(dst_ops->neigh_lookup == NULL))
4086                         dst_ops->neigh_lookup = xfrm_neigh_lookup;
4087                 if (likely(!dst_ops->confirm_neigh))
4088                         dst_ops->confirm_neigh = xfrm_confirm_neigh;
4089                 rcu_assign_pointer(xfrm_policy_afinfo[family], afinfo);
4090         }
4091         spin_unlock(&xfrm_policy_afinfo_lock);
4092
4093         return err;
4094 }
4095 EXPORT_SYMBOL(xfrm_policy_register_afinfo);
4096
4097 void xfrm_policy_unregister_afinfo(const struct xfrm_policy_afinfo *afinfo)
4098 {
4099         struct dst_ops *dst_ops = afinfo->dst_ops;
4100         int i;
4101
4102         for (i = 0; i < ARRAY_SIZE(xfrm_policy_afinfo); i++) {
4103                 if (xfrm_policy_afinfo[i] != afinfo)
4104                         continue;
4105                 RCU_INIT_POINTER(xfrm_policy_afinfo[i], NULL);
4106                 break;
4107         }
4108
4109         synchronize_rcu();
4110
4111         dst_ops->kmem_cachep = NULL;
4112         dst_ops->check = NULL;
4113         dst_ops->negative_advice = NULL;
4114         dst_ops->link_failure = NULL;
4115 }
4116 EXPORT_SYMBOL(xfrm_policy_unregister_afinfo);
4117
4118 void xfrm_if_register_cb(const struct xfrm_if_cb *ifcb)
4119 {
4120         spin_lock(&xfrm_if_cb_lock);
4121         rcu_assign_pointer(xfrm_if_cb, ifcb);
4122         spin_unlock(&xfrm_if_cb_lock);
4123 }
4124 EXPORT_SYMBOL(xfrm_if_register_cb);
4125
4126 void xfrm_if_unregister_cb(void)
4127 {
4128         RCU_INIT_POINTER(xfrm_if_cb, NULL);
4129         synchronize_rcu();
4130 }
4131 EXPORT_SYMBOL(xfrm_if_unregister_cb);
4132
4133 #ifdef CONFIG_XFRM_STATISTICS
4134 static int __net_init xfrm_statistics_init(struct net *net)
4135 {
4136         int rv;
4137         net->mib.xfrm_statistics = alloc_percpu(struct linux_xfrm_mib);
4138         if (!net->mib.xfrm_statistics)
4139                 return -ENOMEM;
4140         rv = xfrm_proc_init(net);
4141         if (rv < 0)
4142                 free_percpu(net->mib.xfrm_statistics);
4143         return rv;
4144 }
4145
4146 static void xfrm_statistics_fini(struct net *net)
4147 {
4148         xfrm_proc_fini(net);
4149         free_percpu(net->mib.xfrm_statistics);
4150 }
4151 #else
4152 static int __net_init xfrm_statistics_init(struct net *net)
4153 {
4154         return 0;
4155 }
4156
4157 static void xfrm_statistics_fini(struct net *net)
4158 {
4159 }
4160 #endif
4161
4162 static int __net_init xfrm_policy_init(struct net *net)
4163 {
4164         unsigned int hmask, sz;
4165         int dir, err;
4166
4167         if (net_eq(net, &init_net)) {
4168                 xfrm_dst_cache = KMEM_CACHE(xfrm_dst, SLAB_HWCACHE_ALIGN | SLAB_PANIC);
4169                 err = rhashtable_init(&xfrm_policy_inexact_table,
4170                                       &xfrm_pol_inexact_params);
4171                 BUG_ON(err);
4172         }
4173
4174         hmask = 8 - 1;
4175         sz = (hmask+1) * sizeof(struct hlist_head);
4176
4177         net->xfrm.policy_byidx = xfrm_hash_alloc(sz);
4178         if (!net->xfrm.policy_byidx)
4179                 goto out_byidx;
4180         net->xfrm.policy_idx_hmask = hmask;
4181
4182         for (dir = 0; dir < XFRM_POLICY_MAX; dir++) {
4183                 struct xfrm_policy_hash *htab;
4184
4185                 net->xfrm.policy_count[dir] = 0;
4186                 net->xfrm.policy_count[XFRM_POLICY_MAX + dir] = 0;
4187                 INIT_HLIST_HEAD(&net->xfrm.policy_inexact[dir]);
4188
4189                 htab = &net->xfrm.policy_bydst[dir];
4190                 htab->table = xfrm_hash_alloc(sz);
4191                 if (!htab->table)
4192                         goto out_bydst;
4193                 htab->hmask = hmask;
4194                 htab->dbits4 = 32;
4195                 htab->sbits4 = 32;
4196                 htab->dbits6 = 128;
4197                 htab->sbits6 = 128;
4198         }
4199         net->xfrm.policy_hthresh.lbits4 = 32;
4200         net->xfrm.policy_hthresh.rbits4 = 32;
4201         net->xfrm.policy_hthresh.lbits6 = 128;
4202         net->xfrm.policy_hthresh.rbits6 = 128;
4203
4204         seqlock_init(&net->xfrm.policy_hthresh.lock);
4205
4206         INIT_LIST_HEAD(&net->xfrm.policy_all);
4207         INIT_LIST_HEAD(&net->xfrm.inexact_bins);
4208         INIT_WORK(&net->xfrm.policy_hash_work, xfrm_hash_resize);
4209         INIT_WORK(&net->xfrm.policy_hthresh.work, xfrm_hash_rebuild);
4210         return 0;
4211
4212 out_bydst:
4213         for (dir--; dir >= 0; dir--) {
4214                 struct xfrm_policy_hash *htab;
4215
4216                 htab = &net->xfrm.policy_bydst[dir];
4217                 xfrm_hash_free(htab->table, sz);
4218         }
4219         xfrm_hash_free(net->xfrm.policy_byidx, sz);
4220 out_byidx:
4221         return -ENOMEM;
4222 }
4223
4224 static void xfrm_policy_fini(struct net *net)
4225 {
4226         struct xfrm_pol_inexact_bin *b, *t;
4227         unsigned int sz;
4228         int dir;
4229
4230         flush_work(&net->xfrm.policy_hash_work);
4231 #ifdef CONFIG_XFRM_SUB_POLICY
4232         xfrm_policy_flush(net, XFRM_POLICY_TYPE_SUB, false);
4233 #endif
4234         xfrm_policy_flush(net, XFRM_POLICY_TYPE_MAIN, false);
4235
4236         WARN_ON(!list_empty(&net->xfrm.policy_all));
4237
4238         for (dir = 0; dir < XFRM_POLICY_MAX; dir++) {
4239                 struct xfrm_policy_hash *htab;
4240
4241                 WARN_ON(!hlist_empty(&net->xfrm.policy_inexact[dir]));
4242
4243                 htab = &net->xfrm.policy_bydst[dir];
4244                 sz = (htab->hmask + 1) * sizeof(struct hlist_head);
4245                 WARN_ON(!hlist_empty(htab->table));
4246                 xfrm_hash_free(htab->table, sz);
4247         }
4248
4249         sz = (net->xfrm.policy_idx_hmask + 1) * sizeof(struct hlist_head);
4250         WARN_ON(!hlist_empty(net->xfrm.policy_byidx));
4251         xfrm_hash_free(net->xfrm.policy_byidx, sz);
4252
4253         spin_lock_bh(&net->xfrm.xfrm_policy_lock);
4254         list_for_each_entry_safe(b, t, &net->xfrm.inexact_bins, inexact_bins)
4255                 __xfrm_policy_inexact_prune_bin(b, true);
4256         spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
4257 }
4258
4259 static int __net_init xfrm_net_init(struct net *net)
4260 {
4261         int rv;
4262
4263         /* Initialize the per-net locks here */
4264         spin_lock_init(&net->xfrm.xfrm_state_lock);
4265         spin_lock_init(&net->xfrm.xfrm_policy_lock);
4266         seqcount_spinlock_init(&net->xfrm.xfrm_policy_hash_generation, &net->xfrm.xfrm_policy_lock);
4267         mutex_init(&net->xfrm.xfrm_cfg_mutex);
4268         net->xfrm.policy_default[XFRM_POLICY_IN] = XFRM_USERPOLICY_ACCEPT;
4269         net->xfrm.policy_default[XFRM_POLICY_FWD] = XFRM_USERPOLICY_ACCEPT;
4270         net->xfrm.policy_default[XFRM_POLICY_OUT] = XFRM_USERPOLICY_ACCEPT;
4271
4272         rv = xfrm_statistics_init(net);
4273         if (rv < 0)
4274                 goto out_statistics;
4275         rv = xfrm_state_init(net);
4276         if (rv < 0)
4277                 goto out_state;
4278         rv = xfrm_policy_init(net);
4279         if (rv < 0)
4280                 goto out_policy;
4281         rv = xfrm_sysctl_init(net);
4282         if (rv < 0)
4283                 goto out_sysctl;
4284
4285         return 0;
4286
4287 out_sysctl:
4288         xfrm_policy_fini(net);
4289 out_policy:
4290         xfrm_state_fini(net);
4291 out_state:
4292         xfrm_statistics_fini(net);
4293 out_statistics:
4294         return rv;
4295 }
4296
4297 static void __net_exit xfrm_net_exit(struct net *net)
4298 {
4299         xfrm_sysctl_fini(net);
4300         xfrm_policy_fini(net);
4301         xfrm_state_fini(net);
4302         xfrm_statistics_fini(net);
4303 }
4304
4305 static struct pernet_operations __net_initdata xfrm_net_ops = {
4306         .init = xfrm_net_init,
4307         .exit = xfrm_net_exit,
4308 };
4309
4310 static const struct flow_dissector_key xfrm_flow_dissector_keys[] = {
4311         {
4312                 .key_id = FLOW_DISSECTOR_KEY_CONTROL,
4313                 .offset = offsetof(struct xfrm_flow_keys, control),
4314         },
4315         {
4316                 .key_id = FLOW_DISSECTOR_KEY_BASIC,
4317                 .offset = offsetof(struct xfrm_flow_keys, basic),
4318         },
4319         {
4320                 .key_id = FLOW_DISSECTOR_KEY_IPV4_ADDRS,
4321                 .offset = offsetof(struct xfrm_flow_keys, addrs.ipv4),
4322         },
4323         {
4324                 .key_id = FLOW_DISSECTOR_KEY_IPV6_ADDRS,
4325                 .offset = offsetof(struct xfrm_flow_keys, addrs.ipv6),
4326         },
4327         {
4328                 .key_id = FLOW_DISSECTOR_KEY_PORTS,
4329                 .offset = offsetof(struct xfrm_flow_keys, ports),
4330         },
4331         {
4332                 .key_id = FLOW_DISSECTOR_KEY_GRE_KEYID,
4333                 .offset = offsetof(struct xfrm_flow_keys, gre),
4334         },
4335         {
4336                 .key_id = FLOW_DISSECTOR_KEY_IP,
4337                 .offset = offsetof(struct xfrm_flow_keys, ip),
4338         },
4339         {
4340                 .key_id = FLOW_DISSECTOR_KEY_ICMP,
4341                 .offset = offsetof(struct xfrm_flow_keys, icmp),
4342         },
4343 };
4344
4345 void __init xfrm_init(void)
4346 {
4347         skb_flow_dissector_init(&xfrm_session_dissector,
4348                                 xfrm_flow_dissector_keys,
4349                                 ARRAY_SIZE(xfrm_flow_dissector_keys));
4350
4351         register_pernet_subsys(&xfrm_net_ops);
4352         xfrm_dev_init();
4353         xfrm_input_init();
4354
4355 #ifdef CONFIG_XFRM_ESPINTCP
4356         espintcp_init();
4357 #endif
4358
4359         register_xfrm_state_bpf();
4360 }
4361
4362 #ifdef CONFIG_AUDITSYSCALL
4363 static void xfrm_audit_common_policyinfo(struct xfrm_policy *xp,
4364                                          struct audit_buffer *audit_buf)
4365 {
4366         struct xfrm_sec_ctx *ctx = xp->security;
4367         struct xfrm_selector *sel = &xp->selector;
4368
4369         if (ctx)
4370                 audit_log_format(audit_buf, " sec_alg=%u sec_doi=%u sec_obj=%s",
4371                                  ctx->ctx_alg, ctx->ctx_doi, ctx->ctx_str);
4372
4373         switch (sel->family) {
4374         case AF_INET:
4375                 audit_log_format(audit_buf, " src=%pI4", &sel->saddr.a4);
4376                 if (sel->prefixlen_s != 32)
4377                         audit_log_format(audit_buf, " src_prefixlen=%d",
4378                                          sel->prefixlen_s);
4379                 audit_log_format(audit_buf, " dst=%pI4", &sel->daddr.a4);
4380                 if (sel->prefixlen_d != 32)
4381                         audit_log_format(audit_buf, " dst_prefixlen=%d",
4382                                          sel->prefixlen_d);
4383                 break;
4384         case AF_INET6:
4385                 audit_log_format(audit_buf, " src=%pI6", sel->saddr.a6);
4386                 if (sel->prefixlen_s != 128)
4387                         audit_log_format(audit_buf, " src_prefixlen=%d",
4388                                          sel->prefixlen_s);
4389                 audit_log_format(audit_buf, " dst=%pI6", sel->daddr.a6);
4390                 if (sel->prefixlen_d != 128)
4391                         audit_log_format(audit_buf, " dst_prefixlen=%d",
4392                                          sel->prefixlen_d);
4393                 break;
4394         }
4395 }
4396
4397 void xfrm_audit_policy_add(struct xfrm_policy *xp, int result, bool task_valid)
4398 {
4399         struct audit_buffer *audit_buf;
4400
4401         audit_buf = xfrm_audit_start("SPD-add");
4402         if (audit_buf == NULL)
4403                 return;
4404         xfrm_audit_helper_usrinfo(task_valid, audit_buf);
4405         audit_log_format(audit_buf, " res=%u", result);
4406         xfrm_audit_common_policyinfo(xp, audit_buf);
4407         audit_log_end(audit_buf);
4408 }
4409 EXPORT_SYMBOL_GPL(xfrm_audit_policy_add);
4410
4411 void xfrm_audit_policy_delete(struct xfrm_policy *xp, int result,
4412                               bool task_valid)
4413 {
4414         struct audit_buffer *audit_buf;
4415
4416         audit_buf = xfrm_audit_start("SPD-delete");
4417         if (audit_buf == NULL)
4418                 return;
4419         xfrm_audit_helper_usrinfo(task_valid, audit_buf);
4420         audit_log_format(audit_buf, " res=%u", result);
4421         xfrm_audit_common_policyinfo(xp, audit_buf);
4422         audit_log_end(audit_buf);
4423 }
4424 EXPORT_SYMBOL_GPL(xfrm_audit_policy_delete);
4425 #endif
4426
4427 #ifdef CONFIG_XFRM_MIGRATE
4428 static bool xfrm_migrate_selector_match(const struct xfrm_selector *sel_cmp,
4429                                         const struct xfrm_selector *sel_tgt)
4430 {
4431         if (sel_cmp->proto == IPSEC_ULPROTO_ANY) {
4432                 if (sel_tgt->family == sel_cmp->family &&
4433                     xfrm_addr_equal(&sel_tgt->daddr, &sel_cmp->daddr,
4434                                     sel_cmp->family) &&
4435                     xfrm_addr_equal(&sel_tgt->saddr, &sel_cmp->saddr,
4436                                     sel_cmp->family) &&
4437                     sel_tgt->prefixlen_d == sel_cmp->prefixlen_d &&
4438                     sel_tgt->prefixlen_s == sel_cmp->prefixlen_s) {
4439                         return true;
4440                 }
4441         } else {
4442                 if (memcmp(sel_tgt, sel_cmp, sizeof(*sel_tgt)) == 0) {
4443                         return true;
4444                 }
4445         }
4446         return false;
4447 }
4448
4449 static struct xfrm_policy *xfrm_migrate_policy_find(const struct xfrm_selector *sel,
4450                                                     u8 dir, u8 type, struct net *net, u32 if_id)
4451 {
4452         struct xfrm_policy *pol, *ret = NULL;
4453         struct hlist_head *chain;
4454         u32 priority = ~0U;
4455
4456         spin_lock_bh(&net->xfrm.xfrm_policy_lock);
4457         chain = policy_hash_direct(net, &sel->daddr, &sel->saddr, sel->family, dir);
4458         hlist_for_each_entry(pol, chain, bydst) {
4459                 if ((if_id == 0 || pol->if_id == if_id) &&
4460                     xfrm_migrate_selector_match(sel, &pol->selector) &&
4461                     pol->type == type) {
4462                         ret = pol;
4463                         priority = ret->priority;
4464                         break;
4465                 }
4466         }
4467         chain = &net->xfrm.policy_inexact[dir];
4468         hlist_for_each_entry(pol, chain, bydst_inexact_list) {
4469                 if ((pol->priority >= priority) && ret)
4470                         break;
4471
4472                 if ((if_id == 0 || pol->if_id == if_id) &&
4473                     xfrm_migrate_selector_match(sel, &pol->selector) &&
4474                     pol->type == type) {
4475                         ret = pol;
4476                         break;
4477                 }
4478         }
4479
4480         xfrm_pol_hold(ret);
4481
4482         spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
4483
4484         return ret;
4485 }
4486
4487 static int migrate_tmpl_match(const struct xfrm_migrate *m, const struct xfrm_tmpl *t)
4488 {
4489         int match = 0;
4490
4491         if (t->mode == m->mode && t->id.proto == m->proto &&
4492             (m->reqid == 0 || t->reqid == m->reqid)) {
4493                 switch (t->mode) {
4494                 case XFRM_MODE_TUNNEL:
4495                 case XFRM_MODE_BEET:
4496                         if (xfrm_addr_equal(&t->id.daddr, &m->old_daddr,
4497                                             m->old_family) &&
4498                             xfrm_addr_equal(&t->saddr, &m->old_saddr,
4499                                             m->old_family)) {
4500                                 match = 1;
4501                         }
4502                         break;
4503                 case XFRM_MODE_TRANSPORT:
4504                         /* in case of transport mode, template does not store
4505                            any IP addresses, hence we just compare mode and
4506                            protocol */
4507                         match = 1;
4508                         break;
4509                 default:
4510                         break;
4511                 }
4512         }
4513         return match;
4514 }
4515
4516 /* update endpoint address(es) of template(s) */
4517 static int xfrm_policy_migrate(struct xfrm_policy *pol,
4518                                struct xfrm_migrate *m, int num_migrate,
4519                                struct netlink_ext_ack *extack)
4520 {
4521         struct xfrm_migrate *mp;
4522         int i, j, n = 0;
4523
4524         write_lock_bh(&pol->lock);
4525         if (unlikely(pol->walk.dead)) {
4526                 /* target policy has been deleted */
4527                 NL_SET_ERR_MSG(extack, "Target policy not found");
4528                 write_unlock_bh(&pol->lock);
4529                 return -ENOENT;
4530         }
4531
4532         for (i = 0; i < pol->xfrm_nr; i++) {
4533                 for (j = 0, mp = m; j < num_migrate; j++, mp++) {
4534                         if (!migrate_tmpl_match(mp, &pol->xfrm_vec[i]))
4535                                 continue;
4536                         n++;
4537                         if (pol->xfrm_vec[i].mode != XFRM_MODE_TUNNEL &&
4538                             pol->xfrm_vec[i].mode != XFRM_MODE_BEET)
4539                                 continue;
4540                         /* update endpoints */
4541                         memcpy(&pol->xfrm_vec[i].id.daddr, &mp->new_daddr,
4542                                sizeof(pol->xfrm_vec[i].id.daddr));
4543                         memcpy(&pol->xfrm_vec[i].saddr, &mp->new_saddr,
4544                                sizeof(pol->xfrm_vec[i].saddr));
4545                         pol->xfrm_vec[i].encap_family = mp->new_family;
4546                         /* flush bundles */
4547                         atomic_inc(&pol->genid);
4548                 }
4549         }
4550
4551         write_unlock_bh(&pol->lock);
4552
4553         if (!n)
4554                 return -ENODATA;
4555
4556         return 0;
4557 }
4558
4559 static int xfrm_migrate_check(const struct xfrm_migrate *m, int num_migrate,
4560                               struct netlink_ext_ack *extack)
4561 {
4562         int i, j;
4563
4564         if (num_migrate < 1 || num_migrate > XFRM_MAX_DEPTH) {
4565                 NL_SET_ERR_MSG(extack, "Invalid number of SAs to migrate, must be 0 < num <= XFRM_MAX_DEPTH (6)");
4566                 return -EINVAL;
4567         }
4568
4569         for (i = 0; i < num_migrate; i++) {
4570                 if (xfrm_addr_any(&m[i].new_daddr, m[i].new_family) ||
4571                     xfrm_addr_any(&m[i].new_saddr, m[i].new_family)) {
4572                         NL_SET_ERR_MSG(extack, "Addresses in the MIGRATE attribute's list cannot be null");
4573                         return -EINVAL;
4574                 }
4575
4576                 /* check if there is any duplicated entry */
4577                 for (j = i + 1; j < num_migrate; j++) {
4578                         if (!memcmp(&m[i].old_daddr, &m[j].old_daddr,
4579                                     sizeof(m[i].old_daddr)) &&
4580                             !memcmp(&m[i].old_saddr, &m[j].old_saddr,
4581                                     sizeof(m[i].old_saddr)) &&
4582                             m[i].proto == m[j].proto &&
4583                             m[i].mode == m[j].mode &&
4584                             m[i].reqid == m[j].reqid &&
4585                             m[i].old_family == m[j].old_family) {
4586                                 NL_SET_ERR_MSG(extack, "Entries in the MIGRATE attribute's list must be unique");
4587                                 return -EINVAL;
4588                         }
4589                 }
4590         }
4591
4592         return 0;
4593 }
4594
4595 int xfrm_migrate(const struct xfrm_selector *sel, u8 dir, u8 type,
4596                  struct xfrm_migrate *m, int num_migrate,
4597                  struct xfrm_kmaddress *k, struct net *net,
4598                  struct xfrm_encap_tmpl *encap, u32 if_id,
4599                  struct netlink_ext_ack *extack)
4600 {
4601         int i, err, nx_cur = 0, nx_new = 0;
4602         struct xfrm_policy *pol = NULL;
4603         struct xfrm_state *x, *xc;
4604         struct xfrm_state *x_cur[XFRM_MAX_DEPTH];
4605         struct xfrm_state *x_new[XFRM_MAX_DEPTH];
4606         struct xfrm_migrate *mp;
4607
4608         /* Stage 0 - sanity checks */
4609         err = xfrm_migrate_check(m, num_migrate, extack);
4610         if (err < 0)
4611                 goto out;
4612
4613         if (dir >= XFRM_POLICY_MAX) {
4614                 NL_SET_ERR_MSG(extack, "Invalid policy direction");
4615                 err = -EINVAL;
4616                 goto out;
4617         }
4618
4619         /* Stage 1 - find policy */
4620         pol = xfrm_migrate_policy_find(sel, dir, type, net, if_id);
4621         if (!pol) {
4622                 NL_SET_ERR_MSG(extack, "Target policy not found");
4623                 err = -ENOENT;
4624                 goto out;
4625         }
4626
4627         /* Stage 2 - find and update state(s) */
4628         for (i = 0, mp = m; i < num_migrate; i++, mp++) {
4629                 if ((x = xfrm_migrate_state_find(mp, net, if_id))) {
4630                         x_cur[nx_cur] = x;
4631                         nx_cur++;
4632                         xc = xfrm_state_migrate(x, mp, encap);
4633                         if (xc) {
4634                                 x_new[nx_new] = xc;
4635                                 nx_new++;
4636                         } else {
4637                                 err = -ENODATA;
4638                                 goto restore_state;
4639                         }
4640                 }
4641         }
4642
4643         /* Stage 3 - update policy */
4644         err = xfrm_policy_migrate(pol, m, num_migrate, extack);
4645         if (err < 0)
4646                 goto restore_state;
4647
4648         /* Stage 4 - delete old state(s) */
4649         if (nx_cur) {
4650                 xfrm_states_put(x_cur, nx_cur);
4651                 xfrm_states_delete(x_cur, nx_cur);
4652         }
4653
4654         /* Stage 5 - announce */
4655         km_migrate(sel, dir, type, m, num_migrate, k, encap);
4656
4657         xfrm_pol_put(pol);
4658
4659         return 0;
4660 out:
4661         return err;
4662
4663 restore_state:
4664         if (pol)
4665                 xfrm_pol_put(pol);
4666         if (nx_cur)
4667                 xfrm_states_put(x_cur, nx_cur);
4668         if (nx_new)
4669                 xfrm_states_delete(x_new, nx_new);
4670
4671         return err;
4672 }
4673 EXPORT_SYMBOL(xfrm_migrate);
4674 #endif