net/core/bpf_sk_storage.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright (c) 2019 Facebook  */
   3 #include <linux/rculist.h>
   4 #include <linux/list.h>
   5 #include <linux/hash.h>
   6 #include <linux/types.h>
   7 #include <linux/spinlock.h>
   8 #include <linux/bpf.h>
   9 #include <linux/btf_ids.h>
  10 #include <linux/bpf_local_storage.h>
  11 #include <net/bpf_sk_storage.h>
  12 #include <net/sock.h>
  13 #include <uapi/linux/sock_diag.h>
  14 #include <uapi/linux/btf.h>
  15
  16 DEFINE_BPF_STORAGE_CACHE(sk_cache);
  17
  18 static int omem_charge(struct sock *sk, unsigned int size)
  19 {
  20         /* same check as in sock_kmalloc() */
  21         if (size <= sysctl_optmem_max &&
  22             atomic_read(&sk->sk_omem_alloc) + size < sysctl_optmem_max) {
  23                 atomic_add(size, &sk->sk_omem_alloc);
  24                 return 0;
  25         }
  26
  27         return -ENOMEM;
  28 }
  29
  30 static struct bpf_local_storage_data *
  31 sk_storage_lookup(struct sock *sk, struct bpf_map *map, bool cacheit_lockit)
  32 {
  33         struct bpf_local_storage *sk_storage;
  34         struct bpf_local_storage_map *smap;
  35
  36         sk_storage = rcu_dereference(sk->sk_bpf_storage);
  37         if (!sk_storage)
  38                 return NULL;
  39
  40         smap = (struct bpf_local_storage_map *)map;
  41         return bpf_local_storage_lookup(sk_storage, smap, cacheit_lockit);
  42 }
  43
  44 static int sk_storage_delete(struct sock *sk, struct bpf_map *map)
  45 {
  46         struct bpf_local_storage_data *sdata;
  47
  48         sdata = sk_storage_lookup(sk, map, false);
  49         if (!sdata)
  50                 return -ENOENT;
  51
  52         bpf_selem_unlink(SELEM(sdata));
  53
  54         return 0;
  55 }
  56
  57 /* Called by __sk_destruct() & bpf_sk_storage_clone() */
  58 void bpf_sk_storage_free(struct sock *sk)
  59 {
  60         struct bpf_local_storage_elem *selem;
  61         struct bpf_local_storage *sk_storage;
  62         bool free_sk_storage = false;
  63         struct hlist_node *n;
  64
  65         rcu_read_lock();
  66         sk_storage = rcu_dereference(sk->sk_bpf_storage);
  67         if (!sk_storage) {
  68                 rcu_read_unlock();
  69                 return;
  70         }
  71
  72         /* Netiher the bpf_prog nor the bpf-map's syscall
  73          * could be modifying the sk_storage->list now.
  74          * Thus, no elem can be added-to or deleted-from the
  75          * sk_storage->list by the bpf_prog or by the bpf-map's syscall.
  76          *
  77          * It is racing with bpf_local_storage_map_free() alone
  78          * when unlinking elem from the sk_storage->list and
  79          * the map's bucket->list.
  80          */
  81         raw_spin_lock_bh(&sk_storage->lock);
  82         hlist_for_each_entry_safe(selem, n, &sk_storage->list, snode) {
  83                 /* Always unlink from map before unlinking from
  84                  * sk_storage.
  85                  */
  86                 bpf_selem_unlink_map(selem);
  87                 free_sk_storage = bpf_selem_unlink_storage_nolock(sk_storage,
  88                                                                   selem, true);
  89         }
  90         raw_spin_unlock_bh(&sk_storage->lock);
  91         rcu_read_unlock();
  92
  93         if (free_sk_storage)
  94                 kfree_rcu(sk_storage, rcu);
  95 }
  96
  97 static void sk_storage_map_free(struct bpf_map *map)
  98 {
  99         struct bpf_local_storage_map *smap;
 100
 101         smap = (struct bpf_local_storage_map *)map;
 102         bpf_local_storage_cache_idx_free(&sk_cache, smap->cache_idx);
 103         bpf_local_storage_map_free(smap);
 104 }
 105
 106 static struct bpf_map *sk_storage_map_alloc(union bpf_attr *attr)
 107 {
 108         struct bpf_local_storage_map *smap;
 109
 110         smap = bpf_local_storage_map_alloc(attr);
 111         if (IS_ERR(smap))
 112                 return ERR_CAST(smap);
 113
 114         smap->cache_idx = bpf_local_storage_cache_idx_get(&sk_cache);
 115         return &smap->map;
 116 }
 117
 118 static int notsupp_get_next_key(struct bpf_map *map, void *key,
 119                                 void *next_key)
 120 {
 121         return -ENOTSUPP;
 122 }
 123
 124 static void *bpf_fd_sk_storage_lookup_elem(struct bpf_map *map, void *key)
 125 {
 126         struct bpf_local_storage_data *sdata;
 127         struct socket *sock;
 128         int fd, err;
 129
 130         fd = *(int *)key;
 131         sock = sockfd_lookup(fd, &err);
 132         if (sock) {
 133                 sdata = sk_storage_lookup(sock->sk, map, true);
 134                 sockfd_put(sock);
 135                 return sdata ? sdata->data : NULL;
 136         }
 137
 138         return ERR_PTR(err);
 139 }
 140
 141 static int bpf_fd_sk_storage_update_elem(struct bpf_map *map, void *key,
 142                                          void *value, u64 map_flags)
 143 {
 144         struct bpf_local_storage_data *sdata;
 145         struct socket *sock;
 146         int fd, err;
 147
 148         fd = *(int *)key;
 149         sock = sockfd_lookup(fd, &err);
 150         if (sock) {
 151                 sdata = bpf_local_storage_update(
 152                         sock->sk, (struct bpf_local_storage_map *)map, value,
 153                         map_flags);
 154                 sockfd_put(sock);
 155                 return PTR_ERR_OR_ZERO(sdata);
 156         }
 157
 158         return err;
 159 }
 160
 161 static int bpf_fd_sk_storage_delete_elem(struct bpf_map *map, void *key)
 162 {
 163         struct socket *sock;
 164         int fd, err;
 165
 166         fd = *(int *)key;
 167         sock = sockfd_lookup(fd, &err);
 168         if (sock) {
 169                 err = sk_storage_delete(sock->sk, map);
 170                 sockfd_put(sock);
 171                 return err;
 172         }
 173
 174         return err;
 175 }
 176
 177 static struct bpf_local_storage_elem *
 178 bpf_sk_storage_clone_elem(struct sock *newsk,
 179                           struct bpf_local_storage_map *smap,
 180                           struct bpf_local_storage_elem *selem)
 181 {
 182         struct bpf_local_storage_elem *copy_selem;
 183
 184         copy_selem = bpf_selem_alloc(smap, newsk, NULL, true);
 185         if (!copy_selem)
 186                 return NULL;
 187
 188         if (map_value_has_spin_lock(&smap->map))
 189                 copy_map_value_locked(&smap->map, SDATA(copy_selem)->data,
 190                                       SDATA(selem)->data, true);
 191         else
 192                 copy_map_value(&smap->map, SDATA(copy_selem)->data,
 193                                SDATA(selem)->data);
 194
 195         return copy_selem;
 196 }
 197
 198 int bpf_sk_storage_clone(const struct sock *sk, struct sock *newsk)
 199 {
 200         struct bpf_local_storage *new_sk_storage = NULL;
 201         struct bpf_local_storage *sk_storage;
 202         struct bpf_local_storage_elem *selem;
 203         int ret = 0;
 204
 205         RCU_INIT_POINTER(newsk->sk_bpf_storage, NULL);
 206
 207         rcu_read_lock();
 208         sk_storage = rcu_dereference(sk->sk_bpf_storage);
 209
 210         if (!sk_storage || hlist_empty(&sk_storage->list))
 211                 goto out;
 212
 213         hlist_for_each_entry_rcu(selem, &sk_storage->list, snode) {
 214                 struct bpf_local_storage_elem *copy_selem;
 215                 struct bpf_local_storage_map *smap;
 216                 struct bpf_map *map;
 217
 218                 smap = rcu_dereference(SDATA(selem)->smap);
 219                 if (!(smap->map.map_flags & BPF_F_CLONE))
 220                         continue;
 221
 222                 /* Note that for lockless listeners adding new element
 223                  * here can race with cleanup in bpf_local_storage_map_free.
 224                  * Try to grab map refcnt to make sure that it's still
 225                  * alive and prevent concurrent removal.
 226                  */
 227                 map = bpf_map_inc_not_zero(&smap->map);
 228                 if (IS_ERR(map))
 229                         continue;
 230
 231                 copy_selem = bpf_sk_storage_clone_elem(newsk, smap, selem);
 232                 if (!copy_selem) {
 233                         ret = -ENOMEM;
 234                         bpf_map_put(map);
 235                         goto out;
 236                 }
 237
 238                 if (new_sk_storage) {
 239                         bpf_selem_link_map(smap, copy_selem);
 240                         bpf_selem_link_storage_nolock(new_sk_storage, copy_selem);
 241                 } else {
 242                         ret = bpf_local_storage_alloc(newsk, smap, copy_selem);
 243                         if (ret) {
 244                                 kfree(copy_selem);
 245                                 atomic_sub(smap->elem_size,
 246                                            &newsk->sk_omem_alloc);
 247                                 bpf_map_put(map);
 248                                 goto out;
 249                         }
 250
 251                         new_sk_storage =
 252                                 rcu_dereference(copy_selem->local_storage);
 253                 }
 254                 bpf_map_put(map);
 255         }
 256
 257 out:
 258         rcu_read_unlock();
 259
 260         /* In case of an error, don't free anything explicitly here, the
 261          * caller is responsible to call bpf_sk_storage_free.
 262          */
 263
 264         return ret;
 265 }
 266
 267 BPF_CALL_4(bpf_sk_storage_get, struct bpf_map *, map, struct sock *, sk,
 268            void *, value, u64, flags)
 269 {
 270         struct bpf_local_storage_data *sdata;
 271
 272         if (!sk || !sk_fullsock(sk) || flags > BPF_SK_STORAGE_GET_F_CREATE)
 273                 return (unsigned long)NULL;
 274
 275         sdata = sk_storage_lookup(sk, map, true);
 276         if (sdata)
 277                 return (unsigned long)sdata->data;
 278
 279         if (flags == BPF_SK_STORAGE_GET_F_CREATE &&
 280             /* Cannot add new elem to a going away sk.
 281              * Otherwise, the new elem may become a leak
 282              * (and also other memory issues during map
 283              *  destruction).
 284              */
 285             refcount_inc_not_zero(&sk->sk_refcnt)) {
 286                 sdata = bpf_local_storage_update(
 287                         sk, (struct bpf_local_storage_map *)map, value,
 288                         BPF_NOEXIST);
 289                 /* sk must be a fullsock (guaranteed by verifier),
 290                  * so sock_gen_put() is unnecessary.
 291                  */
 292                 sock_put(sk);
 293                 return IS_ERR(sdata) ?
 294                         (unsigned long)NULL : (unsigned long)sdata->data;
 295         }
 296
 297         return (unsigned long)NULL;
 298 }
 299
 300 BPF_CALL_2(bpf_sk_storage_delete, struct bpf_map *, map, struct sock *, sk)
 301 {
 302         if (!sk || !sk_fullsock(sk))
 303                 return -EINVAL;
 304
 305         if (refcount_inc_not_zero(&sk->sk_refcnt)) {
 306                 int err;
 307
 308                 err = sk_storage_delete(sk, map);
 309                 sock_put(sk);
 310                 return err;
 311         }
 312
 313         return -ENOENT;
 314 }
 315
 316 static int sk_storage_charge(struct bpf_local_storage_map *smap,
 317                              void *owner, u32 size)
 318 {
 319         return omem_charge(owner, size);
 320 }
 321
 322 static void sk_storage_uncharge(struct bpf_local_storage_map *smap,
 323                                 void *owner, u32 size)
 324 {
 325         struct sock *sk = owner;
 326
 327         atomic_sub(size, &sk->sk_omem_alloc);
 328 }
 329
 330 static struct bpf_local_storage __rcu **
 331 sk_storage_ptr(void *owner)
 332 {
 333         struct sock *sk = owner;
 334
 335         return &sk->sk_bpf_storage;
 336 }
 337
 338 static int sk_storage_map_btf_id;
 339 const struct bpf_map_ops sk_storage_map_ops = {
 340         .map_meta_equal = bpf_map_meta_equal,
 341         .map_alloc_check = bpf_local_storage_map_alloc_check,
 342         .map_alloc = sk_storage_map_alloc,
 343         .map_free = sk_storage_map_free,
 344         .map_get_next_key = notsupp_get_next_key,
 345         .map_lookup_elem = bpf_fd_sk_storage_lookup_elem,
 346         .map_update_elem = bpf_fd_sk_storage_update_elem,
 347         .map_delete_elem = bpf_fd_sk_storage_delete_elem,
 348         .map_check_btf = bpf_local_storage_map_check_btf,
 349         .map_btf_name = "bpf_local_storage_map",
 350         .map_btf_id = &sk_storage_map_btf_id,
 351         .map_local_storage_charge = sk_storage_charge,
 352         .map_local_storage_uncharge = sk_storage_uncharge,
 353         .map_owner_storage_ptr = sk_storage_ptr,
 354 };
 355
 356 const struct bpf_func_proto bpf_sk_storage_get_proto = {
 357         .func           = bpf_sk_storage_get,
 358         .gpl_only       = false,
 359         .ret_type       = RET_PTR_TO_MAP_VALUE_OR_NULL,
 360         .arg1_type      = ARG_CONST_MAP_PTR,
 361         .arg2_type      = ARG_PTR_TO_BTF_ID_SOCK_COMMON,
 362         .arg3_type      = ARG_PTR_TO_MAP_VALUE_OR_NULL,
 363         .arg4_type      = ARG_ANYTHING,
 364 };
 365
 366 const struct bpf_func_proto bpf_sk_storage_get_cg_sock_proto = {
 367         .func           = bpf_sk_storage_get,
 368         .gpl_only       = false,
 369         .ret_type       = RET_PTR_TO_MAP_VALUE_OR_NULL,
 370         .arg1_type      = ARG_CONST_MAP_PTR,
 371         .arg2_type      = ARG_PTR_TO_CTX, /* context is 'struct sock' */
 372         .arg3_type      = ARG_PTR_TO_MAP_VALUE_OR_NULL,
 373         .arg4_type      = ARG_ANYTHING,
 374 };
 375
 376 const struct bpf_func_proto bpf_sk_storage_delete_proto = {
 377         .func           = bpf_sk_storage_delete,
 378         .gpl_only       = false,
 379         .ret_type       = RET_INTEGER,
 380         .arg1_type      = ARG_CONST_MAP_PTR,
 381         .arg2_type      = ARG_PTR_TO_BTF_ID_SOCK_COMMON,
 382 };
 383
 384 struct bpf_sk_storage_diag {
 385         u32 nr_maps;
 386         struct bpf_map *maps[];
 387 };
 388
 389 /* The reply will be like:
 390  * INET_DIAG_BPF_SK_STORAGES (nla_nest)
 391  *      SK_DIAG_BPF_STORAGE (nla_nest)
 392  *              SK_DIAG_BPF_STORAGE_MAP_ID (nla_put_u32)
 393  *              SK_DIAG_BPF_STORAGE_MAP_VALUE (nla_reserve_64bit)
 394  *      SK_DIAG_BPF_STORAGE (nla_nest)
 395  *              SK_DIAG_BPF_STORAGE_MAP_ID (nla_put_u32)
 396  *              SK_DIAG_BPF_STORAGE_MAP_VALUE (nla_reserve_64bit)
 397  *      ....
 398  */
 399 static int nla_value_size(u32 value_size)
 400 {
 401         /* SK_DIAG_BPF_STORAGE (nla_nest)
 402          *      SK_DIAG_BPF_STORAGE_MAP_ID (nla_put_u32)
 403          *      SK_DIAG_BPF_STORAGE_MAP_VALUE (nla_reserve_64bit)
 404          */
 405         return nla_total_size(0) + nla_total_size(sizeof(u32)) +
 406                 nla_total_size_64bit(value_size);
 407 }
 408
 409 void bpf_sk_storage_diag_free(struct bpf_sk_storage_diag *diag)
 410 {
 411         u32 i;
 412
 413         if (!diag)
 414                 return;
 415
 416         for (i = 0; i < diag->nr_maps; i++)
 417                 bpf_map_put(diag->maps[i]);
 418
 419         kfree(diag);
 420 }
 421 EXPORT_SYMBOL_GPL(bpf_sk_storage_diag_free);
 422
 423 static bool diag_check_dup(const struct bpf_sk_storage_diag *diag,
 424                            const struct bpf_map *map)
 425 {
 426         u32 i;
 427
 428         for (i = 0; i < diag->nr_maps; i++) {
 429                 if (diag->maps[i] == map)
 430                         return true;
 431         }
 432
 433         return false;
 434 }
 435
 436 struct bpf_sk_storage_diag *
 437 bpf_sk_storage_diag_alloc(const struct nlattr *nla_stgs)
 438 {
 439         struct bpf_sk_storage_diag *diag;
 440         struct nlattr *nla;
 441         u32 nr_maps = 0;
 442         int rem, err;
 443
 444         /* bpf_local_storage_map is currently limited to CAP_SYS_ADMIN as
 445          * the map_alloc_check() side also does.
 446          */
 447         if (!bpf_capable())
 448                 return ERR_PTR(-EPERM);
 449
 450         nla_for_each_nested(nla, nla_stgs, rem) {
 451                 if (nla_type(nla) == SK_DIAG_BPF_STORAGE_REQ_MAP_FD)
 452                         nr_maps++;
 453         }
 454
 455         diag = kzalloc(sizeof(*diag) + sizeof(diag->maps[0]) * nr_maps,
 456                        GFP_KERNEL);
 457         if (!diag)
 458                 return ERR_PTR(-ENOMEM);
 459
 460         nla_for_each_nested(nla, nla_stgs, rem) {
 461                 struct bpf_map *map;
 462                 int map_fd;
 463
 464                 if (nla_type(nla) != SK_DIAG_BPF_STORAGE_REQ_MAP_FD)
 465                         continue;
 466
 467                 map_fd = nla_get_u32(nla);
 468                 map = bpf_map_get(map_fd);
 469                 if (IS_ERR(map)) {
 470                         err = PTR_ERR(map);
 471                         goto err_free;
 472                 }
 473                 if (map->map_type != BPF_MAP_TYPE_SK_STORAGE) {
 474                         bpf_map_put(map);
 475                         err = -EINVAL;
 476                         goto err_free;
 477                 }
 478                 if (diag_check_dup(diag, map)) {
 479                         bpf_map_put(map);
 480                         err = -EEXIST;
 481                         goto err_free;
 482                 }
 483                 diag->maps[diag->nr_maps++] = map;
 484         }
 485
 486         return diag;
 487
 488 err_free:
 489         bpf_sk_storage_diag_free(diag);
 490         return ERR_PTR(err);
 491 }
 492 EXPORT_SYMBOL_GPL(bpf_sk_storage_diag_alloc);
 493
 494 static int diag_get(struct bpf_local_storage_data *sdata, struct sk_buff *skb)
 495 {
 496         struct nlattr *nla_stg, *nla_value;
 497         struct bpf_local_storage_map *smap;
 498
 499         /* It cannot exceed max nlattr's payload */
 500         BUILD_BUG_ON(U16_MAX - NLA_HDRLEN < BPF_LOCAL_STORAGE_MAX_VALUE_SIZE);
 501
 502         nla_stg = nla_nest_start(skb, SK_DIAG_BPF_STORAGE);
 503         if (!nla_stg)
 504                 return -EMSGSIZE;
 505
 506         smap = rcu_dereference(sdata->smap);
 507         if (nla_put_u32(skb, SK_DIAG_BPF_STORAGE_MAP_ID, smap->map.id))
 508                 goto errout;
 509
 510         nla_value = nla_reserve_64bit(skb, SK_DIAG_BPF_STORAGE_MAP_VALUE,
 511                                       smap->map.value_size,
 512                                       SK_DIAG_BPF_STORAGE_PAD);
 513         if (!nla_value)
 514                 goto errout;
 515
 516         if (map_value_has_spin_lock(&smap->map))
 517                 copy_map_value_locked(&smap->map, nla_data(nla_value),
 518                                       sdata->data, true);
 519         else
 520                 copy_map_value(&smap->map, nla_data(nla_value), sdata->data);
 521
 522         nla_nest_end(skb, nla_stg);
 523         return 0;
 524
 525 errout:
 526         nla_nest_cancel(skb, nla_stg);
 527         return -EMSGSIZE;
 528 }
 529
 530 static int bpf_sk_storage_diag_put_all(struct sock *sk, struct sk_buff *skb,
 531                                        int stg_array_type,
 532                                        unsigned int *res_diag_size)
 533 {
 534         /* stg_array_type (e.g. INET_DIAG_BPF_SK_STORAGES) */
 535         unsigned int diag_size = nla_total_size(0);
 536         struct bpf_local_storage *sk_storage;
 537         struct bpf_local_storage_elem *selem;
 538         struct bpf_local_storage_map *smap;
 539         struct nlattr *nla_stgs;
 540         unsigned int saved_len;
 541         int err = 0;
 542
 543         rcu_read_lock();
 544
 545         sk_storage = rcu_dereference(sk->sk_bpf_storage);
 546         if (!sk_storage || hlist_empty(&sk_storage->list)) {
 547                 rcu_read_unlock();
 548                 return 0;
 549         }
 550
 551         nla_stgs = nla_nest_start(skb, stg_array_type);
 552         if (!nla_stgs)
 553                 /* Continue to learn diag_size */
 554                 err = -EMSGSIZE;
 555
 556         saved_len = skb->len;
 557         hlist_for_each_entry_rcu(selem, &sk_storage->list, snode) {
 558                 smap = rcu_dereference(SDATA(selem)->smap);
 559                 diag_size += nla_value_size(smap->map.value_size);
 560
 561                 if (nla_stgs && diag_get(SDATA(selem), skb))
 562                         /* Continue to learn diag_size */
 563                         err = -EMSGSIZE;
 564         }
 565
 566         rcu_read_unlock();
 567
 568         if (nla_stgs) {
 569                 if (saved_len == skb->len)
 570                         nla_nest_cancel(skb, nla_stgs);
 571                 else
 572                         nla_nest_end(skb, nla_stgs);
 573         }
 574
 575         if (diag_size == nla_total_size(0)) {
 576                 *res_diag_size = 0;
 577                 return 0;
 578         }
 579
 580         *res_diag_size = diag_size;
 581         return err;
 582 }
 583
 584 int bpf_sk_storage_diag_put(struct bpf_sk_storage_diag *diag,
 585                             struct sock *sk, struct sk_buff *skb,
 586                             int stg_array_type,
 587                             unsigned int *res_diag_size)
 588 {
 589         /* stg_array_type (e.g. INET_DIAG_BPF_SK_STORAGES) */
 590         unsigned int diag_size = nla_total_size(0);
 591         struct bpf_local_storage *sk_storage;
 592         struct bpf_local_storage_data *sdata;
 593         struct nlattr *nla_stgs;
 594         unsigned int saved_len;
 595         int err = 0;
 596         u32 i;
 597
 598         *res_diag_size = 0;
 599
 600         /* No map has been specified.  Dump all. */
 601         if (!diag->nr_maps)
 602                 return bpf_sk_storage_diag_put_all(sk, skb, stg_array_type,
 603                                                    res_diag_size);
 604
 605         rcu_read_lock();
 606         sk_storage = rcu_dereference(sk->sk_bpf_storage);
 607         if (!sk_storage || hlist_empty(&sk_storage->list)) {
 608                 rcu_read_unlock();
 609                 return 0;
 610         }
 611
 612         nla_stgs = nla_nest_start(skb, stg_array_type);
 613         if (!nla_stgs)
 614                 /* Continue to learn diag_size */
 615                 err = -EMSGSIZE;
 616
 617         saved_len = skb->len;
 618         for (i = 0; i < diag->nr_maps; i++) {
 619                 sdata = bpf_local_storage_lookup(sk_storage,
 620                                 (struct bpf_local_storage_map *)diag->maps[i],
 621                                 false);
 622
 623                 if (!sdata)
 624                         continue;
 625
 626                 diag_size += nla_value_size(diag->maps[i]->value_size);
 627
 628                 if (nla_stgs && diag_get(sdata, skb))
 629                         /* Continue to learn diag_size */
 630                         err = -EMSGSIZE;
 631         }
 632         rcu_read_unlock();
 633
 634         if (nla_stgs) {
 635                 if (saved_len == skb->len)
 636                         nla_nest_cancel(skb, nla_stgs);
 637                 else
 638                         nla_nest_end(skb, nla_stgs);
 639         }
 640
 641         if (diag_size == nla_total_size(0)) {
 642                 *res_diag_size = 0;
 643                 return 0;
 644         }
 645
 646         *res_diag_size = diag_size;
 647         return err;
 648 }
 649 EXPORT_SYMBOL_GPL(bpf_sk_storage_diag_put);
 650
 651 struct bpf_iter_seq_sk_storage_map_info {
 652         struct bpf_map *map;
 653         unsigned int bucket_id;
 654         unsigned skip_elems;
 655 };
 656
 657 static struct bpf_local_storage_elem *
 658 bpf_sk_storage_map_seq_find_next(struct bpf_iter_seq_sk_storage_map_info *info,
 659                                  struct bpf_local_storage_elem *prev_selem)
 660         __acquires(RCU) __releases(RCU)
 661 {
 662         struct bpf_local_storage *sk_storage;
 663         struct bpf_local_storage_elem *selem;
 664         u32 skip_elems = info->skip_elems;
 665         struct bpf_local_storage_map *smap;
 666         u32 bucket_id = info->bucket_id;
 667         u32 i, count, n_buckets;
 668         struct bpf_local_storage_map_bucket *b;
 669
 670         smap = (struct bpf_local_storage_map *)info->map;
 671         n_buckets = 1U << smap->bucket_log;
 672         if (bucket_id >= n_buckets)
 673                 return NULL;
 674
 675         /* try to find next selem in the same bucket */
 676         selem = prev_selem;
 677         count = 0;
 678         while (selem) {
 679                 selem = hlist_entry_safe(rcu_dereference(hlist_next_rcu(&selem->map_node)),
 680                                          struct bpf_local_storage_elem, map_node);
 681                 if (!selem) {
 682                         /* not found, unlock and go to the next bucket */
 683                         b = &smap->buckets[bucket_id++];
 684                         rcu_read_unlock();
 685                         skip_elems = 0;
 686                         break;
 687                 }
 688                 sk_storage = rcu_dereference(selem->local_storage);
 689                 if (sk_storage) {
 690                         info->skip_elems = skip_elems + count;
 691                         return selem;
 692                 }
 693                 count++;
 694         }
 695
 696         for (i = bucket_id; i < (1U << smap->bucket_log); i++) {
 697                 b = &smap->buckets[i];
 698                 rcu_read_lock();
 699                 count = 0;
 700                 hlist_for_each_entry_rcu(selem, &b->list, map_node) {
 701                         sk_storage = rcu_dereference(selem->local_storage);
 702                         if (sk_storage && count >= skip_elems) {
 703                                 info->bucket_id = i;
 704                                 info->skip_elems = count;
 705                                 return selem;
 706                         }
 707                         count++;
 708                 }
 709                 rcu_read_unlock();
 710                 skip_elems = 0;
 711         }
 712
 713         info->bucket_id = i;
 714         info->skip_elems = 0;
 715         return NULL;
 716 }
 717
 718 static void *bpf_sk_storage_map_seq_start(struct seq_file *seq, loff_t *pos)
 719 {
 720         struct bpf_local_storage_elem *selem;
 721
 722         selem = bpf_sk_storage_map_seq_find_next(seq->private, NULL);
 723         if (!selem)
 724                 return NULL;
 725
 726         if (*pos == 0)
 727                 ++*pos;
 728         return selem;
 729 }
 730
 731 static void *bpf_sk_storage_map_seq_next(struct seq_file *seq, void *v,
 732                                          loff_t *pos)
 733 {
 734         struct bpf_iter_seq_sk_storage_map_info *info = seq->private;
 735
 736         ++*pos;
 737         ++info->skip_elems;
 738         return bpf_sk_storage_map_seq_find_next(seq->private, v);
 739 }
 740
 741 struct bpf_iter__bpf_sk_storage_map {
 742         __bpf_md_ptr(struct bpf_iter_meta *, meta);
 743         __bpf_md_ptr(struct bpf_map *, map);
 744         __bpf_md_ptr(struct sock *, sk);
 745         __bpf_md_ptr(void *, value);
 746 };
 747
 748 DEFINE_BPF_ITER_FUNC(bpf_sk_storage_map, struct bpf_iter_meta *meta,
 749                      struct bpf_map *map, struct sock *sk,
 750                      void *value)
 751
 752 static int __bpf_sk_storage_map_seq_show(struct seq_file *seq,
 753                                          struct bpf_local_storage_elem *selem)
 754 {
 755         struct bpf_iter_seq_sk_storage_map_info *info = seq->private;
 756         struct bpf_iter__bpf_sk_storage_map ctx = {};
 757         struct bpf_local_storage *sk_storage;
 758         struct bpf_iter_meta meta;
 759         struct bpf_prog *prog;
 760         int ret = 0;
 761
 762         meta.seq = seq;
 763         prog = bpf_iter_get_info(&meta, selem == NULL);
 764         if (prog) {
 765                 ctx.meta = &meta;
 766                 ctx.map = info->map;
 767                 if (selem) {
 768                         sk_storage = rcu_dereference(selem->local_storage);
 769                         ctx.sk = sk_storage->owner;
 770                         ctx.value = SDATA(selem)->data;
 771                 }
 772                 ret = bpf_iter_run_prog(prog, &ctx);
 773         }
 774
 775         return ret;
 776 }
 777
 778 static int bpf_sk_storage_map_seq_show(struct seq_file *seq, void *v)
 779 {
 780         return __bpf_sk_storage_map_seq_show(seq, v);
 781 }
 782
 783 static void bpf_sk_storage_map_seq_stop(struct seq_file *seq, void *v)
 784         __releases(RCU)
 785 {
 786         if (!v)
 787                 (void)__bpf_sk_storage_map_seq_show(seq, v);
 788         else
 789                 rcu_read_unlock();
 790 }
 791
 792 static int bpf_iter_init_sk_storage_map(void *priv_data,
 793                                         struct bpf_iter_aux_info *aux)
 794 {
 795         struct bpf_iter_seq_sk_storage_map_info *seq_info = priv_data;
 796
 797         seq_info->map = aux->map;
 798         return 0;
 799 }
 800
 801 static int bpf_iter_attach_map(struct bpf_prog *prog,
 802                                union bpf_iter_link_info *linfo,
 803                                struct bpf_iter_aux_info *aux)
 804 {
 805         struct bpf_map *map;
 806         int err = -EINVAL;
 807
 808         if (!linfo->map.map_fd)
 809                 return -EBADF;
 810
 811         map = bpf_map_get_with_uref(linfo->map.map_fd);
 812         if (IS_ERR(map))
 813                 return PTR_ERR(map);
 814
 815         if (map->map_type != BPF_MAP_TYPE_SK_STORAGE)
 816                 goto put_map;
 817
 818         if (prog->aux->max_rdonly_access > map->value_size) {
 819                 err = -EACCES;
 820                 goto put_map;
 821         }
 822
 823         aux->map = map;
 824         return 0;
 825
 826 put_map:
 827         bpf_map_put_with_uref(map);
 828         return err;
 829 }
 830
 831 static void bpf_iter_detach_map(struct bpf_iter_aux_info *aux)
 832 {
 833         bpf_map_put_with_uref(aux->map);
 834 }
 835
 836 static const struct seq_operations bpf_sk_storage_map_seq_ops = {
 837         .start  = bpf_sk_storage_map_seq_start,
 838         .next   = bpf_sk_storage_map_seq_next,
 839         .stop   = bpf_sk_storage_map_seq_stop,
 840         .show   = bpf_sk_storage_map_seq_show,
 841 };
 842
 843 static const struct bpf_iter_seq_info iter_seq_info = {
 844         .seq_ops                = &bpf_sk_storage_map_seq_ops,
 845         .init_seq_private       = bpf_iter_init_sk_storage_map,
 846         .fini_seq_private       = NULL,
 847         .seq_priv_size          = sizeof(struct bpf_iter_seq_sk_storage_map_info),
 848 };
 849
 850 static struct bpf_iter_reg bpf_sk_storage_map_reg_info = {
 851         .target                 = "bpf_sk_storage_map",
 852         .attach_target          = bpf_iter_attach_map,
 853         .detach_target          = bpf_iter_detach_map,
 854         .show_fdinfo            = bpf_iter_map_show_fdinfo,
 855         .fill_link_info         = bpf_iter_map_fill_link_info,
 856         .ctx_arg_info_size      = 2,
 857         .ctx_arg_info           = {
 858                 { offsetof(struct bpf_iter__bpf_sk_storage_map, sk),
 859                   PTR_TO_BTF_ID_OR_NULL },
 860                 { offsetof(struct bpf_iter__bpf_sk_storage_map, value),
 861                   PTR_TO_RDWR_BUF_OR_NULL },
 862         },
 863         .seq_info               = &iter_seq_info,
 864 };
 865
 866 static int __init bpf_sk_storage_map_iter_init(void)
 867 {
 868         bpf_sk_storage_map_reg_info.ctx_arg_info[0].btf_id =
 869                 btf_sock_ids[BTF_SOCK_TYPE_SOCK];
 870         return bpf_iter_reg_target(&bpf_sk_storage_map_reg_info);
 871 }
 872 late_initcall(bpf_sk_storage_map_iter_init);