kernel/bpf/bpf_inode_storage.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (c) 2019 Facebook
   4  * Copyright 2020 Google LLC.
   5  */
   6
   7 #include <linux/rculist.h>
   8 #include <linux/list.h>
   9 #include <linux/hash.h>
  10 #include <linux/types.h>
  11 #include <linux/spinlock.h>
  12 #include <linux/bpf.h>
  13 #include <linux/bpf_local_storage.h>
  14 #include <net/sock.h>
  15 #include <uapi/linux/sock_diag.h>
  16 #include <uapi/linux/btf.h>
  17 #include <linux/bpf_lsm.h>
  18 #include <linux/btf_ids.h>
  19 #include <linux/fdtable.h>
  20
  21 DEFINE_BPF_STORAGE_CACHE(inode_cache);
  22
  23 static struct bpf_local_storage __rcu **
  24 inode_storage_ptr(void *owner)
  25 {
  26         struct inode *inode = owner;
  27         struct bpf_storage_blob *bsb;
  28
  29         bsb = bpf_inode(inode);
  30         if (!bsb)
  31                 return NULL;
  32         return &bsb->storage;
  33 }
  34
  35 static struct bpf_local_storage_data *inode_storage_lookup(struct inode *inode,
  36                                                            struct bpf_map *map,
  37                                                            bool cacheit_lockit)
  38 {
  39         struct bpf_local_storage *inode_storage;
  40         struct bpf_local_storage_map *smap;
  41         struct bpf_storage_blob *bsb;
  42
  43         bsb = bpf_inode(inode);
  44         if (!bsb)
  45                 return NULL;
  46
  47         inode_storage = rcu_dereference(bsb->storage);
  48         if (!inode_storage)
  49                 return NULL;
  50
  51         smap = (struct bpf_local_storage_map *)map;
  52         return bpf_local_storage_lookup(inode_storage, smap, cacheit_lockit);
  53 }
  54
  55 void bpf_inode_storage_free(struct inode *inode)
  56 {
  57         struct bpf_local_storage_elem *selem;
  58         struct bpf_local_storage *local_storage;
  59         bool free_inode_storage = false;
  60         struct bpf_storage_blob *bsb;
  61         struct hlist_node *n;
  62
  63         bsb = bpf_inode(inode);
  64         if (!bsb)
  65                 return;
  66
  67         rcu_read_lock();
  68
  69         local_storage = rcu_dereference(bsb->storage);
  70         if (!local_storage) {
  71                 rcu_read_unlock();
  72                 return;
  73         }
  74
  75         /* Netiher the bpf_prog nor the bpf-map's syscall
  76          * could be modifying the local_storage->list now.
  77          * Thus, no elem can be added-to or deleted-from the
  78          * local_storage->list by the bpf_prog or by the bpf-map's syscall.
  79          *
  80          * It is racing with bpf_local_storage_map_free() alone
  81          * when unlinking elem from the local_storage->list and
  82          * the map's bucket->list.
  83          */
  84         raw_spin_lock_bh(&local_storage->lock);
  85         hlist_for_each_entry_safe(selem, n, &local_storage->list, snode) {
  86                 /* Always unlink from map before unlinking from
  87                  * local_storage.
  88                  */
  89                 bpf_selem_unlink_map(selem);
  90                 free_inode_storage = bpf_selem_unlink_storage_nolock(
  91                         local_storage, selem, false);
  92         }
  93         raw_spin_unlock_bh(&local_storage->lock);
  94         rcu_read_unlock();
  95
  96         /* free_inoode_storage should always be true as long as
  97          * local_storage->list was non-empty.
  98          */
  99         if (free_inode_storage)
 100                 kfree_rcu(local_storage, rcu);
 101 }
 102
 103 static void *bpf_fd_inode_storage_lookup_elem(struct bpf_map *map, void *key)
 104 {
 105         struct bpf_local_storage_data *sdata;
 106         struct file *f;
 107         int fd;
 108
 109         fd = *(int *)key;
 110         f = fget_raw(fd);
 111         if (!f)
 112                 return NULL;
 113
 114         sdata = inode_storage_lookup(f->f_inode, map, true);
 115         fput(f);
 116         return sdata ? sdata->data : NULL;
 117 }
 118
 119 static int bpf_fd_inode_storage_update_elem(struct bpf_map *map, void *key,
 120                                          void *value, u64 map_flags)
 121 {
 122         struct bpf_local_storage_data *sdata;
 123         struct file *f;
 124         int fd;
 125
 126         fd = *(int *)key;
 127         f = fget_raw(fd);
 128         if (!f || !inode_storage_ptr(f->f_inode))
 129                 return -EBADF;
 130
 131         sdata = bpf_local_storage_update(f->f_inode,
 132                                          (struct bpf_local_storage_map *)map,
 133                                          value, map_flags);
 134         fput(f);
 135         return PTR_ERR_OR_ZERO(sdata);
 136 }
 137
 138 static int inode_storage_delete(struct inode *inode, struct bpf_map *map)
 139 {
 140         struct bpf_local_storage_data *sdata;
 141
 142         sdata = inode_storage_lookup(inode, map, false);
 143         if (!sdata)
 144                 return -ENOENT;
 145
 146         bpf_selem_unlink(SELEM(sdata));
 147
 148         return 0;
 149 }
 150
 151 static int bpf_fd_inode_storage_delete_elem(struct bpf_map *map, void *key)
 152 {
 153         struct file *f;
 154         int fd, err;
 155
 156         fd = *(int *)key;
 157         f = fget_raw(fd);
 158         if (!f)
 159                 return -EBADF;
 160
 161         err = inode_storage_delete(f->f_inode, map);
 162         fput(f);
 163         return err;
 164 }
 165
 166 BPF_CALL_4(bpf_inode_storage_get, struct bpf_map *, map, struct inode *, inode,
 167            void *, value, u64, flags)
 168 {
 169         struct bpf_local_storage_data *sdata;
 170
 171         if (flags & ~(BPF_LOCAL_STORAGE_GET_F_CREATE))
 172                 return (unsigned long)NULL;
 173
 174         /* explicitly check that the inode_storage_ptr is not
 175          * NULL as inode_storage_lookup returns NULL in this case and
 176          * bpf_local_storage_update expects the owner to have a
 177          * valid storage pointer.
 178          */
 179         if (!inode_storage_ptr(inode))
 180                 return (unsigned long)NULL;
 181
 182         sdata = inode_storage_lookup(inode, map, true);
 183         if (sdata)
 184                 return (unsigned long)sdata->data;
 185
 186         /* This helper must only called from where the inode is gurranteed
 187          * to have a refcount and cannot be freed.
 188          */
 189         if (flags & BPF_LOCAL_STORAGE_GET_F_CREATE) {
 190                 sdata = bpf_local_storage_update(
 191                         inode, (struct bpf_local_storage_map *)map, value,
 192                         BPF_NOEXIST);
 193                 return IS_ERR(sdata) ? (unsigned long)NULL :
 194                                              (unsigned long)sdata->data;
 195         }
 196
 197         return (unsigned long)NULL;
 198 }
 199
 200 BPF_CALL_2(bpf_inode_storage_delete,
 201            struct bpf_map *, map, struct inode *, inode)
 202 {
 203         /* This helper must only called from where the inode is gurranteed
 204          * to have a refcount and cannot be freed.
 205          */
 206         return inode_storage_delete(inode, map);
 207 }
 208
 209 static int notsupp_get_next_key(struct bpf_map *map, void *key,
 210                                 void *next_key)
 211 {
 212         return -ENOTSUPP;
 213 }
 214
 215 static struct bpf_map *inode_storage_map_alloc(union bpf_attr *attr)
 216 {
 217         struct bpf_local_storage_map *smap;
 218
 219         smap = bpf_local_storage_map_alloc(attr);
 220         if (IS_ERR(smap))
 221                 return ERR_CAST(smap);
 222
 223         smap->cache_idx = bpf_local_storage_cache_idx_get(&inode_cache);
 224         return &smap->map;
 225 }
 226
 227 static void inode_storage_map_free(struct bpf_map *map)
 228 {
 229         struct bpf_local_storage_map *smap;
 230
 231         smap = (struct bpf_local_storage_map *)map;
 232         bpf_local_storage_cache_idx_free(&inode_cache, smap->cache_idx);
 233         bpf_local_storage_map_free(smap);
 234 }
 235
 236 static int inode_storage_map_btf_id;
 237 const struct bpf_map_ops inode_storage_map_ops = {
 238         .map_meta_equal = bpf_map_meta_equal,
 239         .map_alloc_check = bpf_local_storage_map_alloc_check,
 240         .map_alloc = inode_storage_map_alloc,
 241         .map_free = inode_storage_map_free,
 242         .map_get_next_key = notsupp_get_next_key,
 243         .map_lookup_elem = bpf_fd_inode_storage_lookup_elem,
 244         .map_update_elem = bpf_fd_inode_storage_update_elem,
 245         .map_delete_elem = bpf_fd_inode_storage_delete_elem,
 246         .map_check_btf = bpf_local_storage_map_check_btf,
 247         .map_btf_name = "bpf_local_storage_map",
 248         .map_btf_id = &inode_storage_map_btf_id,
 249         .map_owner_storage_ptr = inode_storage_ptr,
 250 };
 251
 252 BTF_ID_LIST_SINGLE(bpf_inode_storage_btf_ids, struct, inode)
 253
 254 const struct bpf_func_proto bpf_inode_storage_get_proto = {
 255         .func           = bpf_inode_storage_get,
 256         .gpl_only       = false,
 257         .ret_type       = RET_PTR_TO_MAP_VALUE_OR_NULL,
 258         .arg1_type      = ARG_CONST_MAP_PTR,
 259         .arg2_type      = ARG_PTR_TO_BTF_ID,
 260         .arg2_btf_id    = &bpf_inode_storage_btf_ids[0],
 261         .arg3_type      = ARG_PTR_TO_MAP_VALUE_OR_NULL,
 262         .arg4_type      = ARG_ANYTHING,
 263 };
 264
 265 const struct bpf_func_proto bpf_inode_storage_delete_proto = {
 266         .func           = bpf_inode_storage_delete,
 267         .gpl_only       = false,
 268         .ret_type       = RET_INTEGER,
 269         .arg1_type      = ARG_CONST_MAP_PTR,
 270         .arg2_type      = ARG_PTR_TO_BTF_ID,
 271         .arg2_btf_id    = &bpf_inode_storage_btf_ids[0],
 272 };