kernel/bpf/helpers.c

   1 /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
   2  *
   3  * This program is free software; you can redistribute it and/or
   4  * modify it under the terms of version 2 of the GNU General Public
   5  * License as published by the Free Software Foundation.
   6  *
   7  * This program is distributed in the hope that it will be useful, but
   8  * WITHOUT ANY WARRANTY; without even the implied warranty of
   9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  10  * General Public License for more details.
  11  */
  12 #include <linux/bpf.h>
  13 #include <linux/rcupdate.h>
  14 #include <linux/random.h>
  15 #include <linux/smp.h>
  16 #include <linux/topology.h>
  17 #include <linux/ktime.h>
  18 #include <linux/sched.h>
  19 #include <linux/uidgid.h>
  20 #include <linux/filter.h>
  21
  22 /* If kernel subsystem is allowing eBPF programs to call this function,
  23  * inside its own verifier_ops->get_func_proto() callback it should return
  24  * bpf_map_lookup_elem_proto, so that verifier can properly check the arguments
  25  *
  26  * Different map implementations will rely on rcu in map methods
  27  * lookup/update/delete, therefore eBPF programs must run under rcu lock
  28  * if program is allowed to access maps, so check rcu_read_lock_held in
  29  * all three functions.
  30  */
  31 BPF_CALL_2(bpf_map_lookup_elem, struct bpf_map *, map, void *, key)
  32 {
  33         WARN_ON_ONCE(!rcu_read_lock_held());
  34         return (unsigned long) map->ops->map_lookup_elem(map, key);
  35 }
  36
  37 const struct bpf_func_proto bpf_map_lookup_elem_proto = {
  38         .func           = bpf_map_lookup_elem,
  39         .gpl_only       = false,
  40         .pkt_access     = true,
  41         .ret_type       = RET_PTR_TO_MAP_VALUE_OR_NULL,
  42         .arg1_type      = ARG_CONST_MAP_PTR,
  43         .arg2_type      = ARG_PTR_TO_MAP_KEY,
  44 };
  45
  46 BPF_CALL_4(bpf_map_update_elem, struct bpf_map *, map, void *, key,
  47            void *, value, u64, flags)
  48 {
  49         WARN_ON_ONCE(!rcu_read_lock_held());
  50         return map->ops->map_update_elem(map, key, value, flags);
  51 }
  52
  53 const struct bpf_func_proto bpf_map_update_elem_proto = {
  54         .func           = bpf_map_update_elem,
  55         .gpl_only       = false,
  56         .pkt_access     = true,
  57         .ret_type       = RET_INTEGER,
  58         .arg1_type      = ARG_CONST_MAP_PTR,
  59         .arg2_type      = ARG_PTR_TO_MAP_KEY,
  60         .arg3_type      = ARG_PTR_TO_MAP_VALUE,
  61         .arg4_type      = ARG_ANYTHING,
  62 };
  63
  64 BPF_CALL_2(bpf_map_delete_elem, struct bpf_map *, map, void *, key)
  65 {
  66         WARN_ON_ONCE(!rcu_read_lock_held());
  67         return map->ops->map_delete_elem(map, key);
  68 }
  69
  70 const struct bpf_func_proto bpf_map_delete_elem_proto = {
  71         .func           = bpf_map_delete_elem,
  72         .gpl_only       = false,
  73         .pkt_access     = true,
  74         .ret_type       = RET_INTEGER,
  75         .arg1_type      = ARG_CONST_MAP_PTR,
  76         .arg2_type      = ARG_PTR_TO_MAP_KEY,
  77 };
  78
  79 BPF_CALL_3(bpf_map_push_elem, struct bpf_map *, map, void *, value, u64, flags)
  80 {
  81         return map->ops->map_push_elem(map, value, flags);
  82 }
  83
  84 const struct bpf_func_proto bpf_map_push_elem_proto = {
  85         .func           = bpf_map_push_elem,
  86         .gpl_only       = false,
  87         .pkt_access     = true,
  88         .ret_type       = RET_INTEGER,
  89         .arg1_type      = ARG_CONST_MAP_PTR,
  90         .arg2_type      = ARG_PTR_TO_MAP_VALUE,
  91         .arg3_type      = ARG_ANYTHING,
  92 };
  93
  94 BPF_CALL_2(bpf_map_pop_elem, struct bpf_map *, map, void *, value)
  95 {
  96         return map->ops->map_pop_elem(map, value);
  97 }
  98
  99 const struct bpf_func_proto bpf_map_pop_elem_proto = {
 100         .func           = bpf_map_pop_elem,
 101         .gpl_only       = false,
 102         .ret_type       = RET_INTEGER,
 103         .arg1_type      = ARG_CONST_MAP_PTR,
 104         .arg2_type      = ARG_PTR_TO_UNINIT_MAP_VALUE,
 105 };
 106
 107 BPF_CALL_2(bpf_map_peek_elem, struct bpf_map *, map, void *, value)
 108 {
 109         return map->ops->map_peek_elem(map, value);
 110 }
 111
 112 const struct bpf_func_proto bpf_map_peek_elem_proto = {
 113         .func           = bpf_map_pop_elem,
 114         .gpl_only       = false,
 115         .ret_type       = RET_INTEGER,
 116         .arg1_type      = ARG_CONST_MAP_PTR,
 117         .arg2_type      = ARG_PTR_TO_UNINIT_MAP_VALUE,
 118 };
 119
 120 const struct bpf_func_proto bpf_get_prandom_u32_proto = {
 121         .func           = bpf_user_rnd_u32,
 122         .gpl_only       = false,
 123         .ret_type       = RET_INTEGER,
 124 };
 125
 126 BPF_CALL_0(bpf_get_smp_processor_id)
 127 {
 128         return smp_processor_id();
 129 }
 130
 131 const struct bpf_func_proto bpf_get_smp_processor_id_proto = {
 132         .func           = bpf_get_smp_processor_id,
 133         .gpl_only       = false,
 134         .ret_type       = RET_INTEGER,
 135 };
 136
 137 BPF_CALL_0(bpf_get_numa_node_id)
 138 {
 139         return numa_node_id();
 140 }
 141
 142 const struct bpf_func_proto bpf_get_numa_node_id_proto = {
 143         .func           = bpf_get_numa_node_id,
 144         .gpl_only       = false,
 145         .ret_type       = RET_INTEGER,
 146 };
 147
 148 BPF_CALL_0(bpf_ktime_get_ns)
 149 {
 150         /* NMI safe access to clock monotonic */
 151         return ktime_get_mono_fast_ns();
 152 }
 153
 154 const struct bpf_func_proto bpf_ktime_get_ns_proto = {
 155         .func           = bpf_ktime_get_ns,
 156         .gpl_only       = true,
 157         .ret_type       = RET_INTEGER,
 158 };
 159
 160 BPF_CALL_0(bpf_get_current_pid_tgid)
 161 {
 162         struct task_struct *task = current;
 163
 164         if (unlikely(!task))
 165                 return -EINVAL;
 166
 167         return (u64) task->tgid << 32 | task->pid;
 168 }
 169
 170 const struct bpf_func_proto bpf_get_current_pid_tgid_proto = {
 171         .func           = bpf_get_current_pid_tgid,
 172         .gpl_only       = false,
 173         .ret_type       = RET_INTEGER,
 174 };
 175
 176 BPF_CALL_0(bpf_get_current_uid_gid)
 177 {
 178         struct task_struct *task = current;
 179         kuid_t uid;
 180         kgid_t gid;
 181
 182         if (unlikely(!task))
 183                 return -EINVAL;
 184
 185         current_uid_gid(&uid, &gid);
 186         return (u64) from_kgid(&init_user_ns, gid) << 32 |
 187                      from_kuid(&init_user_ns, uid);
 188 }
 189
 190 const struct bpf_func_proto bpf_get_current_uid_gid_proto = {
 191         .func           = bpf_get_current_uid_gid,
 192         .gpl_only       = false,
 193         .ret_type       = RET_INTEGER,
 194 };
 195
 196 BPF_CALL_2(bpf_get_current_comm, char *, buf, u32, size)
 197 {
 198         struct task_struct *task = current;
 199
 200         if (unlikely(!task))
 201                 goto err_clear;
 202
 203         strncpy(buf, task->comm, size);
 204
 205         /* Verifier guarantees that size > 0. For task->comm exceeding
 206          * size, guarantee that buf is %NUL-terminated. Unconditionally
 207          * done here to save the size test.
 208          */
 209         buf[size - 1] = 0;
 210         return 0;
 211 err_clear:
 212         memset(buf, 0, size);
 213         return -EINVAL;
 214 }
 215
 216 const struct bpf_func_proto bpf_get_current_comm_proto = {
 217         .func           = bpf_get_current_comm,
 218         .gpl_only       = false,
 219         .ret_type       = RET_INTEGER,
 220         .arg1_type      = ARG_PTR_TO_UNINIT_MEM,
 221         .arg2_type      = ARG_CONST_SIZE,
 222 };
 223
 224 #if defined(CONFIG_QUEUED_SPINLOCKS) || defined(CONFIG_BPF_ARCH_SPINLOCK)
 225
 226 static inline void __bpf_spin_lock(struct bpf_spin_lock *lock)
 227 {
 228         arch_spinlock_t *l = (void *)lock;
 229         union {
 230                 __u32 val;
 231                 arch_spinlock_t lock;
 232         } u = { .lock = __ARCH_SPIN_LOCK_UNLOCKED };
 233
 234         compiletime_assert(u.val == 0, "__ARCH_SPIN_LOCK_UNLOCKED not 0");
 235         BUILD_BUG_ON(sizeof(*l) != sizeof(__u32));
 236         BUILD_BUG_ON(sizeof(*lock) != sizeof(__u32));
 237         arch_spin_lock(l);
 238 }
 239
 240 static inline void __bpf_spin_unlock(struct bpf_spin_lock *lock)
 241 {
 242         arch_spinlock_t *l = (void *)lock;
 243
 244         arch_spin_unlock(l);
 245 }
 246
 247 #else
 248
 249 static inline void __bpf_spin_lock(struct bpf_spin_lock *lock)
 250 {
 251         atomic_t *l = (void *)lock;
 252
 253         BUILD_BUG_ON(sizeof(*l) != sizeof(*lock));
 254         do {
 255                 atomic_cond_read_relaxed(l, !VAL);
 256         } while (atomic_xchg(l, 1));
 257 }
 258
 259 static inline void __bpf_spin_unlock(struct bpf_spin_lock *lock)
 260 {
 261         atomic_t *l = (void *)lock;
 262
 263         atomic_set_release(l, 0);
 264 }
 265
 266 #endif
 267
 268 static DEFINE_PER_CPU(unsigned long, irqsave_flags);
 269
 270 notrace BPF_CALL_1(bpf_spin_lock, struct bpf_spin_lock *, lock)
 271 {
 272         unsigned long flags;
 273
 274         local_irq_save(flags);
 275         __bpf_spin_lock(lock);
 276         __this_cpu_write(irqsave_flags, flags);
 277         return 0;
 278 }
 279
 280 const struct bpf_func_proto bpf_spin_lock_proto = {
 281         .func           = bpf_spin_lock,
 282         .gpl_only       = false,
 283         .ret_type       = RET_VOID,
 284         .arg1_type      = ARG_PTR_TO_SPIN_LOCK,
 285 };
 286
 287 notrace BPF_CALL_1(bpf_spin_unlock, struct bpf_spin_lock *, lock)
 288 {
 289         unsigned long flags;
 290
 291         flags = __this_cpu_read(irqsave_flags);
 292         __bpf_spin_unlock(lock);
 293         local_irq_restore(flags);
 294         return 0;
 295 }
 296
 297 const struct bpf_func_proto bpf_spin_unlock_proto = {
 298         .func           = bpf_spin_unlock,
 299         .gpl_only       = false,
 300         .ret_type       = RET_VOID,
 301         .arg1_type      = ARG_PTR_TO_SPIN_LOCK,
 302 };
 303
 304 void copy_map_value_locked(struct bpf_map *map, void *dst, void *src,
 305                            bool lock_src)
 306 {
 307         struct bpf_spin_lock *lock;
 308
 309         if (lock_src)
 310                 lock = src + map->spin_lock_off;
 311         else
 312                 lock = dst + map->spin_lock_off;
 313         preempt_disable();
 314         ____bpf_spin_lock(lock);
 315         copy_map_value(map, dst, src);
 316         ____bpf_spin_unlock(lock);
 317         preempt_enable();
 318 }
 319
 320 #ifdef CONFIG_CGROUPS
 321 BPF_CALL_0(bpf_get_current_cgroup_id)
 322 {
 323         struct cgroup *cgrp = task_dfl_cgroup(current);
 324
 325         return cgrp->kn->id.id;
 326 }
 327
 328 const struct bpf_func_proto bpf_get_current_cgroup_id_proto = {
 329         .func           = bpf_get_current_cgroup_id,
 330         .gpl_only       = false,
 331         .ret_type       = RET_INTEGER,
 332 };
 333
 334 #ifdef CONFIG_CGROUP_BPF
 335 DECLARE_PER_CPU(struct bpf_cgroup_storage*,
 336                 bpf_cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]);
 337
 338 BPF_CALL_2(bpf_get_local_storage, struct bpf_map *, map, u64, flags)
 339 {
 340         /* flags argument is not used now,
 341          * but provides an ability to extend the API.
 342          * verifier checks that its value is correct.
 343          */
 344         enum bpf_cgroup_storage_type stype = cgroup_storage_type(map);
 345         struct bpf_cgroup_storage *storage;
 346         void *ptr;
 347
 348         storage = this_cpu_read(bpf_cgroup_storage[stype]);
 349
 350         if (stype == BPF_CGROUP_STORAGE_SHARED)
 351                 ptr = &READ_ONCE(storage->buf)->data[0];
 352         else
 353                 ptr = this_cpu_ptr(storage->percpu_buf);
 354
 355         return (unsigned long)ptr;
 356 }
 357
 358 const struct bpf_func_proto bpf_get_local_storage_proto = {
 359         .func           = bpf_get_local_storage,
 360         .gpl_only       = false,
 361         .ret_type       = RET_PTR_TO_MAP_VALUE,
 362         .arg1_type      = ARG_CONST_MAP_PTR,
 363         .arg2_type      = ARG_ANYTHING,
 364 };
 365 #endif
 366 #endif