bpf: Memcg-based memory accounting for bpf local storage maps

Account memory used by bpf local storage maps:
per-socket, per-inode and per-task storages.

Signed-off-by: Roman Gushchin <guro@fb.com>
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
Link: https://lore.kernel.org/bpf/20201201215900.3569844-16-guro@fb.com

diff --git a/kernel/bpf/bpf_local_storage.c b/kernel/bpf/bpf_local_storage.c
index 5d3a7af..023a3ea 100644 (file)
--- a/kernel/bpf/bpf_local_storage.c
+++ b/kernel/bpf/bpf_local_storage.c
@@ -67,7 +67,8 @@ bpf_selem_alloc(struct bpf_local_storage_map *smap, void *owner,
        if (charge_mem && mem_charge(smap, owner, smap->elem_size))
                return NULL;
 
-       selem = kzalloc(smap->elem_size, GFP_ATOMIC | __GFP_NOWARN);
+       selem = bpf_map_kzalloc(&smap->map, smap->elem_size,
+                               GFP_ATOMIC | __GFP_NOWARN);
        if (selem) {
                if (value)
                        memcpy(SDATA(selem)->data, value, smap->map.value_size);
@@ -264,7 +265,8 @@ int bpf_local_storage_alloc(void *owner,
        if (err)
                return err;
 
-       storage = kzalloc(sizeof(*storage), GFP_ATOMIC | __GFP_NOWARN);
+       storage = bpf_map_kzalloc(&smap->map, sizeof(*storage),
+                                 GFP_ATOMIC | __GFP_NOWARN);
        if (!storage) {
                err = -ENOMEM;
                goto uncharge;
@@ -546,7 +548,7 @@ struct bpf_local_storage_map *bpf_local_storage_map_alloc(union bpf_attr *attr)
        u64 cost;
        int ret;
 
-       smap = kzalloc(sizeof(*smap), GFP_USER | __GFP_NOWARN);
+       smap = kzalloc(sizeof(*smap), GFP_USER | __GFP_NOWARN | __GFP_ACCOUNT);
        if (!smap)
                return ERR_PTR(-ENOMEM);
        bpf_map_init_from_attr(&smap->map, attr);
@@ -564,7 +566,7 @@ struct bpf_local_storage_map *bpf_local_storage_map_alloc(union bpf_attr *attr)
        }
 
        smap->buckets = kvcalloc(sizeof(*smap->buckets), nbuckets,
-                                GFP_USER | __GFP_NOWARN);
+                                GFP_USER | __GFP_NOWARN | __GFP_ACCOUNT);
        if (!smap->buckets) {
                bpf_map_charge_finish(&smap->map.memory);
                kfree(smap);