mm/zswap: change per-cpu mutex and buffer to per-acomp_ctx

First of all, we need to rename acomp_ctx->dstmem field to buffer, since
we are now using for purposes other than compression.

Then we change per-cpu mutex and buffer to per-acomp_ctx, since them
belong to the acomp_ctx and are necessary parts when used in the
compress/decompress contexts.

So we can remove the old per-cpu mutex and dstmem.

Link: https://lkml.kernel.org/r/20231213-zswap-dstmem-v5-5-9382162bbf05@bytedance.com
Signed-off-by: Chengming Zhou <zhouchengming@bytedance.com>
Acked-by: Chris Li <chrisl@kernel.org> (Google)
Reviewed-by: Nhat Pham <nphamcs@gmail.com>
Cc: Barry Song <21cnbao@gmail.com>
Cc: Dan Streetman <ddstreet@ieee.org>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Seth Jennings <sjenning@redhat.com>
Cc: Vitaly Wool <vitaly.wool@konsulko.com>
Cc: Yosry Ahmed <yosryahmed@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h
index efc0c0b..c3e06e2 100644 (file)
--- a/include/linux/cpuhotplug.h
+++ b/include/linux/cpuhotplug.h
@@ -124,7 +124,6 @@ enum cpuhp_state {
        CPUHP_ARM_BL_PREPARE,
        CPUHP_TRACE_RB_PREPARE,
        CPUHP_MM_ZS_PREPARE,
-       CPUHP_MM_ZSWP_MEM_PREPARE,
        CPUHP_MM_ZSWP_POOL_PREPARE,
        CPUHP_KVM_PPC_BOOK3S_PREPARE,
        CPUHP_ZCOMP_PREPARE,

diff --git a/mm/zswap.c b/mm/zswap.c
index 880c33b..12ce5a6 100644 (file)
--- a/mm/zswap.c
+++ b/mm/zswap.c
@@ -161,8 +161,8 @@ struct crypto_acomp_ctx {
        struct crypto_acomp *acomp;
        struct acomp_req *req;
        struct crypto_wait wait;
-       u8 *dstmem;
-       struct mutex *mutex;
+       u8 *buffer;
+       struct mutex mutex;
 };
 
 /*
@@ -688,63 +688,26 @@ static void zswap_alloc_shrinker(struct zswap_pool *pool)
 /*********************************
 * per-cpu code
 **********************************/
-static DEFINE_PER_CPU(u8 *, zswap_dstmem);
-/*
- * If users dynamically change the zpool type and compressor at runtime, i.e.
- * zswap is running, zswap can have more than one zpool on one cpu, but they
- * are sharing dtsmem. So we need this mutex to be per-cpu.
- */
-static DEFINE_PER_CPU(struct mutex *, zswap_mutex);
-
-static int zswap_dstmem_prepare(unsigned int cpu)
-{
-       struct mutex *mutex;
-       u8 *dst;
-
-       dst = kmalloc_node(PAGE_SIZE * 2, GFP_KERNEL, cpu_to_node(cpu));
-       if (!dst)
-               return -ENOMEM;
-
-       mutex = kmalloc_node(sizeof(*mutex), GFP_KERNEL, cpu_to_node(cpu));
-       if (!mutex) {
-               kfree(dst);
-               return -ENOMEM;
-       }
-
-       mutex_init(mutex);
-       per_cpu(zswap_dstmem, cpu) = dst;
-       per_cpu(zswap_mutex, cpu) = mutex;
-       return 0;
-}
-
-static int zswap_dstmem_dead(unsigned int cpu)
-{
-       struct mutex *mutex;
-       u8 *dst;
-
-       mutex = per_cpu(zswap_mutex, cpu);
-       kfree(mutex);
-       per_cpu(zswap_mutex, cpu) = NULL;
-
-       dst = per_cpu(zswap_dstmem, cpu);
-       kfree(dst);
-       per_cpu(zswap_dstmem, cpu) = NULL;
-
-       return 0;
-}
-
 static int zswap_cpu_comp_prepare(unsigned int cpu, struct hlist_node *node)
 {
        struct zswap_pool *pool = hlist_entry(node, struct zswap_pool, node);
        struct crypto_acomp_ctx *acomp_ctx = per_cpu_ptr(pool->acomp_ctx, cpu);
        struct crypto_acomp *acomp;
        struct acomp_req *req;
+       int ret;
+
+       mutex_init(&acomp_ctx->mutex);
+
+       acomp_ctx->buffer = kmalloc_node(PAGE_SIZE * 2, GFP_KERNEL, cpu_to_node(cpu));
+       if (!acomp_ctx->buffer)
+               return -ENOMEM;
 
        acomp = crypto_alloc_acomp_node(pool->tfm_name, 0, 0, cpu_to_node(cpu));
        if (IS_ERR(acomp)) {
                pr_err("could not alloc crypto acomp %s : %ld\n",
                                pool->tfm_name, PTR_ERR(acomp));
-               return PTR_ERR(acomp);
+               ret = PTR_ERR(acomp);
+               goto acomp_fail;
        }
        acomp_ctx->acomp = acomp;
 
@@ -752,8 +715,8 @@ static int zswap_cpu_comp_prepare(unsigned int cpu, struct hlist_node *node)
        if (!req) {
                pr_err("could not alloc crypto acomp_request %s\n",
                       pool->tfm_name);
-               crypto_free_acomp(acomp_ctx->acomp);
-               return -ENOMEM;
+               ret = -ENOMEM;
+               goto req_fail;
        }
        acomp_ctx->req = req;
 
@@ -766,10 +729,13 @@ static int zswap_cpu_comp_prepare(unsigned int cpu, struct hlist_node *node)
        acomp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
                                   crypto_req_done, &acomp_ctx->wait);
 
-       acomp_ctx->mutex = per_cpu(zswap_mutex, cpu);
-       acomp_ctx->dstmem = per_cpu(zswap_dstmem, cpu);
-
        return 0;
+
+req_fail:
+       crypto_free_acomp(acomp_ctx->acomp);
+acomp_fail:
+       kfree(acomp_ctx->buffer);
+       return ret;
 }
 
 static int zswap_cpu_comp_dead(unsigned int cpu, struct hlist_node *node)
@@ -782,6 +748,7 @@ static int zswap_cpu_comp_dead(unsigned int cpu, struct hlist_node *node)
                        acomp_request_free(acomp_ctx->req);
                if (!IS_ERR_OR_NULL(acomp_ctx->acomp))
                        crypto_free_acomp(acomp_ctx->acomp);
+               kfree(acomp_ctx->buffer);
        }
 
        return 0;
@@ -1391,12 +1358,12 @@ static void __zswap_load(struct zswap_entry *entry, struct page *page)
        u8 *src;
 
        acomp_ctx = raw_cpu_ptr(entry->pool->acomp_ctx);
-       mutex_lock(acomp_ctx->mutex);
+       mutex_lock(&acomp_ctx->mutex);
 
        src = zpool_map_handle(zpool, entry->handle, ZPOOL_MM_RO);
        if (!zpool_can_sleep_mapped(zpool)) {
-               memcpy(acomp_ctx->dstmem, src, entry->length);
-               src = acomp_ctx->dstmem;
+               memcpy(acomp_ctx->buffer, src, entry->length);
+               src = acomp_ctx->buffer;
                zpool_unmap_handle(zpool, entry->handle);
        }
 
@@ -1406,7 +1373,7 @@ static void __zswap_load(struct zswap_entry *entry, struct page *page)
        acomp_request_set_params(acomp_ctx->req, &input, &output, entry->length, PAGE_SIZE);
        BUG_ON(crypto_wait_req(crypto_acomp_decompress(acomp_ctx->req), &acomp_ctx->wait));
        BUG_ON(acomp_ctx->req->dlen != PAGE_SIZE);
-       mutex_unlock(acomp_ctx->mutex);
+       mutex_unlock(&acomp_ctx->mutex);
 
        if (zpool_can_sleep_mapped(zpool))
                zpool_unmap_handle(zpool, entry->handle);
@@ -1622,13 +1589,17 @@ bool zswap_store(struct folio *folio)
        /* compress */
        acomp_ctx = raw_cpu_ptr(entry->pool->acomp_ctx);
 
-       mutex_lock(acomp_ctx->mutex);
+       mutex_lock(&acomp_ctx->mutex);
 
-       dst = acomp_ctx->dstmem;
+       dst = acomp_ctx->buffer;
        sg_init_table(&input, 1);
        sg_set_page(&input, page, PAGE_SIZE, 0);
 
-       /* zswap_dstmem is of size (PAGE_SIZE * 2). Reflect same in sg_list */
+       /*
+        * We need PAGE_SIZE * 2 here since there maybe over-compression case,
+        * and hardware-accelerators may won't check the dst buffer size, so
+        * giving the dst buffer with enough length to avoid buffer overflow.
+        */
        sg_init_one(&output, dst, PAGE_SIZE * 2);
        acomp_request_set_params(acomp_ctx->req, &input, &output, PAGE_SIZE, dlen);
        /*
@@ -1668,7 +1639,7 @@ bool zswap_store(struct folio *folio)
        buf = zpool_map_handle(zpool, handle, ZPOOL_MM_WO);
        memcpy(buf, dst, dlen);
        zpool_unmap_handle(zpool, handle);
-       mutex_unlock(acomp_ctx->mutex);
+       mutex_unlock(&acomp_ctx->mutex);
 
        /* populate entry */
        entry->swpentry = swp_entry(type, offset);
@@ -1711,7 +1682,7 @@ insert_entry:
        return true;
 
 put_dstmem:
-       mutex_unlock(acomp_ctx->mutex);
+       mutex_unlock(&acomp_ctx->mutex);
 put_pool:
        zswap_pool_put(entry->pool);
 freepage:
@@ -1886,13 +1857,6 @@ static int zswap_setup(void)
                goto cache_fail;
        }
 
-       ret = cpuhp_setup_state(CPUHP_MM_ZSWP_MEM_PREPARE, "mm/zswap:prepare",
-                               zswap_dstmem_prepare, zswap_dstmem_dead);
-       if (ret) {
-               pr_err("dstmem alloc failed\n");
-               goto dstmem_fail;
-       }
-
        ret = cpuhp_setup_state_multi(CPUHP_MM_ZSWP_POOL_PREPARE,
                                      "mm/zswap_pool:prepare",
                                      zswap_cpu_comp_prepare,
@@ -1924,8 +1888,6 @@ fallback_fail:
        if (pool)
                zswap_pool_destroy(pool);
 hp_fail:
-       cpuhp_remove_state(CPUHP_MM_ZSWP_MEM_PREPARE);
-dstmem_fail:
        kmem_cache_destroy(zswap_entry_cache);
 cache_fail:
        /* if built-in, we aren't unloaded on failure; don't allow use */