1 // SPDX-License-Identifier: GPL-2.0
3 * Manage cache of swap slots to be used for and returned from
6 * Copyright(c) 2016 Intel Corporation.
8 * Author: Tim Chen <tim.c.chen@linux.intel.com>
10 * We allocate the swap slots from the global pool and put
11 * it into local per cpu caches. This has the advantage
12 * of no needing to acquire the swap_info lock every time
15 * There is also opportunity to simply return the slot
16 * to local caches without needing to acquire swap_info
17 * lock. We do not reuse the returned slots directly but
18 * move them back to the global pool in a batch. This
19 * allows the slots to coaellesce and reduce fragmentation.
21 * The swap entry allocated is marked with SWAP_HAS_CACHE
22 * flag in map_count that prevents it from being allocated
23 * again from the global pool.
25 * The swap slots cache is protected by a mutex instead of
26 * a spin lock as when we search for slots with scan_swap_map,
27 * we can possibly sleep.
30 #include <linux/swap_slots.h>
31 #include <linux/cpu.h>
32 #include <linux/cpumask.h>
33 #include <linux/vmalloc.h>
34 #include <linux/mutex.h>
37 static DEFINE_PER_CPU(struct swap_slots_cache, swp_slots);
38 static bool swap_slot_cache_active;
39 bool swap_slot_cache_enabled;
40 static bool swap_slot_cache_initialized;
41 static DEFINE_MUTEX(swap_slots_cache_mutex);
42 /* Serialize swap slots cache enable/disable operations */
43 static DEFINE_MUTEX(swap_slots_cache_enable_mutex);
45 static void __drain_swap_slots_cache(unsigned int type);
46 static void deactivate_swap_slots_cache(void);
47 static void reactivate_swap_slots_cache(void);
49 #define use_swap_slot_cache (swap_slot_cache_active && \
50 swap_slot_cache_enabled && swap_slot_cache_initialized)
51 #define SLOTS_CACHE 0x1
52 #define SLOTS_CACHE_RET 0x2
54 static void deactivate_swap_slots_cache(void)
56 mutex_lock(&swap_slots_cache_mutex);
57 swap_slot_cache_active = false;
58 __drain_swap_slots_cache(SLOTS_CACHE|SLOTS_CACHE_RET);
59 mutex_unlock(&swap_slots_cache_mutex);
62 static void reactivate_swap_slots_cache(void)
64 mutex_lock(&swap_slots_cache_mutex);
65 swap_slot_cache_active = true;
66 mutex_unlock(&swap_slots_cache_mutex);
69 /* Must not be called with cpu hot plug lock */
70 void disable_swap_slots_cache_lock(void)
72 mutex_lock(&swap_slots_cache_enable_mutex);
73 swap_slot_cache_enabled = false;
74 if (swap_slot_cache_initialized) {
75 /* serialize with cpu hotplug operations */
77 __drain_swap_slots_cache(SLOTS_CACHE|SLOTS_CACHE_RET);
82 static void __reenable_swap_slots_cache(void)
84 swap_slot_cache_enabled = has_usable_swap();
87 void reenable_swap_slots_cache_unlock(void)
89 __reenable_swap_slots_cache();
90 mutex_unlock(&swap_slots_cache_enable_mutex);
93 static bool check_cache_active(void)
97 if (!swap_slot_cache_enabled || !swap_slot_cache_initialized)
100 pages = get_nr_swap_pages();
101 if (!swap_slot_cache_active) {
102 if (pages > num_online_cpus() *
103 THRESHOLD_ACTIVATE_SWAP_SLOTS_CACHE)
104 reactivate_swap_slots_cache();
108 /* if global pool of slot caches too low, deactivate cache */
109 if (pages < num_online_cpus() * THRESHOLD_DEACTIVATE_SWAP_SLOTS_CACHE)
110 deactivate_swap_slots_cache();
112 return swap_slot_cache_active;
115 static int alloc_swap_slot_cache(unsigned int cpu)
117 struct swap_slots_cache *cache;
118 swp_entry_t *slots, *slots_ret;
121 * Do allocation outside swap_slots_cache_mutex
122 * as kvzalloc could trigger reclaim and get_swap_page,
123 * which can lock swap_slots_cache_mutex.
125 slots = kvcalloc(SWAP_SLOTS_CACHE_SIZE, sizeof(swp_entry_t),
130 slots_ret = kvcalloc(SWAP_SLOTS_CACHE_SIZE, sizeof(swp_entry_t),
137 mutex_lock(&swap_slots_cache_mutex);
138 cache = &per_cpu(swp_slots, cpu);
139 if (cache->slots || cache->slots_ret)
140 /* cache already allocated */
142 if (!cache->lock_initialized) {
143 mutex_init(&cache->alloc_lock);
144 spin_lock_init(&cache->free_lock);
145 cache->lock_initialized = true;
151 * We initialized alloc_lock and free_lock earlier. We use
152 * !cache->slots or !cache->slots_ret to know if it is safe to acquire
153 * the corresponding lock and use the cache. Memory barrier below
154 * ensures the assumption.
157 cache->slots = slots;
159 cache->slots_ret = slots_ret;
162 mutex_unlock(&swap_slots_cache_mutex);
170 static void drain_slots_cache_cpu(unsigned int cpu, unsigned int type,
173 struct swap_slots_cache *cache;
174 swp_entry_t *slots = NULL;
176 cache = &per_cpu(swp_slots, cpu);
177 if ((type & SLOTS_CACHE) && cache->slots) {
178 mutex_lock(&cache->alloc_lock);
179 swapcache_free_entries(cache->slots + cache->cur, cache->nr);
182 if (free_slots && cache->slots) {
183 kvfree(cache->slots);
186 mutex_unlock(&cache->alloc_lock);
188 if ((type & SLOTS_CACHE_RET) && cache->slots_ret) {
189 spin_lock_irq(&cache->free_lock);
190 swapcache_free_entries(cache->slots_ret, cache->n_ret);
192 if (free_slots && cache->slots_ret) {
193 slots = cache->slots_ret;
194 cache->slots_ret = NULL;
196 spin_unlock_irq(&cache->free_lock);
202 static void __drain_swap_slots_cache(unsigned int type)
207 * This function is called during
208 * 1) swapoff, when we have to make sure no
209 * left over slots are in cache when we remove
211 * 2) disabling of swap slot cache, when we run low
212 * on swap slots when allocating memory and need
213 * to return swap slots to global pool.
215 * We cannot acquire cpu hot plug lock here as
216 * this function can be invoked in the cpu
218 * cpu_up -> lock cpu_hotplug -> cpu hotplug state callback
219 * -> memory allocation -> direct reclaim -> get_swap_page
220 * -> drain_swap_slots_cache
222 * Hence the loop over current online cpu below could miss cpu that
223 * is being brought online but not yet marked as online.
224 * That is okay as we do not schedule and run anything on a
225 * cpu before it has been marked online. Hence, we will not
226 * fill any swap slots in slots cache of such cpu.
227 * There are no slots on such cpu that need to be drained.
229 for_each_online_cpu(cpu)
230 drain_slots_cache_cpu(cpu, type, false);
233 static int free_slot_cache(unsigned int cpu)
235 mutex_lock(&swap_slots_cache_mutex);
236 drain_slots_cache_cpu(cpu, SLOTS_CACHE | SLOTS_CACHE_RET, true);
237 mutex_unlock(&swap_slots_cache_mutex);
241 int enable_swap_slots_cache(void)
245 mutex_lock(&swap_slots_cache_enable_mutex);
246 if (swap_slot_cache_initialized) {
247 __reenable_swap_slots_cache();
251 ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "swap_slots_cache",
252 alloc_swap_slot_cache, free_slot_cache);
253 if (WARN_ONCE(ret < 0, "Cache allocation failed (%s), operating "
254 "without swap slots cache.\n", __func__))
257 swap_slot_cache_initialized = true;
258 __reenable_swap_slots_cache();
260 mutex_unlock(&swap_slots_cache_enable_mutex);
264 /* called with swap slot cache's alloc lock held */
265 static int refill_swap_slots_cache(struct swap_slots_cache *cache)
267 if (!use_swap_slot_cache || cache->nr)
271 if (swap_slot_cache_active)
272 cache->nr = get_swap_pages(SWAP_SLOTS_CACHE_SIZE,
278 int free_swap_slot(swp_entry_t entry)
280 struct swap_slots_cache *cache;
282 cache = raw_cpu_ptr(&swp_slots);
283 if (likely(use_swap_slot_cache && cache->slots_ret)) {
284 spin_lock_irq(&cache->free_lock);
285 /* Swap slots cache may be deactivated before acquiring lock */
286 if (!use_swap_slot_cache || !cache->slots_ret) {
287 spin_unlock_irq(&cache->free_lock);
290 if (cache->n_ret >= SWAP_SLOTS_CACHE_SIZE) {
292 * Return slots to global pool.
293 * The current swap_map value is SWAP_HAS_CACHE.
294 * Set it to 0 to indicate it is available for
295 * allocation in global pool
297 swapcache_free_entries(cache->slots_ret, cache->n_ret);
300 cache->slots_ret[cache->n_ret++] = entry;
301 spin_unlock_irq(&cache->free_lock);
304 swapcache_free_entries(&entry, 1);
310 swp_entry_t get_swap_page(struct page *page)
312 swp_entry_t entry, *pentry;
313 struct swap_slots_cache *cache;
317 if (PageTransHuge(page)) {
318 if (IS_ENABLED(CONFIG_THP_SWAP))
319 get_swap_pages(1, &entry, HPAGE_PMD_NR);
324 * Preemption is allowed here, because we may sleep
325 * in refill_swap_slots_cache(). But it is safe, because
326 * accesses to the per-CPU data structure are protected by the
327 * mutex cache->alloc_lock.
329 * The alloc path here does not touch cache->slots_ret
330 * so cache->free_lock is not taken.
332 cache = raw_cpu_ptr(&swp_slots);
334 if (likely(check_cache_active() && cache->slots)) {
335 mutex_lock(&cache->alloc_lock);
339 pentry = &cache->slots[cache->cur++];
344 if (refill_swap_slots_cache(cache))
348 mutex_unlock(&cache->alloc_lock);
353 get_swap_pages(1, &entry, 1);
355 if (mem_cgroup_try_charge_swap(page, entry)) {
356 put_swap_page(page, entry);