1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /* Volume-level cache cookie handling.
4 * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved.
5 * Written by David Howells (dhowells@redhat.com)
8 #define FSCACHE_DEBUG_LEVEL COOKIE
9 #include <linux/export.h>
10 #include <linux/slab.h>
13 #define fscache_volume_hash_shift 10
14 static struct hlist_bl_head fscache_volume_hash[1 << fscache_volume_hash_shift];
15 static atomic_t fscache_volume_debug_id;
16 static LIST_HEAD(fscache_volumes);
18 static void fscache_create_volume_work(struct work_struct *work);
20 struct fscache_volume *fscache_get_volume(struct fscache_volume *volume,
21 enum fscache_volume_trace where)
25 __refcount_inc(&volume->ref, &ref);
26 trace_fscache_volume(volume->debug_id, ref + 1, where);
30 static void fscache_see_volume(struct fscache_volume *volume,
31 enum fscache_volume_trace where)
33 int ref = refcount_read(&volume->ref);
35 trace_fscache_volume(volume->debug_id, ref, where);
39 * Pin the cache behind a volume so that we can access it.
41 static void __fscache_begin_volume_access(struct fscache_volume *volume,
42 struct fscache_cookie *cookie,
43 enum fscache_access_trace why)
47 n_accesses = atomic_inc_return(&volume->n_accesses);
48 smp_mb__after_atomic();
49 trace_fscache_access_volume(volume->debug_id, cookie ? cookie->debug_id : 0,
50 refcount_read(&volume->ref),
55 * fscache_begin_volume_access - Pin a cache so a volume can be accessed
56 * @volume: The volume cookie
57 * @cookie: A datafile cookie for a tracing reference (or NULL)
58 * @why: An indication of the circumstances of the access for tracing
60 * Attempt to pin the cache to prevent it from going away whilst we're
61 * accessing a volume and returns true if successful. This works as follows:
63 * (1) If the cache tests as not live (state is not FSCACHE_CACHE_IS_ACTIVE),
64 * then we return false to indicate access was not permitted.
66 * (2) If the cache tests as live, then we increment the volume's n_accesses
67 * count and then recheck the cache liveness, ending the access if it
70 * (3) When we end the access, we decrement the volume's n_accesses and wake
71 * up the any waiters if it reaches 0.
73 * (4) Whilst the cache is caching, the volume's n_accesses is kept
74 * artificially incremented to prevent wakeups from happening.
76 * (5) When the cache is taken offline, the state is changed to prevent new
77 * accesses, the volume's n_accesses is decremented and we wait for it to
80 * The datafile @cookie and the @why indicator are merely provided for tracing
83 bool fscache_begin_volume_access(struct fscache_volume *volume,
84 struct fscache_cookie *cookie,
85 enum fscache_access_trace why)
87 if (!fscache_cache_is_live(volume->cache))
89 __fscache_begin_volume_access(volume, cookie, why);
90 if (!fscache_cache_is_live(volume->cache)) {
91 fscache_end_volume_access(volume, cookie, fscache_access_unlive);
98 * fscache_end_volume_access - Unpin a cache at the end of an access.
99 * @volume: The volume cookie
100 * @cookie: A datafile cookie for a tracing reference (or NULL)
101 * @why: An indication of the circumstances of the access for tracing
103 * Unpin a cache volume after we've accessed it. The datafile @cookie and the
104 * @why indicator are merely provided for tracing purposes.
106 void fscache_end_volume_access(struct fscache_volume *volume,
107 struct fscache_cookie *cookie,
108 enum fscache_access_trace why)
112 smp_mb__before_atomic();
113 n_accesses = atomic_dec_return(&volume->n_accesses);
114 trace_fscache_access_volume(volume->debug_id, cookie ? cookie->debug_id : 0,
115 refcount_read(&volume->ref),
118 wake_up_var(&volume->n_accesses);
120 EXPORT_SYMBOL(fscache_end_volume_access);
122 static bool fscache_volume_same(const struct fscache_volume *a,
123 const struct fscache_volume *b)
127 if (a->key_hash != b->key_hash ||
128 a->cache != b->cache ||
129 a->key[0] != b->key[0])
132 klen = round_up(a->key[0] + 1, sizeof(__le32));
133 return memcmp(a->key, b->key, klen) == 0;
136 static bool fscache_is_acquire_pending(struct fscache_volume *volume)
138 return test_bit(FSCACHE_VOLUME_ACQUIRE_PENDING, &volume->flags);
141 static void fscache_wait_on_volume_collision(struct fscache_volume *candidate,
142 unsigned int collidee_debug_id)
144 wait_var_event_timeout(&candidate->flags,
145 !fscache_is_acquire_pending(candidate), 20 * HZ);
146 if (fscache_is_acquire_pending(candidate)) {
147 pr_notice("Potential volume collision new=%08x old=%08x",
148 candidate->debug_id, collidee_debug_id);
149 fscache_stat(&fscache_n_volumes_collision);
150 wait_var_event(&candidate->flags, !fscache_is_acquire_pending(candidate));
155 * Attempt to insert the new volume into the hash. If there's a collision, we
156 * wait for the old volume to complete if it's being relinquished and an error
159 static bool fscache_hash_volume(struct fscache_volume *candidate)
161 struct fscache_volume *cursor;
162 struct hlist_bl_head *h;
163 struct hlist_bl_node *p;
164 unsigned int bucket, collidee_debug_id = 0;
166 bucket = candidate->key_hash & (ARRAY_SIZE(fscache_volume_hash) - 1);
167 h = &fscache_volume_hash[bucket];
170 hlist_bl_for_each_entry(cursor, p, h, hash_link) {
171 if (fscache_volume_same(candidate, cursor)) {
172 if (!test_bit(FSCACHE_VOLUME_RELINQUISHED, &cursor->flags))
174 fscache_see_volume(cursor, fscache_volume_get_hash_collision);
175 set_bit(FSCACHE_VOLUME_COLLIDED_WITH, &cursor->flags);
176 set_bit(FSCACHE_VOLUME_ACQUIRE_PENDING, &candidate->flags);
177 collidee_debug_id = cursor->debug_id;
182 hlist_bl_add_head(&candidate->hash_link, h);
185 if (fscache_is_acquire_pending(candidate))
186 fscache_wait_on_volume_collision(candidate, collidee_debug_id);
190 fscache_see_volume(cursor, fscache_volume_collision);
196 * Allocate and initialise a volume representation cookie.
198 static struct fscache_volume *fscache_alloc_volume(const char *volume_key,
199 const char *cache_name,
200 const void *coherency_data,
201 size_t coherency_len)
203 struct fscache_volume *volume;
204 struct fscache_cache *cache;
208 klen = strlen(volume_key);
215 cache = fscache_lookup_cache(cache_name, false);
219 volume = kzalloc(struct_size(volume, coherency, coherency_len),
224 volume->cache = cache;
225 volume->coherency_len = coherency_len;
227 memcpy(volume->coherency, coherency_data, coherency_len);
228 INIT_LIST_HEAD(&volume->proc_link);
229 INIT_WORK(&volume->work, fscache_create_volume_work);
230 refcount_set(&volume->ref, 1);
231 spin_lock_init(&volume->lock);
233 /* Stick the length on the front of the key and pad it out to make
236 hlen = round_up(1 + klen + 1, sizeof(__le32));
237 key = kzalloc(hlen, GFP_KERNEL);
241 memcpy(key + 1, volume_key, klen);
244 volume->key_hash = fscache_hash(0, key, hlen);
246 volume->debug_id = atomic_inc_return(&fscache_volume_debug_id);
247 down_write(&fscache_addremove_sem);
248 atomic_inc(&cache->n_volumes);
249 list_add_tail(&volume->proc_link, &fscache_volumes);
250 fscache_see_volume(volume, fscache_volume_new_acquire);
251 fscache_stat(&fscache_n_volumes);
252 up_write(&fscache_addremove_sem);
253 _leave(" = v=%x", volume->debug_id);
259 fscache_put_cache(cache, fscache_cache_put_alloc_volume);
260 fscache_stat(&fscache_n_volumes_nomem);
265 * Create a volume's representation on disk. Have a volume ref and a cache
266 * access we have to release.
268 static void fscache_create_volume_work(struct work_struct *work)
270 const struct fscache_cache_ops *ops;
271 struct fscache_volume *volume =
272 container_of(work, struct fscache_volume, work);
274 fscache_see_volume(volume, fscache_volume_see_create_work);
276 ops = volume->cache->ops;
277 if (ops->acquire_volume)
278 ops->acquire_volume(volume);
279 fscache_end_cache_access(volume->cache,
280 fscache_access_acquire_volume_end);
282 clear_bit_unlock(FSCACHE_VOLUME_CREATING, &volume->flags);
283 wake_up_bit(&volume->flags, FSCACHE_VOLUME_CREATING);
284 fscache_put_volume(volume, fscache_volume_put_create_work);
288 * Dispatch a worker thread to create a volume's representation on disk.
290 void fscache_create_volume(struct fscache_volume *volume, bool wait)
292 if (test_and_set_bit(FSCACHE_VOLUME_CREATING, &volume->flags))
294 if (volume->cache_priv)
295 goto no_wait; /* We raced */
296 if (!fscache_begin_cache_access(volume->cache,
297 fscache_access_acquire_volume))
300 fscache_get_volume(volume, fscache_volume_get_create_work);
301 if (!schedule_work(&volume->work))
302 fscache_put_volume(volume, fscache_volume_put_create_work);
306 fscache_see_volume(volume, fscache_volume_wait_create_work);
307 wait_on_bit(&volume->flags, FSCACHE_VOLUME_CREATING,
308 TASK_UNINTERRUPTIBLE);
312 clear_bit_unlock(FSCACHE_VOLUME_CREATING, &volume->flags);
313 wake_up_bit(&volume->flags, FSCACHE_VOLUME_CREATING);
317 * Acquire a volume representation cookie and link it to a (proposed) cache.
319 struct fscache_volume *__fscache_acquire_volume(const char *volume_key,
320 const char *cache_name,
321 const void *coherency_data,
322 size_t coherency_len)
324 struct fscache_volume *volume;
326 volume = fscache_alloc_volume(volume_key, cache_name,
327 coherency_data, coherency_len);
329 return ERR_PTR(-ENOMEM);
331 if (!fscache_hash_volume(volume)) {
332 fscache_put_volume(volume, fscache_volume_put_hash_collision);
333 return ERR_PTR(-EBUSY);
336 fscache_create_volume(volume, false);
339 EXPORT_SYMBOL(__fscache_acquire_volume);
341 static void fscache_wake_pending_volume(struct fscache_volume *volume,
342 struct hlist_bl_head *h)
344 struct fscache_volume *cursor;
345 struct hlist_bl_node *p;
347 hlist_bl_for_each_entry(cursor, p, h, hash_link) {
348 if (fscache_volume_same(cursor, volume)) {
349 fscache_see_volume(cursor, fscache_volume_see_hash_wake);
350 clear_bit(FSCACHE_VOLUME_ACQUIRE_PENDING, &cursor->flags);
351 wake_up_bit(&cursor->flags, FSCACHE_VOLUME_ACQUIRE_PENDING);
358 * Remove a volume cookie from the hash table.
360 static void fscache_unhash_volume(struct fscache_volume *volume)
362 struct hlist_bl_head *h;
365 bucket = volume->key_hash & (ARRAY_SIZE(fscache_volume_hash) - 1);
366 h = &fscache_volume_hash[bucket];
369 hlist_bl_del(&volume->hash_link);
370 if (test_bit(FSCACHE_VOLUME_COLLIDED_WITH, &volume->flags))
371 fscache_wake_pending_volume(volume, h);
376 * Drop a cache's volume attachments.
378 static void fscache_free_volume(struct fscache_volume *volume)
380 struct fscache_cache *cache = volume->cache;
382 if (volume->cache_priv) {
383 __fscache_begin_volume_access(volume, NULL,
384 fscache_access_relinquish_volume);
385 if (volume->cache_priv)
386 cache->ops->free_volume(volume);
387 fscache_end_volume_access(volume, NULL,
388 fscache_access_relinquish_volume_end);
391 down_write(&fscache_addremove_sem);
392 list_del_init(&volume->proc_link);
393 atomic_dec(&volume->cache->n_volumes);
394 up_write(&fscache_addremove_sem);
396 if (!hlist_bl_unhashed(&volume->hash_link))
397 fscache_unhash_volume(volume);
399 trace_fscache_volume(volume->debug_id, 0, fscache_volume_free);
402 fscache_stat_d(&fscache_n_volumes);
403 fscache_put_cache(cache, fscache_cache_put_volume);
407 * Drop a reference to a volume cookie.
409 void fscache_put_volume(struct fscache_volume *volume,
410 enum fscache_volume_trace where)
413 unsigned int debug_id = volume->debug_id;
417 zero = __refcount_dec_and_test(&volume->ref, &ref);
418 trace_fscache_volume(debug_id, ref - 1, where);
420 fscache_free_volume(volume);
425 * Relinquish a volume representation cookie.
427 void __fscache_relinquish_volume(struct fscache_volume *volume,
428 const void *coherency_data,
431 if (WARN_ON(test_and_set_bit(FSCACHE_VOLUME_RELINQUISHED, &volume->flags)))
435 set_bit(FSCACHE_VOLUME_INVALIDATE, &volume->flags);
436 } else if (coherency_data) {
437 memcpy(volume->coherency, coherency_data, volume->coherency_len);
440 fscache_put_volume(volume, fscache_volume_put_relinquish);
442 EXPORT_SYMBOL(__fscache_relinquish_volume);
445 * fscache_withdraw_volume - Withdraw a volume from being cached
446 * @volume: Volume cookie
448 * Withdraw a cache volume from service, waiting for all accesses to complete
451 void fscache_withdraw_volume(struct fscache_volume *volume)
455 _debug("withdraw V=%x", volume->debug_id);
457 /* Allow wakeups on dec-to-0 */
458 n_accesses = atomic_dec_return(&volume->n_accesses);
459 trace_fscache_access_volume(volume->debug_id, 0,
460 refcount_read(&volume->ref),
461 n_accesses, fscache_access_cache_unpin);
463 wait_var_event(&volume->n_accesses,
464 atomic_read(&volume->n_accesses) == 0);
466 EXPORT_SYMBOL(fscache_withdraw_volume);
468 #ifdef CONFIG_PROC_FS
470 * Generate a list of volumes in /proc/fs/fscache/volumes
472 static int fscache_volumes_seq_show(struct seq_file *m, void *v)
474 struct fscache_volume *volume;
476 if (v == &fscache_volumes) {
478 "VOLUME REF nCOOK ACC FL CACHE KEY\n"
479 "======== ===== ===== === == =============== ================\n");
483 volume = list_entry(v, struct fscache_volume, proc_link);
485 "%08x %5d %5d %3d %02lx %-15.15s %s\n",
487 refcount_read(&volume->ref),
488 atomic_read(&volume->n_cookies),
489 atomic_read(&volume->n_accesses),
491 volume->cache->name ?: "-",
496 static void *fscache_volumes_seq_start(struct seq_file *m, loff_t *_pos)
497 __acquires(&fscache_addremove_sem)
499 down_read(&fscache_addremove_sem);
500 return seq_list_start_head(&fscache_volumes, *_pos);
503 static void *fscache_volumes_seq_next(struct seq_file *m, void *v, loff_t *_pos)
505 return seq_list_next(v, &fscache_volumes, _pos);
508 static void fscache_volumes_seq_stop(struct seq_file *m, void *v)
509 __releases(&fscache_addremove_sem)
511 up_read(&fscache_addremove_sem);
514 const struct seq_operations fscache_volumes_seq_ops = {
515 .start = fscache_volumes_seq_start,
516 .next = fscache_volumes_seq_next,
517 .stop = fscache_volumes_seq_stop,
518 .show = fscache_volumes_seq_show,
520 #endif /* CONFIG_PROC_FS */