fs/fscache/cookie.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /* netfs cookie management
   3  *
   4  * Copyright (C) 2004-2007 Red Hat, Inc. All Rights Reserved.
   5  * Written by David Howells (dhowells@redhat.com)
   6  *
   7  * See Documentation/filesystems/caching/netfs-api.rst for more information on
   8  * the netfs API.
   9  */
  10
  11 #define FSCACHE_DEBUG_LEVEL COOKIE
  12 #include <linux/module.h>
  13 #include <linux/slab.h>
  14 #include "internal.h"
  15
  16 struct kmem_cache *fscache_cookie_jar;
  17
  18 static atomic_t fscache_object_debug_id = ATOMIC_INIT(0);
  19
  20 #define fscache_cookie_hash_shift 15
  21 static struct hlist_bl_head fscache_cookie_hash[1 << fscache_cookie_hash_shift];
  22 static LIST_HEAD(fscache_cookies);
  23 static DEFINE_RWLOCK(fscache_cookies_lock);
  24
  25 static int fscache_acquire_non_index_cookie(struct fscache_cookie *cookie,
  26                                             loff_t object_size);
  27 static int fscache_alloc_object(struct fscache_cache *cache,
  28                                 struct fscache_cookie *cookie);
  29 static int fscache_attach_object(struct fscache_cookie *cookie,
  30                                  struct fscache_object *object);
  31
  32 static void fscache_print_cookie(struct fscache_cookie *cookie, char prefix)
  33 {
  34         struct fscache_object *object;
  35         struct hlist_node *o;
  36         const u8 *k;
  37         unsigned loop;
  38
  39         pr_err("%c-cookie c=%08x [p=%08x fl=%lx nc=%u na=%u]\n",
  40                prefix,
  41                cookie->debug_id,
  42                cookie->parent ? cookie->parent->debug_id : 0,
  43                cookie->flags,
  44                atomic_read(&cookie->n_children),
  45                atomic_read(&cookie->n_active));
  46         pr_err("%c-cookie d=%p{%s} n=%p\n",
  47                prefix,
  48                cookie->def,
  49                cookie->def ? cookie->def->name : "?",
  50                cookie->netfs_data);
  51
  52         o = READ_ONCE(cookie->backing_objects.first);
  53         if (o) {
  54                 object = hlist_entry(o, struct fscache_object, cookie_link);
  55                 pr_err("%c-cookie o=%u\n", prefix, object->debug_id);
  56         }
  57
  58         pr_err("%c-key=[%u] '", prefix, cookie->key_len);
  59         k = (cookie->key_len <= sizeof(cookie->inline_key)) ?
  60                 cookie->inline_key : cookie->key;
  61         for (loop = 0; loop < cookie->key_len; loop++)
  62                 pr_cont("%02x", k[loop]);
  63         pr_cont("'\n");
  64 }
  65
  66 void fscache_free_cookie(struct fscache_cookie *cookie)
  67 {
  68         if (cookie) {
  69                 BUG_ON(!hlist_empty(&cookie->backing_objects));
  70                 write_lock(&fscache_cookies_lock);
  71                 list_del(&cookie->proc_link);
  72                 write_unlock(&fscache_cookies_lock);
  73                 if (cookie->aux_len > sizeof(cookie->inline_aux))
  74                         kfree(cookie->aux);
  75                 if (cookie->key_len > sizeof(cookie->inline_key))
  76                         kfree(cookie->key);
  77                 kmem_cache_free(fscache_cookie_jar, cookie);
  78         }
  79 }
  80
  81 /*
  82  * Set the index key in a cookie.  The cookie struct has space for a 16-byte
  83  * key plus length and hash, but if that's not big enough, it's instead a
  84  * pointer to a buffer containing 3 bytes of hash, 1 byte of length and then
  85  * the key data.
  86  */
  87 static int fscache_set_key(struct fscache_cookie *cookie,
  88                            const void *index_key, size_t index_key_len)
  89 {
  90         u32 *buf;
  91         int bufs;
  92
  93         bufs = DIV_ROUND_UP(index_key_len, sizeof(*buf));
  94
  95         if (index_key_len > sizeof(cookie->inline_key)) {
  96                 buf = kcalloc(bufs, sizeof(*buf), GFP_KERNEL);
  97                 if (!buf)
  98                         return -ENOMEM;
  99                 cookie->key = buf;
 100         } else {
 101                 buf = (u32 *)cookie->inline_key;
 102         }
 103
 104         memcpy(buf, index_key, index_key_len);
 105         cookie->key_hash = fscache_hash(0, buf, bufs);
 106         return 0;
 107 }
 108
 109 static long fscache_compare_cookie(const struct fscache_cookie *a,
 110                                    const struct fscache_cookie *b)
 111 {
 112         const void *ka, *kb;
 113
 114         if (a->key_hash != b->key_hash)
 115                 return (long)a->key_hash - (long)b->key_hash;
 116         if (a->parent != b->parent)
 117                 return (long)a->parent - (long)b->parent;
 118         if (a->key_len != b->key_len)
 119                 return (long)a->key_len - (long)b->key_len;
 120         if (a->type != b->type)
 121                 return (long)a->type - (long)b->type;
 122
 123         if (a->key_len <= sizeof(a->inline_key)) {
 124                 ka = &a->inline_key;
 125                 kb = &b->inline_key;
 126         } else {
 127                 ka = a->key;
 128                 kb = b->key;
 129         }
 130         return memcmp(ka, kb, a->key_len);
 131 }
 132
 133 static atomic_t fscache_cookie_debug_id = ATOMIC_INIT(1);
 134
 135 /*
 136  * Allocate a cookie.
 137  */
 138 struct fscache_cookie *fscache_alloc_cookie(
 139         struct fscache_cookie *parent,
 140         const struct fscache_cookie_def *def,
 141         const void *index_key, size_t index_key_len,
 142         const void *aux_data, size_t aux_data_len,
 143         void *netfs_data,
 144         loff_t object_size)
 145 {
 146         struct fscache_cookie *cookie;
 147
 148         /* allocate and initialise a cookie */
 149         cookie = kmem_cache_zalloc(fscache_cookie_jar, GFP_KERNEL);
 150         if (!cookie)
 151                 return NULL;
 152
 153         cookie->key_len = index_key_len;
 154         cookie->aux_len = aux_data_len;
 155
 156         if (fscache_set_key(cookie, index_key, index_key_len) < 0)
 157                 goto nomem;
 158
 159         if (cookie->aux_len <= sizeof(cookie->inline_aux)) {
 160                 memcpy(cookie->inline_aux, aux_data, cookie->aux_len);
 161         } else {
 162                 cookie->aux = kmemdup(aux_data, cookie->aux_len, GFP_KERNEL);
 163                 if (!cookie->aux)
 164                         goto nomem;
 165         }
 166
 167         atomic_set(&cookie->usage, 1);
 168         atomic_set(&cookie->n_children, 0);
 169         cookie->debug_id = atomic_inc_return(&fscache_cookie_debug_id);
 170
 171         /* We keep the active count elevated until relinquishment to prevent an
 172          * attempt to wake up every time the object operations queue quiesces.
 173          */
 174         atomic_set(&cookie->n_active, 1);
 175
 176         cookie->def             = def;
 177         cookie->parent          = parent;
 178         cookie->netfs_data      = netfs_data;
 179         cookie->flags           = (1 << FSCACHE_COOKIE_NO_DATA_YET);
 180         cookie->type            = def->type;
 181         spin_lock_init(&cookie->lock);
 182         spin_lock_init(&cookie->stores_lock);
 183         INIT_HLIST_HEAD(&cookie->backing_objects);
 184
 185         /* radix tree insertion won't use the preallocation pool unless it's
 186          * told it may not wait */
 187         INIT_RADIX_TREE(&cookie->stores, GFP_NOFS & ~__GFP_DIRECT_RECLAIM);
 188
 189         write_lock(&fscache_cookies_lock);
 190         list_add_tail(&cookie->proc_link, &fscache_cookies);
 191         write_unlock(&fscache_cookies_lock);
 192         return cookie;
 193
 194 nomem:
 195         fscache_free_cookie(cookie);
 196         return NULL;
 197 }
 198
 199 /*
 200  * Attempt to insert the new cookie into the hash.  If there's a collision, we
 201  * return the old cookie if it's not in use and an error otherwise.
 202  */
 203 struct fscache_cookie *fscache_hash_cookie(struct fscache_cookie *candidate)
 204 {
 205         struct fscache_cookie *cursor;
 206         struct hlist_bl_head *h;
 207         struct hlist_bl_node *p;
 208         unsigned int bucket;
 209
 210         bucket = candidate->key_hash & (ARRAY_SIZE(fscache_cookie_hash) - 1);
 211         h = &fscache_cookie_hash[bucket];
 212
 213         hlist_bl_lock(h);
 214         hlist_bl_for_each_entry(cursor, p, h, hash_link) {
 215                 if (fscache_compare_cookie(candidate, cursor) == 0)
 216                         goto collision;
 217         }
 218
 219         __set_bit(FSCACHE_COOKIE_ACQUIRED, &candidate->flags);
 220         fscache_cookie_get(candidate->parent, fscache_cookie_get_acquire_parent);
 221         atomic_inc(&candidate->parent->n_children);
 222         hlist_bl_add_head(&candidate->hash_link, h);
 223         hlist_bl_unlock(h);
 224         return candidate;
 225
 226 collision:
 227         if (test_and_set_bit(FSCACHE_COOKIE_ACQUIRED, &cursor->flags)) {
 228                 trace_fscache_cookie(cursor->debug_id, atomic_read(&cursor->usage),
 229                                      fscache_cookie_collision);
 230                 pr_err("Duplicate cookie detected\n");
 231                 fscache_print_cookie(cursor, 'O');
 232                 fscache_print_cookie(candidate, 'N');
 233                 hlist_bl_unlock(h);
 234                 return NULL;
 235         }
 236
 237         fscache_cookie_get(cursor, fscache_cookie_get_reacquire);
 238         hlist_bl_unlock(h);
 239         return cursor;
 240 }
 241
 242 /*
 243  * request a cookie to represent an object (index, datafile, xattr, etc)
 244  * - parent specifies the parent object
 245  *   - the top level index cookie for each netfs is stored in the fscache_netfs
 246  *     struct upon registration
 247  * - def points to the definition
 248  * - the netfs_data will be passed to the functions pointed to in *def
 249  * - all attached caches will be searched to see if they contain this object
 250  * - index objects aren't stored on disk until there's a dependent file that
 251  *   needs storing
 252  * - other objects are stored in a selected cache immediately, and all the
 253  *   indices forming the path to it are instantiated if necessary
 254  * - we never let on to the netfs about errors
 255  *   - we may set a negative cookie pointer, but that's okay
 256  */
 257 struct fscache_cookie *__fscache_acquire_cookie(
 258         struct fscache_cookie *parent,
 259         const struct fscache_cookie_def *def,
 260         const void *index_key, size_t index_key_len,
 261         const void *aux_data, size_t aux_data_len,
 262         void *netfs_data,
 263         loff_t object_size,
 264         bool enable)
 265 {
 266         struct fscache_cookie *candidate, *cookie;
 267
 268         BUG_ON(!def);
 269
 270         _enter("{%s},{%s},%p,%u",
 271                parent ? (char *) parent->def->name : "<no-parent>",
 272                def->name, netfs_data, enable);
 273
 274         if (!index_key || !index_key_len || index_key_len > 255 || aux_data_len > 255)
 275                 return NULL;
 276         if (!aux_data || !aux_data_len) {
 277                 aux_data = NULL;
 278                 aux_data_len = 0;
 279         }
 280
 281         fscache_stat(&fscache_n_acquires);
 282
 283         /* if there's no parent cookie, then we don't create one here either */
 284         if (!parent) {
 285                 fscache_stat(&fscache_n_acquires_null);
 286                 _leave(" [no parent]");
 287                 return NULL;
 288         }
 289
 290         /* validate the definition */
 291         BUG_ON(!def->name[0]);
 292
 293         BUG_ON(def->type == FSCACHE_COOKIE_TYPE_INDEX &&
 294                parent->type != FSCACHE_COOKIE_TYPE_INDEX);
 295
 296         candidate = fscache_alloc_cookie(parent, def,
 297                                          index_key, index_key_len,
 298                                          aux_data, aux_data_len,
 299                                          netfs_data, object_size);
 300         if (!candidate) {
 301                 fscache_stat(&fscache_n_acquires_oom);
 302                 _leave(" [ENOMEM]");
 303                 return NULL;
 304         }
 305
 306         cookie = fscache_hash_cookie(candidate);
 307         if (!cookie) {
 308                 trace_fscache_cookie(candidate->debug_id, 1,
 309                                      fscache_cookie_discard);
 310                 goto out;
 311         }
 312
 313         if (cookie == candidate)
 314                 candidate = NULL;
 315
 316         switch (cookie->type) {
 317         case FSCACHE_COOKIE_TYPE_INDEX:
 318                 fscache_stat(&fscache_n_cookie_index);
 319                 break;
 320         case FSCACHE_COOKIE_TYPE_DATAFILE:
 321                 fscache_stat(&fscache_n_cookie_data);
 322                 break;
 323         default:
 324                 fscache_stat(&fscache_n_cookie_special);
 325                 break;
 326         }
 327
 328         trace_fscache_acquire(cookie);
 329
 330         if (enable) {
 331                 /* if the object is an index then we need do nothing more here
 332                  * - we create indices on disk when we need them as an index
 333                  * may exist in multiple caches */
 334                 if (cookie->type != FSCACHE_COOKIE_TYPE_INDEX) {
 335                         if (fscache_acquire_non_index_cookie(cookie, object_size) == 0) {
 336                                 set_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags);
 337                         } else {
 338                                 atomic_dec(&parent->n_children);
 339                                 fscache_cookie_put(cookie,
 340                                                    fscache_cookie_put_acquire_nobufs);
 341                                 fscache_stat(&fscache_n_acquires_nobufs);
 342                                 _leave(" = NULL");
 343                                 return NULL;
 344                         }
 345                 } else {
 346                         set_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags);
 347                 }
 348         }
 349
 350         fscache_stat(&fscache_n_acquires_ok);
 351
 352 out:
 353         fscache_free_cookie(candidate);
 354         return cookie;
 355 }
 356 EXPORT_SYMBOL(__fscache_acquire_cookie);
 357
 358 /*
 359  * Enable a cookie to permit it to accept new operations.
 360  */
 361 void __fscache_enable_cookie(struct fscache_cookie *cookie,
 362                              const void *aux_data,
 363                              loff_t object_size,
 364                              bool (*can_enable)(void *data),
 365                              void *data)
 366 {
 367         _enter("%x", cookie->debug_id);
 368
 369         trace_fscache_enable(cookie);
 370
 371         wait_on_bit_lock(&cookie->flags, FSCACHE_COOKIE_ENABLEMENT_LOCK,
 372                          TASK_UNINTERRUPTIBLE);
 373
 374         fscache_update_aux(cookie, aux_data);
 375
 376         if (test_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags))
 377                 goto out_unlock;
 378
 379         if (can_enable && !can_enable(data)) {
 380                 /* The netfs decided it didn't want to enable after all */
 381         } else if (cookie->type != FSCACHE_COOKIE_TYPE_INDEX) {
 382                 /* Wait for outstanding disablement to complete */
 383                 __fscache_wait_on_invalidate(cookie);
 384
 385                 if (fscache_acquire_non_index_cookie(cookie, object_size) == 0)
 386                         set_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags);
 387         } else {
 388                 set_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags);
 389         }
 390
 391 out_unlock:
 392         clear_bit_unlock(FSCACHE_COOKIE_ENABLEMENT_LOCK, &cookie->flags);
 393         wake_up_bit(&cookie->flags, FSCACHE_COOKIE_ENABLEMENT_LOCK);
 394 }
 395 EXPORT_SYMBOL(__fscache_enable_cookie);
 396
 397 /*
 398  * acquire a non-index cookie
 399  * - this must make sure the index chain is instantiated and instantiate the
 400  *   object representation too
 401  */
 402 static int fscache_acquire_non_index_cookie(struct fscache_cookie *cookie,
 403                                             loff_t object_size)
 404 {
 405         struct fscache_object *object;
 406         struct fscache_cache *cache;
 407         int ret;
 408
 409         _enter("");
 410
 411         set_bit(FSCACHE_COOKIE_UNAVAILABLE, &cookie->flags);
 412
 413         /* now we need to see whether the backing objects for this cookie yet
 414          * exist, if not there'll be nothing to search */
 415         down_read(&fscache_addremove_sem);
 416
 417         if (list_empty(&fscache_cache_list)) {
 418                 up_read(&fscache_addremove_sem);
 419                 _leave(" = 0 [no caches]");
 420                 return 0;
 421         }
 422
 423         /* select a cache in which to store the object */
 424         cache = fscache_select_cache_for_object(cookie->parent);
 425         if (!cache) {
 426                 up_read(&fscache_addremove_sem);
 427                 fscache_stat(&fscache_n_acquires_no_cache);
 428                 _leave(" = -ENOMEDIUM [no cache]");
 429                 return -ENOMEDIUM;
 430         }
 431
 432         _debug("cache %s", cache->tag->name);
 433
 434         set_bit(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags);
 435
 436         /* ask the cache to allocate objects for this cookie and its parent
 437          * chain */
 438         ret = fscache_alloc_object(cache, cookie);
 439         if (ret < 0) {
 440                 up_read(&fscache_addremove_sem);
 441                 _leave(" = %d", ret);
 442                 return ret;
 443         }
 444
 445         spin_lock(&cookie->lock);
 446         if (hlist_empty(&cookie->backing_objects)) {
 447                 spin_unlock(&cookie->lock);
 448                 goto unavailable;
 449         }
 450
 451         object = hlist_entry(cookie->backing_objects.first,
 452                              struct fscache_object, cookie_link);
 453
 454         fscache_set_store_limit(object, object_size);
 455
 456         /* initiate the process of looking up all the objects in the chain
 457          * (done by fscache_initialise_object()) */
 458         fscache_raise_event(object, FSCACHE_OBJECT_EV_NEW_CHILD);
 459
 460         spin_unlock(&cookie->lock);
 461
 462         /* we may be required to wait for lookup to complete at this point */
 463         if (!fscache_defer_lookup) {
 464                 wait_on_bit(&cookie->flags, FSCACHE_COOKIE_LOOKING_UP,
 465                             TASK_UNINTERRUPTIBLE);
 466                 if (test_bit(FSCACHE_COOKIE_UNAVAILABLE, &cookie->flags))
 467                         goto unavailable;
 468         }
 469
 470         up_read(&fscache_addremove_sem);
 471         _leave(" = 0 [deferred]");
 472         return 0;
 473
 474 unavailable:
 475         up_read(&fscache_addremove_sem);
 476         _leave(" = -ENOBUFS");
 477         return -ENOBUFS;
 478 }
 479
 480 /*
 481  * recursively allocate cache object records for a cookie/cache combination
 482  * - caller must be holding the addremove sem
 483  */
 484 static int fscache_alloc_object(struct fscache_cache *cache,
 485                                 struct fscache_cookie *cookie)
 486 {
 487         struct fscache_object *object;
 488         int ret;
 489
 490         _enter("%s,%x{%s}", cache->tag->name, cookie->debug_id, cookie->def->name);
 491
 492         spin_lock(&cookie->lock);
 493         hlist_for_each_entry(object, &cookie->backing_objects,
 494                              cookie_link) {
 495                 if (object->cache == cache)
 496                         goto object_already_extant;
 497         }
 498         spin_unlock(&cookie->lock);
 499
 500         /* ask the cache to allocate an object (we may end up with duplicate
 501          * objects at this stage, but we sort that out later) */
 502         fscache_stat(&fscache_n_cop_alloc_object);
 503         object = cache->ops->alloc_object(cache, cookie);
 504         fscache_stat_d(&fscache_n_cop_alloc_object);
 505         if (IS_ERR(object)) {
 506                 fscache_stat(&fscache_n_object_no_alloc);
 507                 ret = PTR_ERR(object);
 508                 goto error;
 509         }
 510
 511         ASSERTCMP(object->cookie, ==, cookie);
 512         fscache_stat(&fscache_n_object_alloc);
 513
 514         object->debug_id = atomic_inc_return(&fscache_object_debug_id);
 515
 516         _debug("ALLOC OBJ%x: %s {%lx}",
 517                object->debug_id, cookie->def->name, object->events);
 518
 519         ret = fscache_alloc_object(cache, cookie->parent);
 520         if (ret < 0)
 521                 goto error_put;
 522
 523         /* only attach if we managed to allocate all we needed, otherwise
 524          * discard the object we just allocated and instead use the one
 525          * attached to the cookie */
 526         if (fscache_attach_object(cookie, object) < 0) {
 527                 fscache_stat(&fscache_n_cop_put_object);
 528                 cache->ops->put_object(object, fscache_obj_put_attach_fail);
 529                 fscache_stat_d(&fscache_n_cop_put_object);
 530         }
 531
 532         _leave(" = 0");
 533         return 0;
 534
 535 object_already_extant:
 536         ret = -ENOBUFS;
 537         if (fscache_object_is_dying(object) ||
 538             fscache_cache_is_broken(object)) {
 539                 spin_unlock(&cookie->lock);
 540                 goto error;
 541         }
 542         spin_unlock(&cookie->lock);
 543         _leave(" = 0 [found]");
 544         return 0;
 545
 546 error_put:
 547         fscache_stat(&fscache_n_cop_put_object);
 548         cache->ops->put_object(object, fscache_obj_put_alloc_fail);
 549         fscache_stat_d(&fscache_n_cop_put_object);
 550 error:
 551         _leave(" = %d", ret);
 552         return ret;
 553 }
 554
 555 /*
 556  * attach a cache object to a cookie
 557  */
 558 static int fscache_attach_object(struct fscache_cookie *cookie,
 559                                  struct fscache_object *object)
 560 {
 561         struct fscache_object *p;
 562         struct fscache_cache *cache = object->cache;
 563         int ret;
 564
 565         _enter("{%s},{OBJ%x}", cookie->def->name, object->debug_id);
 566
 567         ASSERTCMP(object->cookie, ==, cookie);
 568
 569         spin_lock(&cookie->lock);
 570
 571         /* there may be multiple initial creations of this object, but we only
 572          * want one */
 573         ret = -EEXIST;
 574         hlist_for_each_entry(p, &cookie->backing_objects, cookie_link) {
 575                 if (p->cache == object->cache) {
 576                         if (fscache_object_is_dying(p))
 577                                 ret = -ENOBUFS;
 578                         goto cant_attach_object;
 579                 }
 580         }
 581
 582         /* pin the parent object */
 583         spin_lock_nested(&cookie->parent->lock, 1);
 584         hlist_for_each_entry(p, &cookie->parent->backing_objects,
 585                              cookie_link) {
 586                 if (p->cache == object->cache) {
 587                         if (fscache_object_is_dying(p)) {
 588                                 ret = -ENOBUFS;
 589                                 spin_unlock(&cookie->parent->lock);
 590                                 goto cant_attach_object;
 591                         }
 592                         object->parent = p;
 593                         spin_lock(&p->lock);
 594                         p->n_children++;
 595                         spin_unlock(&p->lock);
 596                         break;
 597                 }
 598         }
 599         spin_unlock(&cookie->parent->lock);
 600
 601         /* attach to the cache's object list */
 602         if (list_empty(&object->cache_link)) {
 603                 spin_lock(&cache->object_list_lock);
 604                 list_add(&object->cache_link, &cache->object_list);
 605                 spin_unlock(&cache->object_list_lock);
 606         }
 607
 608         /* Attach to the cookie.  The object already has a ref on it. */
 609         hlist_add_head(&object->cookie_link, &cookie->backing_objects);
 610         ret = 0;
 611
 612 cant_attach_object:
 613         spin_unlock(&cookie->lock);
 614         _leave(" = %d", ret);
 615         return ret;
 616 }
 617
 618 /*
 619  * Invalidate an object.  Callable with spinlocks held.
 620  */
 621 void __fscache_invalidate(struct fscache_cookie *cookie)
 622 {
 623         struct fscache_object *object;
 624
 625         _enter("{%s}", cookie->def->name);
 626
 627         fscache_stat(&fscache_n_invalidates);
 628
 629         /* Only permit invalidation of data files.  Invalidating an index will
 630          * require the caller to release all its attachments to the tree rooted
 631          * there, and if it's doing that, it may as well just retire the
 632          * cookie.
 633          */
 634         ASSERTCMP(cookie->type, ==, FSCACHE_COOKIE_TYPE_DATAFILE);
 635
 636         /* If there's an object, we tell the object state machine to handle the
 637          * invalidation on our behalf, otherwise there's nothing to do.
 638          */
 639         if (!hlist_empty(&cookie->backing_objects)) {
 640                 spin_lock(&cookie->lock);
 641
 642                 if (fscache_cookie_enabled(cookie) &&
 643                     !hlist_empty(&cookie->backing_objects) &&
 644                     !test_and_set_bit(FSCACHE_COOKIE_INVALIDATING,
 645                                       &cookie->flags)) {
 646                         object = hlist_entry(cookie->backing_objects.first,
 647                                              struct fscache_object,
 648                                              cookie_link);
 649                         if (fscache_object_is_live(object))
 650                                 fscache_raise_event(
 651                                         object, FSCACHE_OBJECT_EV_INVALIDATE);
 652                 }
 653
 654                 spin_unlock(&cookie->lock);
 655         }
 656
 657         _leave("");
 658 }
 659 EXPORT_SYMBOL(__fscache_invalidate);
 660
 661 /*
 662  * Wait for object invalidation to complete.
 663  */
 664 void __fscache_wait_on_invalidate(struct fscache_cookie *cookie)
 665 {
 666         _enter("%x", cookie->debug_id);
 667
 668         wait_on_bit(&cookie->flags, FSCACHE_COOKIE_INVALIDATING,
 669                     TASK_UNINTERRUPTIBLE);
 670
 671         _leave("");
 672 }
 673 EXPORT_SYMBOL(__fscache_wait_on_invalidate);
 674
 675 /*
 676  * update the index entries backing a cookie
 677  */
 678 void __fscache_update_cookie(struct fscache_cookie *cookie, const void *aux_data)
 679 {
 680         struct fscache_object *object;
 681
 682         fscache_stat(&fscache_n_updates);
 683
 684         if (!cookie) {
 685                 fscache_stat(&fscache_n_updates_null);
 686                 _leave(" [no cookie]");
 687                 return;
 688         }
 689
 690         _enter("{%s}", cookie->def->name);
 691
 692         spin_lock(&cookie->lock);
 693
 694         fscache_update_aux(cookie, aux_data);
 695
 696         if (fscache_cookie_enabled(cookie)) {
 697                 /* update the index entry on disk in each cache backing this
 698                  * cookie.
 699                  */
 700                 hlist_for_each_entry(object,
 701                                      &cookie->backing_objects, cookie_link) {
 702                         fscache_raise_event(object, FSCACHE_OBJECT_EV_UPDATE);
 703                 }
 704         }
 705
 706         spin_unlock(&cookie->lock);
 707         _leave("");
 708 }
 709 EXPORT_SYMBOL(__fscache_update_cookie);
 710
 711 /*
 712  * Disable a cookie to stop it from accepting new requests from the netfs.
 713  */
 714 void __fscache_disable_cookie(struct fscache_cookie *cookie,
 715                               const void *aux_data,
 716                               bool invalidate)
 717 {
 718         struct fscache_object *object;
 719         bool awaken = false;
 720
 721         _enter("%x,%u", cookie->debug_id, invalidate);
 722
 723         trace_fscache_disable(cookie);
 724
 725         ASSERTCMP(atomic_read(&cookie->n_active), >, 0);
 726
 727         if (atomic_read(&cookie->n_children) != 0) {
 728                 pr_err("Cookie '%s' still has children\n",
 729                        cookie->def->name);
 730                 BUG();
 731         }
 732
 733         wait_on_bit_lock(&cookie->flags, FSCACHE_COOKIE_ENABLEMENT_LOCK,
 734                          TASK_UNINTERRUPTIBLE);
 735
 736         fscache_update_aux(cookie, aux_data);
 737
 738         if (!test_and_clear_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags))
 739                 goto out_unlock_enable;
 740
 741         /* If the cookie is being invalidated, wait for that to complete first
 742          * so that we can reuse the flag.
 743          */
 744         __fscache_wait_on_invalidate(cookie);
 745
 746         /* Dispose of the backing objects */
 747         set_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags);
 748
 749         spin_lock(&cookie->lock);
 750         if (!hlist_empty(&cookie->backing_objects)) {
 751                 hlist_for_each_entry(object, &cookie->backing_objects, cookie_link) {
 752                         if (invalidate)
 753                                 set_bit(FSCACHE_OBJECT_RETIRED, &object->flags);
 754                         clear_bit(FSCACHE_OBJECT_PENDING_WRITE, &object->flags);
 755                         fscache_raise_event(object, FSCACHE_OBJECT_EV_KILL);
 756                 }
 757         } else {
 758                 if (test_and_clear_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags))
 759                         awaken = true;
 760         }
 761         spin_unlock(&cookie->lock);
 762         if (awaken)
 763                 wake_up_bit(&cookie->flags, FSCACHE_COOKIE_INVALIDATING);
 764
 765         /* Wait for cessation of activity requiring access to the netfs (when
 766          * n_active reaches 0).  This makes sure outstanding reads and writes
 767          * have completed.
 768          */
 769         if (!atomic_dec_and_test(&cookie->n_active)) {
 770                 wait_var_event(&cookie->n_active,
 771                                !atomic_read(&cookie->n_active));
 772         }
 773
 774         /* Make sure any pending writes are cancelled. */
 775         if (cookie->type != FSCACHE_COOKIE_TYPE_INDEX)
 776                 fscache_invalidate_writes(cookie);
 777
 778         /* Reset the cookie state if it wasn't relinquished */
 779         if (!test_bit(FSCACHE_COOKIE_RELINQUISHED, &cookie->flags)) {
 780                 atomic_inc(&cookie->n_active);
 781                 set_bit(FSCACHE_COOKIE_NO_DATA_YET, &cookie->flags);
 782         }
 783
 784 out_unlock_enable:
 785         clear_bit_unlock(FSCACHE_COOKIE_ENABLEMENT_LOCK, &cookie->flags);
 786         wake_up_bit(&cookie->flags, FSCACHE_COOKIE_ENABLEMENT_LOCK);
 787         _leave("");
 788 }
 789 EXPORT_SYMBOL(__fscache_disable_cookie);
 790
 791 /*
 792  * release a cookie back to the cache
 793  * - the object will be marked as recyclable on disk if retire is true
 794  * - all dependents of this cookie must have already been unregistered
 795  *   (indices/files/pages)
 796  */
 797 void __fscache_relinquish_cookie(struct fscache_cookie *cookie,
 798                                  const void *aux_data,
 799                                  bool retire)
 800 {
 801         fscache_stat(&fscache_n_relinquishes);
 802         if (retire)
 803                 fscache_stat(&fscache_n_relinquishes_retire);
 804
 805         if (!cookie) {
 806                 fscache_stat(&fscache_n_relinquishes_null);
 807                 _leave(" [no cookie]");
 808                 return;
 809         }
 810
 811         _enter("%x{%s,%d},%d",
 812                cookie->debug_id, cookie->def->name,
 813                atomic_read(&cookie->n_active), retire);
 814
 815         trace_fscache_relinquish(cookie, retire);
 816
 817         /* No further netfs-accessing operations on this cookie permitted */
 818         if (test_and_set_bit(FSCACHE_COOKIE_RELINQUISHED, &cookie->flags))
 819                 BUG();
 820
 821         __fscache_disable_cookie(cookie, aux_data, retire);
 822
 823         /* Clear pointers back to the netfs */
 824         cookie->netfs_data      = NULL;
 825         cookie->def             = NULL;
 826         BUG_ON(!radix_tree_empty(&cookie->stores));
 827
 828         if (cookie->parent) {
 829                 ASSERTCMP(atomic_read(&cookie->parent->usage), >, 0);
 830                 ASSERTCMP(atomic_read(&cookie->parent->n_children), >, 0);
 831                 atomic_dec(&cookie->parent->n_children);
 832         }
 833
 834         /* Dispose of the netfs's link to the cookie */
 835         ASSERTCMP(atomic_read(&cookie->usage), >, 0);
 836         fscache_cookie_put(cookie, fscache_cookie_put_relinquish);
 837
 838         _leave("");
 839 }
 840 EXPORT_SYMBOL(__fscache_relinquish_cookie);
 841
 842 /*
 843  * Remove a cookie from the hash table.
 844  */
 845 static void fscache_unhash_cookie(struct fscache_cookie *cookie)
 846 {
 847         struct hlist_bl_head *h;
 848         unsigned int bucket;
 849
 850         bucket = cookie->key_hash & (ARRAY_SIZE(fscache_cookie_hash) - 1);
 851         h = &fscache_cookie_hash[bucket];
 852
 853         hlist_bl_lock(h);
 854         hlist_bl_del(&cookie->hash_link);
 855         hlist_bl_unlock(h);
 856 }
 857
 858 /*
 859  * Drop a reference to a cookie.
 860  */
 861 void fscache_cookie_put(struct fscache_cookie *cookie,
 862                         enum fscache_cookie_trace where)
 863 {
 864         struct fscache_cookie *parent;
 865         int usage;
 866
 867         _enter("%x", cookie->debug_id);
 868
 869         do {
 870                 unsigned int cookie_debug_id = cookie->debug_id;
 871                 usage = atomic_dec_return(&cookie->usage);
 872                 trace_fscache_cookie(cookie_debug_id, usage, where);
 873
 874                 if (usage > 0)
 875                         return;
 876                 BUG_ON(usage < 0);
 877
 878                 parent = cookie->parent;
 879                 fscache_unhash_cookie(cookie);
 880                 fscache_free_cookie(cookie);
 881
 882                 cookie = parent;
 883                 where = fscache_cookie_put_parent;
 884         } while (cookie);
 885
 886         _leave("");
 887 }
 888
 889 /*
 890  * check the consistency between the netfs inode and the backing cache
 891  *
 892  * NOTE: it only serves no-index type
 893  */
 894 int __fscache_check_consistency(struct fscache_cookie *cookie,
 895                                 const void *aux_data)
 896 {
 897         struct fscache_operation *op;
 898         struct fscache_object *object;
 899         bool wake_cookie = false;
 900         int ret;
 901
 902         _enter("%p,", cookie);
 903
 904         ASSERTCMP(cookie->type, ==, FSCACHE_COOKIE_TYPE_DATAFILE);
 905
 906         if (fscache_wait_for_deferred_lookup(cookie) < 0)
 907                 return -ERESTARTSYS;
 908
 909         if (hlist_empty(&cookie->backing_objects))
 910                 return 0;
 911
 912         op = kzalloc(sizeof(*op), GFP_NOIO | __GFP_NOMEMALLOC | __GFP_NORETRY);
 913         if (!op)
 914                 return -ENOMEM;
 915
 916         fscache_operation_init(cookie, op, NULL, NULL, NULL);
 917         op->flags = FSCACHE_OP_MYTHREAD |
 918                 (1 << FSCACHE_OP_WAITING) |
 919                 (1 << FSCACHE_OP_UNUSE_COOKIE);
 920         trace_fscache_page_op(cookie, NULL, op, fscache_page_op_check_consistency);
 921
 922         spin_lock(&cookie->lock);
 923
 924         fscache_update_aux(cookie, aux_data);
 925
 926         if (!fscache_cookie_enabled(cookie) ||
 927             hlist_empty(&cookie->backing_objects))
 928                 goto inconsistent;
 929         object = hlist_entry(cookie->backing_objects.first,
 930                              struct fscache_object, cookie_link);
 931         if (test_bit(FSCACHE_IOERROR, &object->cache->flags))
 932                 goto inconsistent;
 933
 934         op->debug_id = atomic_inc_return(&fscache_op_debug_id);
 935
 936         __fscache_use_cookie(cookie);
 937         if (fscache_submit_op(object, op) < 0)
 938                 goto submit_failed;
 939
 940         /* the work queue now carries its own ref on the object */
 941         spin_unlock(&cookie->lock);
 942
 943         ret = fscache_wait_for_operation_activation(object, op, NULL, NULL);
 944         if (ret == 0) {
 945                 /* ask the cache to honour the operation */
 946                 ret = object->cache->ops->check_consistency(op);
 947                 fscache_op_complete(op, false);
 948         } else if (ret == -ENOBUFS) {
 949                 ret = 0;
 950         }
 951
 952         fscache_put_operation(op);
 953         _leave(" = %d", ret);
 954         return ret;
 955
 956 submit_failed:
 957         wake_cookie = __fscache_unuse_cookie(cookie);
 958 inconsistent:
 959         spin_unlock(&cookie->lock);
 960         if (wake_cookie)
 961                 __fscache_wake_unused_cookie(cookie);
 962         kfree(op);
 963         _leave(" = -ESTALE");
 964         return -ESTALE;
 965 }
 966 EXPORT_SYMBOL(__fscache_check_consistency);
 967
 968 /*
 969  * Generate a list of extant cookies in /proc/fs/fscache/cookies
 970  */
 971 static int fscache_cookies_seq_show(struct seq_file *m, void *v)
 972 {
 973         struct fscache_cookie *cookie;
 974         unsigned int keylen = 0, auxlen = 0;
 975         char _type[3], *type;
 976         u8 *p;
 977
 978         if (v == &fscache_cookies) {
 979                 seq_puts(m,
 980                          "COOKIE   PARENT   USAGE CHILD ACT TY FL  DEF              NETFS_DATA\n"
 981                          "======== ======== ===== ===== === == === ================ ==========\n"
 982                          );
 983                 return 0;
 984         }
 985
 986         cookie = list_entry(v, struct fscache_cookie, proc_link);
 987
 988         switch (cookie->type) {
 989         case 0:
 990                 type = "IX";
 991                 break;
 992         case 1:
 993                 type = "DT";
 994                 break;
 995         default:
 996                 snprintf(_type, sizeof(_type), "%02u",
 997                          cookie->type);
 998                 type = _type;
 999                 break;
1000         }
1001
1002         seq_printf(m,
1003                    "%08x %08x %5u %5u %3u %s %03lx %-16s %px",
1004                    cookie->debug_id,
1005                    cookie->parent ? cookie->parent->debug_id : 0,
1006                    atomic_read(&cookie->usage),
1007                    atomic_read(&cookie->n_children),
1008                    atomic_read(&cookie->n_active),
1009                    type,
1010                    cookie->flags,
1011                    cookie->def->name,
1012                    cookie->netfs_data);
1013
1014         keylen = cookie->key_len;
1015         auxlen = cookie->aux_len;
1016
1017         if (keylen > 0 || auxlen > 0) {
1018                 seq_puts(m, " ");
1019                 p = keylen <= sizeof(cookie->inline_key) ?
1020                         cookie->inline_key : cookie->key;
1021                 for (; keylen > 0; keylen--)
1022                         seq_printf(m, "%02x", *p++);
1023                 if (auxlen > 0) {
1024                         seq_puts(m, ", ");
1025                         p = auxlen <= sizeof(cookie->inline_aux) ?
1026                                 cookie->inline_aux : cookie->aux;
1027                         for (; auxlen > 0; auxlen--)
1028                                 seq_printf(m, "%02x", *p++);
1029                 }
1030         }
1031
1032         seq_puts(m, "\n");
1033         return 0;
1034 }
1035
1036 static void *fscache_cookies_seq_start(struct seq_file *m, loff_t *_pos)
1037         __acquires(fscache_cookies_lock)
1038 {
1039         read_lock(&fscache_cookies_lock);
1040         return seq_list_start_head(&fscache_cookies, *_pos);
1041 }
1042
1043 static void *fscache_cookies_seq_next(struct seq_file *m, void *v, loff_t *_pos)
1044 {
1045         return seq_list_next(v, &fscache_cookies, _pos);
1046 }
1047
1048 static void fscache_cookies_seq_stop(struct seq_file *m, void *v)
1049         __releases(rcu)
1050 {
1051         read_unlock(&fscache_cookies_lock);
1052 }
1053
1054
1055 const struct seq_operations fscache_cookies_seq_ops = {
1056         .start  = fscache_cookies_seq_start,
1057         .next   = fscache_cookies_seq_next,
1058         .stop   = fscache_cookies_seq_stop,
1059         .show   = fscache_cookies_seq_show,
1060 };