fs/fscache/cookie.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /* netfs cookie management
   3  *
   4  * Copyright (C) 2004-2007 Red Hat, Inc. All Rights Reserved.
   5  * Written by David Howells (dhowells@redhat.com)
   6  *
   7  * See Documentation/filesystems/caching/netfs-api.rst for more information on
   8  * the netfs API.
   9  */
  10
  11 #define FSCACHE_DEBUG_LEVEL COOKIE
  12 #include <linux/module.h>
  13 #include <linux/slab.h>
  14 #include "internal.h"
  15
  16 struct kmem_cache *fscache_cookie_jar;
  17
  18 static atomic_t fscache_object_debug_id = ATOMIC_INIT(0);
  19
  20 #define fscache_cookie_hash_shift 15
  21 static struct hlist_bl_head fscache_cookie_hash[1 << fscache_cookie_hash_shift];
  22
  23 static int fscache_acquire_non_index_cookie(struct fscache_cookie *cookie,
  24                                             loff_t object_size);
  25 static int fscache_alloc_object(struct fscache_cache *cache,
  26                                 struct fscache_cookie *cookie);
  27 static int fscache_attach_object(struct fscache_cookie *cookie,
  28                                  struct fscache_object *object);
  29
  30 static void fscache_print_cookie(struct fscache_cookie *cookie, char prefix)
  31 {
  32         struct hlist_node *object;
  33         const u8 *k;
  34         unsigned loop;
  35
  36         pr_err("%c-cookie c=%p [p=%p fl=%lx nc=%u na=%u]\n",
  37                prefix, cookie, cookie->parent, cookie->flags,
  38                atomic_read(&cookie->n_children),
  39                atomic_read(&cookie->n_active));
  40         pr_err("%c-cookie d=%p n=%p\n",
  41                prefix, cookie->def, cookie->netfs_data);
  42
  43         object = READ_ONCE(cookie->backing_objects.first);
  44         if (object)
  45                 pr_err("%c-cookie o=%p\n",
  46                        prefix, hlist_entry(object, struct fscache_object, cookie_link));
  47
  48         pr_err("%c-key=[%u] '", prefix, cookie->key_len);
  49         k = (cookie->key_len <= sizeof(cookie->inline_key)) ?
  50                 cookie->inline_key : cookie->key;
  51         for (loop = 0; loop < cookie->key_len; loop++)
  52                 pr_cont("%02x", k[loop]);
  53         pr_cont("'\n");
  54 }
  55
  56 void fscache_free_cookie(struct fscache_cookie *cookie)
  57 {
  58         if (cookie) {
  59                 BUG_ON(!hlist_empty(&cookie->backing_objects));
  60                 if (cookie->aux_len > sizeof(cookie->inline_aux))
  61                         kfree(cookie->aux);
  62                 if (cookie->key_len > sizeof(cookie->inline_key))
  63                         kfree(cookie->key);
  64                 kmem_cache_free(fscache_cookie_jar, cookie);
  65         }
  66 }
  67
  68 /*
  69  * Set the index key in a cookie.  The cookie struct has space for a 16-byte
  70  * key plus length and hash, but if that's not big enough, it's instead a
  71  * pointer to a buffer containing 3 bytes of hash, 1 byte of length and then
  72  * the key data.
  73  */
  74 static int fscache_set_key(struct fscache_cookie *cookie,
  75                            const void *index_key, size_t index_key_len)
  76 {
  77         unsigned long long h;
  78         u32 *buf;
  79         int bufs;
  80         int i;
  81
  82         bufs = DIV_ROUND_UP(index_key_len, sizeof(*buf));
  83
  84         if (index_key_len > sizeof(cookie->inline_key)) {
  85                 buf = kcalloc(bufs, sizeof(*buf), GFP_KERNEL);
  86                 if (!buf)
  87                         return -ENOMEM;
  88                 cookie->key = buf;
  89         } else {
  90                 buf = (u32 *)cookie->inline_key;
  91         }
  92
  93         memcpy(buf, index_key, index_key_len);
  94
  95         /* Calculate a hash and combine this with the length in the first word
  96          * or first half word
  97          */
  98         h = (unsigned long)cookie->parent;
  99         h += index_key_len + cookie->type;
 100
 101         for (i = 0; i < bufs; i++)
 102                 h += buf[i];
 103
 104         cookie->key_hash = h ^ (h >> 32);
 105         return 0;
 106 }
 107
 108 static long fscache_compare_cookie(const struct fscache_cookie *a,
 109                                    const struct fscache_cookie *b)
 110 {
 111         const void *ka, *kb;
 112
 113         if (a->key_hash != b->key_hash)
 114                 return (long)a->key_hash - (long)b->key_hash;
 115         if (a->parent != b->parent)
 116                 return (long)a->parent - (long)b->parent;
 117         if (a->key_len != b->key_len)
 118                 return (long)a->key_len - (long)b->key_len;
 119         if (a->type != b->type)
 120                 return (long)a->type - (long)b->type;
 121
 122         if (a->key_len <= sizeof(a->inline_key)) {
 123                 ka = &a->inline_key;
 124                 kb = &b->inline_key;
 125         } else {
 126                 ka = a->key;
 127                 kb = b->key;
 128         }
 129         return memcmp(ka, kb, a->key_len);
 130 }
 131
 132 /*
 133  * Allocate a cookie.
 134  */
 135 struct fscache_cookie *fscache_alloc_cookie(
 136         struct fscache_cookie *parent,
 137         const struct fscache_cookie_def *def,
 138         const void *index_key, size_t index_key_len,
 139         const void *aux_data, size_t aux_data_len,
 140         void *netfs_data,
 141         loff_t object_size)
 142 {
 143         struct fscache_cookie *cookie;
 144
 145         /* allocate and initialise a cookie */
 146         cookie = kmem_cache_zalloc(fscache_cookie_jar, GFP_KERNEL);
 147         if (!cookie)
 148                 return NULL;
 149
 150         cookie->key_len = index_key_len;
 151         cookie->aux_len = aux_data_len;
 152
 153         if (fscache_set_key(cookie, index_key, index_key_len) < 0)
 154                 goto nomem;
 155
 156         if (cookie->aux_len <= sizeof(cookie->inline_aux)) {
 157                 memcpy(cookie->inline_aux, aux_data, cookie->aux_len);
 158         } else {
 159                 cookie->aux = kmemdup(aux_data, cookie->aux_len, GFP_KERNEL);
 160                 if (!cookie->aux)
 161                         goto nomem;
 162         }
 163
 164         atomic_set(&cookie->usage, 1);
 165         atomic_set(&cookie->n_children, 0);
 166
 167         /* We keep the active count elevated until relinquishment to prevent an
 168          * attempt to wake up every time the object operations queue quiesces.
 169          */
 170         atomic_set(&cookie->n_active, 1);
 171
 172         cookie->def             = def;
 173         cookie->parent          = parent;
 174         cookie->netfs_data      = netfs_data;
 175         cookie->flags           = (1 << FSCACHE_COOKIE_NO_DATA_YET);
 176         cookie->type            = def->type;
 177         spin_lock_init(&cookie->lock);
 178         spin_lock_init(&cookie->stores_lock);
 179         INIT_HLIST_HEAD(&cookie->backing_objects);
 180
 181         /* radix tree insertion won't use the preallocation pool unless it's
 182          * told it may not wait */
 183         INIT_RADIX_TREE(&cookie->stores, GFP_NOFS & ~__GFP_DIRECT_RECLAIM);
 184         return cookie;
 185
 186 nomem:
 187         fscache_free_cookie(cookie);
 188         return NULL;
 189 }
 190
 191 /*
 192  * Attempt to insert the new cookie into the hash.  If there's a collision, we
 193  * return the old cookie if it's not in use and an error otherwise.
 194  */
 195 struct fscache_cookie *fscache_hash_cookie(struct fscache_cookie *candidate)
 196 {
 197         struct fscache_cookie *cursor;
 198         struct hlist_bl_head *h;
 199         struct hlist_bl_node *p;
 200         unsigned int bucket;
 201
 202         bucket = candidate->key_hash & (ARRAY_SIZE(fscache_cookie_hash) - 1);
 203         h = &fscache_cookie_hash[bucket];
 204
 205         hlist_bl_lock(h);
 206         hlist_bl_for_each_entry(cursor, p, h, hash_link) {
 207                 if (fscache_compare_cookie(candidate, cursor) == 0)
 208                         goto collision;
 209         }
 210
 211         __set_bit(FSCACHE_COOKIE_ACQUIRED, &candidate->flags);
 212         fscache_cookie_get(candidate->parent, fscache_cookie_get_acquire_parent);
 213         atomic_inc(&candidate->parent->n_children);
 214         hlist_bl_add_head(&candidate->hash_link, h);
 215         hlist_bl_unlock(h);
 216         return candidate;
 217
 218 collision:
 219         if (test_and_set_bit(FSCACHE_COOKIE_ACQUIRED, &cursor->flags)) {
 220                 trace_fscache_cookie(cursor, fscache_cookie_collision,
 221                                      atomic_read(&cursor->usage));
 222                 pr_err("Duplicate cookie detected\n");
 223                 fscache_print_cookie(cursor, 'O');
 224                 fscache_print_cookie(candidate, 'N');
 225                 hlist_bl_unlock(h);
 226                 return NULL;
 227         }
 228
 229         fscache_cookie_get(cursor, fscache_cookie_get_reacquire);
 230         hlist_bl_unlock(h);
 231         return cursor;
 232 }
 233
 234 /*
 235  * request a cookie to represent an object (index, datafile, xattr, etc)
 236  * - parent specifies the parent object
 237  *   - the top level index cookie for each netfs is stored in the fscache_netfs
 238  *     struct upon registration
 239  * - def points to the definition
 240  * - the netfs_data will be passed to the functions pointed to in *def
 241  * - all attached caches will be searched to see if they contain this object
 242  * - index objects aren't stored on disk until there's a dependent file that
 243  *   needs storing
 244  * - other objects are stored in a selected cache immediately, and all the
 245  *   indices forming the path to it are instantiated if necessary
 246  * - we never let on to the netfs about errors
 247  *   - we may set a negative cookie pointer, but that's okay
 248  */
 249 struct fscache_cookie *__fscache_acquire_cookie(
 250         struct fscache_cookie *parent,
 251         const struct fscache_cookie_def *def,
 252         const void *index_key, size_t index_key_len,
 253         const void *aux_data, size_t aux_data_len,
 254         void *netfs_data,
 255         loff_t object_size,
 256         bool enable)
 257 {
 258         struct fscache_cookie *candidate, *cookie;
 259
 260         BUG_ON(!def);
 261
 262         _enter("{%s},{%s},%p,%u",
 263                parent ? (char *) parent->def->name : "<no-parent>",
 264                def->name, netfs_data, enable);
 265
 266         if (!index_key || !index_key_len || index_key_len > 255 || aux_data_len > 255)
 267                 return NULL;
 268         if (!aux_data || !aux_data_len) {
 269                 aux_data = NULL;
 270                 aux_data_len = 0;
 271         }
 272
 273         fscache_stat(&fscache_n_acquires);
 274
 275         /* if there's no parent cookie, then we don't create one here either */
 276         if (!parent) {
 277                 fscache_stat(&fscache_n_acquires_null);
 278                 _leave(" [no parent]");
 279                 return NULL;
 280         }
 281
 282         /* validate the definition */
 283         BUG_ON(!def->name[0]);
 284
 285         BUG_ON(def->type == FSCACHE_COOKIE_TYPE_INDEX &&
 286                parent->type != FSCACHE_COOKIE_TYPE_INDEX);
 287
 288         candidate = fscache_alloc_cookie(parent, def,
 289                                          index_key, index_key_len,
 290                                          aux_data, aux_data_len,
 291                                          netfs_data, object_size);
 292         if (!candidate) {
 293                 fscache_stat(&fscache_n_acquires_oom);
 294                 _leave(" [ENOMEM]");
 295                 return NULL;
 296         }
 297
 298         cookie = fscache_hash_cookie(candidate);
 299         if (!cookie) {
 300                 trace_fscache_cookie(candidate, fscache_cookie_discard, 1);
 301                 goto out;
 302         }
 303
 304         if (cookie == candidate)
 305                 candidate = NULL;
 306
 307         switch (cookie->type) {
 308         case FSCACHE_COOKIE_TYPE_INDEX:
 309                 fscache_stat(&fscache_n_cookie_index);
 310                 break;
 311         case FSCACHE_COOKIE_TYPE_DATAFILE:
 312                 fscache_stat(&fscache_n_cookie_data);
 313                 break;
 314         default:
 315                 fscache_stat(&fscache_n_cookie_special);
 316                 break;
 317         }
 318
 319         trace_fscache_acquire(cookie);
 320
 321         if (enable) {
 322                 /* if the object is an index then we need do nothing more here
 323                  * - we create indices on disk when we need them as an index
 324                  * may exist in multiple caches */
 325                 if (cookie->type != FSCACHE_COOKIE_TYPE_INDEX) {
 326                         if (fscache_acquire_non_index_cookie(cookie, object_size) == 0) {
 327                                 set_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags);
 328                         } else {
 329                                 atomic_dec(&parent->n_children);
 330                                 fscache_cookie_put(cookie,
 331                                                    fscache_cookie_put_acquire_nobufs);
 332                                 fscache_stat(&fscache_n_acquires_nobufs);
 333                                 _leave(" = NULL");
 334                                 return NULL;
 335                         }
 336                 } else {
 337                         set_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags);
 338                 }
 339         }
 340
 341         fscache_stat(&fscache_n_acquires_ok);
 342
 343 out:
 344         fscache_free_cookie(candidate);
 345         return cookie;
 346 }
 347 EXPORT_SYMBOL(__fscache_acquire_cookie);
 348
 349 /*
 350  * Enable a cookie to permit it to accept new operations.
 351  */
 352 void __fscache_enable_cookie(struct fscache_cookie *cookie,
 353                              const void *aux_data,
 354                              loff_t object_size,
 355                              bool (*can_enable)(void *data),
 356                              void *data)
 357 {
 358         _enter("%p", cookie);
 359
 360         trace_fscache_enable(cookie);
 361
 362         wait_on_bit_lock(&cookie->flags, FSCACHE_COOKIE_ENABLEMENT_LOCK,
 363                          TASK_UNINTERRUPTIBLE);
 364
 365         fscache_update_aux(cookie, aux_data);
 366
 367         if (test_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags))
 368                 goto out_unlock;
 369
 370         if (can_enable && !can_enable(data)) {
 371                 /* The netfs decided it didn't want to enable after all */
 372         } else if (cookie->type != FSCACHE_COOKIE_TYPE_INDEX) {
 373                 /* Wait for outstanding disablement to complete */
 374                 __fscache_wait_on_invalidate(cookie);
 375
 376                 if (fscache_acquire_non_index_cookie(cookie, object_size) == 0)
 377                         set_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags);
 378         } else {
 379                 set_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags);
 380         }
 381
 382 out_unlock:
 383         clear_bit_unlock(FSCACHE_COOKIE_ENABLEMENT_LOCK, &cookie->flags);
 384         wake_up_bit(&cookie->flags, FSCACHE_COOKIE_ENABLEMENT_LOCK);
 385 }
 386 EXPORT_SYMBOL(__fscache_enable_cookie);
 387
 388 /*
 389  * acquire a non-index cookie
 390  * - this must make sure the index chain is instantiated and instantiate the
 391  *   object representation too
 392  */
 393 static int fscache_acquire_non_index_cookie(struct fscache_cookie *cookie,
 394                                             loff_t object_size)
 395 {
 396         struct fscache_object *object;
 397         struct fscache_cache *cache;
 398         int ret;
 399
 400         _enter("");
 401
 402         set_bit(FSCACHE_COOKIE_UNAVAILABLE, &cookie->flags);
 403
 404         /* now we need to see whether the backing objects for this cookie yet
 405          * exist, if not there'll be nothing to search */
 406         down_read(&fscache_addremove_sem);
 407
 408         if (list_empty(&fscache_cache_list)) {
 409                 up_read(&fscache_addremove_sem);
 410                 _leave(" = 0 [no caches]");
 411                 return 0;
 412         }
 413
 414         /* select a cache in which to store the object */
 415         cache = fscache_select_cache_for_object(cookie->parent);
 416         if (!cache) {
 417                 up_read(&fscache_addremove_sem);
 418                 fscache_stat(&fscache_n_acquires_no_cache);
 419                 _leave(" = -ENOMEDIUM [no cache]");
 420                 return -ENOMEDIUM;
 421         }
 422
 423         _debug("cache %s", cache->tag->name);
 424
 425         set_bit(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags);
 426
 427         /* ask the cache to allocate objects for this cookie and its parent
 428          * chain */
 429         ret = fscache_alloc_object(cache, cookie);
 430         if (ret < 0) {
 431                 up_read(&fscache_addremove_sem);
 432                 _leave(" = %d", ret);
 433                 return ret;
 434         }
 435
 436         spin_lock(&cookie->lock);
 437         if (hlist_empty(&cookie->backing_objects)) {
 438                 spin_unlock(&cookie->lock);
 439                 goto unavailable;
 440         }
 441
 442         object = hlist_entry(cookie->backing_objects.first,
 443                              struct fscache_object, cookie_link);
 444
 445         fscache_set_store_limit(object, object_size);
 446
 447         /* initiate the process of looking up all the objects in the chain
 448          * (done by fscache_initialise_object()) */
 449         fscache_raise_event(object, FSCACHE_OBJECT_EV_NEW_CHILD);
 450
 451         spin_unlock(&cookie->lock);
 452
 453         /* we may be required to wait for lookup to complete at this point */
 454         if (!fscache_defer_lookup) {
 455                 _debug("non-deferred lookup %p", &cookie->flags);
 456                 wait_on_bit(&cookie->flags, FSCACHE_COOKIE_LOOKING_UP,
 457                             TASK_UNINTERRUPTIBLE);
 458                 _debug("complete");
 459                 if (test_bit(FSCACHE_COOKIE_UNAVAILABLE, &cookie->flags))
 460                         goto unavailable;
 461         }
 462
 463         up_read(&fscache_addremove_sem);
 464         _leave(" = 0 [deferred]");
 465         return 0;
 466
 467 unavailable:
 468         up_read(&fscache_addremove_sem);
 469         _leave(" = -ENOBUFS");
 470         return -ENOBUFS;
 471 }
 472
 473 /*
 474  * recursively allocate cache object records for a cookie/cache combination
 475  * - caller must be holding the addremove sem
 476  */
 477 static int fscache_alloc_object(struct fscache_cache *cache,
 478                                 struct fscache_cookie *cookie)
 479 {
 480         struct fscache_object *object;
 481         int ret;
 482
 483         _enter("%p,%p{%s}", cache, cookie, cookie->def->name);
 484
 485         spin_lock(&cookie->lock);
 486         hlist_for_each_entry(object, &cookie->backing_objects,
 487                              cookie_link) {
 488                 if (object->cache == cache)
 489                         goto object_already_extant;
 490         }
 491         spin_unlock(&cookie->lock);
 492
 493         /* ask the cache to allocate an object (we may end up with duplicate
 494          * objects at this stage, but we sort that out later) */
 495         fscache_stat(&fscache_n_cop_alloc_object);
 496         object = cache->ops->alloc_object(cache, cookie);
 497         fscache_stat_d(&fscache_n_cop_alloc_object);
 498         if (IS_ERR(object)) {
 499                 fscache_stat(&fscache_n_object_no_alloc);
 500                 ret = PTR_ERR(object);
 501                 goto error;
 502         }
 503
 504         ASSERTCMP(object->cookie, ==, cookie);
 505         fscache_stat(&fscache_n_object_alloc);
 506
 507         object->debug_id = atomic_inc_return(&fscache_object_debug_id);
 508
 509         _debug("ALLOC OBJ%x: %s {%lx}",
 510                object->debug_id, cookie->def->name, object->events);
 511
 512         ret = fscache_alloc_object(cache, cookie->parent);
 513         if (ret < 0)
 514                 goto error_put;
 515
 516         /* only attach if we managed to allocate all we needed, otherwise
 517          * discard the object we just allocated and instead use the one
 518          * attached to the cookie */
 519         if (fscache_attach_object(cookie, object) < 0) {
 520                 fscache_stat(&fscache_n_cop_put_object);
 521                 cache->ops->put_object(object, fscache_obj_put_attach_fail);
 522                 fscache_stat_d(&fscache_n_cop_put_object);
 523         }
 524
 525         _leave(" = 0");
 526         return 0;
 527
 528 object_already_extant:
 529         ret = -ENOBUFS;
 530         if (fscache_object_is_dying(object) ||
 531             fscache_cache_is_broken(object)) {
 532                 spin_unlock(&cookie->lock);
 533                 goto error;
 534         }
 535         spin_unlock(&cookie->lock);
 536         _leave(" = 0 [found]");
 537         return 0;
 538
 539 error_put:
 540         fscache_stat(&fscache_n_cop_put_object);
 541         cache->ops->put_object(object, fscache_obj_put_alloc_fail);
 542         fscache_stat_d(&fscache_n_cop_put_object);
 543 error:
 544         _leave(" = %d", ret);
 545         return ret;
 546 }
 547
 548 /*
 549  * attach a cache object to a cookie
 550  */
 551 static int fscache_attach_object(struct fscache_cookie *cookie,
 552                                  struct fscache_object *object)
 553 {
 554         struct fscache_object *p;
 555         struct fscache_cache *cache = object->cache;
 556         int ret;
 557
 558         _enter("{%s},{OBJ%x}", cookie->def->name, object->debug_id);
 559
 560         ASSERTCMP(object->cookie, ==, cookie);
 561
 562         spin_lock(&cookie->lock);
 563
 564         /* there may be multiple initial creations of this object, but we only
 565          * want one */
 566         ret = -EEXIST;
 567         hlist_for_each_entry(p, &cookie->backing_objects, cookie_link) {
 568                 if (p->cache == object->cache) {
 569                         if (fscache_object_is_dying(p))
 570                                 ret = -ENOBUFS;
 571                         goto cant_attach_object;
 572                 }
 573         }
 574
 575         /* pin the parent object */
 576         spin_lock_nested(&cookie->parent->lock, 1);
 577         hlist_for_each_entry(p, &cookie->parent->backing_objects,
 578                              cookie_link) {
 579                 if (p->cache == object->cache) {
 580                         if (fscache_object_is_dying(p)) {
 581                                 ret = -ENOBUFS;
 582                                 spin_unlock(&cookie->parent->lock);
 583                                 goto cant_attach_object;
 584                         }
 585                         object->parent = p;
 586                         spin_lock(&p->lock);
 587                         p->n_children++;
 588                         spin_unlock(&p->lock);
 589                         break;
 590                 }
 591         }
 592         spin_unlock(&cookie->parent->lock);
 593
 594         /* attach to the cache's object list */
 595         if (list_empty(&object->cache_link)) {
 596                 spin_lock(&cache->object_list_lock);
 597                 list_add(&object->cache_link, &cache->object_list);
 598                 spin_unlock(&cache->object_list_lock);
 599         }
 600
 601         /* Attach to the cookie.  The object already has a ref on it. */
 602         hlist_add_head(&object->cookie_link, &cookie->backing_objects);
 603
 604         fscache_objlist_add(object);
 605         ret = 0;
 606
 607 cant_attach_object:
 608         spin_unlock(&cookie->lock);
 609         _leave(" = %d", ret);
 610         return ret;
 611 }
 612
 613 /*
 614  * Invalidate an object.  Callable with spinlocks held.
 615  */
 616 void __fscache_invalidate(struct fscache_cookie *cookie)
 617 {
 618         struct fscache_object *object;
 619
 620         _enter("{%s}", cookie->def->name);
 621
 622         fscache_stat(&fscache_n_invalidates);
 623
 624         /* Only permit invalidation of data files.  Invalidating an index will
 625          * require the caller to release all its attachments to the tree rooted
 626          * there, and if it's doing that, it may as well just retire the
 627          * cookie.
 628          */
 629         ASSERTCMP(cookie->type, ==, FSCACHE_COOKIE_TYPE_DATAFILE);
 630
 631         /* If there's an object, we tell the object state machine to handle the
 632          * invalidation on our behalf, otherwise there's nothing to do.
 633          */
 634         if (!hlist_empty(&cookie->backing_objects)) {
 635                 spin_lock(&cookie->lock);
 636
 637                 if (fscache_cookie_enabled(cookie) &&
 638                     !hlist_empty(&cookie->backing_objects) &&
 639                     !test_and_set_bit(FSCACHE_COOKIE_INVALIDATING,
 640                                       &cookie->flags)) {
 641                         object = hlist_entry(cookie->backing_objects.first,
 642                                              struct fscache_object,
 643                                              cookie_link);
 644                         if (fscache_object_is_live(object))
 645                                 fscache_raise_event(
 646                                         object, FSCACHE_OBJECT_EV_INVALIDATE);
 647                 }
 648
 649                 spin_unlock(&cookie->lock);
 650         }
 651
 652         _leave("");
 653 }
 654 EXPORT_SYMBOL(__fscache_invalidate);
 655
 656 /*
 657  * Wait for object invalidation to complete.
 658  */
 659 void __fscache_wait_on_invalidate(struct fscache_cookie *cookie)
 660 {
 661         _enter("%p", cookie);
 662
 663         wait_on_bit(&cookie->flags, FSCACHE_COOKIE_INVALIDATING,
 664                     TASK_UNINTERRUPTIBLE);
 665
 666         _leave("");
 667 }
 668 EXPORT_SYMBOL(__fscache_wait_on_invalidate);
 669
 670 /*
 671  * update the index entries backing a cookie
 672  */
 673 void __fscache_update_cookie(struct fscache_cookie *cookie, const void *aux_data)
 674 {
 675         struct fscache_object *object;
 676
 677         fscache_stat(&fscache_n_updates);
 678
 679         if (!cookie) {
 680                 fscache_stat(&fscache_n_updates_null);
 681                 _leave(" [no cookie]");
 682                 return;
 683         }
 684
 685         _enter("{%s}", cookie->def->name);
 686
 687         spin_lock(&cookie->lock);
 688
 689         fscache_update_aux(cookie, aux_data);
 690
 691         if (fscache_cookie_enabled(cookie)) {
 692                 /* update the index entry on disk in each cache backing this
 693                  * cookie.
 694                  */
 695                 hlist_for_each_entry(object,
 696                                      &cookie->backing_objects, cookie_link) {
 697                         fscache_raise_event(object, FSCACHE_OBJECT_EV_UPDATE);
 698                 }
 699         }
 700
 701         spin_unlock(&cookie->lock);
 702         _leave("");
 703 }
 704 EXPORT_SYMBOL(__fscache_update_cookie);
 705
 706 /*
 707  * Disable a cookie to stop it from accepting new requests from the netfs.
 708  */
 709 void __fscache_disable_cookie(struct fscache_cookie *cookie,
 710                               const void *aux_data,
 711                               bool invalidate)
 712 {
 713         struct fscache_object *object;
 714         bool awaken = false;
 715
 716         _enter("%p,%u", cookie, invalidate);
 717
 718         trace_fscache_disable(cookie);
 719
 720         ASSERTCMP(atomic_read(&cookie->n_active), >, 0);
 721
 722         if (atomic_read(&cookie->n_children) != 0) {
 723                 pr_err("Cookie '%s' still has children\n",
 724                        cookie->def->name);
 725                 BUG();
 726         }
 727
 728         wait_on_bit_lock(&cookie->flags, FSCACHE_COOKIE_ENABLEMENT_LOCK,
 729                          TASK_UNINTERRUPTIBLE);
 730
 731         fscache_update_aux(cookie, aux_data);
 732
 733         if (!test_and_clear_bit(FSCACHE_COOKIE_ENABLED, &cookie->flags))
 734                 goto out_unlock_enable;
 735
 736         /* If the cookie is being invalidated, wait for that to complete first
 737          * so that we can reuse the flag.
 738          */
 739         __fscache_wait_on_invalidate(cookie);
 740
 741         /* Dispose of the backing objects */
 742         set_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags);
 743
 744         spin_lock(&cookie->lock);
 745         if (!hlist_empty(&cookie->backing_objects)) {
 746                 hlist_for_each_entry(object, &cookie->backing_objects, cookie_link) {
 747                         if (invalidate)
 748                                 set_bit(FSCACHE_OBJECT_RETIRED, &object->flags);
 749                         clear_bit(FSCACHE_OBJECT_PENDING_WRITE, &object->flags);
 750                         fscache_raise_event(object, FSCACHE_OBJECT_EV_KILL);
 751                 }
 752         } else {
 753                 if (test_and_clear_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags))
 754                         awaken = true;
 755         }
 756         spin_unlock(&cookie->lock);
 757         if (awaken)
 758                 wake_up_bit(&cookie->flags, FSCACHE_COOKIE_INVALIDATING);
 759
 760         /* Wait for cessation of activity requiring access to the netfs (when
 761          * n_active reaches 0).  This makes sure outstanding reads and writes
 762          * have completed.
 763          */
 764         if (!atomic_dec_and_test(&cookie->n_active)) {
 765                 wait_var_event(&cookie->n_active,
 766                                !atomic_read(&cookie->n_active));
 767         }
 768
 769         /* Make sure any pending writes are cancelled. */
 770         if (cookie->type != FSCACHE_COOKIE_TYPE_INDEX)
 771                 fscache_invalidate_writes(cookie);
 772
 773         /* Reset the cookie state if it wasn't relinquished */
 774         if (!test_bit(FSCACHE_COOKIE_RELINQUISHED, &cookie->flags)) {
 775                 atomic_inc(&cookie->n_active);
 776                 set_bit(FSCACHE_COOKIE_NO_DATA_YET, &cookie->flags);
 777         }
 778
 779 out_unlock_enable:
 780         clear_bit_unlock(FSCACHE_COOKIE_ENABLEMENT_LOCK, &cookie->flags);
 781         wake_up_bit(&cookie->flags, FSCACHE_COOKIE_ENABLEMENT_LOCK);
 782         _leave("");
 783 }
 784 EXPORT_SYMBOL(__fscache_disable_cookie);
 785
 786 /*
 787  * release a cookie back to the cache
 788  * - the object will be marked as recyclable on disk if retire is true
 789  * - all dependents of this cookie must have already been unregistered
 790  *   (indices/files/pages)
 791  */
 792 void __fscache_relinquish_cookie(struct fscache_cookie *cookie,
 793                                  const void *aux_data,
 794                                  bool retire)
 795 {
 796         fscache_stat(&fscache_n_relinquishes);
 797         if (retire)
 798                 fscache_stat(&fscache_n_relinquishes_retire);
 799
 800         if (!cookie) {
 801                 fscache_stat(&fscache_n_relinquishes_null);
 802                 _leave(" [no cookie]");
 803                 return;
 804         }
 805
 806         _enter("%p{%s,%p,%d},%d",
 807                cookie, cookie->def->name, cookie->netfs_data,
 808                atomic_read(&cookie->n_active), retire);
 809
 810         trace_fscache_relinquish(cookie, retire);
 811
 812         /* No further netfs-accessing operations on this cookie permitted */
 813         if (test_and_set_bit(FSCACHE_COOKIE_RELINQUISHED, &cookie->flags))
 814                 BUG();
 815
 816         __fscache_disable_cookie(cookie, aux_data, retire);
 817
 818         /* Clear pointers back to the netfs */
 819         cookie->netfs_data      = NULL;
 820         cookie->def             = NULL;
 821         BUG_ON(!radix_tree_empty(&cookie->stores));
 822
 823         if (cookie->parent) {
 824                 ASSERTCMP(atomic_read(&cookie->parent->usage), >, 0);
 825                 ASSERTCMP(atomic_read(&cookie->parent->n_children), >, 0);
 826                 atomic_dec(&cookie->parent->n_children);
 827         }
 828
 829         /* Dispose of the netfs's link to the cookie */
 830         ASSERTCMP(atomic_read(&cookie->usage), >, 0);
 831         fscache_cookie_put(cookie, fscache_cookie_put_relinquish);
 832
 833         _leave("");
 834 }
 835 EXPORT_SYMBOL(__fscache_relinquish_cookie);
 836
 837 /*
 838  * Remove a cookie from the hash table.
 839  */
 840 static void fscache_unhash_cookie(struct fscache_cookie *cookie)
 841 {
 842         struct hlist_bl_head *h;
 843         unsigned int bucket;
 844
 845         bucket = cookie->key_hash & (ARRAY_SIZE(fscache_cookie_hash) - 1);
 846         h = &fscache_cookie_hash[bucket];
 847
 848         hlist_bl_lock(h);
 849         hlist_bl_del(&cookie->hash_link);
 850         hlist_bl_unlock(h);
 851 }
 852
 853 /*
 854  * Drop a reference to a cookie.
 855  */
 856 void fscache_cookie_put(struct fscache_cookie *cookie,
 857                         enum fscache_cookie_trace where)
 858 {
 859         struct fscache_cookie *parent;
 860         int usage;
 861
 862         _enter("%p", cookie);
 863
 864         do {
 865                 usage = atomic_dec_return(&cookie->usage);
 866                 trace_fscache_cookie(cookie, where, usage);
 867
 868                 if (usage > 0)
 869                         return;
 870                 BUG_ON(usage < 0);
 871
 872                 parent = cookie->parent;
 873                 fscache_unhash_cookie(cookie);
 874                 fscache_free_cookie(cookie);
 875
 876                 cookie = parent;
 877                 where = fscache_cookie_put_parent;
 878         } while (cookie);
 879
 880         _leave("");
 881 }
 882
 883 /*
 884  * check the consistency between the netfs inode and the backing cache
 885  *
 886  * NOTE: it only serves no-index type
 887  */
 888 int __fscache_check_consistency(struct fscache_cookie *cookie,
 889                                 const void *aux_data)
 890 {
 891         struct fscache_operation *op;
 892         struct fscache_object *object;
 893         bool wake_cookie = false;
 894         int ret;
 895
 896         _enter("%p,", cookie);
 897
 898         ASSERTCMP(cookie->type, ==, FSCACHE_COOKIE_TYPE_DATAFILE);
 899
 900         if (fscache_wait_for_deferred_lookup(cookie) < 0)
 901                 return -ERESTARTSYS;
 902
 903         if (hlist_empty(&cookie->backing_objects))
 904                 return 0;
 905
 906         op = kzalloc(sizeof(*op), GFP_NOIO | __GFP_NOMEMALLOC | __GFP_NORETRY);
 907         if (!op)
 908                 return -ENOMEM;
 909
 910         fscache_operation_init(cookie, op, NULL, NULL, NULL);
 911         op->flags = FSCACHE_OP_MYTHREAD |
 912                 (1 << FSCACHE_OP_WAITING) |
 913                 (1 << FSCACHE_OP_UNUSE_COOKIE);
 914         trace_fscache_page_op(cookie, NULL, op, fscache_page_op_check_consistency);
 915
 916         spin_lock(&cookie->lock);
 917
 918         fscache_update_aux(cookie, aux_data);
 919
 920         if (!fscache_cookie_enabled(cookie) ||
 921             hlist_empty(&cookie->backing_objects))
 922                 goto inconsistent;
 923         object = hlist_entry(cookie->backing_objects.first,
 924                              struct fscache_object, cookie_link);
 925         if (test_bit(FSCACHE_IOERROR, &object->cache->flags))
 926                 goto inconsistent;
 927
 928         op->debug_id = atomic_inc_return(&fscache_op_debug_id);
 929
 930         __fscache_use_cookie(cookie);
 931         if (fscache_submit_op(object, op) < 0)
 932                 goto submit_failed;
 933
 934         /* the work queue now carries its own ref on the object */
 935         spin_unlock(&cookie->lock);
 936
 937         ret = fscache_wait_for_operation_activation(object, op, NULL, NULL);
 938         if (ret == 0) {
 939                 /* ask the cache to honour the operation */
 940                 ret = object->cache->ops->check_consistency(op);
 941                 fscache_op_complete(op, false);
 942         } else if (ret == -ENOBUFS) {
 943                 ret = 0;
 944         }
 945
 946         fscache_put_operation(op);
 947         _leave(" = %d", ret);
 948         return ret;
 949
 950 submit_failed:
 951         wake_cookie = __fscache_unuse_cookie(cookie);
 952 inconsistent:
 953         spin_unlock(&cookie->lock);
 954         if (wake_cookie)
 955                 __fscache_wake_unused_cookie(cookie);
 956         kfree(op);
 957         _leave(" = -ESTALE");
 958         return -ESTALE;
 959 }
 960 EXPORT_SYMBOL(__fscache_check_consistency);