fs/erofs/utils.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (C) 2018 HUAWEI, Inc.
   4  *             https://www.huawei.com/
   5  */
   6 #include "internal.h"
   7
   8 struct page *erofs_allocpage(struct page **pagepool, gfp_t gfp)
   9 {
  10         struct page *page = *pagepool;
  11
  12         if (page) {
  13                 DBG_BUGON(page_ref_count(page) != 1);
  14                 *pagepool = (struct page *)page_private(page);
  15         } else {
  16                 page = alloc_page(gfp);
  17         }
  18         return page;
  19 }
  20
  21 void erofs_release_pages(struct page **pagepool)
  22 {
  23         while (*pagepool) {
  24                 struct page *page = *pagepool;
  25
  26                 *pagepool = (struct page *)page_private(page);
  27                 put_page(page);
  28         }
  29 }
  30
  31 #ifdef CONFIG_EROFS_FS_ZIP
  32 /* global shrink count (for all mounted EROFS instances) */
  33 static atomic_long_t erofs_global_shrink_cnt;
  34
  35 static bool erofs_workgroup_get(struct erofs_workgroup *grp)
  36 {
  37         if (lockref_get_not_zero(&grp->lockref))
  38                 return true;
  39
  40         spin_lock(&grp->lockref.lock);
  41         if (__lockref_is_dead(&grp->lockref)) {
  42                 spin_unlock(&grp->lockref.lock);
  43                 return false;
  44         }
  45
  46         if (!grp->lockref.count++)
  47                 atomic_long_dec(&erofs_global_shrink_cnt);
  48         spin_unlock(&grp->lockref.lock);
  49         return true;
  50 }
  51
  52 struct erofs_workgroup *erofs_find_workgroup(struct super_block *sb,
  53                                              pgoff_t index)
  54 {
  55         struct erofs_sb_info *sbi = EROFS_SB(sb);
  56         struct erofs_workgroup *grp;
  57
  58 repeat:
  59         rcu_read_lock();
  60         grp = xa_load(&sbi->managed_pslots, index);
  61         if (grp) {
  62                 if (!erofs_workgroup_get(grp)) {
  63                         /* prefer to relax rcu read side */
  64                         rcu_read_unlock();
  65                         goto repeat;
  66                 }
  67
  68                 DBG_BUGON(index != grp->index);
  69         }
  70         rcu_read_unlock();
  71         return grp;
  72 }
  73
  74 struct erofs_workgroup *erofs_insert_workgroup(struct super_block *sb,
  75                                                struct erofs_workgroup *grp)
  76 {
  77         struct erofs_sb_info *const sbi = EROFS_SB(sb);
  78         struct erofs_workgroup *pre;
  79
  80         DBG_BUGON(grp->lockref.count < 1);
  81 repeat:
  82         xa_lock(&sbi->managed_pslots);
  83         pre = __xa_cmpxchg(&sbi->managed_pslots, grp->index,
  84                            NULL, grp, GFP_KERNEL);
  85         if (pre) {
  86                 if (xa_is_err(pre)) {
  87                         pre = ERR_PTR(xa_err(pre));
  88                 } else if (!erofs_workgroup_get(pre)) {
  89                         /* try to legitimize the current in-tree one */
  90                         xa_unlock(&sbi->managed_pslots);
  91                         cond_resched();
  92                         goto repeat;
  93                 }
  94                 grp = pre;
  95         }
  96         xa_unlock(&sbi->managed_pslots);
  97         return grp;
  98 }
  99
 100 static void  __erofs_workgroup_free(struct erofs_workgroup *grp)
 101 {
 102         atomic_long_dec(&erofs_global_shrink_cnt);
 103         erofs_workgroup_free_rcu(grp);
 104 }
 105
 106 void erofs_workgroup_put(struct erofs_workgroup *grp)
 107 {
 108         if (lockref_put_or_lock(&grp->lockref))
 109                 return;
 110
 111         DBG_BUGON(__lockref_is_dead(&grp->lockref));
 112         if (grp->lockref.count == 1)
 113                 atomic_long_inc(&erofs_global_shrink_cnt);
 114         --grp->lockref.count;
 115         spin_unlock(&grp->lockref.lock);
 116 }
 117
 118 static bool erofs_try_to_release_workgroup(struct erofs_sb_info *sbi,
 119                                            struct erofs_workgroup *grp)
 120 {
 121         int free = false;
 122
 123         spin_lock(&grp->lockref.lock);
 124         if (grp->lockref.count)
 125                 goto out;
 126
 127         /*
 128          * Note that all cached pages should be detached before deleted from
 129          * the XArray. Otherwise some cached pages could be still attached to
 130          * the orphan old workgroup when the new one is available in the tree.
 131          */
 132         if (erofs_try_to_free_all_cached_folios(sbi, grp))
 133                 goto out;
 134
 135         /*
 136          * It's impossible to fail after the workgroup is freezed,
 137          * however in order to avoid some race conditions, add a
 138          * DBG_BUGON to observe this in advance.
 139          */
 140         DBG_BUGON(__xa_erase(&sbi->managed_pslots, grp->index) != grp);
 141
 142         lockref_mark_dead(&grp->lockref);
 143         free = true;
 144 out:
 145         spin_unlock(&grp->lockref.lock);
 146         if (free)
 147                 __erofs_workgroup_free(grp);
 148         return free;
 149 }
 150
 151 static unsigned long erofs_shrink_workstation(struct erofs_sb_info *sbi,
 152                                               unsigned long nr_shrink)
 153 {
 154         struct erofs_workgroup *grp;
 155         unsigned int freed = 0;
 156         unsigned long index;
 157
 158         xa_lock(&sbi->managed_pslots);
 159         xa_for_each(&sbi->managed_pslots, index, grp) {
 160                 /* try to shrink each valid workgroup */
 161                 if (!erofs_try_to_release_workgroup(sbi, grp))
 162                         continue;
 163                 xa_unlock(&sbi->managed_pslots);
 164
 165                 ++freed;
 166                 if (!--nr_shrink)
 167                         return freed;
 168                 xa_lock(&sbi->managed_pslots);
 169         }
 170         xa_unlock(&sbi->managed_pslots);
 171         return freed;
 172 }
 173
 174 /* protected by 'erofs_sb_list_lock' */
 175 static unsigned int shrinker_run_no;
 176
 177 /* protects the mounted 'erofs_sb_list' */
 178 static DEFINE_SPINLOCK(erofs_sb_list_lock);
 179 static LIST_HEAD(erofs_sb_list);
 180
 181 void erofs_shrinker_register(struct super_block *sb)
 182 {
 183         struct erofs_sb_info *sbi = EROFS_SB(sb);
 184
 185         mutex_init(&sbi->umount_mutex);
 186
 187         spin_lock(&erofs_sb_list_lock);
 188         list_add(&sbi->list, &erofs_sb_list);
 189         spin_unlock(&erofs_sb_list_lock);
 190 }
 191
 192 void erofs_shrinker_unregister(struct super_block *sb)
 193 {
 194         struct erofs_sb_info *const sbi = EROFS_SB(sb);
 195
 196         mutex_lock(&sbi->umount_mutex);
 197         /* clean up all remaining workgroups in memory */
 198         erofs_shrink_workstation(sbi, ~0UL);
 199
 200         spin_lock(&erofs_sb_list_lock);
 201         list_del(&sbi->list);
 202         spin_unlock(&erofs_sb_list_lock);
 203         mutex_unlock(&sbi->umount_mutex);
 204 }
 205
 206 static unsigned long erofs_shrink_count(struct shrinker *shrink,
 207                                         struct shrink_control *sc)
 208 {
 209         return atomic_long_read(&erofs_global_shrink_cnt);
 210 }
 211
 212 static unsigned long erofs_shrink_scan(struct shrinker *shrink,
 213                                        struct shrink_control *sc)
 214 {
 215         struct erofs_sb_info *sbi;
 216         struct list_head *p;
 217
 218         unsigned long nr = sc->nr_to_scan;
 219         unsigned int run_no;
 220         unsigned long freed = 0;
 221
 222         spin_lock(&erofs_sb_list_lock);
 223         do {
 224                 run_no = ++shrinker_run_no;
 225         } while (run_no == 0);
 226
 227         /* Iterate over all mounted superblocks and try to shrink them */
 228         p = erofs_sb_list.next;
 229         while (p != &erofs_sb_list) {
 230                 sbi = list_entry(p, struct erofs_sb_info, list);
 231
 232                 /*
 233                  * We move the ones we do to the end of the list, so we stop
 234                  * when we see one we have already done.
 235                  */
 236                 if (sbi->shrinker_run_no == run_no)
 237                         break;
 238
 239                 if (!mutex_trylock(&sbi->umount_mutex)) {
 240                         p = p->next;
 241                         continue;
 242                 }
 243
 244                 spin_unlock(&erofs_sb_list_lock);
 245                 sbi->shrinker_run_no = run_no;
 246
 247                 freed += erofs_shrink_workstation(sbi, nr - freed);
 248
 249                 spin_lock(&erofs_sb_list_lock);
 250                 /* Get the next list element before we move this one */
 251                 p = p->next;
 252
 253                 /*
 254                  * Move this one to the end of the list to provide some
 255                  * fairness.
 256                  */
 257                 list_move_tail(&sbi->list, &erofs_sb_list);
 258                 mutex_unlock(&sbi->umount_mutex);
 259
 260                 if (freed >= nr)
 261                         break;
 262         }
 263         spin_unlock(&erofs_sb_list_lock);
 264         return freed;
 265 }
 266
 267 static struct shrinker *erofs_shrinker_info;
 268
 269 int __init erofs_init_shrinker(void)
 270 {
 271         erofs_shrinker_info = shrinker_alloc(0, "erofs-shrinker");
 272         if (!erofs_shrinker_info)
 273                 return -ENOMEM;
 274
 275         erofs_shrinker_info->count_objects = erofs_shrink_count;
 276         erofs_shrinker_info->scan_objects = erofs_shrink_scan;
 277
 278         shrinker_register(erofs_shrinker_info);
 279
 280         return 0;
 281 }
 282
 283 void erofs_exit_shrinker(void)
 284 {
 285         shrinker_free(erofs_shrinker_info);
 286 }
 287 #endif  /* !CONFIG_EROFS_FS_ZIP */