block/blk-lib.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Functions related to generic helpers functions
   4  */
   5 #include <linux/kernel.h>
   6 #include <linux/module.h>
   7 #include <linux/bio.h>
   8 #include <linux/blkdev.h>
   9 #include <linux/scatterlist.h>
  10
  11 #include "blk.h"
  12
  13 static sector_t bio_discard_limit(struct block_device *bdev, sector_t sector)
  14 {
  15         unsigned int discard_granularity = bdev_discard_granularity(bdev);
  16         sector_t granularity_aligned_sector;
  17
  18         if (bdev_is_partition(bdev))
  19                 sector += bdev->bd_start_sect;
  20
  21         granularity_aligned_sector =
  22                 round_up(sector, discard_granularity >> SECTOR_SHIFT);
  23
  24         /*
  25          * Make sure subsequent bios start aligned to the discard granularity if
  26          * it needs to be split.
  27          */
  28         if (granularity_aligned_sector != sector)
  29                 return granularity_aligned_sector - sector;
  30
  31         /*
  32          * Align the bio size to the discard granularity to make splitting the bio
  33          * at discard granularity boundaries easier in the driver if needed.
  34          */
  35         return round_down(UINT_MAX, discard_granularity) >> SECTOR_SHIFT;
  36 }
  37
  38 static void await_bio_endio(struct bio *bio)
  39 {
  40         complete(bio->bi_private);
  41         bio_put(bio);
  42 }
  43
  44 /*
  45  * await_bio_chain - ends @bio and waits for every chained bio to complete
  46  */
  47 static void await_bio_chain(struct bio *bio)
  48 {
  49         DECLARE_COMPLETION_ONSTACK_MAP(done,
  50                         bio->bi_bdev->bd_disk->lockdep_map);
  51
  52         bio->bi_private = &done;
  53         bio->bi_end_io = await_bio_endio;
  54         bio_endio(bio);
  55         blk_wait_io(&done);
  56 }
  57
  58 int __blkdev_issue_discard(struct block_device *bdev, sector_t sector,
  59                 sector_t nr_sects, gfp_t gfp_mask, struct bio **biop)
  60 {
  61         struct bio *bio = *biop;
  62         sector_t bs_mask;
  63
  64         if (bdev_read_only(bdev))
  65                 return -EPERM;
  66         if (!bdev_max_discard_sectors(bdev))
  67                 return -EOPNOTSUPP;
  68
  69         /* In case the discard granularity isn't set by buggy device driver */
  70         if (WARN_ON_ONCE(!bdev_discard_granularity(bdev))) {
  71                 pr_err_ratelimited("%pg: Error: discard_granularity is 0.\n",
  72                                    bdev);
  73                 return -EOPNOTSUPP;
  74         }
  75
  76         bs_mask = (bdev_logical_block_size(bdev) >> 9) - 1;
  77         if ((sector | nr_sects) & bs_mask)
  78                 return -EINVAL;
  79
  80         if (!nr_sects)
  81                 return -EINVAL;
  82
  83         while (nr_sects) {
  84                 sector_t req_sects =
  85                         min(nr_sects, bio_discard_limit(bdev, sector));
  86
  87                 bio = blk_next_bio(bio, bdev, 0, REQ_OP_DISCARD, gfp_mask);
  88                 bio->bi_iter.bi_sector = sector;
  89                 bio->bi_iter.bi_size = req_sects << 9;
  90                 sector += req_sects;
  91                 nr_sects -= req_sects;
  92
  93                 /*
  94                  * We can loop for a long time in here, if someone does
  95                  * full device discards (like mkfs). Be nice and allow
  96                  * us to schedule out to avoid softlocking if preempt
  97                  * is disabled.
  98                  */
  99                 cond_resched();
 100                 if (fatal_signal_pending(current)) {
 101                         await_bio_chain(bio);
 102                         return -EINTR;
 103                 }
 104         }
 105
 106         *biop = bio;
 107         return 0;
 108 }
 109 EXPORT_SYMBOL(__blkdev_issue_discard);
 110
 111 /**
 112  * blkdev_issue_discard - queue a discard
 113  * @bdev:       blockdev to issue discard for
 114  * @sector:     start sector
 115  * @nr_sects:   number of sectors to discard
 116  * @gfp_mask:   memory allocation flags (for bio_alloc)
 117  *
 118  * Description:
 119  *    Issue a discard request for the sectors in question.
 120  */
 121 int blkdev_issue_discard(struct block_device *bdev, sector_t sector,
 122                 sector_t nr_sects, gfp_t gfp_mask)
 123 {
 124         struct bio *bio = NULL;
 125         struct blk_plug plug;
 126         int ret;
 127
 128         blk_start_plug(&plug);
 129         ret = __blkdev_issue_discard(bdev, sector, nr_sects, gfp_mask, &bio);
 130         if (!ret && bio) {
 131                 ret = submit_bio_wait(bio);
 132                 if (ret == -EOPNOTSUPP)
 133                         ret = 0;
 134                 bio_put(bio);
 135         }
 136         blk_finish_plug(&plug);
 137
 138         return ret;
 139 }
 140 EXPORT_SYMBOL(blkdev_issue_discard);
 141
 142 static int __blkdev_issue_write_zeroes(struct block_device *bdev,
 143                 sector_t sector, sector_t nr_sects, gfp_t gfp_mask,
 144                 struct bio **biop, unsigned flags)
 145 {
 146         struct bio *bio = *biop;
 147         unsigned int max_sectors;
 148
 149         if (bdev_read_only(bdev))
 150                 return -EPERM;
 151
 152         /* Ensure that max_sectors doesn't overflow bi_size */
 153         max_sectors = bdev_write_zeroes_sectors(bdev);
 154
 155         if (max_sectors == 0)
 156                 return -EOPNOTSUPP;
 157
 158         while (nr_sects) {
 159                 unsigned int len = min_t(sector_t, nr_sects, max_sectors);
 160
 161                 bio = blk_next_bio(bio, bdev, 0, REQ_OP_WRITE_ZEROES, gfp_mask);
 162                 bio->bi_iter.bi_sector = sector;
 163                 if (flags & BLKDEV_ZERO_NOUNMAP)
 164                         bio->bi_opf |= REQ_NOUNMAP;
 165
 166                 bio->bi_iter.bi_size = len << SECTOR_SHIFT;
 167                 nr_sects -= len;
 168                 sector += len;
 169                 cond_resched();
 170                 if (fatal_signal_pending(current)) {
 171                         await_bio_chain(bio);
 172                         return -EINTR;
 173                 }
 174         }
 175
 176         *biop = bio;
 177         return 0;
 178 }
 179
 180 /*
 181  * Convert a number of 512B sectors to a number of pages.
 182  * The result is limited to a number of pages that can fit into a BIO.
 183  * Also make sure that the result is always at least 1 (page) for the cases
 184  * where nr_sects is lower than the number of sectors in a page.
 185  */
 186 static unsigned int __blkdev_sectors_to_bio_pages(sector_t nr_sects)
 187 {
 188         sector_t pages = DIV_ROUND_UP_SECTOR_T(nr_sects, PAGE_SIZE / 512);
 189
 190         return min(pages, (sector_t)BIO_MAX_VECS);
 191 }
 192
 193 static int __blkdev_issue_zero_pages(struct block_device *bdev,
 194                 sector_t sector, sector_t nr_sects, gfp_t gfp_mask,
 195                 struct bio **biop)
 196 {
 197         struct bio *bio = *biop;
 198         int bi_size = 0;
 199         unsigned int sz;
 200
 201         if (bdev_read_only(bdev))
 202                 return -EPERM;
 203
 204         while (nr_sects != 0) {
 205                 bio = blk_next_bio(bio, bdev, __blkdev_sectors_to_bio_pages(nr_sects),
 206                                    REQ_OP_WRITE, gfp_mask);
 207                 bio->bi_iter.bi_sector = sector;
 208
 209                 while (nr_sects != 0) {
 210                         sz = min((sector_t) PAGE_SIZE, nr_sects << 9);
 211                         bi_size = bio_add_page(bio, ZERO_PAGE(0), sz, 0);
 212                         nr_sects -= bi_size >> 9;
 213                         sector += bi_size >> 9;
 214                         if (bi_size < sz)
 215                                 break;
 216                 }
 217                 cond_resched();
 218                 if (fatal_signal_pending(current)) {
 219                         await_bio_chain(bio);
 220                         return -EINTR;
 221                 }
 222         }
 223
 224         *biop = bio;
 225         return 0;
 226 }
 227
 228 /**
 229  * __blkdev_issue_zeroout - generate number of zero filed write bios
 230  * @bdev:       blockdev to issue
 231  * @sector:     start sector
 232  * @nr_sects:   number of sectors to write
 233  * @gfp_mask:   memory allocation flags (for bio_alloc)
 234  * @biop:       pointer to anchor bio
 235  * @flags:      controls detailed behavior
 236  *
 237  * Description:
 238  *  Zero-fill a block range, either using hardware offload or by explicitly
 239  *  writing zeroes to the device.
 240  *
 241  *  If a device is using logical block provisioning, the underlying space will
 242  *  not be released if %flags contains BLKDEV_ZERO_NOUNMAP.
 243  *
 244  *  If %flags contains BLKDEV_ZERO_NOFALLBACK, the function will return
 245  *  -EOPNOTSUPP if no explicit hardware offload for zeroing is provided.
 246  */
 247 int __blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,
 248                 sector_t nr_sects, gfp_t gfp_mask, struct bio **biop,
 249                 unsigned flags)
 250 {
 251         int ret;
 252         sector_t bs_mask;
 253
 254         bs_mask = (bdev_logical_block_size(bdev) >> 9) - 1;
 255         if ((sector | nr_sects) & bs_mask)
 256                 return -EINVAL;
 257
 258         ret = __blkdev_issue_write_zeroes(bdev, sector, nr_sects, gfp_mask,
 259                         biop, flags);
 260         if (ret != -EOPNOTSUPP || (flags & BLKDEV_ZERO_NOFALLBACK))
 261                 return ret;
 262
 263         return __blkdev_issue_zero_pages(bdev, sector, nr_sects, gfp_mask,
 264                                          biop);
 265 }
 266 EXPORT_SYMBOL(__blkdev_issue_zeroout);
 267
 268 /**
 269  * blkdev_issue_zeroout - zero-fill a block range
 270  * @bdev:       blockdev to write
 271  * @sector:     start sector
 272  * @nr_sects:   number of sectors to write
 273  * @gfp_mask:   memory allocation flags (for bio_alloc)
 274  * @flags:      controls detailed behavior
 275  *
 276  * Description:
 277  *  Zero-fill a block range, either using hardware offload or by explicitly
 278  *  writing zeroes to the device.  See __blkdev_issue_zeroout() for the
 279  *  valid values for %flags.
 280  */
 281 int blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,
 282                 sector_t nr_sects, gfp_t gfp_mask, unsigned flags)
 283 {
 284         int ret = 0;
 285         sector_t bs_mask;
 286         struct bio *bio;
 287         struct blk_plug plug;
 288         bool try_write_zeroes = !!bdev_write_zeroes_sectors(bdev);
 289
 290         bs_mask = (bdev_logical_block_size(bdev) >> 9) - 1;
 291         if ((sector | nr_sects) & bs_mask)
 292                 return -EINVAL;
 293
 294 retry:
 295         bio = NULL;
 296         blk_start_plug(&plug);
 297         if (try_write_zeroes) {
 298                 ret = __blkdev_issue_write_zeroes(bdev, sector, nr_sects,
 299                                                   gfp_mask, &bio, flags);
 300         } else if (!(flags & BLKDEV_ZERO_NOFALLBACK)) {
 301                 ret = __blkdev_issue_zero_pages(bdev, sector, nr_sects,
 302                                                 gfp_mask, &bio);
 303         } else {
 304                 /* No zeroing offload support */
 305                 ret = -EOPNOTSUPP;
 306         }
 307         if (ret == 0 && bio) {
 308                 ret = submit_bio_wait(bio);
 309                 bio_put(bio);
 310         }
 311         blk_finish_plug(&plug);
 312         if (ret && ret != -EINTR && try_write_zeroes) {
 313                 if (!(flags & BLKDEV_ZERO_NOFALLBACK)) {
 314                         try_write_zeroes = false;
 315                         goto retry;
 316                 }
 317                 if (!bdev_write_zeroes_sectors(bdev)) {
 318                         /*
 319                          * Zeroing offload support was indicated, but the
 320                          * device reported ILLEGAL REQUEST (for some devices
 321                          * there is no non-destructive way to verify whether
 322                          * WRITE ZEROES is actually supported).
 323                          */
 324                         ret = -EOPNOTSUPP;
 325                 }
 326         }
 327
 328         return ret;
 329 }
 330 EXPORT_SYMBOL(blkdev_issue_zeroout);
 331
 332 int blkdev_issue_secure_erase(struct block_device *bdev, sector_t sector,
 333                 sector_t nr_sects, gfp_t gfp)
 334 {
 335         sector_t bs_mask = (bdev_logical_block_size(bdev) >> 9) - 1;
 336         unsigned int max_sectors = bdev_max_secure_erase_sectors(bdev);
 337         struct bio *bio = NULL;
 338         struct blk_plug plug;
 339         int ret = 0;
 340
 341         /* make sure that "len << SECTOR_SHIFT" doesn't overflow */
 342         if (max_sectors > UINT_MAX >> SECTOR_SHIFT)
 343                 max_sectors = UINT_MAX >> SECTOR_SHIFT;
 344         max_sectors &= ~bs_mask;
 345
 346         if (max_sectors == 0)
 347                 return -EOPNOTSUPP;
 348         if ((sector | nr_sects) & bs_mask)
 349                 return -EINVAL;
 350         if (bdev_read_only(bdev))
 351                 return -EPERM;
 352
 353         blk_start_plug(&plug);
 354         while (nr_sects) {
 355                 unsigned int len = min_t(sector_t, nr_sects, max_sectors);
 356
 357                 bio = blk_next_bio(bio, bdev, 0, REQ_OP_SECURE_ERASE, gfp);
 358                 bio->bi_iter.bi_sector = sector;
 359                 bio->bi_iter.bi_size = len << SECTOR_SHIFT;
 360
 361                 sector += len;
 362                 nr_sects -= len;
 363                 cond_resched();
 364                 if (fatal_signal_pending(current)) {
 365                         await_bio_chain(bio);
 366                         ret = -EINTR;
 367                         bio = NULL;
 368                         break;
 369                 }
 370         }
 371         if (bio) {
 372                 ret = submit_bio_wait(bio);
 373                 bio_put(bio);
 374         }
 375         blk_finish_plug(&plug);
 376
 377         return ret;
 378 }
 379 EXPORT_SYMBOL(blkdev_issue_secure_erase);