Merge tag 'for-linus-20180706' of git://git.kernel.dk/linux-block
[linux-2.6-microblaze.git] / drivers / md / dm-zoned-target.c
1 /*
2  * Copyright (C) 2017 Western Digital Corporation or its affiliates.
3  *
4  * This file is released under the GPL.
5  */
6
7 #include "dm-zoned.h"
8
9 #include <linux/module.h>
10
11 #define DM_MSG_PREFIX           "zoned"
12
13 #define DMZ_MIN_BIOS            8192
14
15 /*
16  * Zone BIO context.
17  */
18 struct dmz_bioctx {
19         struct dmz_target       *target;
20         struct dm_zone          *zone;
21         struct bio              *bio;
22         atomic_t                ref;
23         blk_status_t            status;
24 };
25
26 /*
27  * Chunk work descriptor.
28  */
29 struct dm_chunk_work {
30         struct work_struct      work;
31         atomic_t                refcount;
32         struct dmz_target       *target;
33         unsigned int            chunk;
34         struct bio_list         bio_list;
35 };
36
37 /*
38  * Target descriptor.
39  */
40 struct dmz_target {
41         struct dm_dev           *ddev;
42
43         unsigned long           flags;
44
45         /* Zoned block device information */
46         struct dmz_dev          *dev;
47
48         /* For metadata handling */
49         struct dmz_metadata     *metadata;
50
51         /* For reclaim */
52         struct dmz_reclaim      *reclaim;
53
54         /* For chunk work */
55         struct radix_tree_root  chunk_rxtree;
56         struct workqueue_struct *chunk_wq;
57         struct mutex            chunk_lock;
58
59         /* For cloned BIOs to zones */
60         struct bio_set          bio_set;
61
62         /* For flush */
63         spinlock_t              flush_lock;
64         struct bio_list         flush_list;
65         struct delayed_work     flush_work;
66         struct workqueue_struct *flush_wq;
67 };
68
69 /*
70  * Flush intervals (seconds).
71  */
72 #define DMZ_FLUSH_PERIOD        (10 * HZ)
73
74 /*
75  * Target BIO completion.
76  */
77 static inline void dmz_bio_endio(struct bio *bio, blk_status_t status)
78 {
79         struct dmz_bioctx *bioctx = dm_per_bio_data(bio, sizeof(struct dmz_bioctx));
80
81         if (bioctx->status == BLK_STS_OK && status != BLK_STS_OK)
82                 bioctx->status = status;
83         bio_endio(bio);
84 }
85
86 /*
87  * Partial clone read BIO completion callback. This terminates the
88  * target BIO when there are no more references to its context.
89  */
90 static void dmz_read_bio_end_io(struct bio *bio)
91 {
92         struct dmz_bioctx *bioctx = bio->bi_private;
93         blk_status_t status = bio->bi_status;
94
95         bio_put(bio);
96         dmz_bio_endio(bioctx->bio, status);
97 }
98
99 /*
100  * Issue a BIO to a zone. The BIO may only partially process the
101  * original target BIO.
102  */
103 static int dmz_submit_read_bio(struct dmz_target *dmz, struct dm_zone *zone,
104                                struct bio *bio, sector_t chunk_block,
105                                unsigned int nr_blocks)
106 {
107         struct dmz_bioctx *bioctx = dm_per_bio_data(bio, sizeof(struct dmz_bioctx));
108         sector_t sector;
109         struct bio *clone;
110
111         /* BIO remap sector */
112         sector = dmz_start_sect(dmz->metadata, zone) + dmz_blk2sect(chunk_block);
113
114         /* If the read is not partial, there is no need to clone the BIO */
115         if (nr_blocks == dmz_bio_blocks(bio)) {
116                 /* Setup and submit the BIO */
117                 bio->bi_iter.bi_sector = sector;
118                 atomic_inc(&bioctx->ref);
119                 generic_make_request(bio);
120                 return 0;
121         }
122
123         /* Partial BIO: we need to clone the BIO */
124         clone = bio_clone_fast(bio, GFP_NOIO, &dmz->bio_set);
125         if (!clone)
126                 return -ENOMEM;
127
128         /* Setup the clone */
129         clone->bi_iter.bi_sector = sector;
130         clone->bi_iter.bi_size = dmz_blk2sect(nr_blocks) << SECTOR_SHIFT;
131         clone->bi_end_io = dmz_read_bio_end_io;
132         clone->bi_private = bioctx;
133
134         bio_advance(bio, clone->bi_iter.bi_size);
135
136         /* Submit the clone */
137         atomic_inc(&bioctx->ref);
138         generic_make_request(clone);
139
140         return 0;
141 }
142
143 /*
144  * Zero out pages of discarded blocks accessed by a read BIO.
145  */
146 static void dmz_handle_read_zero(struct dmz_target *dmz, struct bio *bio,
147                                  sector_t chunk_block, unsigned int nr_blocks)
148 {
149         unsigned int size = nr_blocks << DMZ_BLOCK_SHIFT;
150
151         /* Clear nr_blocks */
152         swap(bio->bi_iter.bi_size, size);
153         zero_fill_bio(bio);
154         swap(bio->bi_iter.bi_size, size);
155
156         bio_advance(bio, size);
157 }
158
159 /*
160  * Process a read BIO.
161  */
162 static int dmz_handle_read(struct dmz_target *dmz, struct dm_zone *zone,
163                            struct bio *bio)
164 {
165         sector_t chunk_block = dmz_chunk_block(dmz->dev, dmz_bio_block(bio));
166         unsigned int nr_blocks = dmz_bio_blocks(bio);
167         sector_t end_block = chunk_block + nr_blocks;
168         struct dm_zone *rzone, *bzone;
169         int ret;
170
171         /* Read into unmapped chunks need only zeroing the BIO buffer */
172         if (!zone) {
173                 zero_fill_bio(bio);
174                 return 0;
175         }
176
177         dmz_dev_debug(dmz->dev, "READ chunk %llu -> %s zone %u, block %llu, %u blocks",
178                       (unsigned long long)dmz_bio_chunk(dmz->dev, bio),
179                       (dmz_is_rnd(zone) ? "RND" : "SEQ"),
180                       dmz_id(dmz->metadata, zone),
181                       (unsigned long long)chunk_block, nr_blocks);
182
183         /* Check block validity to determine the read location */
184         bzone = zone->bzone;
185         while (chunk_block < end_block) {
186                 nr_blocks = 0;
187                 if (dmz_is_rnd(zone) || chunk_block < zone->wp_block) {
188                         /* Test block validity in the data zone */
189                         ret = dmz_block_valid(dmz->metadata, zone, chunk_block);
190                         if (ret < 0)
191                                 return ret;
192                         if (ret > 0) {
193                                 /* Read data zone blocks */
194                                 nr_blocks = ret;
195                                 rzone = zone;
196                         }
197                 }
198
199                 /*
200                  * No valid blocks found in the data zone.
201                  * Check the buffer zone, if there is one.
202                  */
203                 if (!nr_blocks && bzone) {
204                         ret = dmz_block_valid(dmz->metadata, bzone, chunk_block);
205                         if (ret < 0)
206                                 return ret;
207                         if (ret > 0) {
208                                 /* Read buffer zone blocks */
209                                 nr_blocks = ret;
210                                 rzone = bzone;
211                         }
212                 }
213
214                 if (nr_blocks) {
215                         /* Valid blocks found: read them */
216                         nr_blocks = min_t(unsigned int, nr_blocks, end_block - chunk_block);
217                         ret = dmz_submit_read_bio(dmz, rzone, bio, chunk_block, nr_blocks);
218                         if (ret)
219                                 return ret;
220                         chunk_block += nr_blocks;
221                 } else {
222                         /* No valid block: zeroout the current BIO block */
223                         dmz_handle_read_zero(dmz, bio, chunk_block, 1);
224                         chunk_block++;
225                 }
226         }
227
228         return 0;
229 }
230
231 /*
232  * Issue a write BIO to a zone.
233  */
234 static void dmz_submit_write_bio(struct dmz_target *dmz, struct dm_zone *zone,
235                                  struct bio *bio, sector_t chunk_block,
236                                  unsigned int nr_blocks)
237 {
238         struct dmz_bioctx *bioctx = dm_per_bio_data(bio, sizeof(struct dmz_bioctx));
239
240         /* Setup and submit the BIO */
241         bio_set_dev(bio, dmz->dev->bdev);
242         bio->bi_iter.bi_sector = dmz_start_sect(dmz->metadata, zone) + dmz_blk2sect(chunk_block);
243         atomic_inc(&bioctx->ref);
244         generic_make_request(bio);
245
246         if (dmz_is_seq(zone))
247                 zone->wp_block += nr_blocks;
248 }
249
250 /*
251  * Write blocks directly in a data zone, at the write pointer.
252  * If a buffer zone is assigned, invalidate the blocks written
253  * in place.
254  */
255 static int dmz_handle_direct_write(struct dmz_target *dmz,
256                                    struct dm_zone *zone, struct bio *bio,
257                                    sector_t chunk_block,
258                                    unsigned int nr_blocks)
259 {
260         struct dmz_metadata *zmd = dmz->metadata;
261         struct dm_zone *bzone = zone->bzone;
262         int ret;
263
264         if (dmz_is_readonly(zone))
265                 return -EROFS;
266
267         /* Submit write */
268         dmz_submit_write_bio(dmz, zone, bio, chunk_block, nr_blocks);
269
270         /*
271          * Validate the blocks in the data zone and invalidate
272          * in the buffer zone, if there is one.
273          */
274         ret = dmz_validate_blocks(zmd, zone, chunk_block, nr_blocks);
275         if (ret == 0 && bzone)
276                 ret = dmz_invalidate_blocks(zmd, bzone, chunk_block, nr_blocks);
277
278         return ret;
279 }
280
281 /*
282  * Write blocks in the buffer zone of @zone.
283  * If no buffer zone is assigned yet, get one.
284  * Called with @zone write locked.
285  */
286 static int dmz_handle_buffered_write(struct dmz_target *dmz,
287                                      struct dm_zone *zone, struct bio *bio,
288                                      sector_t chunk_block,
289                                      unsigned int nr_blocks)
290 {
291         struct dmz_metadata *zmd = dmz->metadata;
292         struct dm_zone *bzone;
293         int ret;
294
295         /* Get the buffer zone. One will be allocated if needed */
296         bzone = dmz_get_chunk_buffer(zmd, zone);
297         if (!bzone)
298                 return -ENOSPC;
299
300         if (dmz_is_readonly(bzone))
301                 return -EROFS;
302
303         /* Submit write */
304         dmz_submit_write_bio(dmz, bzone, bio, chunk_block, nr_blocks);
305
306         /*
307          * Validate the blocks in the buffer zone
308          * and invalidate in the data zone.
309          */
310         ret = dmz_validate_blocks(zmd, bzone, chunk_block, nr_blocks);
311         if (ret == 0 && chunk_block < zone->wp_block)
312                 ret = dmz_invalidate_blocks(zmd, zone, chunk_block, nr_blocks);
313
314         return ret;
315 }
316
317 /*
318  * Process a write BIO.
319  */
320 static int dmz_handle_write(struct dmz_target *dmz, struct dm_zone *zone,
321                             struct bio *bio)
322 {
323         sector_t chunk_block = dmz_chunk_block(dmz->dev, dmz_bio_block(bio));
324         unsigned int nr_blocks = dmz_bio_blocks(bio);
325
326         if (!zone)
327                 return -ENOSPC;
328
329         dmz_dev_debug(dmz->dev, "WRITE chunk %llu -> %s zone %u, block %llu, %u blocks",
330                       (unsigned long long)dmz_bio_chunk(dmz->dev, bio),
331                       (dmz_is_rnd(zone) ? "RND" : "SEQ"),
332                       dmz_id(dmz->metadata, zone),
333                       (unsigned long long)chunk_block, nr_blocks);
334
335         if (dmz_is_rnd(zone) || chunk_block == zone->wp_block) {
336                 /*
337                  * zone is a random zone or it is a sequential zone
338                  * and the BIO is aligned to the zone write pointer:
339                  * direct write the zone.
340                  */
341                 return dmz_handle_direct_write(dmz, zone, bio, chunk_block, nr_blocks);
342         }
343
344         /*
345          * This is an unaligned write in a sequential zone:
346          * use buffered write.
347          */
348         return dmz_handle_buffered_write(dmz, zone, bio, chunk_block, nr_blocks);
349 }
350
351 /*
352  * Process a discard BIO.
353  */
354 static int dmz_handle_discard(struct dmz_target *dmz, struct dm_zone *zone,
355                               struct bio *bio)
356 {
357         struct dmz_metadata *zmd = dmz->metadata;
358         sector_t block = dmz_bio_block(bio);
359         unsigned int nr_blocks = dmz_bio_blocks(bio);
360         sector_t chunk_block = dmz_chunk_block(dmz->dev, block);
361         int ret = 0;
362
363         /* For unmapped chunks, there is nothing to do */
364         if (!zone)
365                 return 0;
366
367         if (dmz_is_readonly(zone))
368                 return -EROFS;
369
370         dmz_dev_debug(dmz->dev, "DISCARD chunk %llu -> zone %u, block %llu, %u blocks",
371                       (unsigned long long)dmz_bio_chunk(dmz->dev, bio),
372                       dmz_id(zmd, zone),
373                       (unsigned long long)chunk_block, nr_blocks);
374
375         /*
376          * Invalidate blocks in the data zone and its
377          * buffer zone if one is mapped.
378          */
379         if (dmz_is_rnd(zone) || chunk_block < zone->wp_block)
380                 ret = dmz_invalidate_blocks(zmd, zone, chunk_block, nr_blocks);
381         if (ret == 0 && zone->bzone)
382                 ret = dmz_invalidate_blocks(zmd, zone->bzone,
383                                             chunk_block, nr_blocks);
384         return ret;
385 }
386
387 /*
388  * Process a BIO.
389  */
390 static void dmz_handle_bio(struct dmz_target *dmz, struct dm_chunk_work *cw,
391                            struct bio *bio)
392 {
393         struct dmz_bioctx *bioctx = dm_per_bio_data(bio, sizeof(struct dmz_bioctx));
394         struct dmz_metadata *zmd = dmz->metadata;
395         struct dm_zone *zone;
396         int ret;
397
398         /*
399          * Write may trigger a zone allocation. So make sure the
400          * allocation can succeed.
401          */
402         if (bio_op(bio) == REQ_OP_WRITE)
403                 dmz_schedule_reclaim(dmz->reclaim);
404
405         dmz_lock_metadata(zmd);
406
407         /*
408          * Get the data zone mapping the chunk. There may be no
409          * mapping for read and discard. If a mapping is obtained,
410          + the zone returned will be set to active state.
411          */
412         zone = dmz_get_chunk_mapping(zmd, dmz_bio_chunk(dmz->dev, bio),
413                                      bio_op(bio));
414         if (IS_ERR(zone)) {
415                 ret = PTR_ERR(zone);
416                 goto out;
417         }
418
419         /* Process the BIO */
420         if (zone) {
421                 dmz_activate_zone(zone);
422                 bioctx->zone = zone;
423         }
424
425         switch (bio_op(bio)) {
426         case REQ_OP_READ:
427                 ret = dmz_handle_read(dmz, zone, bio);
428                 break;
429         case REQ_OP_WRITE:
430                 ret = dmz_handle_write(dmz, zone, bio);
431                 break;
432         case REQ_OP_DISCARD:
433         case REQ_OP_WRITE_ZEROES:
434                 ret = dmz_handle_discard(dmz, zone, bio);
435                 break;
436         default:
437                 dmz_dev_err(dmz->dev, "Unsupported BIO operation 0x%x",
438                             bio_op(bio));
439                 ret = -EIO;
440         }
441
442         /*
443          * Release the chunk mapping. This will check that the mapping
444          * is still valid, that is, that the zone used still has valid blocks.
445          */
446         if (zone)
447                 dmz_put_chunk_mapping(zmd, zone);
448 out:
449         dmz_bio_endio(bio, errno_to_blk_status(ret));
450
451         dmz_unlock_metadata(zmd);
452 }
453
454 /*
455  * Increment a chunk reference counter.
456  */
457 static inline void dmz_get_chunk_work(struct dm_chunk_work *cw)
458 {
459         atomic_inc(&cw->refcount);
460 }
461
462 /*
463  * Decrement a chunk work reference count and
464  * free it if it becomes 0.
465  */
466 static void dmz_put_chunk_work(struct dm_chunk_work *cw)
467 {
468         if (atomic_dec_and_test(&cw->refcount)) {
469                 WARN_ON(!bio_list_empty(&cw->bio_list));
470                 radix_tree_delete(&cw->target->chunk_rxtree, cw->chunk);
471                 kfree(cw);
472         }
473 }
474
475 /*
476  * Chunk BIO work function.
477  */
478 static void dmz_chunk_work(struct work_struct *work)
479 {
480         struct dm_chunk_work *cw = container_of(work, struct dm_chunk_work, work);
481         struct dmz_target *dmz = cw->target;
482         struct bio *bio;
483
484         mutex_lock(&dmz->chunk_lock);
485
486         /* Process the chunk BIOs */
487         while ((bio = bio_list_pop(&cw->bio_list))) {
488                 mutex_unlock(&dmz->chunk_lock);
489                 dmz_handle_bio(dmz, cw, bio);
490                 mutex_lock(&dmz->chunk_lock);
491                 dmz_put_chunk_work(cw);
492         }
493
494         /* Queueing the work incremented the work refcount */
495         dmz_put_chunk_work(cw);
496
497         mutex_unlock(&dmz->chunk_lock);
498 }
499
500 /*
501  * Flush work.
502  */
503 static void dmz_flush_work(struct work_struct *work)
504 {
505         struct dmz_target *dmz = container_of(work, struct dmz_target, flush_work.work);
506         struct bio *bio;
507         int ret;
508
509         /* Flush dirty metadata blocks */
510         ret = dmz_flush_metadata(dmz->metadata);
511
512         /* Process queued flush requests */
513         while (1) {
514                 spin_lock(&dmz->flush_lock);
515                 bio = bio_list_pop(&dmz->flush_list);
516                 spin_unlock(&dmz->flush_lock);
517
518                 if (!bio)
519                         break;
520
521                 dmz_bio_endio(bio, errno_to_blk_status(ret));
522         }
523
524         queue_delayed_work(dmz->flush_wq, &dmz->flush_work, DMZ_FLUSH_PERIOD);
525 }
526
527 /*
528  * Get a chunk work and start it to process a new BIO.
529  * If the BIO chunk has no work yet, create one.
530  */
531 static void dmz_queue_chunk_work(struct dmz_target *dmz, struct bio *bio)
532 {
533         unsigned int chunk = dmz_bio_chunk(dmz->dev, bio);
534         struct dm_chunk_work *cw;
535
536         mutex_lock(&dmz->chunk_lock);
537
538         /* Get the BIO chunk work. If one is not active yet, create one */
539         cw = radix_tree_lookup(&dmz->chunk_rxtree, chunk);
540         if (!cw) {
541                 int ret;
542
543                 /* Create a new chunk work */
544                 cw = kmalloc(sizeof(struct dm_chunk_work), GFP_NOIO);
545                 if (!cw)
546                         goto out;
547
548                 INIT_WORK(&cw->work, dmz_chunk_work);
549                 atomic_set(&cw->refcount, 0);
550                 cw->target = dmz;
551                 cw->chunk = chunk;
552                 bio_list_init(&cw->bio_list);
553
554                 ret = radix_tree_insert(&dmz->chunk_rxtree, chunk, cw);
555                 if (unlikely(ret)) {
556                         kfree(cw);
557                         cw = NULL;
558                         goto out;
559                 }
560         }
561
562         bio_list_add(&cw->bio_list, bio);
563         dmz_get_chunk_work(cw);
564
565         if (queue_work(dmz->chunk_wq, &cw->work))
566                 dmz_get_chunk_work(cw);
567 out:
568         mutex_unlock(&dmz->chunk_lock);
569 }
570
571 /*
572  * Process a new BIO.
573  */
574 static int dmz_map(struct dm_target *ti, struct bio *bio)
575 {
576         struct dmz_target *dmz = ti->private;
577         struct dmz_dev *dev = dmz->dev;
578         struct dmz_bioctx *bioctx = dm_per_bio_data(bio, sizeof(struct dmz_bioctx));
579         sector_t sector = bio->bi_iter.bi_sector;
580         unsigned int nr_sectors = bio_sectors(bio);
581         sector_t chunk_sector;
582
583         dmz_dev_debug(dev, "BIO op %d sector %llu + %u => chunk %llu, block %llu, %u blocks",
584                       bio_op(bio), (unsigned long long)sector, nr_sectors,
585                       (unsigned long long)dmz_bio_chunk(dmz->dev, bio),
586                       (unsigned long long)dmz_chunk_block(dmz->dev, dmz_bio_block(bio)),
587                       (unsigned int)dmz_bio_blocks(bio));
588
589         bio_set_dev(bio, dev->bdev);
590
591         if (!nr_sectors && bio_op(bio) != REQ_OP_WRITE)
592                 return DM_MAPIO_REMAPPED;
593
594         /* The BIO should be block aligned */
595         if ((nr_sectors & DMZ_BLOCK_SECTORS_MASK) || (sector & DMZ_BLOCK_SECTORS_MASK))
596                 return DM_MAPIO_KILL;
597
598         /* Initialize the BIO context */
599         bioctx->target = dmz;
600         bioctx->zone = NULL;
601         bioctx->bio = bio;
602         atomic_set(&bioctx->ref, 1);
603         bioctx->status = BLK_STS_OK;
604
605         /* Set the BIO pending in the flush list */
606         if (!nr_sectors && bio_op(bio) == REQ_OP_WRITE) {
607                 spin_lock(&dmz->flush_lock);
608                 bio_list_add(&dmz->flush_list, bio);
609                 spin_unlock(&dmz->flush_lock);
610                 mod_delayed_work(dmz->flush_wq, &dmz->flush_work, 0);
611                 return DM_MAPIO_SUBMITTED;
612         }
613
614         /* Split zone BIOs to fit entirely into a zone */
615         chunk_sector = sector & (dev->zone_nr_sectors - 1);
616         if (chunk_sector + nr_sectors > dev->zone_nr_sectors)
617                 dm_accept_partial_bio(bio, dev->zone_nr_sectors - chunk_sector);
618
619         /* Now ready to handle this BIO */
620         dmz_reclaim_bio_acc(dmz->reclaim);
621         dmz_queue_chunk_work(dmz, bio);
622
623         return DM_MAPIO_SUBMITTED;
624 }
625
626 /*
627  * Completed target BIO processing.
628  */
629 static int dmz_end_io(struct dm_target *ti, struct bio *bio, blk_status_t *error)
630 {
631         struct dmz_bioctx *bioctx = dm_per_bio_data(bio, sizeof(struct dmz_bioctx));
632
633         if (bioctx->status == BLK_STS_OK && *error)
634                 bioctx->status = *error;
635
636         if (!atomic_dec_and_test(&bioctx->ref))
637                 return DM_ENDIO_INCOMPLETE;
638
639         /* Done */
640         bio->bi_status = bioctx->status;
641
642         if (bioctx->zone) {
643                 struct dm_zone *zone = bioctx->zone;
644
645                 if (*error && bio_op(bio) == REQ_OP_WRITE) {
646                         if (dmz_is_seq(zone))
647                                 set_bit(DMZ_SEQ_WRITE_ERR, &zone->flags);
648                 }
649                 dmz_deactivate_zone(zone);
650         }
651
652         return DM_ENDIO_DONE;
653 }
654
655 /*
656  * Get zoned device information.
657  */
658 static int dmz_get_zoned_device(struct dm_target *ti, char *path)
659 {
660         struct dmz_target *dmz = ti->private;
661         struct request_queue *q;
662         struct dmz_dev *dev;
663         sector_t aligned_capacity;
664         int ret;
665
666         /* Get the target device */
667         ret = dm_get_device(ti, path, dm_table_get_mode(ti->table), &dmz->ddev);
668         if (ret) {
669                 ti->error = "Get target device failed";
670                 dmz->ddev = NULL;
671                 return ret;
672         }
673
674         dev = kzalloc(sizeof(struct dmz_dev), GFP_KERNEL);
675         if (!dev) {
676                 ret = -ENOMEM;
677                 goto err;
678         }
679
680         dev->bdev = dmz->ddev->bdev;
681         (void)bdevname(dev->bdev, dev->name);
682
683         if (bdev_zoned_model(dev->bdev) == BLK_ZONED_NONE) {
684                 ti->error = "Not a zoned block device";
685                 ret = -EINVAL;
686                 goto err;
687         }
688
689         q = bdev_get_queue(dev->bdev);
690         dev->capacity = i_size_read(dev->bdev->bd_inode) >> SECTOR_SHIFT;
691         aligned_capacity = dev->capacity & ~(blk_queue_zone_sectors(q) - 1);
692         if (ti->begin ||
693             ((ti->len != dev->capacity) && (ti->len != aligned_capacity))) {
694                 ti->error = "Partial mapping not supported";
695                 ret = -EINVAL;
696                 goto err;
697         }
698
699         dev->zone_nr_sectors = blk_queue_zone_sectors(q);
700         dev->zone_nr_sectors_shift = ilog2(dev->zone_nr_sectors);
701
702         dev->zone_nr_blocks = dmz_sect2blk(dev->zone_nr_sectors);
703         dev->zone_nr_blocks_shift = ilog2(dev->zone_nr_blocks);
704
705         dev->nr_zones = (dev->capacity + dev->zone_nr_sectors - 1)
706                 >> dev->zone_nr_sectors_shift;
707
708         dmz->dev = dev;
709
710         return 0;
711 err:
712         dm_put_device(ti, dmz->ddev);
713         kfree(dev);
714
715         return ret;
716 }
717
718 /*
719  * Cleanup zoned device information.
720  */
721 static void dmz_put_zoned_device(struct dm_target *ti)
722 {
723         struct dmz_target *dmz = ti->private;
724
725         dm_put_device(ti, dmz->ddev);
726         kfree(dmz->dev);
727         dmz->dev = NULL;
728 }
729
730 /*
731  * Setup target.
732  */
733 static int dmz_ctr(struct dm_target *ti, unsigned int argc, char **argv)
734 {
735         struct dmz_target *dmz;
736         struct dmz_dev *dev;
737         int ret;
738
739         /* Check arguments */
740         if (argc != 1) {
741                 ti->error = "Invalid argument count";
742                 return -EINVAL;
743         }
744
745         /* Allocate and initialize the target descriptor */
746         dmz = kzalloc(sizeof(struct dmz_target), GFP_KERNEL);
747         if (!dmz) {
748                 ti->error = "Unable to allocate the zoned target descriptor";
749                 return -ENOMEM;
750         }
751         ti->private = dmz;
752
753         /* Get the target zoned block device */
754         ret = dmz_get_zoned_device(ti, argv[0]);
755         if (ret) {
756                 dmz->ddev = NULL;
757                 goto err;
758         }
759
760         /* Initialize metadata */
761         dev = dmz->dev;
762         ret = dmz_ctr_metadata(dev, &dmz->metadata);
763         if (ret) {
764                 ti->error = "Metadata initialization failed";
765                 goto err_dev;
766         }
767
768         /* Set target (no write same support) */
769         ti->max_io_len = dev->zone_nr_sectors << 9;
770         ti->num_flush_bios = 1;
771         ti->num_discard_bios = 1;
772         ti->num_write_zeroes_bios = 1;
773         ti->per_io_data_size = sizeof(struct dmz_bioctx);
774         ti->flush_supported = true;
775         ti->discards_supported = true;
776         ti->split_discard_bios = true;
777
778         /* The exposed capacity is the number of chunks that can be mapped */
779         ti->len = (sector_t)dmz_nr_chunks(dmz->metadata) << dev->zone_nr_sectors_shift;
780
781         /* Zone BIO */
782         ret = bioset_init(&dmz->bio_set, DMZ_MIN_BIOS, 0, 0);
783         if (ret) {
784                 ti->error = "Create BIO set failed";
785                 goto err_meta;
786         }
787
788         /* Chunk BIO work */
789         mutex_init(&dmz->chunk_lock);
790         INIT_RADIX_TREE(&dmz->chunk_rxtree, GFP_NOIO);
791         dmz->chunk_wq = alloc_workqueue("dmz_cwq_%s", WQ_MEM_RECLAIM | WQ_UNBOUND,
792                                         0, dev->name);
793         if (!dmz->chunk_wq) {
794                 ti->error = "Create chunk workqueue failed";
795                 ret = -ENOMEM;
796                 goto err_bio;
797         }
798
799         /* Flush work */
800         spin_lock_init(&dmz->flush_lock);
801         bio_list_init(&dmz->flush_list);
802         INIT_DELAYED_WORK(&dmz->flush_work, dmz_flush_work);
803         dmz->flush_wq = alloc_ordered_workqueue("dmz_fwq_%s", WQ_MEM_RECLAIM,
804                                                 dev->name);
805         if (!dmz->flush_wq) {
806                 ti->error = "Create flush workqueue failed";
807                 ret = -ENOMEM;
808                 goto err_cwq;
809         }
810         mod_delayed_work(dmz->flush_wq, &dmz->flush_work, DMZ_FLUSH_PERIOD);
811
812         /* Initialize reclaim */
813         ret = dmz_ctr_reclaim(dev, dmz->metadata, &dmz->reclaim);
814         if (ret) {
815                 ti->error = "Zone reclaim initialization failed";
816                 goto err_fwq;
817         }
818
819         dmz_dev_info(dev, "Target device: %llu 512-byte logical sectors (%llu blocks)",
820                      (unsigned long long)ti->len,
821                      (unsigned long long)dmz_sect2blk(ti->len));
822
823         return 0;
824 err_fwq:
825         destroy_workqueue(dmz->flush_wq);
826 err_cwq:
827         destroy_workqueue(dmz->chunk_wq);
828 err_bio:
829         mutex_destroy(&dmz->chunk_lock);
830         bioset_exit(&dmz->bio_set);
831 err_meta:
832         dmz_dtr_metadata(dmz->metadata);
833 err_dev:
834         dmz_put_zoned_device(ti);
835 err:
836         kfree(dmz);
837
838         return ret;
839 }
840
841 /*
842  * Cleanup target.
843  */
844 static void dmz_dtr(struct dm_target *ti)
845 {
846         struct dmz_target *dmz = ti->private;
847
848         flush_workqueue(dmz->chunk_wq);
849         destroy_workqueue(dmz->chunk_wq);
850
851         dmz_dtr_reclaim(dmz->reclaim);
852
853         cancel_delayed_work_sync(&dmz->flush_work);
854         destroy_workqueue(dmz->flush_wq);
855
856         (void) dmz_flush_metadata(dmz->metadata);
857
858         dmz_dtr_metadata(dmz->metadata);
859
860         bioset_exit(&dmz->bio_set);
861
862         dmz_put_zoned_device(ti);
863
864         mutex_destroy(&dmz->chunk_lock);
865
866         kfree(dmz);
867 }
868
869 /*
870  * Setup target request queue limits.
871  */
872 static void dmz_io_hints(struct dm_target *ti, struct queue_limits *limits)
873 {
874         struct dmz_target *dmz = ti->private;
875         unsigned int chunk_sectors = dmz->dev->zone_nr_sectors;
876
877         limits->logical_block_size = DMZ_BLOCK_SIZE;
878         limits->physical_block_size = DMZ_BLOCK_SIZE;
879
880         blk_limits_io_min(limits, DMZ_BLOCK_SIZE);
881         blk_limits_io_opt(limits, DMZ_BLOCK_SIZE);
882
883         limits->discard_alignment = DMZ_BLOCK_SIZE;
884         limits->discard_granularity = DMZ_BLOCK_SIZE;
885         limits->max_discard_sectors = chunk_sectors;
886         limits->max_hw_discard_sectors = chunk_sectors;
887         limits->max_write_zeroes_sectors = chunk_sectors;
888
889         /* FS hint to try to align to the device zone size */
890         limits->chunk_sectors = chunk_sectors;
891         limits->max_sectors = chunk_sectors;
892
893         /* We are exposing a drive-managed zoned block device */
894         limits->zoned = BLK_ZONED_NONE;
895 }
896
897 /*
898  * Pass on ioctl to the backend device.
899  */
900 static int dmz_prepare_ioctl(struct dm_target *ti, struct block_device **bdev)
901 {
902         struct dmz_target *dmz = ti->private;
903
904         *bdev = dmz->dev->bdev;
905
906         return 0;
907 }
908
909 /*
910  * Stop works on suspend.
911  */
912 static void dmz_suspend(struct dm_target *ti)
913 {
914         struct dmz_target *dmz = ti->private;
915
916         flush_workqueue(dmz->chunk_wq);
917         dmz_suspend_reclaim(dmz->reclaim);
918         cancel_delayed_work_sync(&dmz->flush_work);
919 }
920
921 /*
922  * Restart works on resume or if suspend failed.
923  */
924 static void dmz_resume(struct dm_target *ti)
925 {
926         struct dmz_target *dmz = ti->private;
927
928         queue_delayed_work(dmz->flush_wq, &dmz->flush_work, DMZ_FLUSH_PERIOD);
929         dmz_resume_reclaim(dmz->reclaim);
930 }
931
932 static int dmz_iterate_devices(struct dm_target *ti,
933                                iterate_devices_callout_fn fn, void *data)
934 {
935         struct dmz_target *dmz = ti->private;
936         struct dmz_dev *dev = dmz->dev;
937         sector_t capacity = dev->capacity & ~(dev->zone_nr_sectors - 1);
938
939         return fn(ti, dmz->ddev, 0, capacity, data);
940 }
941
942 static struct target_type dmz_type = {
943         .name            = "zoned",
944         .version         = {1, 0, 0},
945         .features        = DM_TARGET_SINGLETON | DM_TARGET_ZONED_HM,
946         .module          = THIS_MODULE,
947         .ctr             = dmz_ctr,
948         .dtr             = dmz_dtr,
949         .map             = dmz_map,
950         .end_io          = dmz_end_io,
951         .io_hints        = dmz_io_hints,
952         .prepare_ioctl   = dmz_prepare_ioctl,
953         .postsuspend     = dmz_suspend,
954         .resume          = dmz_resume,
955         .iterate_devices = dmz_iterate_devices,
956 };
957
958 static int __init dmz_init(void)
959 {
960         return dm_register_target(&dmz_type);
961 }
962
963 static void __exit dmz_exit(void)
964 {
965         dm_unregister_target(&dmz_type);
966 }
967
968 module_init(dmz_init);
969 module_exit(dmz_exit);
970
971 MODULE_DESCRIPTION(DM_NAME " target for zoned block devices");
972 MODULE_AUTHOR("Damien Le Moal <damien.lemoal@wdc.com>");
973 MODULE_LICENSE("GPL");