drivers/md/dm-integrity.c

   1 /*
   2  * Copyright (C) 2016-2017 Red Hat, Inc. All rights reserved.
   3  * Copyright (C) 2016-2017 Milan Broz
   4  * Copyright (C) 2016-2017 Mikulas Patocka
   5  *
   6  * This file is released under the GPL.
   7  */
   8
   9 #include <linux/compiler.h>
  10 #include <linux/module.h>
  11 #include <linux/device-mapper.h>
  12 #include <linux/dm-io.h>
  13 #include <linux/vmalloc.h>
  14 #include <linux/sort.h>
  15 #include <linux/rbtree.h>
  16 #include <linux/delay.h>
  17 #include <linux/random.h>
  18 #include <crypto/hash.h>
  19 #include <crypto/skcipher.h>
  20 #include <linux/async_tx.h>
  21 #include <linux/dm-bufio.h>
  22
  23 #define DM_MSG_PREFIX "integrity"
  24
  25 #define DEFAULT_INTERLEAVE_SECTORS      32768
  26 #define DEFAULT_JOURNAL_SIZE_FACTOR     7
  27 #define DEFAULT_BUFFER_SECTORS          128
  28 #define DEFAULT_JOURNAL_WATERMARK       50
  29 #define DEFAULT_SYNC_MSEC               10000
  30 #define DEFAULT_MAX_JOURNAL_SECTORS     131072
  31 #define MIN_LOG2_INTERLEAVE_SECTORS     3
  32 #define MAX_LOG2_INTERLEAVE_SECTORS     31
  33 #define METADATA_WORKQUEUE_MAX_ACTIVE   16
  34 #define RECALC_SECTORS                  8192
  35 #define RECALC_WRITE_SUPER              16
  36
  37 /*
  38  * Warning - DEBUG_PRINT prints security-sensitive data to the log,
  39  * so it should not be enabled in the official kernel
  40  */
  41 //#define DEBUG_PRINT
  42 //#define INTERNAL_VERIFY
  43
  44 /*
  45  * On disk structures
  46  */
  47
  48 #define SB_MAGIC                        "integrt"
  49 #define SB_VERSION_1                    1
  50 #define SB_VERSION_2                    2
  51 #define SB_SECTORS                      8
  52 #define MAX_SECTORS_PER_BLOCK           8
  53
  54 struct superblock {
  55         __u8 magic[8];
  56         __u8 version;
  57         __u8 log2_interleave_sectors;
  58         __u16 integrity_tag_size;
  59         __u32 journal_sections;
  60         __u64 provided_data_sectors;    /* userspace uses this value */
  61         __u32 flags;
  62         __u8 log2_sectors_per_block;
  63         __u8 pad[3];
  64         __u64 recalc_sector;
  65 };
  66
  67 #define SB_FLAG_HAVE_JOURNAL_MAC        0x1
  68 #define SB_FLAG_RECALCULATING           0x2
  69
  70 #define JOURNAL_ENTRY_ROUNDUP           8
  71
  72 typedef __u64 commit_id_t;
  73 #define JOURNAL_MAC_PER_SECTOR          8
  74
  75 struct journal_entry {
  76         union {
  77                 struct {
  78                         __u32 sector_lo;
  79                         __u32 sector_hi;
  80                 } s;
  81                 __u64 sector;
  82         } u;
  83         commit_id_t last_bytes[0];
  84         /* __u8 tag[0]; */
  85 };
  86
  87 #define journal_entry_tag(ic, je)               ((__u8 *)&(je)->last_bytes[(ic)->sectors_per_block])
  88
  89 #if BITS_PER_LONG == 64
  90 #define journal_entry_set_sector(je, x)         do { smp_wmb(); WRITE_ONCE((je)->u.sector, cpu_to_le64(x)); } while (0)
  91 #define journal_entry_get_sector(je)            le64_to_cpu((je)->u.sector)
  92 #elif defined(CONFIG_LBDAF)
  93 #define journal_entry_set_sector(je, x)         do { (je)->u.s.sector_lo = cpu_to_le32(x); smp_wmb(); WRITE_ONCE((je)->u.s.sector_hi, cpu_to_le32((x) >> 32)); } while (0)
  94 #define journal_entry_get_sector(je)            le64_to_cpu((je)->u.sector)
  95 #else
  96 #define journal_entry_set_sector(je, x)         do { (je)->u.s.sector_lo = cpu_to_le32(x); smp_wmb(); WRITE_ONCE((je)->u.s.sector_hi, cpu_to_le32(0)); } while (0)
  97 #define journal_entry_get_sector(je)            le32_to_cpu((je)->u.s.sector_lo)
  98 #endif
  99 #define journal_entry_is_unused(je)             ((je)->u.s.sector_hi == cpu_to_le32(-1))
 100 #define journal_entry_set_unused(je)            do { ((je)->u.s.sector_hi = cpu_to_le32(-1)); } while (0)
 101 #define journal_entry_is_inprogress(je)         ((je)->u.s.sector_hi == cpu_to_le32(-2))
 102 #define journal_entry_set_inprogress(je)        do { ((je)->u.s.sector_hi = cpu_to_le32(-2)); } while (0)
 103
 104 #define JOURNAL_BLOCK_SECTORS           8
 105 #define JOURNAL_SECTOR_DATA             ((1 << SECTOR_SHIFT) - sizeof(commit_id_t))
 106 #define JOURNAL_MAC_SIZE                (JOURNAL_MAC_PER_SECTOR * JOURNAL_BLOCK_SECTORS)
 107
 108 struct journal_sector {
 109         __u8 entries[JOURNAL_SECTOR_DATA - JOURNAL_MAC_PER_SECTOR];
 110         __u8 mac[JOURNAL_MAC_PER_SECTOR];
 111         commit_id_t commit_id;
 112 };
 113
 114 #define MAX_TAG_SIZE                    (JOURNAL_SECTOR_DATA - JOURNAL_MAC_PER_SECTOR - offsetof(struct journal_entry, last_bytes[MAX_SECTORS_PER_BLOCK]))
 115
 116 #define METADATA_PADDING_SECTORS        8
 117
 118 #define N_COMMIT_IDS                    4
 119
 120 static unsigned char prev_commit_seq(unsigned char seq)
 121 {
 122         return (seq + N_COMMIT_IDS - 1) % N_COMMIT_IDS;
 123 }
 124
 125 static unsigned char next_commit_seq(unsigned char seq)
 126 {
 127         return (seq + 1) % N_COMMIT_IDS;
 128 }
 129
 130 /*
 131  * In-memory structures
 132  */
 133
 134 struct journal_node {
 135         struct rb_node node;
 136         sector_t sector;
 137 };
 138
 139 struct alg_spec {
 140         char *alg_string;
 141         char *key_string;
 142         __u8 *key;
 143         unsigned key_size;
 144 };
 145
 146 struct dm_integrity_c {
 147         struct dm_dev *dev;
 148         struct dm_dev *meta_dev;
 149         unsigned tag_size;
 150         __s8 log2_tag_size;
 151         sector_t start;
 152         mempool_t journal_io_mempool;
 153         struct dm_io_client *io;
 154         struct dm_bufio_client *bufio;
 155         struct workqueue_struct *metadata_wq;
 156         struct superblock *sb;
 157         unsigned journal_pages;
 158         struct page_list *journal;
 159         struct page_list *journal_io;
 160         struct page_list *journal_xor;
 161
 162         struct crypto_skcipher *journal_crypt;
 163         struct scatterlist **journal_scatterlist;
 164         struct scatterlist **journal_io_scatterlist;
 165         struct skcipher_request **sk_requests;
 166
 167         struct crypto_shash *journal_mac;
 168
 169         struct journal_node *journal_tree;
 170         struct rb_root journal_tree_root;
 171
 172         sector_t provided_data_sectors;
 173
 174         unsigned short journal_entry_size;
 175         unsigned char journal_entries_per_sector;
 176         unsigned char journal_section_entries;
 177         unsigned short journal_section_sectors;
 178         unsigned journal_sections;
 179         unsigned journal_entries;
 180         sector_t data_device_sectors;
 181         sector_t meta_device_sectors;
 182         unsigned initial_sectors;
 183         unsigned metadata_run;
 184         __s8 log2_metadata_run;
 185         __u8 log2_buffer_sectors;
 186         __u8 sectors_per_block;
 187
 188         unsigned char mode;
 189         int suspending;
 190
 191         int failed;
 192
 193         struct crypto_shash *internal_hash;
 194
 195         /* these variables are locked with endio_wait.lock */
 196         struct rb_root in_progress;
 197         struct list_head wait_list;
 198         wait_queue_head_t endio_wait;
 199         struct workqueue_struct *wait_wq;
 200
 201         unsigned char commit_seq;
 202         commit_id_t commit_ids[N_COMMIT_IDS];
 203
 204         unsigned committed_section;
 205         unsigned n_committed_sections;
 206
 207         unsigned uncommitted_section;
 208         unsigned n_uncommitted_sections;
 209
 210         unsigned free_section;
 211         unsigned char free_section_entry;
 212         unsigned free_sectors;
 213
 214         unsigned free_sectors_threshold;
 215
 216         struct workqueue_struct *commit_wq;
 217         struct work_struct commit_work;
 218
 219         struct workqueue_struct *writer_wq;
 220         struct work_struct writer_work;
 221
 222         struct workqueue_struct *recalc_wq;
 223         struct work_struct recalc_work;
 224         u8 *recalc_buffer;
 225         u8 *recalc_tags;
 226
 227         struct bio_list flush_bio_list;
 228
 229         unsigned long autocommit_jiffies;
 230         struct timer_list autocommit_timer;
 231         unsigned autocommit_msec;
 232
 233         wait_queue_head_t copy_to_journal_wait;
 234
 235         struct completion crypto_backoff;
 236
 237         bool journal_uptodate;
 238         bool just_formatted;
 239
 240         struct alg_spec internal_hash_alg;
 241         struct alg_spec journal_crypt_alg;
 242         struct alg_spec journal_mac_alg;
 243
 244         atomic64_t number_of_mismatches;
 245 };
 246
 247 struct dm_integrity_range {
 248         sector_t logical_sector;
 249         sector_t n_sectors;
 250         bool waiting;
 251         union {
 252                 struct rb_node node;
 253                 struct {
 254                         struct task_struct *task;
 255                         struct list_head wait_entry;
 256                 };
 257         };
 258 };
 259
 260 struct dm_integrity_io {
 261         struct work_struct work;
 262
 263         struct dm_integrity_c *ic;
 264         bool write;
 265         bool fua;
 266
 267         struct dm_integrity_range range;
 268
 269         sector_t metadata_block;
 270         unsigned metadata_offset;
 271
 272         atomic_t in_flight;
 273         blk_status_t bi_status;
 274
 275         struct completion *completion;
 276
 277         struct gendisk *orig_bi_disk;
 278         u8 orig_bi_partno;
 279         bio_end_io_t *orig_bi_end_io;
 280         struct bio_integrity_payload *orig_bi_integrity;
 281         struct bvec_iter orig_bi_iter;
 282 };
 283
 284 struct journal_completion {
 285         struct dm_integrity_c *ic;
 286         atomic_t in_flight;
 287         struct completion comp;
 288 };
 289
 290 struct journal_io {
 291         struct dm_integrity_range range;
 292         struct journal_completion *comp;
 293 };
 294
 295 static struct kmem_cache *journal_io_cache;
 296
 297 #define JOURNAL_IO_MEMPOOL      32
 298
 299 #ifdef DEBUG_PRINT
 300 #define DEBUG_print(x, ...)     printk(KERN_DEBUG x, ##__VA_ARGS__)
 301 static void __DEBUG_bytes(__u8 *bytes, size_t len, const char *msg, ...)
 302 {
 303         va_list args;
 304         va_start(args, msg);
 305         vprintk(msg, args);
 306         va_end(args);
 307         if (len)
 308                 pr_cont(":");
 309         while (len) {
 310                 pr_cont(" %02x", *bytes);
 311                 bytes++;
 312                 len--;
 313         }
 314         pr_cont("\n");
 315 }
 316 #define DEBUG_bytes(bytes, len, msg, ...)       __DEBUG_bytes(bytes, len, KERN_DEBUG msg, ##__VA_ARGS__)
 317 #else
 318 #define DEBUG_print(x, ...)                     do { } while (0)
 319 #define DEBUG_bytes(bytes, len, msg, ...)       do { } while (0)
 320 #endif
 321
 322 /*
 323  * DM Integrity profile, protection is performed layer above (dm-crypt)
 324  */
 325 static const struct blk_integrity_profile dm_integrity_profile = {
 326         .name                   = "DM-DIF-EXT-TAG",
 327         .generate_fn            = NULL,
 328         .verify_fn              = NULL,
 329 };
 330
 331 static void dm_integrity_map_continue(struct dm_integrity_io *dio, bool from_map);
 332 static void integrity_bio_wait(struct work_struct *w);
 333 static void dm_integrity_dtr(struct dm_target *ti);
 334
 335 static void dm_integrity_io_error(struct dm_integrity_c *ic, const char *msg, int err)
 336 {
 337         if (err == -EILSEQ)
 338                 atomic64_inc(&ic->number_of_mismatches);
 339         if (!cmpxchg(&ic->failed, 0, err))
 340                 DMERR("Error on %s: %d", msg, err);
 341 }
 342
 343 static int dm_integrity_failed(struct dm_integrity_c *ic)
 344 {
 345         return READ_ONCE(ic->failed);
 346 }
 347
 348 static commit_id_t dm_integrity_commit_id(struct dm_integrity_c *ic, unsigned i,
 349                                           unsigned j, unsigned char seq)
 350 {
 351         /*
 352          * Xor the number with section and sector, so that if a piece of
 353          * journal is written at wrong place, it is detected.
 354          */
 355         return ic->commit_ids[seq] ^ cpu_to_le64(((__u64)i << 32) ^ j);
 356 }
 357
 358 static void get_area_and_offset(struct dm_integrity_c *ic, sector_t data_sector,
 359                                 sector_t *area, sector_t *offset)
 360 {
 361         if (!ic->meta_dev) {
 362                 __u8 log2_interleave_sectors = ic->sb->log2_interleave_sectors;
 363                 *area = data_sector >> log2_interleave_sectors;
 364                 *offset = (unsigned)data_sector & ((1U << log2_interleave_sectors) - 1);
 365         } else {
 366                 *area = 0;
 367                 *offset = data_sector;
 368         }
 369 }
 370
 371 #define sector_to_block(ic, n)                                          \
 372 do {                                                                    \
 373         BUG_ON((n) & (unsigned)((ic)->sectors_per_block - 1));          \
 374         (n) >>= (ic)->sb->log2_sectors_per_block;                       \
 375 } while (0)
 376
 377 static __u64 get_metadata_sector_and_offset(struct dm_integrity_c *ic, sector_t area,
 378                                             sector_t offset, unsigned *metadata_offset)
 379 {
 380         __u64 ms;
 381         unsigned mo;
 382
 383         ms = area << ic->sb->log2_interleave_sectors;
 384         if (likely(ic->log2_metadata_run >= 0))
 385                 ms += area << ic->log2_metadata_run;
 386         else
 387                 ms += area * ic->metadata_run;
 388         ms >>= ic->log2_buffer_sectors;
 389
 390         sector_to_block(ic, offset);
 391
 392         if (likely(ic->log2_tag_size >= 0)) {
 393                 ms += offset >> (SECTOR_SHIFT + ic->log2_buffer_sectors - ic->log2_tag_size);
 394                 mo = (offset << ic->log2_tag_size) & ((1U << SECTOR_SHIFT << ic->log2_buffer_sectors) - 1);
 395         } else {
 396                 ms += (__u64)offset * ic->tag_size >> (SECTOR_SHIFT + ic->log2_buffer_sectors);
 397                 mo = (offset * ic->tag_size) & ((1U << SECTOR_SHIFT << ic->log2_buffer_sectors) - 1);
 398         }
 399         *metadata_offset = mo;
 400         return ms;
 401 }
 402
 403 static sector_t get_data_sector(struct dm_integrity_c *ic, sector_t area, sector_t offset)
 404 {
 405         sector_t result;
 406
 407         if (ic->meta_dev)
 408                 return offset;
 409
 410         result = area << ic->sb->log2_interleave_sectors;
 411         if (likely(ic->log2_metadata_run >= 0))
 412                 result += (area + 1) << ic->log2_metadata_run;
 413         else
 414                 result += (area + 1) * ic->metadata_run;
 415
 416         result += (sector_t)ic->initial_sectors + offset;
 417         result += ic->start;
 418
 419         return result;
 420 }
 421
 422 static void wraparound_section(struct dm_integrity_c *ic, unsigned *sec_ptr)
 423 {
 424         if (unlikely(*sec_ptr >= ic->journal_sections))
 425                 *sec_ptr -= ic->journal_sections;
 426 }
 427
 428 static void sb_set_version(struct dm_integrity_c *ic)
 429 {
 430         if (ic->meta_dev || ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING))
 431                 ic->sb->version = SB_VERSION_2;
 432         else
 433                 ic->sb->version = SB_VERSION_1;
 434 }
 435
 436 static int sync_rw_sb(struct dm_integrity_c *ic, int op, int op_flags)
 437 {
 438         struct dm_io_request io_req;
 439         struct dm_io_region io_loc;
 440
 441         io_req.bi_op = op;
 442         io_req.bi_op_flags = op_flags;
 443         io_req.mem.type = DM_IO_KMEM;
 444         io_req.mem.ptr.addr = ic->sb;
 445         io_req.notify.fn = NULL;
 446         io_req.client = ic->io;
 447         io_loc.bdev = ic->meta_dev ? ic->meta_dev->bdev : ic->dev->bdev;
 448         io_loc.sector = ic->start;
 449         io_loc.count = SB_SECTORS;
 450
 451         return dm_io(&io_req, 1, &io_loc, NULL);
 452 }
 453
 454 static void access_journal_check(struct dm_integrity_c *ic, unsigned section, unsigned offset,
 455                                  bool e, const char *function)
 456 {
 457 #if defined(CONFIG_DM_DEBUG) || defined(INTERNAL_VERIFY)
 458         unsigned limit = e ? ic->journal_section_entries : ic->journal_section_sectors;
 459
 460         if (unlikely(section >= ic->journal_sections) ||
 461             unlikely(offset >= limit)) {
 462                 printk(KERN_CRIT "%s: invalid access at (%u,%u), limit (%u,%u)\n",
 463                         function, section, offset, ic->journal_sections, limit);
 464                 BUG();
 465         }
 466 #endif
 467 }
 468
 469 static void page_list_location(struct dm_integrity_c *ic, unsigned section, unsigned offset,
 470                                unsigned *pl_index, unsigned *pl_offset)
 471 {
 472         unsigned sector;
 473
 474         access_journal_check(ic, section, offset, false, "page_list_location");
 475
 476         sector = section * ic->journal_section_sectors + offset;
 477
 478         *pl_index = sector >> (PAGE_SHIFT - SECTOR_SHIFT);
 479         *pl_offset = (sector << SECTOR_SHIFT) & (PAGE_SIZE - 1);
 480 }
 481
 482 static struct journal_sector *access_page_list(struct dm_integrity_c *ic, struct page_list *pl,
 483                                                unsigned section, unsigned offset, unsigned *n_sectors)
 484 {
 485         unsigned pl_index, pl_offset;
 486         char *va;
 487
 488         page_list_location(ic, section, offset, &pl_index, &pl_offset);
 489
 490         if (n_sectors)
 491                 *n_sectors = (PAGE_SIZE - pl_offset) >> SECTOR_SHIFT;
 492
 493         va = lowmem_page_address(pl[pl_index].page);
 494
 495         return (struct journal_sector *)(va + pl_offset);
 496 }
 497
 498 static struct journal_sector *access_journal(struct dm_integrity_c *ic, unsigned section, unsigned offset)
 499 {
 500         return access_page_list(ic, ic->journal, section, offset, NULL);
 501 }
 502
 503 static struct journal_entry *access_journal_entry(struct dm_integrity_c *ic, unsigned section, unsigned n)
 504 {
 505         unsigned rel_sector, offset;
 506         struct journal_sector *js;
 507
 508         access_journal_check(ic, section, n, true, "access_journal_entry");
 509
 510         rel_sector = n % JOURNAL_BLOCK_SECTORS;
 511         offset = n / JOURNAL_BLOCK_SECTORS;
 512
 513         js = access_journal(ic, section, rel_sector);
 514         return (struct journal_entry *)((char *)js + offset * ic->journal_entry_size);
 515 }
 516
 517 static struct journal_sector *access_journal_data(struct dm_integrity_c *ic, unsigned section, unsigned n)
 518 {
 519         n <<= ic->sb->log2_sectors_per_block;
 520
 521         n += JOURNAL_BLOCK_SECTORS;
 522
 523         access_journal_check(ic, section, n, false, "access_journal_data");
 524
 525         return access_journal(ic, section, n);
 526 }
 527
 528 static void section_mac(struct dm_integrity_c *ic, unsigned section, __u8 result[JOURNAL_MAC_SIZE])
 529 {
 530         SHASH_DESC_ON_STACK(desc, ic->journal_mac);
 531         int r;
 532         unsigned j, size;
 533
 534         desc->tfm = ic->journal_mac;
 535         desc->flags = 0;
 536
 537         r = crypto_shash_init(desc);
 538         if (unlikely(r)) {
 539                 dm_integrity_io_error(ic, "crypto_shash_init", r);
 540                 goto err;
 541         }
 542
 543         for (j = 0; j < ic->journal_section_entries; j++) {
 544                 struct journal_entry *je = access_journal_entry(ic, section, j);
 545                 r = crypto_shash_update(desc, (__u8 *)&je->u.sector, sizeof je->u.sector);
 546                 if (unlikely(r)) {
 547                         dm_integrity_io_error(ic, "crypto_shash_update", r);
 548                         goto err;
 549                 }
 550         }
 551
 552         size = crypto_shash_digestsize(ic->journal_mac);
 553
 554         if (likely(size <= JOURNAL_MAC_SIZE)) {
 555                 r = crypto_shash_final(desc, result);
 556                 if (unlikely(r)) {
 557                         dm_integrity_io_error(ic, "crypto_shash_final", r);
 558                         goto err;
 559                 }
 560                 memset(result + size, 0, JOURNAL_MAC_SIZE - size);
 561         } else {
 562                 __u8 digest[HASH_MAX_DIGESTSIZE];
 563
 564                 if (WARN_ON(size > sizeof(digest))) {
 565                         dm_integrity_io_error(ic, "digest_size", -EINVAL);
 566                         goto err;
 567                 }
 568                 r = crypto_shash_final(desc, digest);
 569                 if (unlikely(r)) {
 570                         dm_integrity_io_error(ic, "crypto_shash_final", r);
 571                         goto err;
 572                 }
 573                 memcpy(result, digest, JOURNAL_MAC_SIZE);
 574         }
 575
 576         return;
 577 err:
 578         memset(result, 0, JOURNAL_MAC_SIZE);
 579 }
 580
 581 static void rw_section_mac(struct dm_integrity_c *ic, unsigned section, bool wr)
 582 {
 583         __u8 result[JOURNAL_MAC_SIZE];
 584         unsigned j;
 585
 586         if (!ic->journal_mac)
 587                 return;
 588
 589         section_mac(ic, section, result);
 590
 591         for (j = 0; j < JOURNAL_BLOCK_SECTORS; j++) {
 592                 struct journal_sector *js = access_journal(ic, section, j);
 593
 594                 if (likely(wr))
 595                         memcpy(&js->mac, result + (j * JOURNAL_MAC_PER_SECTOR), JOURNAL_MAC_PER_SECTOR);
 596                 else {
 597                         if (memcmp(&js->mac, result + (j * JOURNAL_MAC_PER_SECTOR), JOURNAL_MAC_PER_SECTOR))
 598                                 dm_integrity_io_error(ic, "journal mac", -EILSEQ);
 599                 }
 600         }
 601 }
 602
 603 static void complete_journal_op(void *context)
 604 {
 605         struct journal_completion *comp = context;
 606         BUG_ON(!atomic_read(&comp->in_flight));
 607         if (likely(atomic_dec_and_test(&comp->in_flight)))
 608                 complete(&comp->comp);
 609 }
 610
 611 static void xor_journal(struct dm_integrity_c *ic, bool encrypt, unsigned section,
 612                         unsigned n_sections, struct journal_completion *comp)
 613 {
 614         struct async_submit_ctl submit;
 615         size_t n_bytes = (size_t)(n_sections * ic->journal_section_sectors) << SECTOR_SHIFT;
 616         unsigned pl_index, pl_offset, section_index;
 617         struct page_list *source_pl, *target_pl;
 618
 619         if (likely(encrypt)) {
 620                 source_pl = ic->journal;
 621                 target_pl = ic->journal_io;
 622         } else {
 623                 source_pl = ic->journal_io;
 624                 target_pl = ic->journal;
 625         }
 626
 627         page_list_location(ic, section, 0, &pl_index, &pl_offset);
 628
 629         atomic_add(roundup(pl_offset + n_bytes, PAGE_SIZE) >> PAGE_SHIFT, &comp->in_flight);
 630
 631         init_async_submit(&submit, ASYNC_TX_XOR_ZERO_DST, NULL, complete_journal_op, comp, NULL);
 632
 633         section_index = pl_index;
 634
 635         do {
 636                 size_t this_step;
 637                 struct page *src_pages[2];
 638                 struct page *dst_page;
 639
 640                 while (unlikely(pl_index == section_index)) {
 641                         unsigned dummy;
 642                         if (likely(encrypt))
 643                                 rw_section_mac(ic, section, true);
 644                         section++;
 645                         n_sections--;
 646                         if (!n_sections)
 647                                 break;
 648                         page_list_location(ic, section, 0, &section_index, &dummy);
 649                 }
 650
 651                 this_step = min(n_bytes, (size_t)PAGE_SIZE - pl_offset);
 652                 dst_page = target_pl[pl_index].page;
 653                 src_pages[0] = source_pl[pl_index].page;
 654                 src_pages[1] = ic->journal_xor[pl_index].page;
 655
 656                 async_xor(dst_page, src_pages, pl_offset, 2, this_step, &submit);
 657
 658                 pl_index++;
 659                 pl_offset = 0;
 660                 n_bytes -= this_step;
 661         } while (n_bytes);
 662
 663         BUG_ON(n_sections);
 664
 665         async_tx_issue_pending_all();
 666 }
 667
 668 static void complete_journal_encrypt(struct crypto_async_request *req, int err)
 669 {
 670         struct journal_completion *comp = req->data;
 671         if (unlikely(err)) {
 672                 if (likely(err == -EINPROGRESS)) {
 673                         complete(&comp->ic->crypto_backoff);
 674                         return;
 675                 }
 676                 dm_integrity_io_error(comp->ic, "asynchronous encrypt", err);
 677         }
 678         complete_journal_op(comp);
 679 }
 680
 681 static bool do_crypt(bool encrypt, struct skcipher_request *req, struct journal_completion *comp)
 682 {
 683         int r;
 684         skcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
 685                                       complete_journal_encrypt, comp);
 686         if (likely(encrypt))
 687                 r = crypto_skcipher_encrypt(req);
 688         else
 689                 r = crypto_skcipher_decrypt(req);
 690         if (likely(!r))
 691                 return false;
 692         if (likely(r == -EINPROGRESS))
 693                 return true;
 694         if (likely(r == -EBUSY)) {
 695                 wait_for_completion(&comp->ic->crypto_backoff);
 696                 reinit_completion(&comp->ic->crypto_backoff);
 697                 return true;
 698         }
 699         dm_integrity_io_error(comp->ic, "encrypt", r);
 700         return false;
 701 }
 702
 703 static void crypt_journal(struct dm_integrity_c *ic, bool encrypt, unsigned section,
 704                           unsigned n_sections, struct journal_completion *comp)
 705 {
 706         struct scatterlist **source_sg;
 707         struct scatterlist **target_sg;
 708
 709         atomic_add(2, &comp->in_flight);
 710
 711         if (likely(encrypt)) {
 712                 source_sg = ic->journal_scatterlist;
 713                 target_sg = ic->journal_io_scatterlist;
 714         } else {
 715                 source_sg = ic->journal_io_scatterlist;
 716                 target_sg = ic->journal_scatterlist;
 717         }
 718
 719         do {
 720                 struct skcipher_request *req;
 721                 unsigned ivsize;
 722                 char *iv;
 723
 724                 if (likely(encrypt))
 725                         rw_section_mac(ic, section, true);
 726
 727                 req = ic->sk_requests[section];
 728                 ivsize = crypto_skcipher_ivsize(ic->journal_crypt);
 729                 iv = req->iv;
 730
 731                 memcpy(iv, iv + ivsize, ivsize);
 732
 733                 req->src = source_sg[section];
 734                 req->dst = target_sg[section];
 735
 736                 if (unlikely(do_crypt(encrypt, req, comp)))
 737                         atomic_inc(&comp->in_flight);
 738
 739                 section++;
 740                 n_sections--;
 741         } while (n_sections);
 742
 743         atomic_dec(&comp->in_flight);
 744         complete_journal_op(comp);
 745 }
 746
 747 static void encrypt_journal(struct dm_integrity_c *ic, bool encrypt, unsigned section,
 748                             unsigned n_sections, struct journal_completion *comp)
 749 {
 750         if (ic->journal_xor)
 751                 return xor_journal(ic, encrypt, section, n_sections, comp);
 752         else
 753                 return crypt_journal(ic, encrypt, section, n_sections, comp);
 754 }
 755
 756 static void complete_journal_io(unsigned long error, void *context)
 757 {
 758         struct journal_completion *comp = context;
 759         if (unlikely(error != 0))
 760                 dm_integrity_io_error(comp->ic, "writing journal", -EIO);
 761         complete_journal_op(comp);
 762 }
 763
 764 static void rw_journal_sectors(struct dm_integrity_c *ic, int op, int op_flags,
 765                                unsigned sector, unsigned n_sectors, struct journal_completion *comp)
 766 {
 767         struct dm_io_request io_req;
 768         struct dm_io_region io_loc;
 769         unsigned pl_index, pl_offset;
 770         int r;
 771
 772         if (unlikely(dm_integrity_failed(ic))) {
 773                 if (comp)
 774                         complete_journal_io(-1UL, comp);
 775                 return;
 776         }
 777
 778         pl_index = sector >> (PAGE_SHIFT - SECTOR_SHIFT);
 779         pl_offset = (sector << SECTOR_SHIFT) & (PAGE_SIZE - 1);
 780
 781         io_req.bi_op = op;
 782         io_req.bi_op_flags = op_flags;
 783         io_req.mem.type = DM_IO_PAGE_LIST;
 784         if (ic->journal_io)
 785                 io_req.mem.ptr.pl = &ic->journal_io[pl_index];
 786         else
 787                 io_req.mem.ptr.pl = &ic->journal[pl_index];
 788         io_req.mem.offset = pl_offset;
 789         if (likely(comp != NULL)) {
 790                 io_req.notify.fn = complete_journal_io;
 791                 io_req.notify.context = comp;
 792         } else {
 793                 io_req.notify.fn = NULL;
 794         }
 795         io_req.client = ic->io;
 796         io_loc.bdev = ic->meta_dev ? ic->meta_dev->bdev : ic->dev->bdev;
 797         io_loc.sector = ic->start + SB_SECTORS + sector;
 798         io_loc.count = n_sectors;
 799
 800         r = dm_io(&io_req, 1, &io_loc, NULL);
 801         if (unlikely(r)) {
 802                 dm_integrity_io_error(ic, op == REQ_OP_READ ? "reading journal" : "writing journal", r);
 803                 if (comp) {
 804                         WARN_ONCE(1, "asynchronous dm_io failed: %d", r);
 805                         complete_journal_io(-1UL, comp);
 806                 }
 807         }
 808 }
 809
 810 static void rw_journal(struct dm_integrity_c *ic, int op, int op_flags, unsigned section,
 811                        unsigned n_sections, struct journal_completion *comp)
 812 {
 813         unsigned sector, n_sectors;
 814
 815         sector = section * ic->journal_section_sectors;
 816         n_sectors = n_sections * ic->journal_section_sectors;
 817
 818         rw_journal_sectors(ic, op, op_flags, sector, n_sectors, comp);
 819 }
 820
 821 static void write_journal(struct dm_integrity_c *ic, unsigned commit_start, unsigned commit_sections)
 822 {
 823         struct journal_completion io_comp;
 824         struct journal_completion crypt_comp_1;
 825         struct journal_completion crypt_comp_2;
 826         unsigned i;
 827
 828         io_comp.ic = ic;
 829         init_completion(&io_comp.comp);
 830
 831         if (commit_start + commit_sections <= ic->journal_sections) {
 832                 io_comp.in_flight = (atomic_t)ATOMIC_INIT(1);
 833                 if (ic->journal_io) {
 834                         crypt_comp_1.ic = ic;
 835                         init_completion(&crypt_comp_1.comp);
 836                         crypt_comp_1.in_flight = (atomic_t)ATOMIC_INIT(0);
 837                         encrypt_journal(ic, true, commit_start, commit_sections, &crypt_comp_1);
 838                         wait_for_completion_io(&crypt_comp_1.comp);
 839                 } else {
 840                         for (i = 0; i < commit_sections; i++)
 841                                 rw_section_mac(ic, commit_start + i, true);
 842                 }
 843                 rw_journal(ic, REQ_OP_WRITE, REQ_FUA | REQ_SYNC, commit_start,
 844                            commit_sections, &io_comp);
 845         } else {
 846                 unsigned to_end;
 847                 io_comp.in_flight = (atomic_t)ATOMIC_INIT(2);
 848                 to_end = ic->journal_sections - commit_start;
 849                 if (ic->journal_io) {
 850                         crypt_comp_1.ic = ic;
 851                         init_completion(&crypt_comp_1.comp);
 852                         crypt_comp_1.in_flight = (atomic_t)ATOMIC_INIT(0);
 853                         encrypt_journal(ic, true, commit_start, to_end, &crypt_comp_1);
 854                         if (try_wait_for_completion(&crypt_comp_1.comp)) {
 855                                 rw_journal(ic, REQ_OP_WRITE, REQ_FUA, commit_start, to_end, &io_comp);
 856                                 reinit_completion(&crypt_comp_1.comp);
 857                                 crypt_comp_1.in_flight = (atomic_t)ATOMIC_INIT(0);
 858                                 encrypt_journal(ic, true, 0, commit_sections - to_end, &crypt_comp_1);
 859                                 wait_for_completion_io(&crypt_comp_1.comp);
 860                         } else {
 861                                 crypt_comp_2.ic = ic;
 862                                 init_completion(&crypt_comp_2.comp);
 863                                 crypt_comp_2.in_flight = (atomic_t)ATOMIC_INIT(0);
 864                                 encrypt_journal(ic, true, 0, commit_sections - to_end, &crypt_comp_2);
 865                                 wait_for_completion_io(&crypt_comp_1.comp);
 866                                 rw_journal(ic, REQ_OP_WRITE, REQ_FUA, commit_start, to_end, &io_comp);
 867                                 wait_for_completion_io(&crypt_comp_2.comp);
 868                         }
 869                 } else {
 870                         for (i = 0; i < to_end; i++)
 871                                 rw_section_mac(ic, commit_start + i, true);
 872                         rw_journal(ic, REQ_OP_WRITE, REQ_FUA, commit_start, to_end, &io_comp);
 873                         for (i = 0; i < commit_sections - to_end; i++)
 874                                 rw_section_mac(ic, i, true);
 875                 }
 876                 rw_journal(ic, REQ_OP_WRITE, REQ_FUA, 0, commit_sections - to_end, &io_comp);
 877         }
 878
 879         wait_for_completion_io(&io_comp.comp);
 880 }
 881
 882 static void copy_from_journal(struct dm_integrity_c *ic, unsigned section, unsigned offset,
 883                               unsigned n_sectors, sector_t target, io_notify_fn fn, void *data)
 884 {
 885         struct dm_io_request io_req;
 886         struct dm_io_region io_loc;
 887         int r;
 888         unsigned sector, pl_index, pl_offset;
 889
 890         BUG_ON((target | n_sectors | offset) & (unsigned)(ic->sectors_per_block - 1));
 891
 892         if (unlikely(dm_integrity_failed(ic))) {
 893                 fn(-1UL, data);
 894                 return;
 895         }
 896
 897         sector = section * ic->journal_section_sectors + JOURNAL_BLOCK_SECTORS + offset;
 898
 899         pl_index = sector >> (PAGE_SHIFT - SECTOR_SHIFT);
 900         pl_offset = (sector << SECTOR_SHIFT) & (PAGE_SIZE - 1);
 901
 902         io_req.bi_op = REQ_OP_WRITE;
 903         io_req.bi_op_flags = 0;
 904         io_req.mem.type = DM_IO_PAGE_LIST;
 905         io_req.mem.ptr.pl = &ic->journal[pl_index];
 906         io_req.mem.offset = pl_offset;
 907         io_req.notify.fn = fn;
 908         io_req.notify.context = data;
 909         io_req.client = ic->io;
 910         io_loc.bdev = ic->dev->bdev;
 911         io_loc.sector = target;
 912         io_loc.count = n_sectors;
 913
 914         r = dm_io(&io_req, 1, &io_loc, NULL);
 915         if (unlikely(r)) {
 916                 WARN_ONCE(1, "asynchronous dm_io failed: %d", r);
 917                 fn(-1UL, data);
 918         }
 919 }
 920
 921 static bool ranges_overlap(struct dm_integrity_range *range1, struct dm_integrity_range *range2)
 922 {
 923         return range1->logical_sector < range2->logical_sector + range2->n_sectors &&
 924                range1->logical_sector + range1->n_sectors > range2->logical_sector;
 925 }
 926
 927 static bool add_new_range(struct dm_integrity_c *ic, struct dm_integrity_range *new_range, bool check_waiting)
 928 {
 929         struct rb_node **n = &ic->in_progress.rb_node;
 930         struct rb_node *parent;
 931
 932         BUG_ON((new_range->logical_sector | new_range->n_sectors) & (unsigned)(ic->sectors_per_block - 1));
 933
 934         if (likely(check_waiting)) {
 935                 struct dm_integrity_range *range;
 936                 list_for_each_entry(range, &ic->wait_list, wait_entry) {
 937                         if (unlikely(ranges_overlap(range, new_range)))
 938                                 return false;
 939                 }
 940         }
 941
 942         parent = NULL;
 943
 944         while (*n) {
 945                 struct dm_integrity_range *range = container_of(*n, struct dm_integrity_range, node);
 946
 947                 parent = *n;
 948                 if (new_range->logical_sector + new_range->n_sectors <= range->logical_sector) {
 949                         n = &range->node.rb_left;
 950                 } else if (new_range->logical_sector >= range->logical_sector + range->n_sectors) {
 951                         n = &range->node.rb_right;
 952                 } else {
 953                         return false;
 954                 }
 955         }
 956
 957         rb_link_node(&new_range->node, parent, n);
 958         rb_insert_color(&new_range->node, &ic->in_progress);
 959
 960         return true;
 961 }
 962
 963 static void remove_range_unlocked(struct dm_integrity_c *ic, struct dm_integrity_range *range)
 964 {
 965         rb_erase(&range->node, &ic->in_progress);
 966         while (unlikely(!list_empty(&ic->wait_list))) {
 967                 struct dm_integrity_range *last_range =
 968                         list_first_entry(&ic->wait_list, struct dm_integrity_range, wait_entry);
 969                 struct task_struct *last_range_task;
 970                 last_range_task = last_range->task;
 971                 list_del(&last_range->wait_entry);
 972                 if (!add_new_range(ic, last_range, false)) {
 973                         last_range->task = last_range_task;
 974                         list_add(&last_range->wait_entry, &ic->wait_list);
 975                         break;
 976                 }
 977                 last_range->waiting = false;
 978                 wake_up_process(last_range_task);
 979         }
 980 }
 981
 982 static void remove_range(struct dm_integrity_c *ic, struct dm_integrity_range *range)
 983 {
 984         unsigned long flags;
 985
 986         spin_lock_irqsave(&ic->endio_wait.lock, flags);
 987         remove_range_unlocked(ic, range);
 988         spin_unlock_irqrestore(&ic->endio_wait.lock, flags);
 989 }
 990
 991 static void wait_and_add_new_range(struct dm_integrity_c *ic, struct dm_integrity_range *new_range)
 992 {
 993         new_range->waiting = true;
 994         list_add_tail(&new_range->wait_entry, &ic->wait_list);
 995         new_range->task = current;
 996         do {
 997                 __set_current_state(TASK_UNINTERRUPTIBLE);
 998                 spin_unlock_irq(&ic->endio_wait.lock);
 999                 io_schedule();
1000                 spin_lock_irq(&ic->endio_wait.lock);
1001         } while (unlikely(new_range->waiting));
1002 }
1003
1004 static void add_new_range_and_wait(struct dm_integrity_c *ic, struct dm_integrity_range *new_range)
1005 {
1006         if (unlikely(!add_new_range(ic, new_range, true)))
1007                 wait_and_add_new_range(ic, new_range);
1008 }
1009
1010 static void init_journal_node(struct journal_node *node)
1011 {
1012         RB_CLEAR_NODE(&node->node);
1013         node->sector = (sector_t)-1;
1014 }
1015
1016 static void add_journal_node(struct dm_integrity_c *ic, struct journal_node *node, sector_t sector)
1017 {
1018         struct rb_node **link;
1019         struct rb_node *parent;
1020
1021         node->sector = sector;
1022         BUG_ON(!RB_EMPTY_NODE(&node->node));
1023
1024         link = &ic->journal_tree_root.rb_node;
1025         parent = NULL;
1026
1027         while (*link) {
1028                 struct journal_node *j;
1029                 parent = *link;
1030                 j = container_of(parent, struct journal_node, node);
1031                 if (sector < j->sector)
1032                         link = &j->node.rb_left;
1033                 else
1034                         link = &j->node.rb_right;
1035         }
1036
1037         rb_link_node(&node->node, parent, link);
1038         rb_insert_color(&node->node, &ic->journal_tree_root);
1039 }
1040
1041 static void remove_journal_node(struct dm_integrity_c *ic, struct journal_node *node)
1042 {
1043         BUG_ON(RB_EMPTY_NODE(&node->node));
1044         rb_erase(&node->node, &ic->journal_tree_root);
1045         init_journal_node(node);
1046 }
1047
1048 #define NOT_FOUND       (-1U)
1049
1050 static unsigned find_journal_node(struct dm_integrity_c *ic, sector_t sector, sector_t *next_sector)
1051 {
1052         struct rb_node *n = ic->journal_tree_root.rb_node;
1053         unsigned found = NOT_FOUND;
1054         *next_sector = (sector_t)-1;
1055         while (n) {
1056                 struct journal_node *j = container_of(n, struct journal_node, node);
1057                 if (sector == j->sector) {
1058                         found = j - ic->journal_tree;
1059                 }
1060                 if (sector < j->sector) {
1061                         *next_sector = j->sector;
1062                         n = j->node.rb_left;
1063                 } else {
1064                         n = j->node.rb_right;
1065                 }
1066         }
1067
1068         return found;
1069 }
1070
1071 static bool test_journal_node(struct dm_integrity_c *ic, unsigned pos, sector_t sector)
1072 {
1073         struct journal_node *node, *next_node;
1074         struct rb_node *next;
1075
1076         if (unlikely(pos >= ic->journal_entries))
1077                 return false;
1078         node = &ic->journal_tree[pos];
1079         if (unlikely(RB_EMPTY_NODE(&node->node)))
1080                 return false;
1081         if (unlikely(node->sector != sector))
1082                 return false;
1083
1084         next = rb_next(&node->node);
1085         if (unlikely(!next))
1086                 return true;
1087
1088         next_node = container_of(next, struct journal_node, node);
1089         return next_node->sector != sector;
1090 }
1091
1092 static bool find_newer_committed_node(struct dm_integrity_c *ic, struct journal_node *node)
1093 {
1094         struct rb_node *next;
1095         struct journal_node *next_node;
1096         unsigned next_section;
1097
1098         BUG_ON(RB_EMPTY_NODE(&node->node));
1099
1100         next = rb_next(&node->node);
1101         if (unlikely(!next))
1102                 return false;
1103
1104         next_node = container_of(next, struct journal_node, node);
1105
1106         if (next_node->sector != node->sector)
1107                 return false;
1108
1109         next_section = (unsigned)(next_node - ic->journal_tree) / ic->journal_section_entries;
1110         if (next_section >= ic->committed_section &&
1111             next_section < ic->committed_section + ic->n_committed_sections)
1112                 return true;
1113         if (next_section + ic->journal_sections < ic->committed_section + ic->n_committed_sections)
1114                 return true;
1115
1116         return false;
1117 }
1118
1119 #define TAG_READ        0
1120 #define TAG_WRITE       1
1121 #define TAG_CMP         2
1122
1123 static int dm_integrity_rw_tag(struct dm_integrity_c *ic, unsigned char *tag, sector_t *metadata_block,
1124                                unsigned *metadata_offset, unsigned total_size, int op)
1125 {
1126         do {
1127                 unsigned char *data, *dp;
1128                 struct dm_buffer *b;
1129                 unsigned to_copy;
1130                 int r;
1131
1132                 r = dm_integrity_failed(ic);
1133                 if (unlikely(r))
1134                         return r;
1135
1136                 data = dm_bufio_read(ic->bufio, *metadata_block, &b);
1137                 if (IS_ERR(data))
1138                         return PTR_ERR(data);
1139
1140                 to_copy = min((1U << SECTOR_SHIFT << ic->log2_buffer_sectors) - *metadata_offset, total_size);
1141                 dp = data + *metadata_offset;
1142                 if (op == TAG_READ) {
1143                         memcpy(tag, dp, to_copy);
1144                 } else if (op == TAG_WRITE) {
1145                         memcpy(dp, tag, to_copy);
1146                         dm_bufio_mark_partial_buffer_dirty(b, *metadata_offset, *metadata_offset + to_copy);
1147                 } else  {
1148                         /* e.g.: op == TAG_CMP */
1149                         if (unlikely(memcmp(dp, tag, to_copy))) {
1150                                 unsigned i;
1151
1152                                 for (i = 0; i < to_copy; i++) {
1153                                         if (dp[i] != tag[i])
1154                                                 break;
1155                                         total_size--;
1156                                 }
1157                                 dm_bufio_release(b);
1158                                 return total_size;
1159                         }
1160                 }
1161                 dm_bufio_release(b);
1162
1163                 tag += to_copy;
1164                 *metadata_offset += to_copy;
1165                 if (unlikely(*metadata_offset == 1U << SECTOR_SHIFT << ic->log2_buffer_sectors)) {
1166                         (*metadata_block)++;
1167                         *metadata_offset = 0;
1168                 }
1169                 total_size -= to_copy;
1170         } while (unlikely(total_size));
1171
1172         return 0;
1173 }
1174
1175 static void dm_integrity_flush_buffers(struct dm_integrity_c *ic)
1176 {
1177         int r;
1178         r = dm_bufio_write_dirty_buffers(ic->bufio);
1179         if (unlikely(r))
1180                 dm_integrity_io_error(ic, "writing tags", r);
1181 }
1182
1183 static void sleep_on_endio_wait(struct dm_integrity_c *ic)
1184 {
1185         DECLARE_WAITQUEUE(wait, current);
1186         __add_wait_queue(&ic->endio_wait, &wait);
1187         __set_current_state(TASK_UNINTERRUPTIBLE);
1188         spin_unlock_irq(&ic->endio_wait.lock);
1189         io_schedule();
1190         spin_lock_irq(&ic->endio_wait.lock);
1191         __remove_wait_queue(&ic->endio_wait, &wait);
1192 }
1193
1194 static void autocommit_fn(struct timer_list *t)
1195 {
1196         struct dm_integrity_c *ic = from_timer(ic, t, autocommit_timer);
1197
1198         if (likely(!dm_integrity_failed(ic)))
1199                 queue_work(ic->commit_wq, &ic->commit_work);
1200 }
1201
1202 static void schedule_autocommit(struct dm_integrity_c *ic)
1203 {
1204         if (!timer_pending(&ic->autocommit_timer))
1205                 mod_timer(&ic->autocommit_timer, jiffies + ic->autocommit_jiffies);
1206 }
1207
1208 static void submit_flush_bio(struct dm_integrity_c *ic, struct dm_integrity_io *dio)
1209 {
1210         struct bio *bio;
1211         unsigned long flags;
1212
1213         spin_lock_irqsave(&ic->endio_wait.lock, flags);
1214         bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));
1215         bio_list_add(&ic->flush_bio_list, bio);
1216         spin_unlock_irqrestore(&ic->endio_wait.lock, flags);
1217
1218         queue_work(ic->commit_wq, &ic->commit_work);
1219 }
1220
1221 static void do_endio(struct dm_integrity_c *ic, struct bio *bio)
1222 {
1223         int r = dm_integrity_failed(ic);
1224         if (unlikely(r) && !bio->bi_status)
1225                 bio->bi_status = errno_to_blk_status(r);
1226         bio_endio(bio);
1227 }
1228
1229 static void do_endio_flush(struct dm_integrity_c *ic, struct dm_integrity_io *dio)
1230 {
1231         struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));
1232
1233         if (unlikely(dio->fua) && likely(!bio->bi_status) && likely(!dm_integrity_failed(ic)))
1234                 submit_flush_bio(ic, dio);
1235         else
1236                 do_endio(ic, bio);
1237 }
1238
1239 static void dec_in_flight(struct dm_integrity_io *dio)
1240 {
1241         if (atomic_dec_and_test(&dio->in_flight)) {
1242                 struct dm_integrity_c *ic = dio->ic;
1243                 struct bio *bio;
1244
1245                 remove_range(ic, &dio->range);
1246
1247                 if (unlikely(dio->write))
1248                         schedule_autocommit(ic);
1249
1250                 bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));
1251
1252                 if (unlikely(dio->bi_status) && !bio->bi_status)
1253                         bio->bi_status = dio->bi_status;
1254                 if (likely(!bio->bi_status) && unlikely(bio_sectors(bio) != dio->range.n_sectors)) {
1255                         dio->range.logical_sector += dio->range.n_sectors;
1256                         bio_advance(bio, dio->range.n_sectors << SECTOR_SHIFT);
1257                         INIT_WORK(&dio->work, integrity_bio_wait);
1258                         queue_work(ic->wait_wq, &dio->work);
1259                         return;
1260                 }
1261                 do_endio_flush(ic, dio);
1262         }
1263 }
1264
1265 static void integrity_end_io(struct bio *bio)
1266 {
1267         struct dm_integrity_io *dio = dm_per_bio_data(bio, sizeof(struct dm_integrity_io));
1268
1269         bio->bi_iter = dio->orig_bi_iter;
1270         bio->bi_disk = dio->orig_bi_disk;
1271         bio->bi_partno = dio->orig_bi_partno;
1272         if (dio->orig_bi_integrity) {
1273                 bio->bi_integrity = dio->orig_bi_integrity;
1274                 bio->bi_opf |= REQ_INTEGRITY;
1275         }
1276         bio->bi_end_io = dio->orig_bi_end_io;
1277
1278         if (dio->completion)
1279                 complete(dio->completion);
1280
1281         dec_in_flight(dio);
1282 }
1283
1284 static void integrity_sector_checksum(struct dm_integrity_c *ic, sector_t sector,
1285                                       const char *data, char *result)
1286 {
1287         __u64 sector_le = cpu_to_le64(sector);
1288         SHASH_DESC_ON_STACK(req, ic->internal_hash);
1289         int r;
1290         unsigned digest_size;
1291
1292         req->tfm = ic->internal_hash;
1293         req->flags = 0;
1294
1295         r = crypto_shash_init(req);
1296         if (unlikely(r < 0)) {
1297                 dm_integrity_io_error(ic, "crypto_shash_init", r);
1298                 goto failed;
1299         }
1300
1301         r = crypto_shash_update(req, (const __u8 *)&sector_le, sizeof sector_le);
1302         if (unlikely(r < 0)) {
1303                 dm_integrity_io_error(ic, "crypto_shash_update", r);
1304                 goto failed;
1305         }
1306
1307         r = crypto_shash_update(req, data, ic->sectors_per_block << SECTOR_SHIFT);
1308         if (unlikely(r < 0)) {
1309                 dm_integrity_io_error(ic, "crypto_shash_update", r);
1310                 goto failed;
1311         }
1312
1313         r = crypto_shash_final(req, result);
1314         if (unlikely(r < 0)) {
1315                 dm_integrity_io_error(ic, "crypto_shash_final", r);
1316                 goto failed;
1317         }
1318
1319         digest_size = crypto_shash_digestsize(ic->internal_hash);
1320         if (unlikely(digest_size < ic->tag_size))
1321                 memset(result + digest_size, 0, ic->tag_size - digest_size);
1322
1323         return;
1324
1325 failed:
1326         /* this shouldn't happen anyway, the hash functions have no reason to fail */
1327         get_random_bytes(result, ic->tag_size);
1328 }
1329
1330 static void integrity_metadata(struct work_struct *w)
1331 {
1332         struct dm_integrity_io *dio = container_of(w, struct dm_integrity_io, work);
1333         struct dm_integrity_c *ic = dio->ic;
1334
1335         int r;
1336
1337         if (ic->internal_hash) {
1338                 struct bvec_iter iter;
1339                 struct bio_vec bv;
1340                 unsigned digest_size = crypto_shash_digestsize(ic->internal_hash);
1341                 struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));
1342                 char *checksums;
1343                 unsigned extra_space = unlikely(digest_size > ic->tag_size) ? digest_size - ic->tag_size : 0;
1344                 char checksums_onstack[HASH_MAX_DIGESTSIZE];
1345                 unsigned sectors_to_process = dio->range.n_sectors;
1346                 sector_t sector = dio->range.logical_sector;
1347
1348                 if (unlikely(ic->mode == 'R'))
1349                         goto skip_io;
1350
1351                 checksums = kmalloc((PAGE_SIZE >> SECTOR_SHIFT >> ic->sb->log2_sectors_per_block) * ic->tag_size + extra_space,
1352                                     GFP_NOIO | __GFP_NORETRY | __GFP_NOWARN);
1353                 if (!checksums) {
1354                         checksums = checksums_onstack;
1355                         if (WARN_ON(extra_space &&
1356                                     digest_size > sizeof(checksums_onstack))) {
1357                                 r = -EINVAL;
1358                                 goto error;
1359                         }
1360                 }
1361
1362                 __bio_for_each_segment(bv, bio, iter, dio->orig_bi_iter) {
1363                         unsigned pos;
1364                         char *mem, *checksums_ptr;
1365
1366 again:
1367                         mem = (char *)kmap_atomic(bv.bv_page) + bv.bv_offset;
1368                         pos = 0;
1369                         checksums_ptr = checksums;
1370                         do {
1371                                 integrity_sector_checksum(ic, sector, mem + pos, checksums_ptr);
1372                                 checksums_ptr += ic->tag_size;
1373                                 sectors_to_process -= ic->sectors_per_block;
1374                                 pos += ic->sectors_per_block << SECTOR_SHIFT;
1375                                 sector += ic->sectors_per_block;
1376                         } while (pos < bv.bv_len && sectors_to_process && checksums != checksums_onstack);
1377                         kunmap_atomic(mem);
1378
1379                         r = dm_integrity_rw_tag(ic, checksums, &dio->metadata_block, &dio->metadata_offset,
1380                                                 checksums_ptr - checksums, !dio->write ? TAG_CMP : TAG_WRITE);
1381                         if (unlikely(r)) {
1382                                 if (r > 0) {
1383                                         DMERR_LIMIT("Checksum failed at sector 0x%llx",
1384                                                     (unsigned long long)(sector - ((r + ic->tag_size - 1) / ic->tag_size)));
1385                                         r = -EILSEQ;
1386                                         atomic64_inc(&ic->number_of_mismatches);
1387                                 }
1388                                 if (likely(checksums != checksums_onstack))
1389                                         kfree(checksums);
1390                                 goto error;
1391                         }
1392
1393                         if (!sectors_to_process)
1394                                 break;
1395
1396                         if (unlikely(pos < bv.bv_len)) {
1397                                 bv.bv_offset += pos;
1398                                 bv.bv_len -= pos;
1399                                 goto again;
1400                         }
1401                 }
1402
1403                 if (likely(checksums != checksums_onstack))
1404                         kfree(checksums);
1405         } else {
1406                 struct bio_integrity_payload *bip = dio->orig_bi_integrity;
1407
1408                 if (bip) {
1409                         struct bio_vec biv;
1410                         struct bvec_iter iter;
1411                         unsigned data_to_process = dio->range.n_sectors;
1412                         sector_to_block(ic, data_to_process);
1413                         data_to_process *= ic->tag_size;
1414
1415                         bip_for_each_vec(biv, bip, iter) {
1416                                 unsigned char *tag;
1417                                 unsigned this_len;
1418
1419                                 BUG_ON(PageHighMem(biv.bv_page));
1420                                 tag = lowmem_page_address(biv.bv_page) + biv.bv_offset;
1421                                 this_len = min(biv.bv_len, data_to_process);
1422                                 r = dm_integrity_rw_tag(ic, tag, &dio->metadata_block, &dio->metadata_offset,
1423                                                         this_len, !dio->write ? TAG_READ : TAG_WRITE);
1424                                 if (unlikely(r))
1425                                         goto error;
1426                                 data_to_process -= this_len;
1427                                 if (!data_to_process)
1428                                         break;
1429                         }
1430                 }
1431         }
1432 skip_io:
1433         dec_in_flight(dio);
1434         return;
1435 error:
1436         dio->bi_status = errno_to_blk_status(r);
1437         dec_in_flight(dio);
1438 }
1439
1440 static int dm_integrity_map(struct dm_target *ti, struct bio *bio)
1441 {
1442         struct dm_integrity_c *ic = ti->private;
1443         struct dm_integrity_io *dio = dm_per_bio_data(bio, sizeof(struct dm_integrity_io));
1444         struct bio_integrity_payload *bip;
1445
1446         sector_t area, offset;
1447
1448         dio->ic = ic;
1449         dio->bi_status = 0;
1450
1451         if (unlikely(bio->bi_opf & REQ_PREFLUSH)) {
1452                 submit_flush_bio(ic, dio);
1453                 return DM_MAPIO_SUBMITTED;
1454         }
1455
1456         dio->range.logical_sector = dm_target_offset(ti, bio->bi_iter.bi_sector);
1457         dio->write = bio_op(bio) == REQ_OP_WRITE;
1458         dio->fua = dio->write && bio->bi_opf & REQ_FUA;
1459         if (unlikely(dio->fua)) {
1460                 /*
1461                  * Don't pass down the FUA flag because we have to flush
1462                  * disk cache anyway.
1463                  */
1464                 bio->bi_opf &= ~REQ_FUA;
1465         }
1466         if (unlikely(dio->range.logical_sector + bio_sectors(bio) > ic->provided_data_sectors)) {
1467                 DMERR("Too big sector number: 0x%llx + 0x%x > 0x%llx",
1468                       (unsigned long long)dio->range.logical_sector, bio_sectors(bio),
1469                       (unsigned long long)ic->provided_data_sectors);
1470                 return DM_MAPIO_KILL;
1471         }
1472         if (unlikely((dio->range.logical_sector | bio_sectors(bio)) & (unsigned)(ic->sectors_per_block - 1))) {
1473                 DMERR("Bio not aligned on %u sectors: 0x%llx, 0x%x",
1474                       ic->sectors_per_block,
1475                       (unsigned long long)dio->range.logical_sector, bio_sectors(bio));
1476                 return DM_MAPIO_KILL;
1477         }
1478
1479         if (ic->sectors_per_block > 1) {
1480                 struct bvec_iter iter;
1481                 struct bio_vec bv;
1482                 bio_for_each_segment(bv, bio, iter) {
1483                         if (unlikely(bv.bv_len & ((ic->sectors_per_block << SECTOR_SHIFT) - 1))) {
1484                                 DMERR("Bio vector (%u,%u) is not aligned on %u-sector boundary",
1485                                         bv.bv_offset, bv.bv_len, ic->sectors_per_block);
1486                                 return DM_MAPIO_KILL;
1487                         }
1488                 }
1489         }
1490
1491         bip = bio_integrity(bio);
1492         if (!ic->internal_hash) {
1493                 if (bip) {
1494                         unsigned wanted_tag_size = bio_sectors(bio) >> ic->sb->log2_sectors_per_block;
1495                         if (ic->log2_tag_size >= 0)
1496                                 wanted_tag_size <<= ic->log2_tag_size;
1497                         else
1498                                 wanted_tag_size *= ic->tag_size;
1499                         if (unlikely(wanted_tag_size != bip->bip_iter.bi_size)) {
1500                                 DMERR("Invalid integrity data size %u, expected %u", bip->bip_iter.bi_size, wanted_tag_size);
1501                                 return DM_MAPIO_KILL;
1502                         }
1503                 }
1504         } else {
1505                 if (unlikely(bip != NULL)) {
1506                         DMERR("Unexpected integrity data when using internal hash");
1507                         return DM_MAPIO_KILL;
1508                 }
1509         }
1510
1511         if (unlikely(ic->mode == 'R') && unlikely(dio->write))
1512                 return DM_MAPIO_KILL;
1513
1514         get_area_and_offset(ic, dio->range.logical_sector, &area, &offset);
1515         dio->metadata_block = get_metadata_sector_and_offset(ic, area, offset, &dio->metadata_offset);
1516         bio->bi_iter.bi_sector = get_data_sector(ic, area, offset);
1517
1518         dm_integrity_map_continue(dio, true);
1519         return DM_MAPIO_SUBMITTED;
1520 }
1521
1522 static bool __journal_read_write(struct dm_integrity_io *dio, struct bio *bio,
1523                                  unsigned journal_section, unsigned journal_entry)
1524 {
1525         struct dm_integrity_c *ic = dio->ic;
1526         sector_t logical_sector;
1527         unsigned n_sectors;
1528
1529         logical_sector = dio->range.logical_sector;
1530         n_sectors = dio->range.n_sectors;
1531         do {
1532                 struct bio_vec bv = bio_iovec(bio);
1533                 char *mem;
1534
1535                 if (unlikely(bv.bv_len >> SECTOR_SHIFT > n_sectors))
1536                         bv.bv_len = n_sectors << SECTOR_SHIFT;
1537                 n_sectors -= bv.bv_len >> SECTOR_SHIFT;
1538                 bio_advance_iter(bio, &bio->bi_iter, bv.bv_len);
1539 retry_kmap:
1540                 mem = kmap_atomic(bv.bv_page);
1541                 if (likely(dio->write))
1542                         flush_dcache_page(bv.bv_page);
1543
1544                 do {
1545                         struct journal_entry *je = access_journal_entry(ic, journal_section, journal_entry);
1546
1547                         if (unlikely(!dio->write)) {
1548                                 struct journal_sector *js;
1549                                 char *mem_ptr;
1550                                 unsigned s;
1551
1552                                 if (unlikely(journal_entry_is_inprogress(je))) {
1553                                         flush_dcache_page(bv.bv_page);
1554                                         kunmap_atomic(mem);
1555
1556                                         __io_wait_event(ic->copy_to_journal_wait, !journal_entry_is_inprogress(je));
1557                                         goto retry_kmap;
1558                                 }
1559                                 smp_rmb();
1560                                 BUG_ON(journal_entry_get_sector(je) != logical_sector);
1561                                 js = access_journal_data(ic, journal_section, journal_entry);
1562                                 mem_ptr = mem + bv.bv_offset;
1563                                 s = 0;
1564                                 do {
1565                                         memcpy(mem_ptr, js, JOURNAL_SECTOR_DATA);
1566                                         *(commit_id_t *)(mem_ptr + JOURNAL_SECTOR_DATA) = je->last_bytes[s];
1567                                         js++;
1568                                         mem_ptr += 1 << SECTOR_SHIFT;
1569                                 } while (++s < ic->sectors_per_block);
1570 #ifdef INTERNAL_VERIFY
1571                                 if (ic->internal_hash) {
1572                                         char checksums_onstack[max(HASH_MAX_DIGESTSIZE, MAX_TAG_SIZE)];
1573
1574                                         integrity_sector_checksum(ic, logical_sector, mem + bv.bv_offset, checksums_onstack);
1575                                         if (unlikely(memcmp(checksums_onstack, journal_entry_tag(ic, je), ic->tag_size))) {
1576                                                 DMERR_LIMIT("Checksum failed when reading from journal, at sector 0x%llx",
1577                                                             (unsigned long long)logical_sector);
1578                                         }
1579                                 }
1580 #endif
1581                         }
1582
1583                         if (!ic->internal_hash) {
1584                                 struct bio_integrity_payload *bip = bio_integrity(bio);
1585                                 unsigned tag_todo = ic->tag_size;
1586                                 char *tag_ptr = journal_entry_tag(ic, je);
1587
1588                                 if (bip) do {
1589                                         struct bio_vec biv = bvec_iter_bvec(bip->bip_vec, bip->bip_iter);
1590                                         unsigned tag_now = min(biv.bv_len, tag_todo);
1591                                         char *tag_addr;
1592                                         BUG_ON(PageHighMem(biv.bv_page));
1593                                         tag_addr = lowmem_page_address(biv.bv_page) + biv.bv_offset;
1594                                         if (likely(dio->write))
1595                                                 memcpy(tag_ptr, tag_addr, tag_now);
1596                                         else
1597                                                 memcpy(tag_addr, tag_ptr, tag_now);
1598                                         bvec_iter_advance(bip->bip_vec, &bip->bip_iter, tag_now);
1599                                         tag_ptr += tag_now;
1600                                         tag_todo -= tag_now;
1601                                 } while (unlikely(tag_todo)); else {
1602                                         if (likely(dio->write))
1603                                                 memset(tag_ptr, 0, tag_todo);
1604                                 }
1605                         }
1606
1607                         if (likely(dio->write)) {
1608                                 struct journal_sector *js;
1609                                 unsigned s;
1610
1611                                 js = access_journal_data(ic, journal_section, journal_entry);
1612                                 memcpy(js, mem + bv.bv_offset, ic->sectors_per_block << SECTOR_SHIFT);
1613
1614                                 s = 0;
1615                                 do {
1616                                         je->last_bytes[s] = js[s].commit_id;
1617                                 } while (++s < ic->sectors_per_block);
1618
1619                                 if (ic->internal_hash) {
1620                                         unsigned digest_size = crypto_shash_digestsize(ic->internal_hash);
1621                                         if (unlikely(digest_size > ic->tag_size)) {
1622                                                 char checksums_onstack[HASH_MAX_DIGESTSIZE];
1623                                                 integrity_sector_checksum(ic, logical_sector, (char *)js, checksums_onstack);
1624                                                 memcpy(journal_entry_tag(ic, je), checksums_onstack, ic->tag_size);
1625                                         } else
1626                                                 integrity_sector_checksum(ic, logical_sector, (char *)js, journal_entry_tag(ic, je));
1627                                 }
1628
1629                                 journal_entry_set_sector(je, logical_sector);
1630                         }
1631                         logical_sector += ic->sectors_per_block;
1632
1633                         journal_entry++;
1634                         if (unlikely(journal_entry == ic->journal_section_entries)) {
1635                                 journal_entry = 0;
1636                                 journal_section++;
1637                                 wraparound_section(ic, &journal_section);
1638                         }
1639
1640                         bv.bv_offset += ic->sectors_per_block << SECTOR_SHIFT;
1641                 } while (bv.bv_len -= ic->sectors_per_block << SECTOR_SHIFT);
1642
1643                 if (unlikely(!dio->write))
1644                         flush_dcache_page(bv.bv_page);
1645                 kunmap_atomic(mem);
1646         } while (n_sectors);
1647
1648         if (likely(dio->write)) {
1649                 smp_mb();
1650                 if (unlikely(waitqueue_active(&ic->copy_to_journal_wait)))
1651                         wake_up(&ic->copy_to_journal_wait);
1652                 if (READ_ONCE(ic->free_sectors) <= ic->free_sectors_threshold) {
1653                         queue_work(ic->commit_wq, &ic->commit_work);
1654                 } else {
1655                         schedule_autocommit(ic);
1656                 }
1657         } else {
1658                 remove_range(ic, &dio->range);
1659         }
1660
1661         if (unlikely(bio->bi_iter.bi_size)) {
1662                 sector_t area, offset;
1663
1664                 dio->range.logical_sector = logical_sector;
1665                 get_area_and_offset(ic, dio->range.logical_sector, &area, &offset);
1666                 dio->metadata_block = get_metadata_sector_and_offset(ic, area, offset, &dio->metadata_offset);
1667                 return true;
1668         }
1669
1670         return false;
1671 }
1672
1673 static void dm_integrity_map_continue(struct dm_integrity_io *dio, bool from_map)
1674 {
1675         struct dm_integrity_c *ic = dio->ic;
1676         struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));
1677         unsigned journal_section, journal_entry;
1678         unsigned journal_read_pos;
1679         struct completion read_comp;
1680         bool need_sync_io = ic->internal_hash && !dio->write;
1681
1682         if (need_sync_io && from_map) {
1683                 INIT_WORK(&dio->work, integrity_bio_wait);
1684                 queue_work(ic->metadata_wq, &dio->work);
1685                 return;
1686         }
1687
1688 lock_retry:
1689         spin_lock_irq(&ic->endio_wait.lock);
1690 retry:
1691         if (unlikely(dm_integrity_failed(ic))) {
1692                 spin_unlock_irq(&ic->endio_wait.lock);
1693                 do_endio(ic, bio);
1694                 return;
1695         }
1696         dio->range.n_sectors = bio_sectors(bio);
1697         journal_read_pos = NOT_FOUND;
1698         if (likely(ic->mode == 'J')) {
1699                 if (dio->write) {
1700                         unsigned next_entry, i, pos;
1701                         unsigned ws, we, range_sectors;
1702
1703                         dio->range.n_sectors = min(dio->range.n_sectors,
1704                                                    (sector_t)ic->free_sectors << ic->sb->log2_sectors_per_block);
1705                         if (unlikely(!dio->range.n_sectors)) {
1706                                 if (from_map)
1707                                         goto offload_to_thread;
1708                                 sleep_on_endio_wait(ic);
1709                                 goto retry;
1710                         }
1711                         range_sectors = dio->range.n_sectors >> ic->sb->log2_sectors_per_block;
1712                         ic->free_sectors -= range_sectors;
1713                         journal_section = ic->free_section;
1714                         journal_entry = ic->free_section_entry;
1715
1716                         next_entry = ic->free_section_entry + range_sectors;
1717                         ic->free_section_entry = next_entry % ic->journal_section_entries;
1718                         ic->free_section += next_entry / ic->journal_section_entries;
1719                         ic->n_uncommitted_sections += next_entry / ic->journal_section_entries;
1720                         wraparound_section(ic, &ic->free_section);
1721
1722                         pos = journal_section * ic->journal_section_entries + journal_entry;
1723                         ws = journal_section;
1724                         we = journal_entry;
1725                         i = 0;
1726                         do {
1727                                 struct journal_entry *je;
1728
1729                                 add_journal_node(ic, &ic->journal_tree[pos], dio->range.logical_sector + i);
1730                                 pos++;
1731                                 if (unlikely(pos >= ic->journal_entries))
1732                                         pos = 0;
1733
1734                                 je = access_journal_entry(ic, ws, we);
1735                                 BUG_ON(!journal_entry_is_unused(je));
1736                                 journal_entry_set_inprogress(je);
1737                                 we++;
1738                                 if (unlikely(we == ic->journal_section_entries)) {
1739                                         we = 0;
1740                                         ws++;
1741                                         wraparound_section(ic, &ws);
1742                                 }
1743                         } while ((i += ic->sectors_per_block) < dio->range.n_sectors);
1744
1745                         spin_unlock_irq(&ic->endio_wait.lock);
1746                         goto journal_read_write;
1747                 } else {
1748                         sector_t next_sector;
1749                         journal_read_pos = find_journal_node(ic, dio->range.logical_sector, &next_sector);
1750                         if (likely(journal_read_pos == NOT_FOUND)) {
1751                                 if (unlikely(dio->range.n_sectors > next_sector - dio->range.logical_sector))
1752                                         dio->range.n_sectors = next_sector - dio->range.logical_sector;
1753                         } else {
1754                                 unsigned i;
1755                                 unsigned jp = journal_read_pos + 1;
1756                                 for (i = ic->sectors_per_block; i < dio->range.n_sectors; i += ic->sectors_per_block, jp++) {
1757                                         if (!test_journal_node(ic, jp, dio->range.logical_sector + i))
1758                                                 break;
1759                                 }
1760                                 dio->range.n_sectors = i;
1761                         }
1762                 }
1763         }
1764         if (unlikely(!add_new_range(ic, &dio->range, true))) {
1765                 /*
1766                  * We must not sleep in the request routine because it could
1767                  * stall bios on current->bio_list.
1768                  * So, we offload the bio to a workqueue if we have to sleep.
1769                  */
1770                 if (from_map) {
1771 offload_to_thread:
1772                         spin_unlock_irq(&ic->endio_wait.lock);
1773                         INIT_WORK(&dio->work, integrity_bio_wait);
1774                         queue_work(ic->wait_wq, &dio->work);
1775                         return;
1776                 }
1777                 wait_and_add_new_range(ic, &dio->range);
1778         }
1779         spin_unlock_irq(&ic->endio_wait.lock);
1780
1781         if (unlikely(journal_read_pos != NOT_FOUND)) {
1782                 journal_section = journal_read_pos / ic->journal_section_entries;
1783                 journal_entry = journal_read_pos % ic->journal_section_entries;
1784                 goto journal_read_write;
1785         }
1786
1787         dio->in_flight = (atomic_t)ATOMIC_INIT(2);
1788
1789         if (need_sync_io) {
1790                 init_completion(&read_comp);
1791                 dio->completion = &read_comp;
1792         } else
1793                 dio->completion = NULL;
1794
1795         dio->orig_bi_iter = bio->bi_iter;
1796
1797         dio->orig_bi_disk = bio->bi_disk;
1798         dio->orig_bi_partno = bio->bi_partno;
1799         bio_set_dev(bio, ic->dev->bdev);
1800
1801         dio->orig_bi_integrity = bio_integrity(bio);
1802         bio->bi_integrity = NULL;
1803         bio->bi_opf &= ~REQ_INTEGRITY;
1804
1805         dio->orig_bi_end_io = bio->bi_end_io;
1806         bio->bi_end_io = integrity_end_io;
1807
1808         bio->bi_iter.bi_size = dio->range.n_sectors << SECTOR_SHIFT;
1809         generic_make_request(bio);
1810
1811         if (need_sync_io) {
1812                 wait_for_completion_io(&read_comp);
1813                 if (unlikely(ic->recalc_wq != NULL) &&
1814                     ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING) &&
1815                     dio->range.logical_sector + dio->range.n_sectors > le64_to_cpu(ic->sb->recalc_sector))
1816                         goto skip_check;
1817                 if (likely(!bio->bi_status))
1818                         integrity_metadata(&dio->work);
1819                 else
1820 skip_check:
1821                         dec_in_flight(dio);
1822
1823         } else {
1824                 INIT_WORK(&dio->work, integrity_metadata);
1825                 queue_work(ic->metadata_wq, &dio->work);
1826         }
1827
1828         return;
1829
1830 journal_read_write:
1831         if (unlikely(__journal_read_write(dio, bio, journal_section, journal_entry)))
1832                 goto lock_retry;
1833
1834         do_endio_flush(ic, dio);
1835 }
1836
1837
1838 static void integrity_bio_wait(struct work_struct *w)
1839 {
1840         struct dm_integrity_io *dio = container_of(w, struct dm_integrity_io, work);
1841
1842         dm_integrity_map_continue(dio, false);
1843 }
1844
1845 static void pad_uncommitted(struct dm_integrity_c *ic)
1846 {
1847         if (ic->free_section_entry) {
1848                 ic->free_sectors -= ic->journal_section_entries - ic->free_section_entry;
1849                 ic->free_section_entry = 0;
1850                 ic->free_section++;
1851                 wraparound_section(ic, &ic->free_section);
1852                 ic->n_uncommitted_sections++;
1853         }
1854         WARN_ON(ic->journal_sections * ic->journal_section_entries !=
1855                 (ic->n_uncommitted_sections + ic->n_committed_sections) * ic->journal_section_entries + ic->free_sectors);
1856 }
1857
1858 static void integrity_commit(struct work_struct *w)
1859 {
1860         struct dm_integrity_c *ic = container_of(w, struct dm_integrity_c, commit_work);
1861         unsigned commit_start, commit_sections;
1862         unsigned i, j, n;
1863         struct bio *flushes;
1864
1865         del_timer(&ic->autocommit_timer);
1866
1867         spin_lock_irq(&ic->endio_wait.lock);
1868         flushes = bio_list_get(&ic->flush_bio_list);
1869         if (unlikely(ic->mode != 'J')) {
1870                 spin_unlock_irq(&ic->endio_wait.lock);
1871                 dm_integrity_flush_buffers(ic);
1872                 goto release_flush_bios;
1873         }
1874
1875         pad_uncommitted(ic);
1876         commit_start = ic->uncommitted_section;
1877         commit_sections = ic->n_uncommitted_sections;
1878         spin_unlock_irq(&ic->endio_wait.lock);
1879
1880         if (!commit_sections)
1881                 goto release_flush_bios;
1882
1883         i = commit_start;
1884         for (n = 0; n < commit_sections; n++) {
1885                 for (j = 0; j < ic->journal_section_entries; j++) {
1886                         struct journal_entry *je;
1887                         je = access_journal_entry(ic, i, j);
1888                         io_wait_event(ic->copy_to_journal_wait, !journal_entry_is_inprogress(je));
1889                 }
1890                 for (j = 0; j < ic->journal_section_sectors; j++) {
1891                         struct journal_sector *js;
1892                         js = access_journal(ic, i, j);
1893                         js->commit_id = dm_integrity_commit_id(ic, i, j, ic->commit_seq);
1894                 }
1895                 i++;
1896                 if (unlikely(i >= ic->journal_sections))
1897                         ic->commit_seq = next_commit_seq(ic->commit_seq);
1898                 wraparound_section(ic, &i);
1899         }
1900         smp_rmb();
1901
1902         write_journal(ic, commit_start, commit_sections);
1903
1904         spin_lock_irq(&ic->endio_wait.lock);
1905         ic->uncommitted_section += commit_sections;
1906         wraparound_section(ic, &ic->uncommitted_section);
1907         ic->n_uncommitted_sections -= commit_sections;
1908         ic->n_committed_sections += commit_sections;
1909         spin_unlock_irq(&ic->endio_wait.lock);
1910
1911         if (READ_ONCE(ic->free_sectors) <= ic->free_sectors_threshold)
1912                 queue_work(ic->writer_wq, &ic->writer_work);
1913
1914 release_flush_bios:
1915         while (flushes) {
1916                 struct bio *next = flushes->bi_next;
1917                 flushes->bi_next = NULL;
1918                 do_endio(ic, flushes);
1919                 flushes = next;
1920         }
1921 }
1922
1923 static void complete_copy_from_journal(unsigned long error, void *context)
1924 {
1925         struct journal_io *io = context;
1926         struct journal_completion *comp = io->comp;
1927         struct dm_integrity_c *ic = comp->ic;
1928         remove_range(ic, &io->range);
1929         mempool_free(io, &ic->journal_io_mempool);
1930         if (unlikely(error != 0))
1931                 dm_integrity_io_error(ic, "copying from journal", -EIO);
1932         complete_journal_op(comp);
1933 }
1934
1935 static void restore_last_bytes(struct dm_integrity_c *ic, struct journal_sector *js,
1936                                struct journal_entry *je)
1937 {
1938         unsigned s = 0;
1939         do {
1940                 js->commit_id = je->last_bytes[s];
1941                 js++;
1942         } while (++s < ic->sectors_per_block);
1943 }
1944
1945 static void do_journal_write(struct dm_integrity_c *ic, unsigned write_start,
1946                              unsigned write_sections, bool from_replay)
1947 {
1948         unsigned i, j, n;
1949         struct journal_completion comp;
1950         struct blk_plug plug;
1951
1952         blk_start_plug(&plug);
1953
1954         comp.ic = ic;
1955         comp.in_flight = (atomic_t)ATOMIC_INIT(1);
1956         init_completion(&comp.comp);
1957
1958         i = write_start;
1959         for (n = 0; n < write_sections; n++, i++, wraparound_section(ic, &i)) {
1960 #ifndef INTERNAL_VERIFY
1961                 if (unlikely(from_replay))
1962 #endif
1963                         rw_section_mac(ic, i, false);
1964                 for (j = 0; j < ic->journal_section_entries; j++) {
1965                         struct journal_entry *je = access_journal_entry(ic, i, j);
1966                         sector_t sec, area, offset;
1967                         unsigned k, l, next_loop;
1968                         sector_t metadata_block;
1969                         unsigned metadata_offset;
1970                         struct journal_io *io;
1971
1972                         if (journal_entry_is_unused(je))
1973                                 continue;
1974                         BUG_ON(unlikely(journal_entry_is_inprogress(je)) && !from_replay);
1975                         sec = journal_entry_get_sector(je);
1976                         if (unlikely(from_replay)) {
1977                                 if (unlikely(sec & (unsigned)(ic->sectors_per_block - 1))) {
1978                                         dm_integrity_io_error(ic, "invalid sector in journal", -EIO);
1979                                         sec &= ~(sector_t)(ic->sectors_per_block - 1);
1980                                 }
1981                         }
1982                         get_area_and_offset(ic, sec, &area, &offset);
1983                         restore_last_bytes(ic, access_journal_data(ic, i, j), je);
1984                         for (k = j + 1; k < ic->journal_section_entries; k++) {
1985                                 struct journal_entry *je2 = access_journal_entry(ic, i, k);
1986                                 sector_t sec2, area2, offset2;
1987                                 if (journal_entry_is_unused(je2))
1988                                         break;
1989                                 BUG_ON(unlikely(journal_entry_is_inprogress(je2)) && !from_replay);
1990                                 sec2 = journal_entry_get_sector(je2);
1991                                 get_area_and_offset(ic, sec2, &area2, &offset2);
1992                                 if (area2 != area || offset2 != offset + ((k - j) << ic->sb->log2_sectors_per_block))
1993                                         break;
1994                                 restore_last_bytes(ic, access_journal_data(ic, i, k), je2);
1995                         }
1996                         next_loop = k - 1;
1997
1998                         io = mempool_alloc(&ic->journal_io_mempool, GFP_NOIO);
1999                         io->comp = &comp;
2000                         io->range.logical_sector = sec;
2001                         io->range.n_sectors = (k - j) << ic->sb->log2_sectors_per_block;
2002
2003                         spin_lock_irq(&ic->endio_wait.lock);
2004                         add_new_range_and_wait(ic, &io->range);
2005
2006                         if (likely(!from_replay)) {
2007                                 struct journal_node *section_node = &ic->journal_tree[i * ic->journal_section_entries];
2008
2009                                 /* don't write if there is newer committed sector */
2010                                 while (j < k && find_newer_committed_node(ic, &section_node[j])) {
2011                                         struct journal_entry *je2 = access_journal_entry(ic, i, j);
2012
2013                                         journal_entry_set_unused(je2);
2014                                         remove_journal_node(ic, &section_node[j]);
2015                                         j++;
2016                                         sec += ic->sectors_per_block;
2017                                         offset += ic->sectors_per_block;
2018                                 }
2019                                 while (j < k && find_newer_committed_node(ic, &section_node[k - 1])) {
2020                                         struct journal_entry *je2 = access_journal_entry(ic, i, k - 1);
2021
2022                                         journal_entry_set_unused(je2);
2023                                         remove_journal_node(ic, &section_node[k - 1]);
2024                                         k--;
2025                                 }
2026                                 if (j == k) {
2027                                         remove_range_unlocked(ic, &io->range);
2028                                         spin_unlock_irq(&ic->endio_wait.lock);
2029                                         mempool_free(io, &ic->journal_io_mempool);
2030                                         goto skip_io;
2031                                 }
2032                                 for (l = j; l < k; l++) {
2033                                         remove_journal_node(ic, &section_node[l]);
2034                                 }
2035                         }
2036                         spin_unlock_irq(&ic->endio_wait.lock);
2037
2038                         metadata_block = get_metadata_sector_and_offset(ic, area, offset, &metadata_offset);
2039                         for (l = j; l < k; l++) {
2040                                 int r;
2041                                 struct journal_entry *je2 = access_journal_entry(ic, i, l);
2042
2043                                 if (
2044 #ifndef INTERNAL_VERIFY
2045                                     unlikely(from_replay) &&
2046 #endif
2047                                     ic->internal_hash) {
2048                                         char test_tag[max_t(size_t, HASH_MAX_DIGESTSIZE, MAX_TAG_SIZE)];
2049
2050                                         integrity_sector_checksum(ic, sec + ((l - j) << ic->sb->log2_sectors_per_block),
2051                                                                   (char *)access_journal_data(ic, i, l), test_tag);
2052                                         if (unlikely(memcmp(test_tag, journal_entry_tag(ic, je2), ic->tag_size)))
2053                                                 dm_integrity_io_error(ic, "tag mismatch when replaying journal", -EILSEQ);
2054                                 }
2055
2056                                 journal_entry_set_unused(je2);
2057                                 r = dm_integrity_rw_tag(ic, journal_entry_tag(ic, je2), &metadata_block, &metadata_offset,
2058                                                         ic->tag_size, TAG_WRITE);
2059                                 if (unlikely(r)) {
2060                                         dm_integrity_io_error(ic, "reading tags", r);
2061                                 }
2062                         }
2063
2064                         atomic_inc(&comp.in_flight);
2065                         copy_from_journal(ic, i, j << ic->sb->log2_sectors_per_block,
2066                                           (k - j) << ic->sb->log2_sectors_per_block,
2067                                           get_data_sector(ic, area, offset),
2068                                           complete_copy_from_journal, io);
2069 skip_io:
2070                         j = next_loop;
2071                 }
2072         }
2073
2074         dm_bufio_write_dirty_buffers_async(ic->bufio);
2075
2076         blk_finish_plug(&plug);
2077
2078         complete_journal_op(&comp);
2079         wait_for_completion_io(&comp.comp);
2080
2081         dm_integrity_flush_buffers(ic);
2082 }
2083
2084 static void integrity_writer(struct work_struct *w)
2085 {
2086         struct dm_integrity_c *ic = container_of(w, struct dm_integrity_c, writer_work);
2087         unsigned write_start, write_sections;
2088
2089         unsigned prev_free_sectors;
2090
2091         /* the following test is not needed, but it tests the replay code */
2092         if (READ_ONCE(ic->suspending) && !ic->meta_dev)
2093                 return;
2094
2095         spin_lock_irq(&ic->endio_wait.lock);
2096         write_start = ic->committed_section;
2097         write_sections = ic->n_committed_sections;
2098         spin_unlock_irq(&ic->endio_wait.lock);
2099
2100         if (!write_sections)
2101                 return;
2102
2103         do_journal_write(ic, write_start, write_sections, false);
2104
2105         spin_lock_irq(&ic->endio_wait.lock);
2106
2107         ic->committed_section += write_sections;
2108         wraparound_section(ic, &ic->committed_section);
2109         ic->n_committed_sections -= write_sections;
2110
2111         prev_free_sectors = ic->free_sectors;
2112         ic->free_sectors += write_sections * ic->journal_section_entries;
2113         if (unlikely(!prev_free_sectors))
2114                 wake_up_locked(&ic->endio_wait);
2115
2116         spin_unlock_irq(&ic->endio_wait.lock);
2117 }
2118
2119 static void recalc_write_super(struct dm_integrity_c *ic)
2120 {
2121         int r;
2122
2123         dm_integrity_flush_buffers(ic);
2124         if (dm_integrity_failed(ic))
2125                 return;
2126
2127         sb_set_version(ic);
2128         r = sync_rw_sb(ic, REQ_OP_WRITE, 0);
2129         if (unlikely(r))
2130                 dm_integrity_io_error(ic, "writing superblock", r);
2131 }
2132
2133 static void integrity_recalc(struct work_struct *w)
2134 {
2135         struct dm_integrity_c *ic = container_of(w, struct dm_integrity_c, recalc_work);
2136         struct dm_integrity_range range;
2137         struct dm_io_request io_req;
2138         struct dm_io_region io_loc;
2139         sector_t area, offset;
2140         sector_t metadata_block;
2141         unsigned metadata_offset;
2142         __u8 *t;
2143         unsigned i;
2144         int r;
2145         unsigned super_counter = 0;
2146
2147         spin_lock_irq(&ic->endio_wait.lock);
2148
2149 next_chunk:
2150
2151         if (unlikely(READ_ONCE(ic->suspending)))
2152                 goto unlock_ret;
2153
2154         range.logical_sector = le64_to_cpu(ic->sb->recalc_sector);
2155         if (unlikely(range.logical_sector >= ic->provided_data_sectors))
2156                 goto unlock_ret;
2157
2158         get_area_and_offset(ic, range.logical_sector, &area, &offset);
2159         range.n_sectors = min((sector_t)RECALC_SECTORS, ic->provided_data_sectors - range.logical_sector);
2160         if (!ic->meta_dev)
2161                 range.n_sectors = min(range.n_sectors, ((sector_t)1U << ic->sb->log2_interleave_sectors) - (unsigned)offset);
2162
2163         add_new_range_and_wait(ic, &range);
2164
2165         spin_unlock_irq(&ic->endio_wait.lock);
2166
2167         if (unlikely(++super_counter == RECALC_WRITE_SUPER)) {
2168                 recalc_write_super(ic);
2169                 super_counter = 0;
2170         }
2171
2172         if (unlikely(dm_integrity_failed(ic)))
2173                 goto err;
2174
2175         io_req.bi_op = REQ_OP_READ;
2176         io_req.bi_op_flags = 0;
2177         io_req.mem.type = DM_IO_VMA;
2178         io_req.mem.ptr.addr = ic->recalc_buffer;
2179         io_req.notify.fn = NULL;
2180         io_req.client = ic->io;
2181         io_loc.bdev = ic->dev->bdev;
2182         io_loc.sector = get_data_sector(ic, area, offset);
2183         io_loc.count = range.n_sectors;
2184
2185         r = dm_io(&io_req, 1, &io_loc, NULL);
2186         if (unlikely(r)) {
2187                 dm_integrity_io_error(ic, "reading data", r);
2188                 goto err;
2189         }
2190
2191         t = ic->recalc_tags;
2192         for (i = 0; i < range.n_sectors; i += ic->sectors_per_block) {
2193                 integrity_sector_checksum(ic, range.logical_sector + i, ic->recalc_buffer + (i << SECTOR_SHIFT), t);
2194                 t += ic->tag_size;
2195         }
2196
2197         metadata_block = get_metadata_sector_and_offset(ic, area, offset, &metadata_offset);
2198
2199         r = dm_integrity_rw_tag(ic, ic->recalc_tags, &metadata_block, &metadata_offset, t - ic->recalc_tags, TAG_WRITE);
2200         if (unlikely(r)) {
2201                 dm_integrity_io_error(ic, "writing tags", r);
2202                 goto err;
2203         }
2204
2205         spin_lock_irq(&ic->endio_wait.lock);
2206         remove_range_unlocked(ic, &range);
2207         ic->sb->recalc_sector = cpu_to_le64(range.logical_sector + range.n_sectors);
2208         goto next_chunk;
2209
2210 err:
2211         remove_range(ic, &range);
2212         return;
2213
2214 unlock_ret:
2215         spin_unlock_irq(&ic->endio_wait.lock);
2216
2217         recalc_write_super(ic);
2218 }
2219
2220 static void init_journal(struct dm_integrity_c *ic, unsigned start_section,
2221                          unsigned n_sections, unsigned char commit_seq)
2222 {
2223         unsigned i, j, n;
2224
2225         if (!n_sections)
2226                 return;
2227
2228         for (n = 0; n < n_sections; n++) {
2229                 i = start_section + n;
2230                 wraparound_section(ic, &i);
2231                 for (j = 0; j < ic->journal_section_sectors; j++) {
2232                         struct journal_sector *js = access_journal(ic, i, j);
2233                         memset(&js->entries, 0, JOURNAL_SECTOR_DATA);
2234                         js->commit_id = dm_integrity_commit_id(ic, i, j, commit_seq);
2235                 }
2236                 for (j = 0; j < ic->journal_section_entries; j++) {
2237                         struct journal_entry *je = access_journal_entry(ic, i, j);
2238                         journal_entry_set_unused(je);
2239                 }
2240         }
2241
2242         write_journal(ic, start_section, n_sections);
2243 }
2244
2245 static int find_commit_seq(struct dm_integrity_c *ic, unsigned i, unsigned j, commit_id_t id)
2246 {
2247         unsigned char k;
2248         for (k = 0; k < N_COMMIT_IDS; k++) {
2249                 if (dm_integrity_commit_id(ic, i, j, k) == id)
2250                         return k;
2251         }
2252         dm_integrity_io_error(ic, "journal commit id", -EIO);
2253         return -EIO;
2254 }
2255
2256 static void replay_journal(struct dm_integrity_c *ic)
2257 {
2258         unsigned i, j;
2259         bool used_commit_ids[N_COMMIT_IDS];
2260         unsigned max_commit_id_sections[N_COMMIT_IDS];
2261         unsigned write_start, write_sections;
2262         unsigned continue_section;
2263         bool journal_empty;
2264         unsigned char unused, last_used, want_commit_seq;
2265
2266         if (ic->mode == 'R')
2267                 return;
2268
2269         if (ic->journal_uptodate)
2270                 return;
2271
2272         last_used = 0;
2273         write_start = 0;
2274
2275         if (!ic->just_formatted) {
2276                 DEBUG_print("reading journal\n");
2277                 rw_journal(ic, REQ_OP_READ, 0, 0, ic->journal_sections, NULL);
2278                 if (ic->journal_io)
2279                         DEBUG_bytes(lowmem_page_address(ic->journal_io[0].page), 64, "read journal");
2280                 if (ic->journal_io) {
2281                         struct journal_completion crypt_comp;
2282                         crypt_comp.ic = ic;
2283                         init_completion(&crypt_comp.comp);
2284                         crypt_comp.in_flight = (atomic_t)ATOMIC_INIT(0);
2285                         encrypt_journal(ic, false, 0, ic->journal_sections, &crypt_comp);
2286                         wait_for_completion(&crypt_comp.comp);
2287                 }
2288                 DEBUG_bytes(lowmem_page_address(ic->journal[0].page), 64, "decrypted journal");
2289         }
2290
2291         if (dm_integrity_failed(ic))
2292                 goto clear_journal;
2293
2294         journal_empty = true;
2295         memset(used_commit_ids, 0, sizeof used_commit_ids);
2296         memset(max_commit_id_sections, 0, sizeof max_commit_id_sections);
2297         for (i = 0; i < ic->journal_sections; i++) {
2298                 for (j = 0; j < ic->journal_section_sectors; j++) {
2299                         int k;
2300                         struct journal_sector *js = access_journal(ic, i, j);
2301                         k = find_commit_seq(ic, i, j, js->commit_id);
2302                         if (k < 0)
2303                                 goto clear_journal;
2304                         used_commit_ids[k] = true;
2305                         max_commit_id_sections[k] = i;
2306                 }
2307                 if (journal_empty) {
2308                         for (j = 0; j < ic->journal_section_entries; j++) {
2309                                 struct journal_entry *je = access_journal_entry(ic, i, j);
2310                                 if (!journal_entry_is_unused(je)) {
2311                                         journal_empty = false;
2312                                         break;
2313                                 }
2314                         }
2315                 }
2316         }
2317
2318         if (!used_commit_ids[N_COMMIT_IDS - 1]) {
2319                 unused = N_COMMIT_IDS - 1;
2320                 while (unused && !used_commit_ids[unused - 1])
2321                         unused--;
2322         } else {
2323                 for (unused = 0; unused < N_COMMIT_IDS; unused++)
2324                         if (!used_commit_ids[unused])
2325                                 break;
2326                 if (unused == N_COMMIT_IDS) {
2327                         dm_integrity_io_error(ic, "journal commit ids", -EIO);
2328                         goto clear_journal;
2329                 }
2330         }
2331         DEBUG_print("first unused commit seq %d [%d,%d,%d,%d]\n",
2332                     unused, used_commit_ids[0], used_commit_ids[1],
2333                     used_commit_ids[2], used_commit_ids[3]);
2334
2335         last_used = prev_commit_seq(unused);
2336         want_commit_seq = prev_commit_seq(last_used);
2337
2338         if (!used_commit_ids[want_commit_seq] && used_commit_ids[prev_commit_seq(want_commit_seq)])
2339                 journal_empty = true;
2340
2341         write_start = max_commit_id_sections[last_used] + 1;
2342         if (unlikely(write_start >= ic->journal_sections))
2343                 want_commit_seq = next_commit_seq(want_commit_seq);
2344         wraparound_section(ic, &write_start);
2345
2346         i = write_start;
2347         for (write_sections = 0; write_sections < ic->journal_sections; write_sections++) {
2348                 for (j = 0; j < ic->journal_section_sectors; j++) {
2349                         struct journal_sector *js = access_journal(ic, i, j);
2350
2351                         if (js->commit_id != dm_integrity_commit_id(ic, i, j, want_commit_seq)) {
2352                                 /*
2353                                  * This could be caused by crash during writing.
2354                                  * We won't replay the inconsistent part of the
2355                                  * journal.
2356                                  */
2357                                 DEBUG_print("commit id mismatch at position (%u, %u): %d != %d\n",
2358                                             i, j, find_commit_seq(ic, i, j, js->commit_id), want_commit_seq);
2359                                 goto brk;
2360                         }
2361                 }
2362                 i++;
2363                 if (unlikely(i >= ic->journal_sections))
2364                         want_commit_seq = next_commit_seq(want_commit_seq);
2365                 wraparound_section(ic, &i);
2366         }
2367 brk:
2368
2369         if (!journal_empty) {
2370                 DEBUG_print("replaying %u sections, starting at %u, commit seq %d\n",
2371                             write_sections, write_start, want_commit_seq);
2372                 do_journal_write(ic, write_start, write_sections, true);
2373         }
2374
2375         if (write_sections == ic->journal_sections && (ic->mode == 'J' || journal_empty)) {
2376                 continue_section = write_start;
2377                 ic->commit_seq = want_commit_seq;
2378                 DEBUG_print("continuing from section %u, commit seq %d\n", write_start, ic->commit_seq);
2379         } else {
2380                 unsigned s;
2381                 unsigned char erase_seq;
2382 clear_journal:
2383                 DEBUG_print("clearing journal\n");
2384
2385                 erase_seq = prev_commit_seq(prev_commit_seq(last_used));
2386                 s = write_start;
2387                 init_journal(ic, s, 1, erase_seq);
2388                 s++;
2389                 wraparound_section(ic, &s);
2390                 if (ic->journal_sections >= 2) {
2391                         init_journal(ic, s, ic->journal_sections - 2, erase_seq);
2392                         s += ic->journal_sections - 2;
2393                         wraparound_section(ic, &s);
2394                         init_journal(ic, s, 1, erase_seq);
2395                 }
2396
2397                 continue_section = 0;
2398                 ic->commit_seq = next_commit_seq(erase_seq);
2399         }
2400
2401         ic->committed_section = continue_section;
2402         ic->n_committed_sections = 0;
2403
2404         ic->uncommitted_section = continue_section;
2405         ic->n_uncommitted_sections = 0;
2406
2407         ic->free_section = continue_section;
2408         ic->free_section_entry = 0;
2409         ic->free_sectors = ic->journal_entries;
2410
2411         ic->journal_tree_root = RB_ROOT;
2412         for (i = 0; i < ic->journal_entries; i++)
2413                 init_journal_node(&ic->journal_tree[i]);
2414 }
2415
2416 static void dm_integrity_postsuspend(struct dm_target *ti)
2417 {
2418         struct dm_integrity_c *ic = (struct dm_integrity_c *)ti->private;
2419
2420         del_timer_sync(&ic->autocommit_timer);
2421
2422         WRITE_ONCE(ic->suspending, 1);
2423
2424         if (ic->recalc_wq)
2425                 drain_workqueue(ic->recalc_wq);
2426
2427         queue_work(ic->commit_wq, &ic->commit_work);
2428         drain_workqueue(ic->commit_wq);
2429
2430         if (ic->mode == 'J') {
2431                 if (ic->meta_dev)
2432                         queue_work(ic->writer_wq, &ic->writer_work);
2433                 drain_workqueue(ic->writer_wq);
2434                 dm_integrity_flush_buffers(ic);
2435         }
2436
2437         WRITE_ONCE(ic->suspending, 0);
2438
2439         BUG_ON(!RB_EMPTY_ROOT(&ic->in_progress));
2440
2441         ic->journal_uptodate = true;
2442 }
2443
2444 static void dm_integrity_resume(struct dm_target *ti)
2445 {
2446         struct dm_integrity_c *ic = (struct dm_integrity_c *)ti->private;
2447
2448         replay_journal(ic);
2449
2450         if (ic->recalc_wq && ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING)) {
2451                 __u64 recalc_pos = le64_to_cpu(ic->sb->recalc_sector);
2452                 if (recalc_pos < ic->provided_data_sectors) {
2453                         queue_work(ic->recalc_wq, &ic->recalc_work);
2454                 } else if (recalc_pos > ic->provided_data_sectors) {
2455                         ic->sb->recalc_sector = cpu_to_le64(ic->provided_data_sectors);
2456                         recalc_write_super(ic);
2457                 }
2458         }
2459 }
2460
2461 static void dm_integrity_status(struct dm_target *ti, status_type_t type,
2462                                 unsigned status_flags, char *result, unsigned maxlen)
2463 {
2464         struct dm_integrity_c *ic = (struct dm_integrity_c *)ti->private;
2465         unsigned arg_count;
2466         size_t sz = 0;
2467
2468         switch (type) {
2469         case STATUSTYPE_INFO:
2470                 DMEMIT("%llu %llu",
2471                         (unsigned long long)atomic64_read(&ic->number_of_mismatches),
2472                         (unsigned long long)ic->provided_data_sectors);
2473                 if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING))
2474                         DMEMIT(" %llu", (unsigned long long)le64_to_cpu(ic->sb->recalc_sector));
2475                 else
2476                         DMEMIT(" -");
2477                 break;
2478
2479         case STATUSTYPE_TABLE: {
2480                 __u64 watermark_percentage = (__u64)(ic->journal_entries - ic->free_sectors_threshold) * 100;
2481                 watermark_percentage += ic->journal_entries / 2;
2482                 do_div(watermark_percentage, ic->journal_entries);
2483                 arg_count = 3;
2484                 arg_count += !!ic->meta_dev;
2485                 arg_count += ic->sectors_per_block != 1;
2486                 arg_count += !!(ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING));
2487                 arg_count += ic->mode == 'J';
2488                 arg_count += ic->mode == 'J';
2489                 arg_count += !!ic->internal_hash_alg.alg_string;
2490                 arg_count += !!ic->journal_crypt_alg.alg_string;
2491                 arg_count += !!ic->journal_mac_alg.alg_string;
2492                 DMEMIT("%s %llu %u %c %u", ic->dev->name, (unsigned long long)ic->start,
2493                        ic->tag_size, ic->mode, arg_count);
2494                 if (ic->meta_dev)
2495                         DMEMIT(" meta_device:%s", ic->meta_dev->name);
2496                 if (ic->sectors_per_block != 1)
2497                         DMEMIT(" block_size:%u", ic->sectors_per_block << SECTOR_SHIFT);
2498                 if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING))
2499                         DMEMIT(" recalculate");
2500                 DMEMIT(" journal_sectors:%u", ic->initial_sectors - SB_SECTORS);
2501                 DMEMIT(" interleave_sectors:%u", 1U << ic->sb->log2_interleave_sectors);
2502                 DMEMIT(" buffer_sectors:%u", 1U << ic->log2_buffer_sectors);
2503                 if (ic->mode == 'J') {
2504                         DMEMIT(" journal_watermark:%u", (unsigned)watermark_percentage);
2505                         DMEMIT(" commit_time:%u", ic->autocommit_msec);
2506                 }
2507
2508 #define EMIT_ALG(a, n)                                                  \
2509                 do {                                                    \
2510                         if (ic->a.alg_string) {                         \
2511                                 DMEMIT(" %s:%s", n, ic->a.alg_string);  \
2512                                 if (ic->a.key_string)                   \
2513                                         DMEMIT(":%s", ic->a.key_string);\
2514                         }                                               \
2515                 } while (0)
2516                 EMIT_ALG(internal_hash_alg, "internal_hash");
2517                 EMIT_ALG(journal_crypt_alg, "journal_crypt");
2518                 EMIT_ALG(journal_mac_alg, "journal_mac");
2519                 break;
2520         }
2521         }
2522 }
2523
2524 static int dm_integrity_iterate_devices(struct dm_target *ti,
2525                                         iterate_devices_callout_fn fn, void *data)
2526 {
2527         struct dm_integrity_c *ic = ti->private;
2528
2529         if (!ic->meta_dev)
2530                 return fn(ti, ic->dev, ic->start + ic->initial_sectors + ic->metadata_run, ti->len, data);
2531         else
2532                 return fn(ti, ic->dev, 0, ti->len, data);
2533 }
2534
2535 static void dm_integrity_io_hints(struct dm_target *ti, struct queue_limits *limits)
2536 {
2537         struct dm_integrity_c *ic = ti->private;
2538
2539         if (ic->sectors_per_block > 1) {
2540                 limits->logical_block_size = ic->sectors_per_block << SECTOR_SHIFT;
2541                 limits->physical_block_size = ic->sectors_per_block << SECTOR_SHIFT;
2542                 blk_limits_io_min(limits, ic->sectors_per_block << SECTOR_SHIFT);
2543         }
2544 }
2545
2546 static void calculate_journal_section_size(struct dm_integrity_c *ic)
2547 {
2548         unsigned sector_space = JOURNAL_SECTOR_DATA;
2549
2550         ic->journal_sections = le32_to_cpu(ic->sb->journal_sections);
2551         ic->journal_entry_size = roundup(offsetof(struct journal_entry, last_bytes[ic->sectors_per_block]) + ic->tag_size,
2552                                          JOURNAL_ENTRY_ROUNDUP);
2553
2554         if (ic->sb->flags & cpu_to_le32(SB_FLAG_HAVE_JOURNAL_MAC))
2555                 sector_space -= JOURNAL_MAC_PER_SECTOR;
2556         ic->journal_entries_per_sector = sector_space / ic->journal_entry_size;
2557         ic->journal_section_entries = ic->journal_entries_per_sector * JOURNAL_BLOCK_SECTORS;
2558         ic->journal_section_sectors = (ic->journal_section_entries << ic->sb->log2_sectors_per_block) + JOURNAL_BLOCK_SECTORS;
2559         ic->journal_entries = ic->journal_section_entries * ic->journal_sections;
2560 }
2561
2562 static int calculate_device_limits(struct dm_integrity_c *ic)
2563 {
2564         __u64 initial_sectors;
2565
2566         calculate_journal_section_size(ic);
2567         initial_sectors = SB_SECTORS + (__u64)ic->journal_section_sectors * ic->journal_sections;
2568         if (initial_sectors + METADATA_PADDING_SECTORS >= ic->meta_device_sectors || initial_sectors > UINT_MAX)
2569                 return -EINVAL;
2570         ic->initial_sectors = initial_sectors;
2571
2572         if (!ic->meta_dev) {
2573                 sector_t last_sector, last_area, last_offset;
2574
2575                 ic->metadata_run = roundup((__u64)ic->tag_size << (ic->sb->log2_interleave_sectors - ic->sb->log2_sectors_per_block),
2576                                            (__u64)(1 << SECTOR_SHIFT << METADATA_PADDING_SECTORS)) >> SECTOR_SHIFT;
2577                 if (!(ic->metadata_run & (ic->metadata_run - 1)))
2578                         ic->log2_metadata_run = __ffs(ic->metadata_run);
2579                 else
2580                         ic->log2_metadata_run = -1;
2581
2582                 get_area_and_offset(ic, ic->provided_data_sectors - 1, &last_area, &last_offset);
2583                 last_sector = get_data_sector(ic, last_area, last_offset);
2584                 if (last_sector < ic->start || last_sector >= ic->meta_device_sectors)
2585                         return -EINVAL;
2586         } else {
2587                 __u64 meta_size = (ic->provided_data_sectors >> ic->sb->log2_sectors_per_block) * ic->tag_size;
2588                 meta_size = (meta_size + ((1U << (ic->log2_buffer_sectors + SECTOR_SHIFT)) - 1))
2589                                 >> (ic->log2_buffer_sectors + SECTOR_SHIFT);
2590                 meta_size <<= ic->log2_buffer_sectors;
2591                 if (ic->initial_sectors + meta_size < ic->initial_sectors ||
2592                     ic->initial_sectors + meta_size > ic->meta_device_sectors)
2593                         return -EINVAL;
2594                 ic->metadata_run = 1;
2595                 ic->log2_metadata_run = 0;
2596         }
2597
2598         return 0;
2599 }
2600
2601 static int initialize_superblock(struct dm_integrity_c *ic, unsigned journal_sectors, unsigned interleave_sectors)
2602 {
2603         unsigned journal_sections;
2604         int test_bit;
2605
2606         memset(ic->sb, 0, SB_SECTORS << SECTOR_SHIFT);
2607         memcpy(ic->sb->magic, SB_MAGIC, 8);
2608         ic->sb->integrity_tag_size = cpu_to_le16(ic->tag_size);
2609         ic->sb->log2_sectors_per_block = __ffs(ic->sectors_per_block);
2610         if (ic->journal_mac_alg.alg_string)
2611                 ic->sb->flags |= cpu_to_le32(SB_FLAG_HAVE_JOURNAL_MAC);
2612
2613         calculate_journal_section_size(ic);
2614         journal_sections = journal_sectors / ic->journal_section_sectors;
2615         if (!journal_sections)
2616                 journal_sections = 1;
2617
2618         if (!ic->meta_dev) {
2619                 ic->sb->journal_sections = cpu_to_le32(journal_sections);
2620                 if (!interleave_sectors)
2621                         interleave_sectors = DEFAULT_INTERLEAVE_SECTORS;
2622                 ic->sb->log2_interleave_sectors = __fls(interleave_sectors);
2623                 ic->sb->log2_interleave_sectors = max((__u8)MIN_LOG2_INTERLEAVE_SECTORS, ic->sb->log2_interleave_sectors);
2624                 ic->sb->log2_interleave_sectors = min((__u8)MAX_LOG2_INTERLEAVE_SECTORS, ic->sb->log2_interleave_sectors);
2625
2626                 ic->provided_data_sectors = 0;
2627                 for (test_bit = fls64(ic->meta_device_sectors) - 1; test_bit >= 3; test_bit--) {
2628                         __u64 prev_data_sectors = ic->provided_data_sectors;
2629
2630                         ic->provided_data_sectors |= (sector_t)1 << test_bit;
2631                         if (calculate_device_limits(ic))
2632                                 ic->provided_data_sectors = prev_data_sectors;
2633                 }
2634                 if (!ic->provided_data_sectors)
2635                         return -EINVAL;
2636         } else {
2637                 ic->sb->log2_interleave_sectors = 0;
2638                 ic->provided_data_sectors = ic->data_device_sectors;
2639                 ic->provided_data_sectors &= ~(sector_t)(ic->sectors_per_block - 1);
2640
2641 try_smaller_buffer:
2642                 ic->sb->journal_sections = cpu_to_le32(0);
2643                 for (test_bit = fls(journal_sections) - 1; test_bit >= 0; test_bit--) {
2644                         __u32 prev_journal_sections = le32_to_cpu(ic->sb->journal_sections);
2645                         __u32 test_journal_sections = prev_journal_sections | (1U << test_bit);
2646                         if (test_journal_sections > journal_sections)
2647                                 continue;
2648                         ic->sb->journal_sections = cpu_to_le32(test_journal_sections);
2649                         if (calculate_device_limits(ic))
2650                                 ic->sb->journal_sections = cpu_to_le32(prev_journal_sections);
2651
2652                 }
2653                 if (!le32_to_cpu(ic->sb->journal_sections)) {
2654                         if (ic->log2_buffer_sectors > 3) {
2655                                 ic->log2_buffer_sectors--;
2656                                 goto try_smaller_buffer;
2657                         }
2658                         return -EINVAL;
2659                 }
2660         }
2661
2662         ic->sb->provided_data_sectors = cpu_to_le64(ic->provided_data_sectors);
2663
2664         sb_set_version(ic);
2665
2666         return 0;
2667 }
2668
2669 static void dm_integrity_set(struct dm_target *ti, struct dm_integrity_c *ic)
2670 {
2671         struct gendisk *disk = dm_disk(dm_table_get_md(ti->table));
2672         struct blk_integrity bi;
2673
2674         memset(&bi, 0, sizeof(bi));
2675         bi.profile = &dm_integrity_profile;
2676         bi.tuple_size = ic->tag_size;
2677         bi.tag_size = bi.tuple_size;
2678         bi.interval_exp = ic->sb->log2_sectors_per_block + SECTOR_SHIFT;
2679
2680         blk_integrity_register(disk, &bi);
2681         blk_queue_max_integrity_segments(disk->queue, UINT_MAX);
2682 }
2683
2684 static void dm_integrity_free_page_list(struct page_list *pl)
2685 {
2686         unsigned i;
2687
2688         if (!pl)
2689                 return;
2690         for (i = 0; pl[i].page; i++)
2691                 __free_page(pl[i].page);
2692         kvfree(pl);
2693 }
2694
2695 static struct page_list *dm_integrity_alloc_page_list(unsigned n_pages)
2696 {
2697         struct page_list *pl;
2698         unsigned i;
2699
2700         pl = kvmalloc_array(n_pages + 1, sizeof(struct page_list), GFP_KERNEL | __GFP_ZERO);
2701         if (!pl)
2702                 return NULL;
2703
2704         for (i = 0; i < n_pages; i++) {
2705                 pl[i].page = alloc_page(GFP_KERNEL);
2706                 if (!pl[i].page) {
2707                         dm_integrity_free_page_list(pl);
2708                         return NULL;
2709                 }
2710                 if (i)
2711                         pl[i - 1].next = &pl[i];
2712         }
2713         pl[i].page = NULL;
2714         pl[i].next = NULL;
2715
2716         return pl;
2717 }
2718
2719 static void dm_integrity_free_journal_scatterlist(struct dm_integrity_c *ic, struct scatterlist **sl)
2720 {
2721         unsigned i;
2722         for (i = 0; i < ic->journal_sections; i++)
2723                 kvfree(sl[i]);
2724         kvfree(sl);
2725 }
2726
2727 static struct scatterlist **dm_integrity_alloc_journal_scatterlist(struct dm_integrity_c *ic, struct page_list *pl)
2728 {
2729         struct scatterlist **sl;
2730         unsigned i;
2731
2732         sl = kvmalloc_array(ic->journal_sections,
2733                             sizeof(struct scatterlist *),
2734                             GFP_KERNEL | __GFP_ZERO);
2735         if (!sl)
2736                 return NULL;
2737
2738         for (i = 0; i < ic->journal_sections; i++) {
2739                 struct scatterlist *s;
2740                 unsigned start_index, start_offset;
2741                 unsigned end_index, end_offset;
2742                 unsigned n_pages;
2743                 unsigned idx;
2744
2745                 page_list_location(ic, i, 0, &start_index, &start_offset);
2746                 page_list_location(ic, i, ic->journal_section_sectors - 1, &end_index, &end_offset);
2747
2748                 n_pages = (end_index - start_index + 1);
2749
2750                 s = kvmalloc_array(n_pages, sizeof(struct scatterlist),
2751                                    GFP_KERNEL);
2752                 if (!s) {
2753                         dm_integrity_free_journal_scatterlist(ic, sl);
2754                         return NULL;
2755                 }
2756
2757                 sg_init_table(s, n_pages);
2758                 for (idx = start_index; idx <= end_index; idx++) {
2759                         char *va = lowmem_page_address(pl[idx].page);
2760                         unsigned start = 0, end = PAGE_SIZE;
2761                         if (idx == start_index)
2762                                 start = start_offset;
2763                         if (idx == end_index)
2764                                 end = end_offset + (1 << SECTOR_SHIFT);
2765                         sg_set_buf(&s[idx - start_index], va + start, end - start);
2766                 }
2767
2768                 sl[i] = s;
2769         }
2770
2771         return sl;
2772 }
2773
2774 static void free_alg(struct alg_spec *a)
2775 {
2776         kzfree(a->alg_string);
2777         kzfree(a->key);
2778         memset(a, 0, sizeof *a);
2779 }
2780
2781 static int get_alg_and_key(const char *arg, struct alg_spec *a, char **error, char *error_inval)
2782 {
2783         char *k;
2784
2785         free_alg(a);
2786
2787         a->alg_string = kstrdup(strchr(arg, ':') + 1, GFP_KERNEL);
2788         if (!a->alg_string)
2789                 goto nomem;
2790
2791         k = strchr(a->alg_string, ':');
2792         if (k) {
2793                 *k = 0;
2794                 a->key_string = k + 1;
2795                 if (strlen(a->key_string) & 1)
2796                         goto inval;
2797
2798                 a->key_size = strlen(a->key_string) / 2;
2799                 a->key = kmalloc(a->key_size, GFP_KERNEL);
2800                 if (!a->key)
2801                         goto nomem;
2802                 if (hex2bin(a->key, a->key_string, a->key_size))
2803                         goto inval;
2804         }
2805
2806         return 0;
2807 inval:
2808         *error = error_inval;
2809         return -EINVAL;
2810 nomem:
2811         *error = "Out of memory for an argument";
2812         return -ENOMEM;
2813 }
2814
2815 static int get_mac(struct crypto_shash **hash, struct alg_spec *a, char **error,
2816                    char *error_alg, char *error_key)
2817 {
2818         int r;
2819
2820         if (a->alg_string) {
2821                 *hash = crypto_alloc_shash(a->alg_string, 0, 0);
2822                 if (IS_ERR(*hash)) {
2823                         *error = error_alg;
2824                         r = PTR_ERR(*hash);
2825                         *hash = NULL;
2826                         return r;
2827                 }
2828
2829                 if (a->key) {
2830                         r = crypto_shash_setkey(*hash, a->key, a->key_size);
2831                         if (r) {
2832                                 *error = error_key;
2833                                 return r;
2834                         }
2835                 } else if (crypto_shash_get_flags(*hash) & CRYPTO_TFM_NEED_KEY) {
2836                         *error = error_key;
2837                         return -ENOKEY;
2838                 }
2839         }
2840
2841         return 0;
2842 }
2843
2844 static int create_journal(struct dm_integrity_c *ic, char **error)
2845 {
2846         int r = 0;
2847         unsigned i;
2848         __u64 journal_pages, journal_desc_size, journal_tree_size;
2849         unsigned char *crypt_data = NULL, *crypt_iv = NULL;
2850         struct skcipher_request *req = NULL;
2851
2852         ic->commit_ids[0] = cpu_to_le64(0x1111111111111111ULL);
2853         ic->commit_ids[1] = cpu_to_le64(0x2222222222222222ULL);
2854         ic->commit_ids[2] = cpu_to_le64(0x3333333333333333ULL);
2855         ic->commit_ids[3] = cpu_to_le64(0x4444444444444444ULL);
2856
2857         journal_pages = roundup((__u64)ic->journal_sections * ic->journal_section_sectors,
2858                                 PAGE_SIZE >> SECTOR_SHIFT) >> (PAGE_SHIFT - SECTOR_SHIFT);
2859         journal_desc_size = journal_pages * sizeof(struct page_list);
2860         if (journal_pages >= totalram_pages() - totalhigh_pages() || journal_desc_size > ULONG_MAX) {
2861                 *error = "Journal doesn't fit into memory";
2862                 r = -ENOMEM;
2863                 goto bad;
2864         }
2865         ic->journal_pages = journal_pages;
2866
2867         ic->journal = dm_integrity_alloc_page_list(ic->journal_pages);
2868         if (!ic->journal) {
2869                 *error = "Could not allocate memory for journal";
2870                 r = -ENOMEM;
2871                 goto bad;
2872         }
2873         if (ic->journal_crypt_alg.alg_string) {
2874                 unsigned ivsize, blocksize;
2875                 struct journal_completion comp;
2876
2877                 comp.ic = ic;
2878                 ic->journal_crypt = crypto_alloc_skcipher(ic->journal_crypt_alg.alg_string, 0, 0);
2879                 if (IS_ERR(ic->journal_crypt)) {
2880                         *error = "Invalid journal cipher";
2881                         r = PTR_ERR(ic->journal_crypt);
2882                         ic->journal_crypt = NULL;
2883                         goto bad;
2884                 }
2885                 ivsize = crypto_skcipher_ivsize(ic->journal_crypt);
2886                 blocksize = crypto_skcipher_blocksize(ic->journal_crypt);
2887
2888                 if (ic->journal_crypt_alg.key) {
2889                         r = crypto_skcipher_setkey(ic->journal_crypt, ic->journal_crypt_alg.key,
2890                                                    ic->journal_crypt_alg.key_size);
2891                         if (r) {
2892                                 *error = "Error setting encryption key";
2893                                 goto bad;
2894                         }
2895                 }
2896                 DEBUG_print("cipher %s, block size %u iv size %u\n",
2897                             ic->journal_crypt_alg.alg_string, blocksize, ivsize);
2898
2899                 ic->journal_io = dm_integrity_alloc_page_list(ic->journal_pages);
2900                 if (!ic->journal_io) {
2901                         *error = "Could not allocate memory for journal io";
2902                         r = -ENOMEM;
2903                         goto bad;
2904                 }
2905
2906                 if (blocksize == 1) {
2907                         struct scatterlist *sg;
2908
2909                         req = skcipher_request_alloc(ic->journal_crypt, GFP_KERNEL);
2910                         if (!req) {
2911                                 *error = "Could not allocate crypt request";
2912                                 r = -ENOMEM;
2913                                 goto bad;
2914                         }
2915
2916                         crypt_iv = kmalloc(ivsize, GFP_KERNEL);
2917                         if (!crypt_iv) {
2918                                 *error = "Could not allocate iv";
2919                                 r = -ENOMEM;
2920                                 goto bad;
2921                         }
2922
2923                         ic->journal_xor = dm_integrity_alloc_page_list(ic->journal_pages);
2924                         if (!ic->journal_xor) {
2925                                 *error = "Could not allocate memory for journal xor";
2926                                 r = -ENOMEM;
2927                                 goto bad;
2928                         }
2929
2930                         sg = kvmalloc_array(ic->journal_pages + 1,
2931                                             sizeof(struct scatterlist),
2932                                             GFP_KERNEL);
2933                         if (!sg) {
2934                                 *error = "Unable to allocate sg list";
2935                                 r = -ENOMEM;
2936                                 goto bad;
2937                         }
2938                         sg_init_table(sg, ic->journal_pages + 1);
2939                         for (i = 0; i < ic->journal_pages; i++) {
2940                                 char *va = lowmem_page_address(ic->journal_xor[i].page);
2941                                 clear_page(va);
2942                                 sg_set_buf(&sg[i], va, PAGE_SIZE);
2943                         }
2944                         sg_set_buf(&sg[i], &ic->commit_ids, sizeof ic->commit_ids);
2945                         memset(crypt_iv, 0x00, ivsize);
2946
2947                         skcipher_request_set_crypt(req, sg, sg, PAGE_SIZE * ic->journal_pages + sizeof ic->commit_ids, crypt_iv);
2948                         init_completion(&comp.comp);
2949                         comp.in_flight = (atomic_t)ATOMIC_INIT(1);
2950                         if (do_crypt(true, req, &comp))
2951                                 wait_for_completion(&comp.comp);
2952                         kvfree(sg);
2953                         r = dm_integrity_failed(ic);
2954                         if (r) {
2955                                 *error = "Unable to encrypt journal";
2956                                 goto bad;
2957                         }
2958                         DEBUG_bytes(lowmem_page_address(ic->journal_xor[0].page), 64, "xor data");
2959
2960                         crypto_free_skcipher(ic->journal_crypt);
2961                         ic->journal_crypt = NULL;
2962                 } else {
2963                         unsigned crypt_len = roundup(ivsize, blocksize);
2964
2965                         req = skcipher_request_alloc(ic->journal_crypt, GFP_KERNEL);
2966                         if (!req) {
2967                                 *error = "Could not allocate crypt request";
2968                                 r = -ENOMEM;
2969                                 goto bad;
2970                         }
2971
2972                         crypt_iv = kmalloc(ivsize, GFP_KERNEL);
2973                         if (!crypt_iv) {
2974                                 *error = "Could not allocate iv";
2975                                 r = -ENOMEM;
2976                                 goto bad;
2977                         }
2978
2979                         crypt_data = kmalloc(crypt_len, GFP_KERNEL);
2980                         if (!crypt_data) {
2981                                 *error = "Unable to allocate crypt data";
2982                                 r = -ENOMEM;
2983                                 goto bad;
2984                         }
2985
2986                         ic->journal_scatterlist = dm_integrity_alloc_journal_scatterlist(ic, ic->journal);
2987                         if (!ic->journal_scatterlist) {
2988                                 *error = "Unable to allocate sg list";
2989                                 r = -ENOMEM;
2990                                 goto bad;
2991                         }
2992                         ic->journal_io_scatterlist = dm_integrity_alloc_journal_scatterlist(ic, ic->journal_io);
2993                         if (!ic->journal_io_scatterlist) {
2994                                 *error = "Unable to allocate sg list";
2995                                 r = -ENOMEM;
2996                                 goto bad;
2997                         }
2998                         ic->sk_requests = kvmalloc_array(ic->journal_sections,
2999                                                          sizeof(struct skcipher_request *),
3000                                                          GFP_KERNEL | __GFP_ZERO);
3001                         if (!ic->sk_requests) {
3002                                 *error = "Unable to allocate sk requests";
3003                                 r = -ENOMEM;
3004                                 goto bad;
3005                         }
3006                         for (i = 0; i < ic->journal_sections; i++) {
3007                                 struct scatterlist sg;
3008                                 struct skcipher_request *section_req;
3009                                 __u32 section_le = cpu_to_le32(i);
3010
3011                                 memset(crypt_iv, 0x00, ivsize);
3012                                 memset(crypt_data, 0x00, crypt_len);
3013                                 memcpy(crypt_data, &section_le, min((size_t)crypt_len, sizeof(section_le)));
3014
3015                                 sg_init_one(&sg, crypt_data, crypt_len);
3016                                 skcipher_request_set_crypt(req, &sg, &sg, crypt_len, crypt_iv);
3017                                 init_completion(&comp.comp);
3018                                 comp.in_flight = (atomic_t)ATOMIC_INIT(1);
3019                                 if (do_crypt(true, req, &comp))
3020                                         wait_for_completion(&comp.comp);
3021
3022                                 r = dm_integrity_failed(ic);
3023                                 if (r) {
3024                                         *error = "Unable to generate iv";
3025                                         goto bad;
3026                                 }
3027
3028                                 section_req = skcipher_request_alloc(ic->journal_crypt, GFP_KERNEL);
3029                                 if (!section_req) {
3030                                         *error = "Unable to allocate crypt request";
3031                                         r = -ENOMEM;
3032                                         goto bad;
3033                                 }
3034                                 section_req->iv = kmalloc_array(ivsize, 2,
3035                                                                 GFP_KERNEL);
3036                                 if (!section_req->iv) {
3037                                         skcipher_request_free(section_req);
3038                                         *error = "Unable to allocate iv";
3039                                         r = -ENOMEM;
3040                                         goto bad;
3041                                 }
3042                                 memcpy(section_req->iv + ivsize, crypt_data, ivsize);
3043                                 section_req->cryptlen = (size_t)ic->journal_section_sectors << SECTOR_SHIFT;
3044                                 ic->sk_requests[i] = section_req;
3045                                 DEBUG_bytes(crypt_data, ivsize, "iv(%u)", i);
3046                         }
3047                 }
3048         }
3049
3050         for (i = 0; i < N_COMMIT_IDS; i++) {
3051                 unsigned j;
3052 retest_commit_id:
3053                 for (j = 0; j < i; j++) {
3054                         if (ic->commit_ids[j] == ic->commit_ids[i]) {
3055                                 ic->commit_ids[i] = cpu_to_le64(le64_to_cpu(ic->commit_ids[i]) + 1);
3056                                 goto retest_commit_id;
3057                         }
3058                 }
3059                 DEBUG_print("commit id %u: %016llx\n", i, ic->commit_ids[i]);
3060         }
3061
3062         journal_tree_size = (__u64)ic->journal_entries * sizeof(struct journal_node);
3063         if (journal_tree_size > ULONG_MAX) {
3064                 *error = "Journal doesn't fit into memory";
3065                 r = -ENOMEM;
3066                 goto bad;
3067         }
3068         ic->journal_tree = kvmalloc(journal_tree_size, GFP_KERNEL);
3069         if (!ic->journal_tree) {
3070                 *error = "Could not allocate memory for journal tree";
3071                 r = -ENOMEM;
3072         }
3073 bad:
3074         kfree(crypt_data);
3075         kfree(crypt_iv);
3076         skcipher_request_free(req);
3077
3078         return r;
3079 }
3080
3081 /*
3082  * Construct a integrity mapping
3083  *
3084  * Arguments:
3085  *      device
3086  *      offset from the start of the device
3087  *      tag size
3088  *      D - direct writes, J - journal writes, R - recovery mode
3089  *      number of optional arguments
3090  *      optional arguments:
3091  *              journal_sectors
3092  *              interleave_sectors
3093  *              buffer_sectors
3094  *              journal_watermark
3095  *              commit_time
3096  *              meta_device
3097  *              block_size
3098  *              internal_hash
3099  *              journal_crypt
3100  *              journal_mac
3101  *              recalculate
3102  */
3103 static int dm_integrity_ctr(struct dm_target *ti, unsigned argc, char **argv)
3104 {
3105         struct dm_integrity_c *ic;
3106         char dummy;
3107         int r;
3108         unsigned extra_args;
3109         struct dm_arg_set as;
3110         static const struct dm_arg _args[] = {
3111                 {0, 9, "Invalid number of feature args"},
3112         };
3113         unsigned journal_sectors, interleave_sectors, buffer_sectors, journal_watermark, sync_msec;
3114         bool recalculate;
3115         bool should_write_sb;
3116         __u64 threshold;
3117         unsigned long long start;
3118
3119 #define DIRECT_ARGUMENTS        4
3120
3121         if (argc <= DIRECT_ARGUMENTS) {
3122                 ti->error = "Invalid argument count";
3123                 return -EINVAL;
3124         }
3125
3126         ic = kzalloc(sizeof(struct dm_integrity_c), GFP_KERNEL);
3127         if (!ic) {
3128                 ti->error = "Cannot allocate integrity context";
3129                 return -ENOMEM;
3130         }
3131         ti->private = ic;
3132         ti->per_io_data_size = sizeof(struct dm_integrity_io);
3133
3134         ic->in_progress = RB_ROOT;
3135         INIT_LIST_HEAD(&ic->wait_list);
3136         init_waitqueue_head(&ic->endio_wait);
3137         bio_list_init(&ic->flush_bio_list);
3138         init_waitqueue_head(&ic->copy_to_journal_wait);
3139         init_completion(&ic->crypto_backoff);
3140         atomic64_set(&ic->number_of_mismatches, 0);
3141
3142         r = dm_get_device(ti, argv[0], dm_table_get_mode(ti->table), &ic->dev);
3143         if (r) {
3144                 ti->error = "Device lookup failed";
3145                 goto bad;
3146         }
3147
3148         if (sscanf(argv[1], "%llu%c", &start, &dummy) != 1 || start != (sector_t)start) {
3149                 ti->error = "Invalid starting offset";
3150                 r = -EINVAL;
3151                 goto bad;
3152         }
3153         ic->start = start;
3154
3155         if (strcmp(argv[2], "-")) {
3156                 if (sscanf(argv[2], "%u%c", &ic->tag_size, &dummy) != 1 || !ic->tag_size) {
3157                         ti->error = "Invalid tag size";
3158                         r = -EINVAL;
3159                         goto bad;
3160                 }
3161         }
3162
3163         if (!strcmp(argv[3], "J") || !strcmp(argv[3], "D") || !strcmp(argv[3], "R"))
3164                 ic->mode = argv[3][0];
3165         else {
3166                 ti->error = "Invalid mode (expecting J, D, R)";
3167                 r = -EINVAL;
3168                 goto bad;
3169         }
3170
3171         journal_sectors = 0;
3172         interleave_sectors = DEFAULT_INTERLEAVE_SECTORS;
3173         buffer_sectors = DEFAULT_BUFFER_SECTORS;
3174         journal_watermark = DEFAULT_JOURNAL_WATERMARK;
3175         sync_msec = DEFAULT_SYNC_MSEC;
3176         recalculate = false;
3177         ic->sectors_per_block = 1;
3178
3179         as.argc = argc - DIRECT_ARGUMENTS;
3180         as.argv = argv + DIRECT_ARGUMENTS;
3181         r = dm_read_arg_group(_args, &as, &extra_args, &ti->error);
3182         if (r)
3183                 goto bad;
3184
3185         while (extra_args--) {
3186                 const char *opt_string;
3187                 unsigned val;
3188                 opt_string = dm_shift_arg(&as);
3189                 if (!opt_string) {
3190                         r = -EINVAL;
3191                         ti->error = "Not enough feature arguments";
3192                         goto bad;
3193                 }
3194                 if (sscanf(opt_string, "journal_sectors:%u%c", &val, &dummy) == 1)
3195                         journal_sectors = val ? val : 1;
3196                 else if (sscanf(opt_string, "interleave_sectors:%u%c", &val, &dummy) == 1)
3197                         interleave_sectors = val;
3198                 else if (sscanf(opt_string, "buffer_sectors:%u%c", &val, &dummy) == 1)
3199                         buffer_sectors = val;
3200                 else if (sscanf(opt_string, "journal_watermark:%u%c", &val, &dummy) == 1 && val <= 100)
3201                         journal_watermark = val;
3202                 else if (sscanf(opt_string, "commit_time:%u%c", &val, &dummy) == 1)
3203                         sync_msec = val;
3204                 else if (!strncmp(opt_string, "meta_device:", strlen("meta_device:"))) {
3205                         if (ic->meta_dev) {
3206                                 dm_put_device(ti, ic->meta_dev);
3207                                 ic->meta_dev = NULL;
3208                         }
3209                         r = dm_get_device(ti, strchr(opt_string, ':') + 1, dm_table_get_mode(ti->table), &ic->meta_dev);
3210                         if (r) {
3211                                 ti->error = "Device lookup failed";
3212                                 goto bad;
3213                         }
3214                 } else if (sscanf(opt_string, "block_size:%u%c", &val, &dummy) == 1) {
3215                         if (val < 1 << SECTOR_SHIFT ||
3216                             val > MAX_SECTORS_PER_BLOCK << SECTOR_SHIFT ||
3217                             (val & (val -1))) {
3218                                 r = -EINVAL;
3219                                 ti->error = "Invalid block_size argument";
3220                                 goto bad;
3221                         }
3222                         ic->sectors_per_block = val >> SECTOR_SHIFT;
3223                 } else if (!strncmp(opt_string, "internal_hash:", strlen("internal_hash:"))) {
3224                         r = get_alg_and_key(opt_string, &ic->internal_hash_alg, &ti->error,
3225                                             "Invalid internal_hash argument");
3226                         if (r)
3227                                 goto bad;
3228                 } else if (!strncmp(opt_string, "journal_crypt:", strlen("journal_crypt:"))) {
3229                         r = get_alg_and_key(opt_string, &ic->journal_crypt_alg, &ti->error,
3230                                             "Invalid journal_crypt argument");
3231                         if (r)
3232                                 goto bad;
3233                 } else if (!strncmp(opt_string, "journal_mac:", strlen("journal_mac:"))) {
3234                         r = get_alg_and_key(opt_string, &ic->journal_mac_alg,  &ti->error,
3235                                             "Invalid journal_mac argument");
3236                         if (r)
3237                                 goto bad;
3238                 } else if (!strcmp(opt_string, "recalculate")) {
3239                         recalculate = true;
3240                 } else {
3241                         r = -EINVAL;
3242                         ti->error = "Invalid argument";
3243                         goto bad;
3244                 }
3245         }
3246
3247         ic->data_device_sectors = i_size_read(ic->dev->bdev->bd_inode) >> SECTOR_SHIFT;
3248         if (!ic->meta_dev)
3249                 ic->meta_device_sectors = ic->data_device_sectors;
3250         else
3251                 ic->meta_device_sectors = i_size_read(ic->meta_dev->bdev->bd_inode) >> SECTOR_SHIFT;
3252
3253         if (!journal_sectors) {
3254                 journal_sectors = min((sector_t)DEFAULT_MAX_JOURNAL_SECTORS,
3255                         ic->data_device_sectors >> DEFAULT_JOURNAL_SIZE_FACTOR);
3256         }
3257
3258         if (!buffer_sectors)
3259                 buffer_sectors = 1;
3260         ic->log2_buffer_sectors = min((int)__fls(buffer_sectors), 31 - SECTOR_SHIFT);
3261
3262         r = get_mac(&ic->internal_hash, &ic->internal_hash_alg, &ti->error,
3263                     "Invalid internal hash", "Error setting internal hash key");
3264         if (r)
3265                 goto bad;
3266
3267         r = get_mac(&ic->journal_mac, &ic->journal_mac_alg, &ti->error,
3268                     "Invalid journal mac", "Error setting journal mac key");
3269         if (r)
3270                 goto bad;
3271
3272         if (!ic->tag_size) {
3273                 if (!ic->internal_hash) {
3274                         ti->error = "Unknown tag size";
3275                         r = -EINVAL;
3276                         goto bad;
3277                 }
3278                 ic->tag_size = crypto_shash_digestsize(ic->internal_hash);
3279         }
3280         if (ic->tag_size > MAX_TAG_SIZE) {
3281                 ti->error = "Too big tag size";
3282                 r = -EINVAL;
3283                 goto bad;
3284         }
3285         if (!(ic->tag_size & (ic->tag_size - 1)))
3286                 ic->log2_tag_size = __ffs(ic->tag_size);
3287         else
3288                 ic->log2_tag_size = -1;
3289
3290         ic->autocommit_jiffies = msecs_to_jiffies(sync_msec);
3291         ic->autocommit_msec = sync_msec;
3292         timer_setup(&ic->autocommit_timer, autocommit_fn, 0);
3293
3294         ic->io = dm_io_client_create();
3295         if (IS_ERR(ic->io)) {
3296                 r = PTR_ERR(ic->io);
3297                 ic->io = NULL;
3298                 ti->error = "Cannot allocate dm io";
3299                 goto bad;
3300         }
3301
3302         r = mempool_init_slab_pool(&ic->journal_io_mempool, JOURNAL_IO_MEMPOOL, journal_io_cache);
3303         if (r) {
3304                 ti->error = "Cannot allocate mempool";
3305                 goto bad;
3306         }
3307
3308         ic->metadata_wq = alloc_workqueue("dm-integrity-metadata",
3309                                           WQ_MEM_RECLAIM, METADATA_WORKQUEUE_MAX_ACTIVE);
3310         if (!ic->metadata_wq) {
3311                 ti->error = "Cannot allocate workqueue";
3312                 r = -ENOMEM;
3313                 goto bad;
3314         }
3315
3316         /*
3317          * If this workqueue were percpu, it would cause bio reordering
3318          * and reduced performance.
3319          */
3320         ic->wait_wq = alloc_workqueue("dm-integrity-wait", WQ_MEM_RECLAIM | WQ_UNBOUND, 1);
3321         if (!ic->wait_wq) {
3322                 ti->error = "Cannot allocate workqueue";
3323                 r = -ENOMEM;
3324                 goto bad;
3325         }
3326
3327         ic->commit_wq = alloc_workqueue("dm-integrity-commit", WQ_MEM_RECLAIM, 1);
3328         if (!ic->commit_wq) {
3329                 ti->error = "Cannot allocate workqueue";
3330                 r = -ENOMEM;
3331                 goto bad;
3332         }
3333         INIT_WORK(&ic->commit_work, integrity_commit);
3334
3335         if (ic->mode == 'J') {
3336                 ic->writer_wq = alloc_workqueue("dm-integrity-writer", WQ_MEM_RECLAIM, 1);
3337                 if (!ic->writer_wq) {
3338                         ti->error = "Cannot allocate workqueue";
3339                         r = -ENOMEM;
3340                         goto bad;
3341                 }
3342                 INIT_WORK(&ic->writer_work, integrity_writer);
3343         }
3344
3345         ic->sb = alloc_pages_exact(SB_SECTORS << SECTOR_SHIFT, GFP_KERNEL);
3346         if (!ic->sb) {
3347                 r = -ENOMEM;
3348                 ti->error = "Cannot allocate superblock area";
3349                 goto bad;
3350         }
3351
3352         r = sync_rw_sb(ic, REQ_OP_READ, 0);
3353         if (r) {
3354                 ti->error = "Error reading superblock";
3355                 goto bad;
3356         }
3357         should_write_sb = false;
3358         if (memcmp(ic->sb->magic, SB_MAGIC, 8)) {
3359                 if (ic->mode != 'R') {
3360                         if (memchr_inv(ic->sb, 0, SB_SECTORS << SECTOR_SHIFT)) {
3361                                 r = -EINVAL;
3362                                 ti->error = "The device is not initialized";
3363                                 goto bad;
3364                         }
3365                 }
3366
3367                 r = initialize_superblock(ic, journal_sectors, interleave_sectors);
3368                 if (r) {
3369                         ti->error = "Could not initialize superblock";
3370                         goto bad;
3371                 }
3372                 if (ic->mode != 'R')
3373                         should_write_sb = true;
3374         }
3375
3376         if (!ic->sb->version || ic->sb->version > SB_VERSION_2) {
3377                 r = -EINVAL;
3378                 ti->error = "Unknown version";
3379                 goto bad;
3380         }
3381         if (le16_to_cpu(ic->sb->integrity_tag_size) != ic->tag_size) {
3382                 r = -EINVAL;
3383                 ti->error = "Tag size doesn't match the information in superblock";
3384                 goto bad;
3385         }
3386         if (ic->sb->log2_sectors_per_block != __ffs(ic->sectors_per_block)) {
3387                 r = -EINVAL;
3388                 ti->error = "Block size doesn't match the information in superblock";
3389                 goto bad;
3390         }
3391         if (!le32_to_cpu(ic->sb->journal_sections)) {
3392                 r = -EINVAL;
3393                 ti->error = "Corrupted superblock, journal_sections is 0";
3394                 goto bad;
3395         }
3396         /* make sure that ti->max_io_len doesn't overflow */
3397         if (!ic->meta_dev) {
3398                 if (ic->sb->log2_interleave_sectors < MIN_LOG2_INTERLEAVE_SECTORS ||
3399                     ic->sb->log2_interleave_sectors > MAX_LOG2_INTERLEAVE_SECTORS) {
3400                         r = -EINVAL;
3401                         ti->error = "Invalid interleave_sectors in the superblock";
3402                         goto bad;
3403                 }
3404         } else {
3405                 if (ic->sb->log2_interleave_sectors) {
3406                         r = -EINVAL;
3407                         ti->error = "Invalid interleave_sectors in the superblock";
3408                         goto bad;
3409                 }
3410         }
3411         ic->provided_data_sectors = le64_to_cpu(ic->sb->provided_data_sectors);
3412         if (ic->provided_data_sectors != le64_to_cpu(ic->sb->provided_data_sectors)) {
3413                 /* test for overflow */
3414                 r = -EINVAL;
3415                 ti->error = "The superblock has 64-bit device size, but the kernel was compiled with 32-bit sectors";
3416                 goto bad;
3417         }
3418         if (!!(ic->sb->flags & cpu_to_le32(SB_FLAG_HAVE_JOURNAL_MAC)) != !!ic->journal_mac_alg.alg_string) {
3419                 r = -EINVAL;
3420                 ti->error = "Journal mac mismatch";
3421                 goto bad;
3422         }
3423
3424 try_smaller_buffer:
3425         r = calculate_device_limits(ic);
3426         if (r) {
3427                 if (ic->meta_dev) {
3428                         if (ic->log2_buffer_sectors > 3) {
3429                                 ic->log2_buffer_sectors--;
3430                                 goto try_smaller_buffer;
3431                         }
3432                 }
3433                 ti->error = "The device is too small";
3434                 goto bad;
3435         }
3436         if (!ic->meta_dev)
3437                 ic->log2_buffer_sectors = min(ic->log2_buffer_sectors, (__u8)__ffs(ic->metadata_run));
3438
3439         if (ti->len > ic->provided_data_sectors) {
3440                 r = -EINVAL;
3441                 ti->error = "Not enough provided sectors for requested mapping size";
3442                 goto bad;
3443         }
3444
3445
3446         threshold = (__u64)ic->journal_entries * (100 - journal_watermark);
3447         threshold += 50;
3448         do_div(threshold, 100);
3449         ic->free_sectors_threshold = threshold;
3450
3451         DEBUG_print("initialized:\n");
3452         DEBUG_print("   integrity_tag_size %u\n", le16_to_cpu(ic->sb->integrity_tag_size));
3453         DEBUG_print("   journal_entry_size %u\n", ic->journal_entry_size);
3454         DEBUG_print("   journal_entries_per_sector %u\n", ic->journal_entries_per_sector);
3455         DEBUG_print("   journal_section_entries %u\n", ic->journal_section_entries);
3456         DEBUG_print("   journal_section_sectors %u\n", ic->journal_section_sectors);
3457         DEBUG_print("   journal_sections %u\n", (unsigned)le32_to_cpu(ic->sb->journal_sections));
3458         DEBUG_print("   journal_entries %u\n", ic->journal_entries);
3459         DEBUG_print("   log2_interleave_sectors %d\n", ic->sb->log2_interleave_sectors);
3460         DEBUG_print("   data_device_sectors 0x%llx\n", (unsigned long long)ic->data_device_sectors);
3461         DEBUG_print("   initial_sectors 0x%x\n", ic->initial_sectors);
3462         DEBUG_print("   metadata_run 0x%x\n", ic->metadata_run);
3463         DEBUG_print("   log2_metadata_run %d\n", ic->log2_metadata_run);
3464         DEBUG_print("   provided_data_sectors 0x%llx (%llu)\n", (unsigned long long)ic->provided_data_sectors,
3465                     (unsigned long long)ic->provided_data_sectors);
3466         DEBUG_print("   log2_buffer_sectors %u\n", ic->log2_buffer_sectors);
3467
3468         if (recalculate && !(ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING))) {
3469                 ic->sb->flags |= cpu_to_le32(SB_FLAG_RECALCULATING);
3470                 ic->sb->recalc_sector = cpu_to_le64(0);
3471         }
3472
3473         if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING)) {
3474                 if (!ic->internal_hash) {
3475                         r = -EINVAL;
3476                         ti->error = "Recalculate is only valid with internal hash";
3477                         goto bad;
3478                 }
3479                 ic->recalc_wq = alloc_workqueue("dm-integrity-recalc", WQ_MEM_RECLAIM, 1);
3480                 if (!ic->recalc_wq ) {
3481                         ti->error = "Cannot allocate workqueue";
3482                         r = -ENOMEM;
3483                         goto bad;
3484                 }
3485                 INIT_WORK(&ic->recalc_work, integrity_recalc);
3486                 ic->recalc_buffer = vmalloc(RECALC_SECTORS << SECTOR_SHIFT);
3487                 if (!ic->recalc_buffer) {
3488                         ti->error = "Cannot allocate buffer for recalculating";
3489                         r = -ENOMEM;
3490                         goto bad;
3491                 }
3492                 ic->recalc_tags = kvmalloc_array(RECALC_SECTORS >> ic->sb->log2_sectors_per_block,
3493                                                  ic->tag_size, GFP_KERNEL);
3494                 if (!ic->recalc_tags) {
3495                         ti->error = "Cannot allocate tags for recalculating";
3496                         r = -ENOMEM;
3497                         goto bad;
3498                 }
3499         }
3500
3501         ic->bufio = dm_bufio_client_create(ic->meta_dev ? ic->meta_dev->bdev : ic->dev->bdev,
3502                         1U << (SECTOR_SHIFT + ic->log2_buffer_sectors), 1, 0, NULL, NULL);
3503         if (IS_ERR(ic->bufio)) {
3504                 r = PTR_ERR(ic->bufio);
3505                 ti->error = "Cannot initialize dm-bufio";
3506                 ic->bufio = NULL;
3507                 goto bad;
3508         }
3509         dm_bufio_set_sector_offset(ic->bufio, ic->start + ic->initial_sectors);
3510
3511         if (ic->mode != 'R') {
3512                 r = create_journal(ic, &ti->error);
3513                 if (r)
3514                         goto bad;
3515         }
3516
3517         if (should_write_sb) {
3518                 int r;
3519
3520                 init_journal(ic, 0, ic->journal_sections, 0);
3521                 r = dm_integrity_failed(ic);
3522                 if (unlikely(r)) {
3523                         ti->error = "Error initializing journal";
3524                         goto bad;
3525                 }
3526                 r = sync_rw_sb(ic, REQ_OP_WRITE, REQ_FUA);
3527                 if (r) {
3528                         ti->error = "Error initializing superblock";
3529                         goto bad;
3530                 }
3531                 ic->just_formatted = true;
3532         }
3533
3534         if (!ic->meta_dev) {
3535                 r = dm_set_target_max_io_len(ti, 1U << ic->sb->log2_interleave_sectors);
3536                 if (r)
3537                         goto bad;
3538         }
3539
3540         if (!ic->internal_hash)
3541                 dm_integrity_set(ti, ic);
3542
3543         ti->num_flush_bios = 1;
3544         ti->flush_supported = true;
3545
3546         return 0;
3547 bad:
3548         dm_integrity_dtr(ti);
3549         return r;
3550 }
3551
3552 static void dm_integrity_dtr(struct dm_target *ti)
3553 {
3554         struct dm_integrity_c *ic = ti->private;
3555
3556         BUG_ON(!RB_EMPTY_ROOT(&ic->in_progress));
3557         BUG_ON(!list_empty(&ic->wait_list));
3558
3559         if (ic->metadata_wq)
3560                 destroy_workqueue(ic->metadata_wq);
3561         if (ic->wait_wq)
3562                 destroy_workqueue(ic->wait_wq);
3563         if (ic->commit_wq)
3564                 destroy_workqueue(ic->commit_wq);
3565         if (ic->writer_wq)
3566                 destroy_workqueue(ic->writer_wq);
3567         if (ic->recalc_wq)
3568                 destroy_workqueue(ic->recalc_wq);
3569         vfree(ic->recalc_buffer);
3570         kvfree(ic->recalc_tags);
3571         if (ic->bufio)
3572                 dm_bufio_client_destroy(ic->bufio);
3573         mempool_exit(&ic->journal_io_mempool);
3574         if (ic->io)
3575                 dm_io_client_destroy(ic->io);
3576         if (ic->dev)
3577                 dm_put_device(ti, ic->dev);
3578         if (ic->meta_dev)
3579                 dm_put_device(ti, ic->meta_dev);
3580         dm_integrity_free_page_list(ic->journal);
3581         dm_integrity_free_page_list(ic->journal_io);
3582         dm_integrity_free_page_list(ic->journal_xor);
3583         if (ic->journal_scatterlist)
3584                 dm_integrity_free_journal_scatterlist(ic, ic->journal_scatterlist);
3585         if (ic->journal_io_scatterlist)
3586                 dm_integrity_free_journal_scatterlist(ic, ic->journal_io_scatterlist);
3587         if (ic->sk_requests) {
3588                 unsigned i;
3589
3590                 for (i = 0; i < ic->journal_sections; i++) {
3591                         struct skcipher_request *req = ic->sk_requests[i];
3592                         if (req) {
3593                                 kzfree(req->iv);
3594                                 skcipher_request_free(req);
3595                         }
3596                 }
3597                 kvfree(ic->sk_requests);
3598         }
3599         kvfree(ic->journal_tree);
3600         if (ic->sb)
3601                 free_pages_exact(ic->sb, SB_SECTORS << SECTOR_SHIFT);
3602
3603         if (ic->internal_hash)
3604                 crypto_free_shash(ic->internal_hash);
3605         free_alg(&ic->internal_hash_alg);
3606
3607         if (ic->journal_crypt)
3608                 crypto_free_skcipher(ic->journal_crypt);
3609         free_alg(&ic->journal_crypt_alg);
3610
3611         if (ic->journal_mac)
3612                 crypto_free_shash(ic->journal_mac);
3613         free_alg(&ic->journal_mac_alg);
3614
3615         kfree(ic);
3616 }
3617
3618 static struct target_type integrity_target = {
3619         .name                   = "integrity",
3620         .version                = {1, 2, 0},
3621         .module                 = THIS_MODULE,
3622         .features               = DM_TARGET_SINGLETON | DM_TARGET_INTEGRITY,
3623         .ctr                    = dm_integrity_ctr,
3624         .dtr                    = dm_integrity_dtr,
3625         .map                    = dm_integrity_map,
3626         .postsuspend            = dm_integrity_postsuspend,
3627         .resume                 = dm_integrity_resume,
3628         .status                 = dm_integrity_status,
3629         .iterate_devices        = dm_integrity_iterate_devices,
3630         .io_hints               = dm_integrity_io_hints,
3631 };
3632
3633 static int __init dm_integrity_init(void)
3634 {
3635         int r;
3636
3637         journal_io_cache = kmem_cache_create("integrity_journal_io",
3638                                              sizeof(struct journal_io), 0, 0, NULL);
3639         if (!journal_io_cache) {
3640                 DMERR("can't allocate journal io cache");
3641                 return -ENOMEM;
3642         }
3643
3644         r = dm_register_target(&integrity_target);
3645
3646         if (r < 0)
3647                 DMERR("register failed %d", r);
3648
3649         return r;
3650 }
3651
3652 static void __exit dm_integrity_exit(void)
3653 {
3654         dm_unregister_target(&integrity_target);
3655         kmem_cache_destroy(journal_io_cache);
3656 }
3657
3658 module_init(dm_integrity_init);
3659 module_exit(dm_integrity_exit);
3660
3661 MODULE_AUTHOR("Milan Broz");
3662 MODULE_AUTHOR("Mikulas Patocka");
3663 MODULE_DESCRIPTION(DM_NAME " target for integrity tags extension");
3664 MODULE_LICENSE("GPL");