lib/iov_iter.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 #include <crypto/hash.h>
   3 #include <linux/export.h>
   4 #include <linux/bvec.h>
   5 #include <linux/fault-inject-usercopy.h>
   6 #include <linux/uio.h>
   7 #include <linux/pagemap.h>
   8 #include <linux/highmem.h>
   9 #include <linux/slab.h>
  10 #include <linux/vmalloc.h>
  11 #include <linux/splice.h>
  12 #include <linux/compat.h>
  13 #include <net/checksum.h>
  14 #include <linux/scatterlist.h>
  15 #include <linux/instrumented.h>
  16
  17 #define PIPE_PARANOIA /* for now */
  18
  19 #define iterate_iovec(i, n, __v, __p, skip, STEP) {     \
  20         size_t left;                                    \
  21         size_t wanted = n;                              \
  22         __p = i->iov;                                   \
  23         __v.iov_len = min(n, __p->iov_len - skip);      \
  24         if (likely(__v.iov_len)) {                      \
  25                 __v.iov_base = __p->iov_base + skip;    \
  26                 left = (STEP);                          \
  27                 __v.iov_len -= left;                    \
  28                 skip += __v.iov_len;                    \
  29                 n -= __v.iov_len;                       \
  30         } else {                                        \
  31                 left = 0;                               \
  32         }                                               \
  33         while (unlikely(!left && n)) {                  \
  34                 __p++;                                  \
  35                 __v.iov_len = min(n, __p->iov_len);     \
  36                 if (unlikely(!__v.iov_len))             \
  37                         continue;                       \
  38                 __v.iov_base = __p->iov_base;           \
  39                 left = (STEP);                          \
  40                 __v.iov_len -= left;                    \
  41                 skip = __v.iov_len;                     \
  42                 n -= __v.iov_len;                       \
  43         }                                               \
  44         n = wanted - n;                                 \
  45 }
  46
  47 #define iterate_kvec(i, n, __v, __p, skip, STEP) {      \
  48         size_t wanted = n;                              \
  49         __p = i->kvec;                                  \
  50         __v.iov_len = min(n, __p->iov_len - skip);      \
  51         if (likely(__v.iov_len)) {                      \
  52                 __v.iov_base = __p->iov_base + skip;    \
  53                 (void)(STEP);                           \
  54                 skip += __v.iov_len;                    \
  55                 n -= __v.iov_len;                       \
  56         }                                               \
  57         while (unlikely(n)) {                           \
  58                 __p++;                                  \
  59                 __v.iov_len = min(n, __p->iov_len);     \
  60                 if (unlikely(!__v.iov_len))             \
  61                         continue;                       \
  62                 __v.iov_base = __p->iov_base;           \
  63                 (void)(STEP);                           \
  64                 skip = __v.iov_len;                     \
  65                 n -= __v.iov_len;                       \
  66         }                                               \
  67         n = wanted;                                     \
  68 }
  69
  70 #define iterate_bvec(i, n, __v, __bi, skip, STEP) {     \
  71         struct bvec_iter __start;                       \
  72         __start.bi_size = n;                            \
  73         __start.bi_bvec_done = skip;                    \
  74         __start.bi_idx = 0;                             \
  75         for_each_bvec(__v, i->bvec, __bi, __start) {    \
  76                 (void)(STEP);                           \
  77         }                                               \
  78 }
  79
  80 #define iterate_xarray(i, n, __v, skip, STEP) {         \
  81         struct page *head = NULL;                               \
  82         size_t wanted = n, seg, offset;                         \
  83         loff_t start = i->xarray_start + skip;                  \
  84         pgoff_t index = start >> PAGE_SHIFT;                    \
  85         int j;                                                  \
  86                                                                 \
  87         XA_STATE(xas, i->xarray, index);                        \
  88                                                                 \
  89         rcu_read_lock();                                                \
  90         xas_for_each(&xas, head, ULONG_MAX) {                           \
  91                 if (xas_retry(&xas, head))                              \
  92                         continue;                                       \
  93                 if (WARN_ON(xa_is_value(head)))                         \
  94                         break;                                          \
  95                 if (WARN_ON(PageHuge(head)))                            \
  96                         break;                                          \
  97                 for (j = (head->index < index) ? index - head->index : 0; \
  98                      j < thp_nr_pages(head); j++) {                     \
  99                         __v.bv_page = head + j;                         \
 100                         offset = (i->xarray_start + skip) & ~PAGE_MASK; \
 101                         seg = PAGE_SIZE - offset;                       \
 102                         __v.bv_offset = offset;                         \
 103                         __v.bv_len = min(n, seg);                       \
 104                         (void)(STEP);                                   \
 105                         n -= __v.bv_len;                                \
 106                         skip += __v.bv_len;                             \
 107                         if (n == 0)                                     \
 108                                 break;                                  \
 109                 }                                                       \
 110                 if (n == 0)                                             \
 111                         break;                                          \
 112         }                                                       \
 113         rcu_read_unlock();                                      \
 114         n = wanted - n;                                         \
 115 }
 116
 117 #define iterate_all_kinds(i, n, v, I, B, K, X) {                \
 118         if (likely(n)) {                                        \
 119                 size_t skip = i->iov_offset;                    \
 120                 if (unlikely(i->type & ITER_BVEC)) {            \
 121                         struct bio_vec v;                       \
 122                         struct bvec_iter __bi;                  \
 123                         iterate_bvec(i, n, v, __bi, skip, (B))  \
 124                 } else if (unlikely(i->type & ITER_KVEC)) {     \
 125                         const struct kvec *kvec;                \
 126                         struct kvec v;                          \
 127                         iterate_kvec(i, n, v, kvec, skip, (K))  \
 128                 } else if (unlikely(i->type & ITER_DISCARD)) {  \
 129                 } else if (unlikely(i->type & ITER_XARRAY)) {   \
 130                         struct bio_vec v;                       \
 131                         iterate_xarray(i, n, v, skip, (X));     \
 132                 } else {                                        \
 133                         const struct iovec *iov;                \
 134                         struct iovec v;                         \
 135                         iterate_iovec(i, n, v, iov, skip, (I))  \
 136                 }                                               \
 137         }                                                       \
 138 }
 139
 140 #define iterate_and_advance(i, n, v, I, B, K, X) {              \
 141         if (unlikely(i->count < n))                             \
 142                 n = i->count;                                   \
 143         if (i->count) {                                         \
 144                 size_t skip = i->iov_offset;                    \
 145                 if (unlikely(i->type & ITER_BVEC)) {            \
 146                         const struct bio_vec *bvec = i->bvec;   \
 147                         struct bio_vec v;                       \
 148                         struct bvec_iter __bi;                  \
 149                         iterate_bvec(i, n, v, __bi, skip, (B))  \
 150                         i->bvec = __bvec_iter_bvec(i->bvec, __bi);      \
 151                         i->nr_segs -= i->bvec - bvec;           \
 152                         skip = __bi.bi_bvec_done;               \
 153                 } else if (unlikely(i->type & ITER_KVEC)) {     \
 154                         const struct kvec *kvec;                \
 155                         struct kvec v;                          \
 156                         iterate_kvec(i, n, v, kvec, skip, (K))  \
 157                         if (skip == kvec->iov_len) {            \
 158                                 kvec++;                         \
 159                                 skip = 0;                       \
 160                         }                                       \
 161                         i->nr_segs -= kvec - i->kvec;           \
 162                         i->kvec = kvec;                         \
 163                 } else if (unlikely(i->type & ITER_DISCARD)) {  \
 164                         skip += n;                              \
 165                 } else if (unlikely(i->type & ITER_XARRAY)) {   \
 166                         struct bio_vec v;                       \
 167                         iterate_xarray(i, n, v, skip, (X))      \
 168                 } else {                                        \
 169                         const struct iovec *iov;                \
 170                         struct iovec v;                         \
 171                         iterate_iovec(i, n, v, iov, skip, (I))  \
 172                         if (skip == iov->iov_len) {             \
 173                                 iov++;                          \
 174                                 skip = 0;                       \
 175                         }                                       \
 176                         i->nr_segs -= iov - i->iov;             \
 177                         i->iov = iov;                           \
 178                 }                                               \
 179                 i->count -= n;                                  \
 180                 i->iov_offset = skip;                           \
 181         }                                                       \
 182 }
 183
 184 static int copyout(void __user *to, const void *from, size_t n)
 185 {
 186         if (should_fail_usercopy())
 187                 return n;
 188         if (access_ok(to, n)) {
 189                 instrument_copy_to_user(to, from, n);
 190                 n = raw_copy_to_user(to, from, n);
 191         }
 192         return n;
 193 }
 194
 195 static int copyin(void *to, const void __user *from, size_t n)
 196 {
 197         if (should_fail_usercopy())
 198                 return n;
 199         if (access_ok(from, n)) {
 200                 instrument_copy_from_user(to, from, n);
 201                 n = raw_copy_from_user(to, from, n);
 202         }
 203         return n;
 204 }
 205
 206 static size_t copy_page_to_iter_iovec(struct page *page, size_t offset, size_t bytes,
 207                          struct iov_iter *i)
 208 {
 209         size_t skip, copy, left, wanted;
 210         const struct iovec *iov;
 211         char __user *buf;
 212         void *kaddr, *from;
 213
 214         if (unlikely(bytes > i->count))
 215                 bytes = i->count;
 216
 217         if (unlikely(!bytes))
 218                 return 0;
 219
 220         might_fault();
 221         wanted = bytes;
 222         iov = i->iov;
 223         skip = i->iov_offset;
 224         buf = iov->iov_base + skip;
 225         copy = min(bytes, iov->iov_len - skip);
 226
 227         if (IS_ENABLED(CONFIG_HIGHMEM) && !fault_in_pages_writeable(buf, copy)) {
 228                 kaddr = kmap_atomic(page);
 229                 from = kaddr + offset;
 230
 231                 /* first chunk, usually the only one */
 232                 left = copyout(buf, from, copy);
 233                 copy -= left;
 234                 skip += copy;
 235                 from += copy;
 236                 bytes -= copy;
 237
 238                 while (unlikely(!left && bytes)) {
 239                         iov++;
 240                         buf = iov->iov_base;
 241                         copy = min(bytes, iov->iov_len);
 242                         left = copyout(buf, from, copy);
 243                         copy -= left;
 244                         skip = copy;
 245                         from += copy;
 246                         bytes -= copy;
 247                 }
 248                 if (likely(!bytes)) {
 249                         kunmap_atomic(kaddr);
 250                         goto done;
 251                 }
 252                 offset = from - kaddr;
 253                 buf += copy;
 254                 kunmap_atomic(kaddr);
 255                 copy = min(bytes, iov->iov_len - skip);
 256         }
 257         /* Too bad - revert to non-atomic kmap */
 258
 259         kaddr = kmap(page);
 260         from = kaddr + offset;
 261         left = copyout(buf, from, copy);
 262         copy -= left;
 263         skip += copy;
 264         from += copy;
 265         bytes -= copy;
 266         while (unlikely(!left && bytes)) {
 267                 iov++;
 268                 buf = iov->iov_base;
 269                 copy = min(bytes, iov->iov_len);
 270                 left = copyout(buf, from, copy);
 271                 copy -= left;
 272                 skip = copy;
 273                 from += copy;
 274                 bytes -= copy;
 275         }
 276         kunmap(page);
 277
 278 done:
 279         if (skip == iov->iov_len) {
 280                 iov++;
 281                 skip = 0;
 282         }
 283         i->count -= wanted - bytes;
 284         i->nr_segs -= iov - i->iov;
 285         i->iov = iov;
 286         i->iov_offset = skip;
 287         return wanted - bytes;
 288 }
 289
 290 static size_t copy_page_from_iter_iovec(struct page *page, size_t offset, size_t bytes,
 291                          struct iov_iter *i)
 292 {
 293         size_t skip, copy, left, wanted;
 294         const struct iovec *iov;
 295         char __user *buf;
 296         void *kaddr, *to;
 297
 298         if (unlikely(bytes > i->count))
 299                 bytes = i->count;
 300
 301         if (unlikely(!bytes))
 302                 return 0;
 303
 304         might_fault();
 305         wanted = bytes;
 306         iov = i->iov;
 307         skip = i->iov_offset;
 308         buf = iov->iov_base + skip;
 309         copy = min(bytes, iov->iov_len - skip);
 310
 311         if (IS_ENABLED(CONFIG_HIGHMEM) && !fault_in_pages_readable(buf, copy)) {
 312                 kaddr = kmap_atomic(page);
 313                 to = kaddr + offset;
 314
 315                 /* first chunk, usually the only one */
 316                 left = copyin(to, buf, copy);
 317                 copy -= left;
 318                 skip += copy;
 319                 to += copy;
 320                 bytes -= copy;
 321
 322                 while (unlikely(!left && bytes)) {
 323                         iov++;
 324                         buf = iov->iov_base;
 325                         copy = min(bytes, iov->iov_len);
 326                         left = copyin(to, buf, copy);
 327                         copy -= left;
 328                         skip = copy;
 329                         to += copy;
 330                         bytes -= copy;
 331                 }
 332                 if (likely(!bytes)) {
 333                         kunmap_atomic(kaddr);
 334                         goto done;
 335                 }
 336                 offset = to - kaddr;
 337                 buf += copy;
 338                 kunmap_atomic(kaddr);
 339                 copy = min(bytes, iov->iov_len - skip);
 340         }
 341         /* Too bad - revert to non-atomic kmap */
 342
 343         kaddr = kmap(page);
 344         to = kaddr + offset;
 345         left = copyin(to, buf, copy);
 346         copy -= left;
 347         skip += copy;
 348         to += copy;
 349         bytes -= copy;
 350         while (unlikely(!left && bytes)) {
 351                 iov++;
 352                 buf = iov->iov_base;
 353                 copy = min(bytes, iov->iov_len);
 354                 left = copyin(to, buf, copy);
 355                 copy -= left;
 356                 skip = copy;
 357                 to += copy;
 358                 bytes -= copy;
 359         }
 360         kunmap(page);
 361
 362 done:
 363         if (skip == iov->iov_len) {
 364                 iov++;
 365                 skip = 0;
 366         }
 367         i->count -= wanted - bytes;
 368         i->nr_segs -= iov - i->iov;
 369         i->iov = iov;
 370         i->iov_offset = skip;
 371         return wanted - bytes;
 372 }
 373
 374 #ifdef PIPE_PARANOIA
 375 static bool sanity(const struct iov_iter *i)
 376 {
 377         struct pipe_inode_info *pipe = i->pipe;
 378         unsigned int p_head = pipe->head;
 379         unsigned int p_tail = pipe->tail;
 380         unsigned int p_mask = pipe->ring_size - 1;
 381         unsigned int p_occupancy = pipe_occupancy(p_head, p_tail);
 382         unsigned int i_head = i->head;
 383         unsigned int idx;
 384
 385         if (i->iov_offset) {
 386                 struct pipe_buffer *p;
 387                 if (unlikely(p_occupancy == 0))
 388                         goto Bad;       // pipe must be non-empty
 389                 if (unlikely(i_head != p_head - 1))
 390                         goto Bad;       // must be at the last buffer...
 391
 392                 p = &pipe->bufs[i_head & p_mask];
 393                 if (unlikely(p->offset + p->len != i->iov_offset))
 394                         goto Bad;       // ... at the end of segment
 395         } else {
 396                 if (i_head != p_head)
 397                         goto Bad;       // must be right after the last buffer
 398         }
 399         return true;
 400 Bad:
 401         printk(KERN_ERR "idx = %d, offset = %zd\n", i_head, i->iov_offset);
 402         printk(KERN_ERR "head = %d, tail = %d, buffers = %d\n",
 403                         p_head, p_tail, pipe->ring_size);
 404         for (idx = 0; idx < pipe->ring_size; idx++)
 405                 printk(KERN_ERR "[%p %p %d %d]\n",
 406                         pipe->bufs[idx].ops,
 407                         pipe->bufs[idx].page,
 408                         pipe->bufs[idx].offset,
 409                         pipe->bufs[idx].len);
 410         WARN_ON(1);
 411         return false;
 412 }
 413 #else
 414 #define sanity(i) true
 415 #endif
 416
 417 static size_t copy_page_to_iter_pipe(struct page *page, size_t offset, size_t bytes,
 418                          struct iov_iter *i)
 419 {
 420         struct pipe_inode_info *pipe = i->pipe;
 421         struct pipe_buffer *buf;
 422         unsigned int p_tail = pipe->tail;
 423         unsigned int p_mask = pipe->ring_size - 1;
 424         unsigned int i_head = i->head;
 425         size_t off;
 426
 427         if (unlikely(bytes > i->count))
 428                 bytes = i->count;
 429
 430         if (unlikely(!bytes))
 431                 return 0;
 432
 433         if (!sanity(i))
 434                 return 0;
 435
 436         off = i->iov_offset;
 437         buf = &pipe->bufs[i_head & p_mask];
 438         if (off) {
 439                 if (offset == off && buf->page == page) {
 440                         /* merge with the last one */
 441                         buf->len += bytes;
 442                         i->iov_offset += bytes;
 443                         goto out;
 444                 }
 445                 i_head++;
 446                 buf = &pipe->bufs[i_head & p_mask];
 447         }
 448         if (pipe_full(i_head, p_tail, pipe->max_usage))
 449                 return 0;
 450
 451         buf->ops = &page_cache_pipe_buf_ops;
 452         get_page(page);
 453         buf->page = page;
 454         buf->offset = offset;
 455         buf->len = bytes;
 456
 457         pipe->head = i_head + 1;
 458         i->iov_offset = offset + bytes;
 459         i->head = i_head;
 460 out:
 461         i->count -= bytes;
 462         return bytes;
 463 }
 464
 465 /*
 466  * Fault in one or more iovecs of the given iov_iter, to a maximum length of
 467  * bytes.  For each iovec, fault in each page that constitutes the iovec.
 468  *
 469  * Return 0 on success, or non-zero if the memory could not be accessed (i.e.
 470  * because it is an invalid address).
 471  */
 472 int iov_iter_fault_in_readable(struct iov_iter *i, size_t bytes)
 473 {
 474         size_t skip = i->iov_offset;
 475         const struct iovec *iov;
 476         int err;
 477         struct iovec v;
 478
 479         if (iter_is_iovec(i)) {
 480                 iterate_iovec(i, bytes, v, iov, skip, ({
 481                         err = fault_in_pages_readable(v.iov_base, v.iov_len);
 482                         if (unlikely(err))
 483                         return err;
 484                 0;}))
 485         }
 486         return 0;
 487 }
 488 EXPORT_SYMBOL(iov_iter_fault_in_readable);
 489
 490 void iov_iter_init(struct iov_iter *i, unsigned int direction,
 491                         const struct iovec *iov, unsigned long nr_segs,
 492                         size_t count)
 493 {
 494         WARN_ON(direction & ~(READ | WRITE));
 495         direction &= READ | WRITE;
 496
 497         /* It will get better.  Eventually... */
 498         if (uaccess_kernel()) {
 499                 i->type = ITER_KVEC | direction;
 500                 i->kvec = (struct kvec *)iov;
 501         } else {
 502                 i->type = ITER_IOVEC | direction;
 503                 i->iov = iov;
 504         }
 505         i->nr_segs = nr_segs;
 506         i->iov_offset = 0;
 507         i->count = count;
 508 }
 509 EXPORT_SYMBOL(iov_iter_init);
 510
 511 static inline bool allocated(struct pipe_buffer *buf)
 512 {
 513         return buf->ops == &default_pipe_buf_ops;
 514 }
 515
 516 static inline void data_start(const struct iov_iter *i,
 517                               unsigned int *iter_headp, size_t *offp)
 518 {
 519         unsigned int p_mask = i->pipe->ring_size - 1;
 520         unsigned int iter_head = i->head;
 521         size_t off = i->iov_offset;
 522
 523         if (off && (!allocated(&i->pipe->bufs[iter_head & p_mask]) ||
 524                     off == PAGE_SIZE)) {
 525                 iter_head++;
 526                 off = 0;
 527         }
 528         *iter_headp = iter_head;
 529         *offp = off;
 530 }
 531
 532 static size_t push_pipe(struct iov_iter *i, size_t size,
 533                         int *iter_headp, size_t *offp)
 534 {
 535         struct pipe_inode_info *pipe = i->pipe;
 536         unsigned int p_tail = pipe->tail;
 537         unsigned int p_mask = pipe->ring_size - 1;
 538         unsigned int iter_head;
 539         size_t off;
 540         ssize_t left;
 541
 542         if (unlikely(size > i->count))
 543                 size = i->count;
 544         if (unlikely(!size))
 545                 return 0;
 546
 547         left = size;
 548         data_start(i, &iter_head, &off);
 549         *iter_headp = iter_head;
 550         *offp = off;
 551         if (off) {
 552                 left -= PAGE_SIZE - off;
 553                 if (left <= 0) {
 554                         pipe->bufs[iter_head & p_mask].len += size;
 555                         return size;
 556                 }
 557                 pipe->bufs[iter_head & p_mask].len = PAGE_SIZE;
 558                 iter_head++;
 559         }
 560         while (!pipe_full(iter_head, p_tail, pipe->max_usage)) {
 561                 struct pipe_buffer *buf = &pipe->bufs[iter_head & p_mask];
 562                 struct page *page = alloc_page(GFP_USER);
 563                 if (!page)
 564                         break;
 565
 566                 buf->ops = &default_pipe_buf_ops;
 567                 buf->page = page;
 568                 buf->offset = 0;
 569                 buf->len = min_t(ssize_t, left, PAGE_SIZE);
 570                 left -= buf->len;
 571                 iter_head++;
 572                 pipe->head = iter_head;
 573
 574                 if (left == 0)
 575                         return size;
 576         }
 577         return size - left;
 578 }
 579
 580 static size_t copy_pipe_to_iter(const void *addr, size_t bytes,
 581                                 struct iov_iter *i)
 582 {
 583         struct pipe_inode_info *pipe = i->pipe;
 584         unsigned int p_mask = pipe->ring_size - 1;
 585         unsigned int i_head;
 586         size_t n, off;
 587
 588         if (!sanity(i))
 589                 return 0;
 590
 591         bytes = n = push_pipe(i, bytes, &i_head, &off);
 592         if (unlikely(!n))
 593                 return 0;
 594         do {
 595                 size_t chunk = min_t(size_t, n, PAGE_SIZE - off);
 596                 memcpy_to_page(pipe->bufs[i_head & p_mask].page, off, addr, chunk);
 597                 i->head = i_head;
 598                 i->iov_offset = off + chunk;
 599                 n -= chunk;
 600                 addr += chunk;
 601                 off = 0;
 602                 i_head++;
 603         } while (n);
 604         i->count -= bytes;
 605         return bytes;
 606 }
 607
 608 static __wsum csum_and_memcpy(void *to, const void *from, size_t len,
 609                               __wsum sum, size_t off)
 610 {
 611         __wsum next = csum_partial_copy_nocheck(from, to, len);
 612         return csum_block_add(sum, next, off);
 613 }
 614
 615 static size_t csum_and_copy_to_pipe_iter(const void *addr, size_t bytes,
 616                                          struct csum_state *csstate,
 617                                          struct iov_iter *i)
 618 {
 619         struct pipe_inode_info *pipe = i->pipe;
 620         unsigned int p_mask = pipe->ring_size - 1;
 621         __wsum sum = csstate->csum;
 622         size_t off = csstate->off;
 623         unsigned int i_head;
 624         size_t n, r;
 625
 626         if (!sanity(i))
 627                 return 0;
 628
 629         bytes = n = push_pipe(i, bytes, &i_head, &r);
 630         if (unlikely(!n))
 631                 return 0;
 632         do {
 633                 size_t chunk = min_t(size_t, n, PAGE_SIZE - r);
 634                 char *p = kmap_atomic(pipe->bufs[i_head & p_mask].page);
 635                 sum = csum_and_memcpy(p + r, addr, chunk, sum, off);
 636                 kunmap_atomic(p);
 637                 i->head = i_head;
 638                 i->iov_offset = r + chunk;
 639                 n -= chunk;
 640                 off += chunk;
 641                 addr += chunk;
 642                 r = 0;
 643                 i_head++;
 644         } while (n);
 645         i->count -= bytes;
 646         csstate->csum = sum;
 647         csstate->off = off;
 648         return bytes;
 649 }
 650
 651 size_t _copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i)
 652 {
 653         const char *from = addr;
 654         if (unlikely(iov_iter_is_pipe(i)))
 655                 return copy_pipe_to_iter(addr, bytes, i);
 656         if (iter_is_iovec(i))
 657                 might_fault();
 658         iterate_and_advance(i, bytes, v,
 659                 copyout(v.iov_base, (from += v.iov_len) - v.iov_len, v.iov_len),
 660                 memcpy_to_page(v.bv_page, v.bv_offset,
 661                                (from += v.bv_len) - v.bv_len, v.bv_len),
 662                 memcpy(v.iov_base, (from += v.iov_len) - v.iov_len, v.iov_len),
 663                 memcpy_to_page(v.bv_page, v.bv_offset,
 664                                (from += v.bv_len) - v.bv_len, v.bv_len)
 665         )
 666
 667         return bytes;
 668 }
 669 EXPORT_SYMBOL(_copy_to_iter);
 670
 671 #ifdef CONFIG_ARCH_HAS_COPY_MC
 672 static int copyout_mc(void __user *to, const void *from, size_t n)
 673 {
 674         if (access_ok(to, n)) {
 675                 instrument_copy_to_user(to, from, n);
 676                 n = copy_mc_to_user((__force void *) to, from, n);
 677         }
 678         return n;
 679 }
 680
 681 static unsigned long copy_mc_to_page(struct page *page, size_t offset,
 682                 const char *from, size_t len)
 683 {
 684         unsigned long ret;
 685         char *to;
 686
 687         to = kmap_atomic(page);
 688         ret = copy_mc_to_kernel(to + offset, from, len);
 689         kunmap_atomic(to);
 690
 691         return ret;
 692 }
 693
 694 static size_t copy_mc_pipe_to_iter(const void *addr, size_t bytes,
 695                                 struct iov_iter *i)
 696 {
 697         struct pipe_inode_info *pipe = i->pipe;
 698         unsigned int p_mask = pipe->ring_size - 1;
 699         unsigned int i_head;
 700         size_t n, off, xfer = 0;
 701
 702         if (!sanity(i))
 703                 return 0;
 704
 705         bytes = n = push_pipe(i, bytes, &i_head, &off);
 706         if (unlikely(!n))
 707                 return 0;
 708         do {
 709                 size_t chunk = min_t(size_t, n, PAGE_SIZE - off);
 710                 unsigned long rem;
 711
 712                 rem = copy_mc_to_page(pipe->bufs[i_head & p_mask].page,
 713                                             off, addr, chunk);
 714                 i->head = i_head;
 715                 i->iov_offset = off + chunk - rem;
 716                 xfer += chunk - rem;
 717                 if (rem)
 718                         break;
 719                 n -= chunk;
 720                 addr += chunk;
 721                 off = 0;
 722                 i_head++;
 723         } while (n);
 724         i->count -= xfer;
 725         return xfer;
 726 }
 727
 728 /**
 729  * _copy_mc_to_iter - copy to iter with source memory error exception handling
 730  * @addr: source kernel address
 731  * @bytes: total transfer length
 732  * @iter: destination iterator
 733  *
 734  * The pmem driver deploys this for the dax operation
 735  * (dax_copy_to_iter()) for dax reads (bypass page-cache and the
 736  * block-layer). Upon #MC read(2) aborts and returns EIO or the bytes
 737  * successfully copied.
 738  *
 739  * The main differences between this and typical _copy_to_iter().
 740  *
 741  * * Typical tail/residue handling after a fault retries the copy
 742  *   byte-by-byte until the fault happens again. Re-triggering machine
 743  *   checks is potentially fatal so the implementation uses source
 744  *   alignment and poison alignment assumptions to avoid re-triggering
 745  *   hardware exceptions.
 746  *
 747  * * ITER_KVEC, ITER_PIPE, and ITER_BVEC can return short copies.
 748  *   Compare to copy_to_iter() where only ITER_IOVEC attempts might return
 749  *   a short copy.
 750  */
 751 size_t _copy_mc_to_iter(const void *addr, size_t bytes, struct iov_iter *i)
 752 {
 753         const char *from = addr;
 754         unsigned long rem, curr_addr, s_addr = (unsigned long) addr;
 755
 756         if (unlikely(iov_iter_is_pipe(i)))
 757                 return copy_mc_pipe_to_iter(addr, bytes, i);
 758         if (iter_is_iovec(i))
 759                 might_fault();
 760         iterate_and_advance(i, bytes, v,
 761                 copyout_mc(v.iov_base, (from += v.iov_len) - v.iov_len,
 762                            v.iov_len),
 763                 ({
 764                 rem = copy_mc_to_page(v.bv_page, v.bv_offset,
 765                                       (from += v.bv_len) - v.bv_len, v.bv_len);
 766                 if (rem) {
 767                         curr_addr = (unsigned long) from;
 768                         bytes = curr_addr - s_addr - rem;
 769                         return bytes;
 770                 }
 771                 }),
 772                 ({
 773                 rem = copy_mc_to_kernel(v.iov_base, (from += v.iov_len)
 774                                         - v.iov_len, v.iov_len);
 775                 if (rem) {
 776                         curr_addr = (unsigned long) from;
 777                         bytes = curr_addr - s_addr - rem;
 778                         return bytes;
 779                 }
 780                 }),
 781                 ({
 782                 rem = copy_mc_to_page(v.bv_page, v.bv_offset,
 783                                       (from += v.bv_len) - v.bv_len, v.bv_len);
 784                 if (rem) {
 785                         curr_addr = (unsigned long) from;
 786                         bytes = curr_addr - s_addr - rem;
 787                         rcu_read_unlock();
 788                         i->iov_offset += bytes;
 789                         i->count -= bytes;
 790                         return bytes;
 791                 }
 792                 })
 793         )
 794
 795         return bytes;
 796 }
 797 EXPORT_SYMBOL_GPL(_copy_mc_to_iter);
 798 #endif /* CONFIG_ARCH_HAS_COPY_MC */
 799
 800 size_t _copy_from_iter(void *addr, size_t bytes, struct iov_iter *i)
 801 {
 802         char *to = addr;
 803         if (unlikely(iov_iter_is_pipe(i))) {
 804                 WARN_ON(1);
 805                 return 0;
 806         }
 807         if (iter_is_iovec(i))
 808                 might_fault();
 809         iterate_and_advance(i, bytes, v,
 810                 copyin((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len),
 811                 memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
 812                                  v.bv_offset, v.bv_len),
 813                 memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len),
 814                 memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
 815                                  v.bv_offset, v.bv_len)
 816         )
 817
 818         return bytes;
 819 }
 820 EXPORT_SYMBOL(_copy_from_iter);
 821
 822 bool _copy_from_iter_full(void *addr, size_t bytes, struct iov_iter *i)
 823 {
 824         char *to = addr;
 825         if (unlikely(iov_iter_is_pipe(i))) {
 826                 WARN_ON(1);
 827                 return false;
 828         }
 829         if (unlikely(i->count < bytes))
 830                 return false;
 831
 832         if (iter_is_iovec(i))
 833                 might_fault();
 834         iterate_all_kinds(i, bytes, v, ({
 835                 if (copyin((to += v.iov_len) - v.iov_len,
 836                                       v.iov_base, v.iov_len))
 837                         return false;
 838                 0;}),
 839                 memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
 840                                  v.bv_offset, v.bv_len),
 841                 memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len),
 842                 memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
 843                                  v.bv_offset, v.bv_len)
 844         )
 845
 846         iov_iter_advance(i, bytes);
 847         return true;
 848 }
 849 EXPORT_SYMBOL(_copy_from_iter_full);
 850
 851 size_t _copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i)
 852 {
 853         char *to = addr;
 854         if (unlikely(iov_iter_is_pipe(i))) {
 855                 WARN_ON(1);
 856                 return 0;
 857         }
 858         iterate_and_advance(i, bytes, v,
 859                 __copy_from_user_inatomic_nocache((to += v.iov_len) - v.iov_len,
 860                                          v.iov_base, v.iov_len),
 861                 memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
 862                                  v.bv_offset, v.bv_len),
 863                 memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len),
 864                 memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
 865                                  v.bv_offset, v.bv_len)
 866         )
 867
 868         return bytes;
 869 }
 870 EXPORT_SYMBOL(_copy_from_iter_nocache);
 871
 872 #ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE
 873 /**
 874  * _copy_from_iter_flushcache - write destination through cpu cache
 875  * @addr: destination kernel address
 876  * @bytes: total transfer length
 877  * @iter: source iterator
 878  *
 879  * The pmem driver arranges for filesystem-dax to use this facility via
 880  * dax_copy_from_iter() for ensuring that writes to persistent memory
 881  * are flushed through the CPU cache. It is differentiated from
 882  * _copy_from_iter_nocache() in that guarantees all data is flushed for
 883  * all iterator types. The _copy_from_iter_nocache() only attempts to
 884  * bypass the cache for the ITER_IOVEC case, and on some archs may use
 885  * instructions that strand dirty-data in the cache.
 886  */
 887 size_t _copy_from_iter_flushcache(void *addr, size_t bytes, struct iov_iter *i)
 888 {
 889         char *to = addr;
 890         if (unlikely(iov_iter_is_pipe(i))) {
 891                 WARN_ON(1);
 892                 return 0;
 893         }
 894         iterate_and_advance(i, bytes, v,
 895                 __copy_from_user_flushcache((to += v.iov_len) - v.iov_len,
 896                                          v.iov_base, v.iov_len),
 897                 memcpy_page_flushcache((to += v.bv_len) - v.bv_len, v.bv_page,
 898                                  v.bv_offset, v.bv_len),
 899                 memcpy_flushcache((to += v.iov_len) - v.iov_len, v.iov_base,
 900                         v.iov_len),
 901                 memcpy_page_flushcache((to += v.bv_len) - v.bv_len, v.bv_page,
 902                                  v.bv_offset, v.bv_len)
 903         )
 904
 905         return bytes;
 906 }
 907 EXPORT_SYMBOL_GPL(_copy_from_iter_flushcache);
 908 #endif
 909
 910 bool _copy_from_iter_full_nocache(void *addr, size_t bytes, struct iov_iter *i)
 911 {
 912         char *to = addr;
 913         if (unlikely(iov_iter_is_pipe(i))) {
 914                 WARN_ON(1);
 915                 return false;
 916         }
 917         if (unlikely(i->count < bytes))
 918                 return false;
 919         iterate_all_kinds(i, bytes, v, ({
 920                 if (__copy_from_user_inatomic_nocache((to += v.iov_len) - v.iov_len,
 921                                              v.iov_base, v.iov_len))
 922                         return false;
 923                 0;}),
 924                 memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
 925                                  v.bv_offset, v.bv_len),
 926                 memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len),
 927                 memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
 928                                  v.bv_offset, v.bv_len)
 929         )
 930
 931         iov_iter_advance(i, bytes);
 932         return true;
 933 }
 934 EXPORT_SYMBOL(_copy_from_iter_full_nocache);
 935
 936 static inline bool page_copy_sane(struct page *page, size_t offset, size_t n)
 937 {
 938         struct page *head;
 939         size_t v = n + offset;
 940
 941         /*
 942          * The general case needs to access the page order in order
 943          * to compute the page size.
 944          * However, we mostly deal with order-0 pages and thus can
 945          * avoid a possible cache line miss for requests that fit all
 946          * page orders.
 947          */
 948         if (n <= v && v <= PAGE_SIZE)
 949                 return true;
 950
 951         head = compound_head(page);
 952         v += (page - head) << PAGE_SHIFT;
 953
 954         if (likely(n <= v && v <= (page_size(head))))
 955                 return true;
 956         WARN_ON(1);
 957         return false;
 958 }
 959
 960 static size_t __copy_page_to_iter(struct page *page, size_t offset, size_t bytes,
 961                          struct iov_iter *i)
 962 {
 963         if (i->type & (ITER_BVEC | ITER_KVEC | ITER_XARRAY)) {
 964                 void *kaddr = kmap_atomic(page);
 965                 size_t wanted = copy_to_iter(kaddr + offset, bytes, i);
 966                 kunmap_atomic(kaddr);
 967                 return wanted;
 968         } else if (unlikely(iov_iter_is_discard(i))) {
 969                 if (unlikely(i->count < bytes))
 970                         bytes = i->count;
 971                 i->count -= bytes;
 972                 return bytes;
 973         } else if (likely(!iov_iter_is_pipe(i)))
 974                 return copy_page_to_iter_iovec(page, offset, bytes, i);
 975         else
 976                 return copy_page_to_iter_pipe(page, offset, bytes, i);
 977 }
 978
 979 size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes,
 980                          struct iov_iter *i)
 981 {
 982         size_t res = 0;
 983         if (unlikely(!page_copy_sane(page, offset, bytes)))
 984                 return 0;
 985         page += offset / PAGE_SIZE; // first subpage
 986         offset %= PAGE_SIZE;
 987         while (1) {
 988                 size_t n = __copy_page_to_iter(page, offset,
 989                                 min(bytes, (size_t)PAGE_SIZE - offset), i);
 990                 res += n;
 991                 bytes -= n;
 992                 if (!bytes || !n)
 993                         break;
 994                 offset += n;
 995                 if (offset == PAGE_SIZE) {
 996                         page++;
 997                         offset = 0;
 998                 }
 999         }
1000         return res;
1001 }
1002 EXPORT_SYMBOL(copy_page_to_iter);
1003
1004 size_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes,
1005                          struct iov_iter *i)
1006 {
1007         if (unlikely(!page_copy_sane(page, offset, bytes)))
1008                 return 0;
1009         if (unlikely(iov_iter_is_pipe(i) || iov_iter_is_discard(i))) {
1010                 WARN_ON(1);
1011                 return 0;
1012         }
1013         if (i->type & (ITER_BVEC | ITER_KVEC | ITER_XARRAY)) {
1014                 void *kaddr = kmap_atomic(page);
1015                 size_t wanted = _copy_from_iter(kaddr + offset, bytes, i);
1016                 kunmap_atomic(kaddr);
1017                 return wanted;
1018         } else
1019                 return copy_page_from_iter_iovec(page, offset, bytes, i);
1020 }
1021 EXPORT_SYMBOL(copy_page_from_iter);
1022
1023 static size_t pipe_zero(size_t bytes, struct iov_iter *i)
1024 {
1025         struct pipe_inode_info *pipe = i->pipe;
1026         unsigned int p_mask = pipe->ring_size - 1;
1027         unsigned int i_head;
1028         size_t n, off;
1029
1030         if (!sanity(i))
1031                 return 0;
1032
1033         bytes = n = push_pipe(i, bytes, &i_head, &off);
1034         if (unlikely(!n))
1035                 return 0;
1036
1037         do {
1038                 size_t chunk = min_t(size_t, n, PAGE_SIZE - off);
1039                 memzero_page(pipe->bufs[i_head & p_mask].page, off, chunk);
1040                 i->head = i_head;
1041                 i->iov_offset = off + chunk;
1042                 n -= chunk;
1043                 off = 0;
1044                 i_head++;
1045         } while (n);
1046         i->count -= bytes;
1047         return bytes;
1048 }
1049
1050 size_t iov_iter_zero(size_t bytes, struct iov_iter *i)
1051 {
1052         if (unlikely(iov_iter_is_pipe(i)))
1053                 return pipe_zero(bytes, i);
1054         iterate_and_advance(i, bytes, v,
1055                 clear_user(v.iov_base, v.iov_len),
1056                 memzero_page(v.bv_page, v.bv_offset, v.bv_len),
1057                 memset(v.iov_base, 0, v.iov_len),
1058                 memzero_page(v.bv_page, v.bv_offset, v.bv_len)
1059         )
1060
1061         return bytes;
1062 }
1063 EXPORT_SYMBOL(iov_iter_zero);
1064
1065 size_t iov_iter_copy_from_user_atomic(struct page *page,
1066                 struct iov_iter *i, unsigned long offset, size_t bytes)
1067 {
1068         char *kaddr = kmap_atomic(page), *p = kaddr + offset;
1069         if (unlikely(!page_copy_sane(page, offset, bytes))) {
1070                 kunmap_atomic(kaddr);
1071                 return 0;
1072         }
1073         if (unlikely(iov_iter_is_pipe(i) || iov_iter_is_discard(i))) {
1074                 kunmap_atomic(kaddr);
1075                 WARN_ON(1);
1076                 return 0;
1077         }
1078         iterate_all_kinds(i, bytes, v,
1079                 copyin((p += v.iov_len) - v.iov_len, v.iov_base, v.iov_len),
1080                 memcpy_from_page((p += v.bv_len) - v.bv_len, v.bv_page,
1081                                  v.bv_offset, v.bv_len),
1082                 memcpy((p += v.iov_len) - v.iov_len, v.iov_base, v.iov_len),
1083                 memcpy_from_page((p += v.bv_len) - v.bv_len, v.bv_page,
1084                                  v.bv_offset, v.bv_len)
1085         )
1086         kunmap_atomic(kaddr);
1087         return bytes;
1088 }
1089 EXPORT_SYMBOL(iov_iter_copy_from_user_atomic);
1090
1091 static inline void pipe_truncate(struct iov_iter *i)
1092 {
1093         struct pipe_inode_info *pipe = i->pipe;
1094         unsigned int p_tail = pipe->tail;
1095         unsigned int p_head = pipe->head;
1096         unsigned int p_mask = pipe->ring_size - 1;
1097
1098         if (!pipe_empty(p_head, p_tail)) {
1099                 struct pipe_buffer *buf;
1100                 unsigned int i_head = i->head;
1101                 size_t off = i->iov_offset;
1102
1103                 if (off) {
1104                         buf = &pipe->bufs[i_head & p_mask];
1105                         buf->len = off - buf->offset;
1106                         i_head++;
1107                 }
1108                 while (p_head != i_head) {
1109                         p_head--;
1110                         pipe_buf_release(pipe, &pipe->bufs[p_head & p_mask]);
1111                 }
1112
1113                 pipe->head = p_head;
1114         }
1115 }
1116
1117 static void pipe_advance(struct iov_iter *i, size_t size)
1118 {
1119         struct pipe_inode_info *pipe = i->pipe;
1120         if (size) {
1121                 struct pipe_buffer *buf;
1122                 unsigned int p_mask = pipe->ring_size - 1;
1123                 unsigned int i_head = i->head;
1124                 size_t off = i->iov_offset, left = size;
1125
1126                 if (off) /* make it relative to the beginning of buffer */
1127                         left += off - pipe->bufs[i_head & p_mask].offset;
1128                 while (1) {
1129                         buf = &pipe->bufs[i_head & p_mask];
1130                         if (left <= buf->len)
1131                                 break;
1132                         left -= buf->len;
1133                         i_head++;
1134                 }
1135                 i->head = i_head;
1136                 i->iov_offset = buf->offset + left;
1137         }
1138         i->count -= size;
1139         /* ... and discard everything past that point */
1140         pipe_truncate(i);
1141 }
1142
1143 static void iov_iter_bvec_advance(struct iov_iter *i, size_t size)
1144 {
1145         struct bvec_iter bi;
1146
1147         bi.bi_size = i->count;
1148         bi.bi_bvec_done = i->iov_offset;
1149         bi.bi_idx = 0;
1150         bvec_iter_advance(i->bvec, &bi, size);
1151
1152         i->bvec += bi.bi_idx;
1153         i->nr_segs -= bi.bi_idx;
1154         i->count = bi.bi_size;
1155         i->iov_offset = bi.bi_bvec_done;
1156 }
1157
1158 void iov_iter_advance(struct iov_iter *i, size_t size)
1159 {
1160         if (unlikely(i->count < size))
1161                 size = i->count;
1162         if (unlikely(iov_iter_is_pipe(i))) {
1163                 pipe_advance(i, size);
1164                 return;
1165         }
1166         if (unlikely(iov_iter_is_discard(i))) {
1167                 i->count -= size;
1168                 return;
1169         }
1170         if (unlikely(iov_iter_is_xarray(i))) {
1171                 i->iov_offset += size;
1172                 i->count -= size;
1173                 return;
1174         }
1175         if (iov_iter_is_bvec(i)) {
1176                 iov_iter_bvec_advance(i, size);
1177                 return;
1178         }
1179         iterate_and_advance(i, size, v, 0, 0, 0, 0)
1180 }
1181 EXPORT_SYMBOL(iov_iter_advance);
1182
1183 void iov_iter_revert(struct iov_iter *i, size_t unroll)
1184 {
1185         if (!unroll)
1186                 return;
1187         if (WARN_ON(unroll > MAX_RW_COUNT))
1188                 return;
1189         i->count += unroll;
1190         if (unlikely(iov_iter_is_pipe(i))) {
1191                 struct pipe_inode_info *pipe = i->pipe;
1192                 unsigned int p_mask = pipe->ring_size - 1;
1193                 unsigned int i_head = i->head;
1194                 size_t off = i->iov_offset;
1195                 while (1) {
1196                         struct pipe_buffer *b = &pipe->bufs[i_head & p_mask];
1197                         size_t n = off - b->offset;
1198                         if (unroll < n) {
1199                                 off -= unroll;
1200                                 break;
1201                         }
1202                         unroll -= n;
1203                         if (!unroll && i_head == i->start_head) {
1204                                 off = 0;
1205                                 break;
1206                         }
1207                         i_head--;
1208                         b = &pipe->bufs[i_head & p_mask];
1209                         off = b->offset + b->len;
1210                 }
1211                 i->iov_offset = off;
1212                 i->head = i_head;
1213                 pipe_truncate(i);
1214                 return;
1215         }
1216         if (unlikely(iov_iter_is_discard(i)))
1217                 return;
1218         if (unroll <= i->iov_offset) {
1219                 i->iov_offset -= unroll;
1220                 return;
1221         }
1222         unroll -= i->iov_offset;
1223         if (iov_iter_is_xarray(i)) {
1224                 BUG(); /* We should never go beyond the start of the specified
1225                         * range since we might then be straying into pages that
1226                         * aren't pinned.
1227                         */
1228         } else if (iov_iter_is_bvec(i)) {
1229                 const struct bio_vec *bvec = i->bvec;
1230                 while (1) {
1231                         size_t n = (--bvec)->bv_len;
1232                         i->nr_segs++;
1233                         if (unroll <= n) {
1234                                 i->bvec = bvec;
1235                                 i->iov_offset = n - unroll;
1236                                 return;
1237                         }
1238                         unroll -= n;
1239                 }
1240         } else { /* same logics for iovec and kvec */
1241                 const struct iovec *iov = i->iov;
1242                 while (1) {
1243                         size_t n = (--iov)->iov_len;
1244                         i->nr_segs++;
1245                         if (unroll <= n) {
1246                                 i->iov = iov;
1247                                 i->iov_offset = n - unroll;
1248                                 return;
1249                         }
1250                         unroll -= n;
1251                 }
1252         }
1253 }
1254 EXPORT_SYMBOL(iov_iter_revert);
1255
1256 /*
1257  * Return the count of just the current iov_iter segment.
1258  */
1259 size_t iov_iter_single_seg_count(const struct iov_iter *i)
1260 {
1261         if (unlikely(iov_iter_is_pipe(i)))
1262                 return i->count;        // it is a silly place, anyway
1263         if (i->nr_segs == 1)
1264                 return i->count;
1265         if (unlikely(iov_iter_is_discard(i) || iov_iter_is_xarray(i)))
1266                 return i->count;
1267         if (iov_iter_is_bvec(i))
1268                 return min(i->count, i->bvec->bv_len - i->iov_offset);
1269         else
1270                 return min(i->count, i->iov->iov_len - i->iov_offset);
1271 }
1272 EXPORT_SYMBOL(iov_iter_single_seg_count);
1273
1274 void iov_iter_kvec(struct iov_iter *i, unsigned int direction,
1275                         const struct kvec *kvec, unsigned long nr_segs,
1276                         size_t count)
1277 {
1278         WARN_ON(direction & ~(READ | WRITE));
1279         i->type = ITER_KVEC | (direction & (READ | WRITE));
1280         i->kvec = kvec;
1281         i->nr_segs = nr_segs;
1282         i->iov_offset = 0;
1283         i->count = count;
1284 }
1285 EXPORT_SYMBOL(iov_iter_kvec);
1286
1287 void iov_iter_bvec(struct iov_iter *i, unsigned int direction,
1288                         const struct bio_vec *bvec, unsigned long nr_segs,
1289                         size_t count)
1290 {
1291         WARN_ON(direction & ~(READ | WRITE));
1292         i->type = ITER_BVEC | (direction & (READ | WRITE));
1293         i->bvec = bvec;
1294         i->nr_segs = nr_segs;
1295         i->iov_offset = 0;
1296         i->count = count;
1297 }
1298 EXPORT_SYMBOL(iov_iter_bvec);
1299
1300 void iov_iter_pipe(struct iov_iter *i, unsigned int direction,
1301                         struct pipe_inode_info *pipe,
1302                         size_t count)
1303 {
1304         BUG_ON(direction != READ);
1305         WARN_ON(pipe_full(pipe->head, pipe->tail, pipe->ring_size));
1306         i->type = ITER_PIPE | READ;
1307         i->pipe = pipe;
1308         i->head = pipe->head;
1309         i->iov_offset = 0;
1310         i->count = count;
1311         i->start_head = i->head;
1312 }
1313 EXPORT_SYMBOL(iov_iter_pipe);
1314
1315 /**
1316  * iov_iter_xarray - Initialise an I/O iterator to use the pages in an xarray
1317  * @i: The iterator to initialise.
1318  * @direction: The direction of the transfer.
1319  * @xarray: The xarray to access.
1320  * @start: The start file position.
1321  * @count: The size of the I/O buffer in bytes.
1322  *
1323  * Set up an I/O iterator to either draw data out of the pages attached to an
1324  * inode or to inject data into those pages.  The pages *must* be prevented
1325  * from evaporation, either by taking a ref on them or locking them by the
1326  * caller.
1327  */
1328 void iov_iter_xarray(struct iov_iter *i, unsigned int direction,
1329                      struct xarray *xarray, loff_t start, size_t count)
1330 {
1331         BUG_ON(direction & ~1);
1332         i->type = ITER_XARRAY | (direction & (READ | WRITE));
1333         i->xarray = xarray;
1334         i->xarray_start = start;
1335         i->count = count;
1336         i->iov_offset = 0;
1337 }
1338 EXPORT_SYMBOL(iov_iter_xarray);
1339
1340 /**
1341  * iov_iter_discard - Initialise an I/O iterator that discards data
1342  * @i: The iterator to initialise.
1343  * @direction: The direction of the transfer.
1344  * @count: The size of the I/O buffer in bytes.
1345  *
1346  * Set up an I/O iterator that just discards everything that's written to it.
1347  * It's only available as a READ iterator.
1348  */
1349 void iov_iter_discard(struct iov_iter *i, unsigned int direction, size_t count)
1350 {
1351         BUG_ON(direction != READ);
1352         i->type = ITER_DISCARD | READ;
1353         i->count = count;
1354         i->iov_offset = 0;
1355 }
1356 EXPORT_SYMBOL(iov_iter_discard);
1357
1358 unsigned long iov_iter_alignment(const struct iov_iter *i)
1359 {
1360         unsigned long res = 0;
1361         size_t size = i->count;
1362
1363         if (unlikely(iov_iter_is_pipe(i))) {
1364                 unsigned int p_mask = i->pipe->ring_size - 1;
1365
1366                 if (size && i->iov_offset && allocated(&i->pipe->bufs[i->head & p_mask]))
1367                         return size | i->iov_offset;
1368                 return size;
1369         }
1370         if (unlikely(iov_iter_is_xarray(i)))
1371                 return (i->xarray_start + i->iov_offset) | i->count;
1372         iterate_all_kinds(i, size, v,
1373                 (res |= (unsigned long)v.iov_base | v.iov_len, 0),
1374                 res |= v.bv_offset | v.bv_len,
1375                 res |= (unsigned long)v.iov_base | v.iov_len,
1376                 res |= v.bv_offset | v.bv_len
1377         )
1378         return res;
1379 }
1380 EXPORT_SYMBOL(iov_iter_alignment);
1381
1382 unsigned long iov_iter_gap_alignment(const struct iov_iter *i)
1383 {
1384         unsigned long res = 0;
1385         size_t size = i->count;
1386
1387         if (unlikely(iov_iter_is_pipe(i) || iov_iter_is_discard(i))) {
1388                 WARN_ON(1);
1389                 return ~0U;
1390         }
1391
1392         iterate_all_kinds(i, size, v,
1393                 (res |= (!res ? 0 : (unsigned long)v.iov_base) |
1394                         (size != v.iov_len ? size : 0), 0),
1395                 (res |= (!res ? 0 : (unsigned long)v.bv_offset) |
1396                         (size != v.bv_len ? size : 0)),
1397                 (res |= (!res ? 0 : (unsigned long)v.iov_base) |
1398                         (size != v.iov_len ? size : 0)),
1399                 (res |= (!res ? 0 : (unsigned long)v.bv_offset) |
1400                         (size != v.bv_len ? size : 0))
1401                 );
1402         return res;
1403 }
1404 EXPORT_SYMBOL(iov_iter_gap_alignment);
1405
1406 static inline ssize_t __pipe_get_pages(struct iov_iter *i,
1407                                 size_t maxsize,
1408                                 struct page **pages,
1409                                 int iter_head,
1410                                 size_t *start)
1411 {
1412         struct pipe_inode_info *pipe = i->pipe;
1413         unsigned int p_mask = pipe->ring_size - 1;
1414         ssize_t n = push_pipe(i, maxsize, &iter_head, start);
1415         if (!n)
1416                 return -EFAULT;
1417
1418         maxsize = n;
1419         n += *start;
1420         while (n > 0) {
1421                 get_page(*pages++ = pipe->bufs[iter_head & p_mask].page);
1422                 iter_head++;
1423                 n -= PAGE_SIZE;
1424         }
1425
1426         return maxsize;
1427 }
1428
1429 static ssize_t pipe_get_pages(struct iov_iter *i,
1430                    struct page **pages, size_t maxsize, unsigned maxpages,
1431                    size_t *start)
1432 {
1433         unsigned int iter_head, npages;
1434         size_t capacity;
1435
1436         if (!maxsize)
1437                 return 0;
1438
1439         if (!sanity(i))
1440                 return -EFAULT;
1441
1442         data_start(i, &iter_head, start);
1443         /* Amount of free space: some of this one + all after this one */
1444         npages = pipe_space_for_user(iter_head, i->pipe->tail, i->pipe);
1445         capacity = min(npages, maxpages) * PAGE_SIZE - *start;
1446
1447         return __pipe_get_pages(i, min(maxsize, capacity), pages, iter_head, start);
1448 }
1449
1450 static ssize_t iter_xarray_populate_pages(struct page **pages, struct xarray *xa,
1451                                           pgoff_t index, unsigned int nr_pages)
1452 {
1453         XA_STATE(xas, xa, index);
1454         struct page *page;
1455         unsigned int ret = 0;
1456
1457         rcu_read_lock();
1458         for (page = xas_load(&xas); page; page = xas_next(&xas)) {
1459                 if (xas_retry(&xas, page))
1460                         continue;
1461
1462                 /* Has the page moved or been split? */
1463                 if (unlikely(page != xas_reload(&xas))) {
1464                         xas_reset(&xas);
1465                         continue;
1466                 }
1467
1468                 pages[ret] = find_subpage(page, xas.xa_index);
1469                 get_page(pages[ret]);
1470                 if (++ret == nr_pages)
1471                         break;
1472         }
1473         rcu_read_unlock();
1474         return ret;
1475 }
1476
1477 static ssize_t iter_xarray_get_pages(struct iov_iter *i,
1478                                      struct page **pages, size_t maxsize,
1479                                      unsigned maxpages, size_t *_start_offset)
1480 {
1481         unsigned nr, offset;
1482         pgoff_t index, count;
1483         size_t size = maxsize, actual;
1484         loff_t pos;
1485
1486         if (!size || !maxpages)
1487                 return 0;
1488
1489         pos = i->xarray_start + i->iov_offset;
1490         index = pos >> PAGE_SHIFT;
1491         offset = pos & ~PAGE_MASK;
1492         *_start_offset = offset;
1493
1494         count = 1;
1495         if (size > PAGE_SIZE - offset) {
1496                 size -= PAGE_SIZE - offset;
1497                 count += size >> PAGE_SHIFT;
1498                 size &= ~PAGE_MASK;
1499                 if (size)
1500                         count++;
1501         }
1502
1503         if (count > maxpages)
1504                 count = maxpages;
1505
1506         nr = iter_xarray_populate_pages(pages, i->xarray, index, count);
1507         if (nr == 0)
1508                 return 0;
1509
1510         actual = PAGE_SIZE * nr;
1511         actual -= offset;
1512         if (nr == count && size > 0) {
1513                 unsigned last_offset = (nr > 1) ? 0 : offset;
1514                 actual -= PAGE_SIZE - (last_offset + size);
1515         }
1516         return actual;
1517 }
1518
1519 ssize_t iov_iter_get_pages(struct iov_iter *i,
1520                    struct page **pages, size_t maxsize, unsigned maxpages,
1521                    size_t *start)
1522 {
1523         if (maxsize > i->count)
1524                 maxsize = i->count;
1525
1526         if (unlikely(iov_iter_is_pipe(i)))
1527                 return pipe_get_pages(i, pages, maxsize, maxpages, start);
1528         if (unlikely(iov_iter_is_xarray(i)))
1529                 return iter_xarray_get_pages(i, pages, maxsize, maxpages, start);
1530         if (unlikely(iov_iter_is_discard(i)))
1531                 return -EFAULT;
1532
1533         iterate_all_kinds(i, maxsize, v, ({
1534                 unsigned long addr = (unsigned long)v.iov_base;
1535                 size_t len = v.iov_len + (*start = addr & (PAGE_SIZE - 1));
1536                 int n;
1537                 int res;
1538
1539                 if (len > maxpages * PAGE_SIZE)
1540                         len = maxpages * PAGE_SIZE;
1541                 addr &= ~(PAGE_SIZE - 1);
1542                 n = DIV_ROUND_UP(len, PAGE_SIZE);
1543                 res = get_user_pages_fast(addr, n,
1544                                 iov_iter_rw(i) != WRITE ?  FOLL_WRITE : 0,
1545                                 pages);
1546                 if (unlikely(res < 0))
1547                         return res;
1548                 return (res == n ? len : res * PAGE_SIZE) - *start;
1549         0;}),({
1550                 /* can't be more than PAGE_SIZE */
1551                 *start = v.bv_offset;
1552                 get_page(*pages = v.bv_page);
1553                 return v.bv_len;
1554         }),({
1555                 return -EFAULT;
1556         }),
1557         0
1558         )
1559         return 0;
1560 }
1561 EXPORT_SYMBOL(iov_iter_get_pages);
1562
1563 static struct page **get_pages_array(size_t n)
1564 {
1565         return kvmalloc_array(n, sizeof(struct page *), GFP_KERNEL);
1566 }
1567
1568 static ssize_t pipe_get_pages_alloc(struct iov_iter *i,
1569                    struct page ***pages, size_t maxsize,
1570                    size_t *start)
1571 {
1572         struct page **p;
1573         unsigned int iter_head, npages;
1574         ssize_t n;
1575
1576         if (!maxsize)
1577                 return 0;
1578
1579         if (!sanity(i))
1580                 return -EFAULT;
1581
1582         data_start(i, &iter_head, start);
1583         /* Amount of free space: some of this one + all after this one */
1584         npages = pipe_space_for_user(iter_head, i->pipe->tail, i->pipe);
1585         n = npages * PAGE_SIZE - *start;
1586         if (maxsize > n)
1587                 maxsize = n;
1588         else
1589                 npages = DIV_ROUND_UP(maxsize + *start, PAGE_SIZE);
1590         p = get_pages_array(npages);
1591         if (!p)
1592                 return -ENOMEM;
1593         n = __pipe_get_pages(i, maxsize, p, iter_head, start);
1594         if (n > 0)
1595                 *pages = p;
1596         else
1597                 kvfree(p);
1598         return n;
1599 }
1600
1601 static ssize_t iter_xarray_get_pages_alloc(struct iov_iter *i,
1602                                            struct page ***pages, size_t maxsize,
1603                                            size_t *_start_offset)
1604 {
1605         struct page **p;
1606         unsigned nr, offset;
1607         pgoff_t index, count;
1608         size_t size = maxsize, actual;
1609         loff_t pos;
1610
1611         if (!size)
1612                 return 0;
1613
1614         pos = i->xarray_start + i->iov_offset;
1615         index = pos >> PAGE_SHIFT;
1616         offset = pos & ~PAGE_MASK;
1617         *_start_offset = offset;
1618
1619         count = 1;
1620         if (size > PAGE_SIZE - offset) {
1621                 size -= PAGE_SIZE - offset;
1622                 count += size >> PAGE_SHIFT;
1623                 size &= ~PAGE_MASK;
1624                 if (size)
1625                         count++;
1626         }
1627
1628         p = get_pages_array(count);
1629         if (!p)
1630                 return -ENOMEM;
1631         *pages = p;
1632
1633         nr = iter_xarray_populate_pages(p, i->xarray, index, count);
1634         if (nr == 0)
1635                 return 0;
1636
1637         actual = PAGE_SIZE * nr;
1638         actual -= offset;
1639         if (nr == count && size > 0) {
1640                 unsigned last_offset = (nr > 1) ? 0 : offset;
1641                 actual -= PAGE_SIZE - (last_offset + size);
1642         }
1643         return actual;
1644 }
1645
1646 ssize_t iov_iter_get_pages_alloc(struct iov_iter *i,
1647                    struct page ***pages, size_t maxsize,
1648                    size_t *start)
1649 {
1650         struct page **p;
1651
1652         if (maxsize > i->count)
1653                 maxsize = i->count;
1654
1655         if (unlikely(iov_iter_is_pipe(i)))
1656                 return pipe_get_pages_alloc(i, pages, maxsize, start);
1657         if (unlikely(iov_iter_is_xarray(i)))
1658                 return iter_xarray_get_pages_alloc(i, pages, maxsize, start);
1659         if (unlikely(iov_iter_is_discard(i)))
1660                 return -EFAULT;
1661
1662         iterate_all_kinds(i, maxsize, v, ({
1663                 unsigned long addr = (unsigned long)v.iov_base;
1664                 size_t len = v.iov_len + (*start = addr & (PAGE_SIZE - 1));
1665                 int n;
1666                 int res;
1667
1668                 addr &= ~(PAGE_SIZE - 1);
1669                 n = DIV_ROUND_UP(len, PAGE_SIZE);
1670                 p = get_pages_array(n);
1671                 if (!p)
1672                         return -ENOMEM;
1673                 res = get_user_pages_fast(addr, n,
1674                                 iov_iter_rw(i) != WRITE ?  FOLL_WRITE : 0, p);
1675                 if (unlikely(res < 0)) {
1676                         kvfree(p);
1677                         return res;
1678                 }
1679                 *pages = p;
1680                 return (res == n ? len : res * PAGE_SIZE) - *start;
1681         0;}),({
1682                 /* can't be more than PAGE_SIZE */
1683                 *start = v.bv_offset;
1684                 *pages = p = get_pages_array(1);
1685                 if (!p)
1686                         return -ENOMEM;
1687                 get_page(*p = v.bv_page);
1688                 return v.bv_len;
1689         }),({
1690                 return -EFAULT;
1691         }), 0
1692         )
1693         return 0;
1694 }
1695 EXPORT_SYMBOL(iov_iter_get_pages_alloc);
1696
1697 size_t csum_and_copy_from_iter(void *addr, size_t bytes, __wsum *csum,
1698                                struct iov_iter *i)
1699 {
1700         char *to = addr;
1701         __wsum sum, next;
1702         size_t off = 0;
1703         sum = *csum;
1704         if (unlikely(iov_iter_is_pipe(i) || iov_iter_is_discard(i))) {
1705                 WARN_ON(1);
1706                 return 0;
1707         }
1708         iterate_and_advance(i, bytes, v, ({
1709                 next = csum_and_copy_from_user(v.iov_base,
1710                                                (to += v.iov_len) - v.iov_len,
1711                                                v.iov_len);
1712                 if (next) {
1713                         sum = csum_block_add(sum, next, off);
1714                         off += v.iov_len;
1715                 }
1716                 next ? 0 : v.iov_len;
1717         }), ({
1718                 char *p = kmap_atomic(v.bv_page);
1719                 sum = csum_and_memcpy((to += v.bv_len) - v.bv_len,
1720                                       p + v.bv_offset, v.bv_len,
1721                                       sum, off);
1722                 kunmap_atomic(p);
1723                 off += v.bv_len;
1724         }),({
1725                 sum = csum_and_memcpy((to += v.iov_len) - v.iov_len,
1726                                       v.iov_base, v.iov_len,
1727                                       sum, off);
1728                 off += v.iov_len;
1729         }), ({
1730                 char *p = kmap_atomic(v.bv_page);
1731                 sum = csum_and_memcpy((to += v.bv_len) - v.bv_len,
1732                                       p + v.bv_offset, v.bv_len,
1733                                       sum, off);
1734                 kunmap_atomic(p);
1735                 off += v.bv_len;
1736         })
1737         )
1738         *csum = sum;
1739         return bytes;
1740 }
1741 EXPORT_SYMBOL(csum_and_copy_from_iter);
1742
1743 bool csum_and_copy_from_iter_full(void *addr, size_t bytes, __wsum *csum,
1744                                struct iov_iter *i)
1745 {
1746         char *to = addr;
1747         __wsum sum, next;
1748         size_t off = 0;
1749         sum = *csum;
1750         if (unlikely(iov_iter_is_pipe(i) || iov_iter_is_discard(i))) {
1751                 WARN_ON(1);
1752                 return false;
1753         }
1754         if (unlikely(i->count < bytes))
1755                 return false;
1756         iterate_all_kinds(i, bytes, v, ({
1757                 next = csum_and_copy_from_user(v.iov_base,
1758                                                (to += v.iov_len) - v.iov_len,
1759                                                v.iov_len);
1760                 if (!next)
1761                         return false;
1762                 sum = csum_block_add(sum, next, off);
1763                 off += v.iov_len;
1764                 0;
1765         }), ({
1766                 char *p = kmap_atomic(v.bv_page);
1767                 sum = csum_and_memcpy((to += v.bv_len) - v.bv_len,
1768                                       p + v.bv_offset, v.bv_len,
1769                                       sum, off);
1770                 kunmap_atomic(p);
1771                 off += v.bv_len;
1772         }),({
1773                 sum = csum_and_memcpy((to += v.iov_len) - v.iov_len,
1774                                       v.iov_base, v.iov_len,
1775                                       sum, off);
1776                 off += v.iov_len;
1777         }), ({
1778                 char *p = kmap_atomic(v.bv_page);
1779                 sum = csum_and_memcpy((to += v.bv_len) - v.bv_len,
1780                                       p + v.bv_offset, v.bv_len,
1781                                       sum, off);
1782                 kunmap_atomic(p);
1783                 off += v.bv_len;
1784         })
1785         )
1786         *csum = sum;
1787         iov_iter_advance(i, bytes);
1788         return true;
1789 }
1790 EXPORT_SYMBOL(csum_and_copy_from_iter_full);
1791
1792 size_t csum_and_copy_to_iter(const void *addr, size_t bytes, void *_csstate,
1793                              struct iov_iter *i)
1794 {
1795         struct csum_state *csstate = _csstate;
1796         const char *from = addr;
1797         __wsum sum, next;
1798         size_t off;
1799
1800         if (unlikely(iov_iter_is_pipe(i)))
1801                 return csum_and_copy_to_pipe_iter(addr, bytes, _csstate, i);
1802
1803         sum = csstate->csum;
1804         off = csstate->off;
1805         if (unlikely(iov_iter_is_discard(i))) {
1806                 WARN_ON(1);     /* for now */
1807                 return 0;
1808         }
1809         iterate_and_advance(i, bytes, v, ({
1810                 next = csum_and_copy_to_user((from += v.iov_len) - v.iov_len,
1811                                              v.iov_base,
1812                                              v.iov_len);
1813                 if (next) {
1814                         sum = csum_block_add(sum, next, off);
1815                         off += v.iov_len;
1816                 }
1817                 next ? 0 : v.iov_len;
1818         }), ({
1819                 char *p = kmap_atomic(v.bv_page);
1820                 sum = csum_and_memcpy(p + v.bv_offset,
1821                                       (from += v.bv_len) - v.bv_len,
1822                                       v.bv_len, sum, off);
1823                 kunmap_atomic(p);
1824                 off += v.bv_len;
1825         }),({
1826                 sum = csum_and_memcpy(v.iov_base,
1827                                      (from += v.iov_len) - v.iov_len,
1828                                      v.iov_len, sum, off);
1829                 off += v.iov_len;
1830         }), ({
1831                 char *p = kmap_atomic(v.bv_page);
1832                 sum = csum_and_memcpy(p + v.bv_offset,
1833                                       (from += v.bv_len) - v.bv_len,
1834                                       v.bv_len, sum, off);
1835                 kunmap_atomic(p);
1836                 off += v.bv_len;
1837         })
1838         )
1839         csstate->csum = sum;
1840         csstate->off = off;
1841         return bytes;
1842 }
1843 EXPORT_SYMBOL(csum_and_copy_to_iter);
1844
1845 size_t hash_and_copy_to_iter(const void *addr, size_t bytes, void *hashp,
1846                 struct iov_iter *i)
1847 {
1848 #ifdef CONFIG_CRYPTO_HASH
1849         struct ahash_request *hash = hashp;
1850         struct scatterlist sg;
1851         size_t copied;
1852
1853         copied = copy_to_iter(addr, bytes, i);
1854         sg_init_one(&sg, addr, copied);
1855         ahash_request_set_crypt(hash, &sg, NULL, copied);
1856         crypto_ahash_update(hash);
1857         return copied;
1858 #else
1859         return 0;
1860 #endif
1861 }
1862 EXPORT_SYMBOL(hash_and_copy_to_iter);
1863
1864 int iov_iter_npages(const struct iov_iter *i, int maxpages)
1865 {
1866         size_t size = i->count;
1867         int npages = 0;
1868
1869         if (!size)
1870                 return 0;
1871         if (unlikely(iov_iter_is_discard(i)))
1872                 return 0;
1873
1874         if (unlikely(iov_iter_is_pipe(i))) {
1875                 struct pipe_inode_info *pipe = i->pipe;
1876                 unsigned int iter_head;
1877                 size_t off;
1878
1879                 if (!sanity(i))
1880                         return 0;
1881
1882                 data_start(i, &iter_head, &off);
1883                 /* some of this one + all after this one */
1884                 npages = pipe_space_for_user(iter_head, pipe->tail, pipe);
1885                 if (npages >= maxpages)
1886                         return maxpages;
1887         } else if (unlikely(iov_iter_is_xarray(i))) {
1888                 unsigned offset;
1889
1890                 offset = (i->xarray_start + i->iov_offset) & ~PAGE_MASK;
1891
1892                 npages = 1;
1893                 if (size > PAGE_SIZE - offset) {
1894                         size -= PAGE_SIZE - offset;
1895                         npages += size >> PAGE_SHIFT;
1896                         size &= ~PAGE_MASK;
1897                         if (size)
1898                                 npages++;
1899                 }
1900                 if (npages >= maxpages)
1901                         return maxpages;
1902         } else iterate_all_kinds(i, size, v, ({
1903                 unsigned long p = (unsigned long)v.iov_base;
1904                 npages += DIV_ROUND_UP(p + v.iov_len, PAGE_SIZE)
1905                         - p / PAGE_SIZE;
1906                 if (npages >= maxpages)
1907                         return maxpages;
1908         0;}),({
1909                 npages++;
1910                 if (npages >= maxpages)
1911                         return maxpages;
1912         }),({
1913                 unsigned long p = (unsigned long)v.iov_base;
1914                 npages += DIV_ROUND_UP(p + v.iov_len, PAGE_SIZE)
1915                         - p / PAGE_SIZE;
1916                 if (npages >= maxpages)
1917                         return maxpages;
1918         }),
1919         0
1920         )
1921         return npages;
1922 }
1923 EXPORT_SYMBOL(iov_iter_npages);
1924
1925 const void *dup_iter(struct iov_iter *new, struct iov_iter *old, gfp_t flags)
1926 {
1927         *new = *old;
1928         if (unlikely(iov_iter_is_pipe(new))) {
1929                 WARN_ON(1);
1930                 return NULL;
1931         }
1932         if (unlikely(iov_iter_is_discard(new) || iov_iter_is_xarray(new)))
1933                 return NULL;
1934         if (iov_iter_is_bvec(new))
1935                 return new->bvec = kmemdup(new->bvec,
1936                                     new->nr_segs * sizeof(struct bio_vec),
1937                                     flags);
1938         else
1939                 /* iovec and kvec have identical layout */
1940                 return new->iov = kmemdup(new->iov,
1941                                    new->nr_segs * sizeof(struct iovec),
1942                                    flags);
1943 }
1944 EXPORT_SYMBOL(dup_iter);
1945
1946 static int copy_compat_iovec_from_user(struct iovec *iov,
1947                 const struct iovec __user *uvec, unsigned long nr_segs)
1948 {
1949         const struct compat_iovec __user *uiov =
1950                 (const struct compat_iovec __user *)uvec;
1951         int ret = -EFAULT, i;
1952
1953         if (!user_access_begin(uiov, nr_segs * sizeof(*uiov)))
1954                 return -EFAULT;
1955
1956         for (i = 0; i < nr_segs; i++) {
1957                 compat_uptr_t buf;
1958                 compat_ssize_t len;
1959
1960                 unsafe_get_user(len, &uiov[i].iov_len, uaccess_end);
1961                 unsafe_get_user(buf, &uiov[i].iov_base, uaccess_end);
1962
1963                 /* check for compat_size_t not fitting in compat_ssize_t .. */
1964                 if (len < 0) {
1965                         ret = -EINVAL;
1966                         goto uaccess_end;
1967                 }
1968                 iov[i].iov_base = compat_ptr(buf);
1969                 iov[i].iov_len = len;
1970         }
1971
1972         ret = 0;
1973 uaccess_end:
1974         user_access_end();
1975         return ret;
1976 }
1977
1978 static int copy_iovec_from_user(struct iovec *iov,
1979                 const struct iovec __user *uvec, unsigned long nr_segs)
1980 {
1981         unsigned long seg;
1982
1983         if (copy_from_user(iov, uvec, nr_segs * sizeof(*uvec)))
1984                 return -EFAULT;
1985         for (seg = 0; seg < nr_segs; seg++) {
1986                 if ((ssize_t)iov[seg].iov_len < 0)
1987                         return -EINVAL;
1988         }
1989
1990         return 0;
1991 }
1992
1993 struct iovec *iovec_from_user(const struct iovec __user *uvec,
1994                 unsigned long nr_segs, unsigned long fast_segs,
1995                 struct iovec *fast_iov, bool compat)
1996 {
1997         struct iovec *iov = fast_iov;
1998         int ret;
1999
2000         /*
2001          * SuS says "The readv() function *may* fail if the iovcnt argument was
2002          * less than or equal to 0, or greater than {IOV_MAX}.  Linux has
2003          * traditionally returned zero for zero segments, so...
2004          */
2005         if (nr_segs == 0)
2006                 return iov;
2007         if (nr_segs > UIO_MAXIOV)
2008                 return ERR_PTR(-EINVAL);
2009         if (nr_segs > fast_segs) {
2010                 iov = kmalloc_array(nr_segs, sizeof(struct iovec), GFP_KERNEL);
2011                 if (!iov)
2012                         return ERR_PTR(-ENOMEM);
2013         }
2014
2015         if (compat)
2016                 ret = copy_compat_iovec_from_user(iov, uvec, nr_segs);
2017         else
2018                 ret = copy_iovec_from_user(iov, uvec, nr_segs);
2019         if (ret) {
2020                 if (iov != fast_iov)
2021                         kfree(iov);
2022                 return ERR_PTR(ret);
2023         }
2024
2025         return iov;
2026 }
2027
2028 ssize_t __import_iovec(int type, const struct iovec __user *uvec,
2029                  unsigned nr_segs, unsigned fast_segs, struct iovec **iovp,
2030                  struct iov_iter *i, bool compat)
2031 {
2032         ssize_t total_len = 0;
2033         unsigned long seg;
2034         struct iovec *iov;
2035
2036         iov = iovec_from_user(uvec, nr_segs, fast_segs, *iovp, compat);
2037         if (IS_ERR(iov)) {
2038                 *iovp = NULL;
2039                 return PTR_ERR(iov);
2040         }
2041
2042         /*
2043          * According to the Single Unix Specification we should return EINVAL if
2044          * an element length is < 0 when cast to ssize_t or if the total length
2045          * would overflow the ssize_t return value of the system call.
2046          *
2047          * Linux caps all read/write calls to MAX_RW_COUNT, and avoids the
2048          * overflow case.
2049          */
2050         for (seg = 0; seg < nr_segs; seg++) {
2051                 ssize_t len = (ssize_t)iov[seg].iov_len;
2052
2053                 if (!access_ok(iov[seg].iov_base, len)) {
2054                         if (iov != *iovp)
2055                                 kfree(iov);
2056                         *iovp = NULL;
2057                         return -EFAULT;
2058                 }
2059
2060                 if (len > MAX_RW_COUNT - total_len) {
2061                         len = MAX_RW_COUNT - total_len;
2062                         iov[seg].iov_len = len;
2063                 }
2064                 total_len += len;
2065         }
2066
2067         iov_iter_init(i, type, iov, nr_segs, total_len);
2068         if (iov == *iovp)
2069                 *iovp = NULL;
2070         else
2071                 *iovp = iov;
2072         return total_len;
2073 }
2074
2075 /**
2076  * import_iovec() - Copy an array of &struct iovec from userspace
2077  *     into the kernel, check that it is valid, and initialize a new
2078  *     &struct iov_iter iterator to access it.
2079  *
2080  * @type: One of %READ or %WRITE.
2081  * @uvec: Pointer to the userspace array.
2082  * @nr_segs: Number of elements in userspace array.
2083  * @fast_segs: Number of elements in @iov.
2084  * @iovp: (input and output parameter) Pointer to pointer to (usually small
2085  *     on-stack) kernel array.
2086  * @i: Pointer to iterator that will be initialized on success.
2087  *
2088  * If the array pointed to by *@iov is large enough to hold all @nr_segs,
2089  * then this function places %NULL in *@iov on return. Otherwise, a new
2090  * array will be allocated and the result placed in *@iov. This means that
2091  * the caller may call kfree() on *@iov regardless of whether the small
2092  * on-stack array was used or not (and regardless of whether this function
2093  * returns an error or not).
2094  *
2095  * Return: Negative error code on error, bytes imported on success
2096  */
2097 ssize_t import_iovec(int type, const struct iovec __user *uvec,
2098                  unsigned nr_segs, unsigned fast_segs,
2099                  struct iovec **iovp, struct iov_iter *i)
2100 {
2101         return __import_iovec(type, uvec, nr_segs, fast_segs, iovp, i,
2102                               in_compat_syscall());
2103 }
2104 EXPORT_SYMBOL(import_iovec);
2105
2106 int import_single_range(int rw, void __user *buf, size_t len,
2107                  struct iovec *iov, struct iov_iter *i)
2108 {
2109         if (len > MAX_RW_COUNT)
2110                 len = MAX_RW_COUNT;
2111         if (unlikely(!access_ok(buf, len)))
2112                 return -EFAULT;
2113
2114         iov->iov_base = buf;
2115         iov->iov_len = len;
2116         iov_iter_init(i, rw, iov, 1, len);
2117         return 0;
2118 }
2119 EXPORT_SYMBOL(import_single_range);