1 // SPDX-License-Identifier: GPL-2.0-only
2 #include <linux/export.h>
3 #include <linux/bvec.h>
5 #include <linux/pagemap.h>
6 #include <linux/slab.h>
7 #include <linux/vmalloc.h>
8 #include <linux/splice.h>
9 #include <net/checksum.h>
10 #include <linux/scatterlist.h>
12 #define PIPE_PARANOIA /* for now */
14 #define iterate_iovec(i, n, __v, __p, skip, STEP) { \
18 __v.iov_len = min(n, __p->iov_len - skip); \
19 if (likely(__v.iov_len)) { \
20 __v.iov_base = __p->iov_base + skip; \
22 __v.iov_len -= left; \
23 skip += __v.iov_len; \
28 while (unlikely(!left && n)) { \
30 __v.iov_len = min(n, __p->iov_len); \
31 if (unlikely(!__v.iov_len)) \
33 __v.iov_base = __p->iov_base; \
35 __v.iov_len -= left; \
42 #define iterate_kvec(i, n, __v, __p, skip, STEP) { \
45 __v.iov_len = min(n, __p->iov_len - skip); \
46 if (likely(__v.iov_len)) { \
47 __v.iov_base = __p->iov_base + skip; \
49 skip += __v.iov_len; \
52 while (unlikely(n)) { \
54 __v.iov_len = min(n, __p->iov_len); \
55 if (unlikely(!__v.iov_len)) \
57 __v.iov_base = __p->iov_base; \
65 #define iterate_bvec(i, n, __v, __bi, skip, STEP) { \
66 struct bvec_iter __start; \
67 __start.bi_size = n; \
68 __start.bi_bvec_done = skip; \
70 for_each_bvec(__v, i->bvec, __bi, __start) { \
77 #define iterate_all_kinds(i, n, v, I, B, K) { \
79 size_t skip = i->iov_offset; \
80 if (unlikely(i->type & ITER_BVEC)) { \
82 struct bvec_iter __bi; \
83 iterate_bvec(i, n, v, __bi, skip, (B)) \
84 } else if (unlikely(i->type & ITER_KVEC)) { \
85 const struct kvec *kvec; \
87 iterate_kvec(i, n, v, kvec, skip, (K)) \
88 } else if (unlikely(i->type & ITER_DISCARD)) { \
90 const struct iovec *iov; \
92 iterate_iovec(i, n, v, iov, skip, (I)) \
97 #define iterate_and_advance(i, n, v, I, B, K) { \
98 if (unlikely(i->count < n)) \
101 size_t skip = i->iov_offset; \
102 if (unlikely(i->type & ITER_BVEC)) { \
103 const struct bio_vec *bvec = i->bvec; \
105 struct bvec_iter __bi; \
106 iterate_bvec(i, n, v, __bi, skip, (B)) \
107 i->bvec = __bvec_iter_bvec(i->bvec, __bi); \
108 i->nr_segs -= i->bvec - bvec; \
109 skip = __bi.bi_bvec_done; \
110 } else if (unlikely(i->type & ITER_KVEC)) { \
111 const struct kvec *kvec; \
113 iterate_kvec(i, n, v, kvec, skip, (K)) \
114 if (skip == kvec->iov_len) { \
118 i->nr_segs -= kvec - i->kvec; \
120 } else if (unlikely(i->type & ITER_DISCARD)) { \
123 const struct iovec *iov; \
125 iterate_iovec(i, n, v, iov, skip, (I)) \
126 if (skip == iov->iov_len) { \
130 i->nr_segs -= iov - i->iov; \
134 i->iov_offset = skip; \
138 static int copyout(void __user *to, const void *from, size_t n)
140 if (access_ok(to, n)) {
141 kasan_check_read(from, n);
142 n = raw_copy_to_user(to, from, n);
147 static int copyin(void *to, const void __user *from, size_t n)
149 if (access_ok(from, n)) {
150 kasan_check_write(to, n);
151 n = raw_copy_from_user(to, from, n);
156 static size_t copy_page_to_iter_iovec(struct page *page, size_t offset, size_t bytes,
159 size_t skip, copy, left, wanted;
160 const struct iovec *iov;
164 if (unlikely(bytes > i->count))
167 if (unlikely(!bytes))
173 skip = i->iov_offset;
174 buf = iov->iov_base + skip;
175 copy = min(bytes, iov->iov_len - skip);
177 if (IS_ENABLED(CONFIG_HIGHMEM) && !fault_in_pages_writeable(buf, copy)) {
178 kaddr = kmap_atomic(page);
179 from = kaddr + offset;
181 /* first chunk, usually the only one */
182 left = copyout(buf, from, copy);
188 while (unlikely(!left && bytes)) {
191 copy = min(bytes, iov->iov_len);
192 left = copyout(buf, from, copy);
198 if (likely(!bytes)) {
199 kunmap_atomic(kaddr);
202 offset = from - kaddr;
204 kunmap_atomic(kaddr);
205 copy = min(bytes, iov->iov_len - skip);
207 /* Too bad - revert to non-atomic kmap */
210 from = kaddr + offset;
211 left = copyout(buf, from, copy);
216 while (unlikely(!left && bytes)) {
219 copy = min(bytes, iov->iov_len);
220 left = copyout(buf, from, copy);
229 if (skip == iov->iov_len) {
233 i->count -= wanted - bytes;
234 i->nr_segs -= iov - i->iov;
236 i->iov_offset = skip;
237 return wanted - bytes;
240 static size_t copy_page_from_iter_iovec(struct page *page, size_t offset, size_t bytes,
243 size_t skip, copy, left, wanted;
244 const struct iovec *iov;
248 if (unlikely(bytes > i->count))
251 if (unlikely(!bytes))
257 skip = i->iov_offset;
258 buf = iov->iov_base + skip;
259 copy = min(bytes, iov->iov_len - skip);
261 if (IS_ENABLED(CONFIG_HIGHMEM) && !fault_in_pages_readable(buf, copy)) {
262 kaddr = kmap_atomic(page);
265 /* first chunk, usually the only one */
266 left = copyin(to, buf, copy);
272 while (unlikely(!left && bytes)) {
275 copy = min(bytes, iov->iov_len);
276 left = copyin(to, buf, copy);
282 if (likely(!bytes)) {
283 kunmap_atomic(kaddr);
288 kunmap_atomic(kaddr);
289 copy = min(bytes, iov->iov_len - skip);
291 /* Too bad - revert to non-atomic kmap */
295 left = copyin(to, buf, copy);
300 while (unlikely(!left && bytes)) {
303 copy = min(bytes, iov->iov_len);
304 left = copyin(to, buf, copy);
313 if (skip == iov->iov_len) {
317 i->count -= wanted - bytes;
318 i->nr_segs -= iov - i->iov;
320 i->iov_offset = skip;
321 return wanted - bytes;
325 static bool sanity(const struct iov_iter *i)
327 struct pipe_inode_info *pipe = i->pipe;
328 unsigned int p_head = pipe->head;
329 unsigned int p_tail = pipe->tail;
330 unsigned int p_mask = pipe->ring_size - 1;
331 unsigned int p_occupancy = pipe_occupancy(p_head, p_tail);
332 unsigned int i_head = i->head;
336 struct pipe_buffer *p;
337 if (unlikely(p_occupancy == 0))
338 goto Bad; // pipe must be non-empty
339 if (unlikely(i_head != p_head - 1))
340 goto Bad; // must be at the last buffer...
342 p = &pipe->bufs[i_head & p_mask];
343 if (unlikely(p->offset + p->len != i->iov_offset))
344 goto Bad; // ... at the end of segment
346 if (i_head != p_head)
347 goto Bad; // must be right after the last buffer
351 printk(KERN_ERR "idx = %d, offset = %zd\n", i_head, i->iov_offset);
352 printk(KERN_ERR "head = %d, tail = %d, buffers = %d\n",
353 p_head, p_tail, pipe->ring_size);
354 for (idx = 0; idx < pipe->ring_size; idx++)
355 printk(KERN_ERR "[%p %p %d %d]\n",
357 pipe->bufs[idx].page,
358 pipe->bufs[idx].offset,
359 pipe->bufs[idx].len);
364 #define sanity(i) true
367 static size_t copy_page_to_iter_pipe(struct page *page, size_t offset, size_t bytes,
370 struct pipe_inode_info *pipe = i->pipe;
371 struct pipe_buffer *buf;
372 unsigned int p_tail = pipe->tail;
373 unsigned int p_mask = pipe->ring_size - 1;
374 unsigned int i_head = i->head;
377 if (unlikely(bytes > i->count))
380 if (unlikely(!bytes))
387 buf = &pipe->bufs[i_head & p_mask];
389 if (offset == off && buf->page == page) {
390 /* merge with the last one */
392 i->iov_offset += bytes;
396 buf = &pipe->bufs[i_head & p_mask];
398 if (pipe_full(i_head, p_tail, pipe->max_usage))
401 buf->ops = &page_cache_pipe_buf_ops;
404 buf->offset = offset;
407 pipe->head = i_head + 1;
408 i->iov_offset = offset + bytes;
416 * Fault in one or more iovecs of the given iov_iter, to a maximum length of
417 * bytes. For each iovec, fault in each page that constitutes the iovec.
419 * Return 0 on success, or non-zero if the memory could not be accessed (i.e.
420 * because it is an invalid address).
422 int iov_iter_fault_in_readable(struct iov_iter *i, size_t bytes)
424 size_t skip = i->iov_offset;
425 const struct iovec *iov;
429 if (!(i->type & (ITER_BVEC|ITER_KVEC))) {
430 iterate_iovec(i, bytes, v, iov, skip, ({
431 err = fault_in_pages_readable(v.iov_base, v.iov_len);
438 EXPORT_SYMBOL(iov_iter_fault_in_readable);
440 void iov_iter_init(struct iov_iter *i, unsigned int direction,
441 const struct iovec *iov, unsigned long nr_segs,
444 WARN_ON(direction & ~(READ | WRITE));
445 direction &= READ | WRITE;
447 /* It will get better. Eventually... */
448 if (uaccess_kernel()) {
449 i->type = ITER_KVEC | direction;
450 i->kvec = (struct kvec *)iov;
452 i->type = ITER_IOVEC | direction;
455 i->nr_segs = nr_segs;
459 EXPORT_SYMBOL(iov_iter_init);
461 static void memcpy_from_page(char *to, struct page *page, size_t offset, size_t len)
463 char *from = kmap_atomic(page);
464 memcpy(to, from + offset, len);
468 static void memcpy_to_page(struct page *page, size_t offset, const char *from, size_t len)
470 char *to = kmap_atomic(page);
471 memcpy(to + offset, from, len);
475 static void memzero_page(struct page *page, size_t offset, size_t len)
477 char *addr = kmap_atomic(page);
478 memset(addr + offset, 0, len);
482 static inline bool allocated(struct pipe_buffer *buf)
484 return buf->ops == &default_pipe_buf_ops;
487 static inline void data_start(const struct iov_iter *i,
488 unsigned int *iter_headp, size_t *offp)
490 unsigned int p_mask = i->pipe->ring_size - 1;
491 unsigned int iter_head = i->head;
492 size_t off = i->iov_offset;
494 if (off && (!allocated(&i->pipe->bufs[iter_head & p_mask]) ||
499 *iter_headp = iter_head;
503 static size_t push_pipe(struct iov_iter *i, size_t size,
504 int *iter_headp, size_t *offp)
506 struct pipe_inode_info *pipe = i->pipe;
507 unsigned int p_tail = pipe->tail;
508 unsigned int p_mask = pipe->ring_size - 1;
509 unsigned int iter_head;
513 if (unlikely(size > i->count))
519 data_start(i, &iter_head, &off);
520 *iter_headp = iter_head;
523 left -= PAGE_SIZE - off;
525 pipe->bufs[iter_head & p_mask].len += size;
528 pipe->bufs[iter_head & p_mask].len = PAGE_SIZE;
531 while (!pipe_full(iter_head, p_tail, pipe->max_usage)) {
532 struct pipe_buffer *buf = &pipe->bufs[iter_head & p_mask];
533 struct page *page = alloc_page(GFP_USER);
537 buf->ops = &default_pipe_buf_ops;
540 buf->len = min_t(ssize_t, left, PAGE_SIZE);
543 pipe->head = iter_head;
551 static size_t copy_pipe_to_iter(const void *addr, size_t bytes,
554 struct pipe_inode_info *pipe = i->pipe;
555 unsigned int p_mask = pipe->ring_size - 1;
562 bytes = n = push_pipe(i, bytes, &i_head, &off);
566 size_t chunk = min_t(size_t, n, PAGE_SIZE - off);
567 memcpy_to_page(pipe->bufs[i_head & p_mask].page, off, addr, chunk);
569 i->iov_offset = off + chunk;
579 static __wsum csum_and_memcpy(void *to, const void *from, size_t len,
580 __wsum sum, size_t off)
582 __wsum next = csum_partial_copy_nocheck(from, to, len, 0);
583 return csum_block_add(sum, next, off);
586 static size_t csum_and_copy_to_pipe_iter(const void *addr, size_t bytes,
587 __wsum *csum, struct iov_iter *i)
589 struct pipe_inode_info *pipe = i->pipe;
590 unsigned int p_mask = pipe->ring_size - 1;
599 bytes = n = push_pipe(i, bytes, &i_head, &r);
603 size_t chunk = min_t(size_t, n, PAGE_SIZE - r);
604 char *p = kmap_atomic(pipe->bufs[i_head & p_mask].page);
605 sum = csum_and_memcpy(p + r, addr, chunk, sum, off);
608 i->iov_offset = r + chunk;
620 size_t _copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i)
622 const char *from = addr;
623 if (unlikely(iov_iter_is_pipe(i)))
624 return copy_pipe_to_iter(addr, bytes, i);
625 if (iter_is_iovec(i))
627 iterate_and_advance(i, bytes, v,
628 copyout(v.iov_base, (from += v.iov_len) - v.iov_len, v.iov_len),
629 memcpy_to_page(v.bv_page, v.bv_offset,
630 (from += v.bv_len) - v.bv_len, v.bv_len),
631 memcpy(v.iov_base, (from += v.iov_len) - v.iov_len, v.iov_len)
636 EXPORT_SYMBOL(_copy_to_iter);
638 #ifdef CONFIG_ARCH_HAS_UACCESS_MCSAFE
639 static int copyout_mcsafe(void __user *to, const void *from, size_t n)
641 if (access_ok(to, n)) {
642 kasan_check_read(from, n);
643 n = copy_to_user_mcsafe((__force void *) to, from, n);
648 static unsigned long memcpy_mcsafe_to_page(struct page *page, size_t offset,
649 const char *from, size_t len)
654 to = kmap_atomic(page);
655 ret = memcpy_mcsafe(to + offset, from, len);
661 static size_t copy_pipe_to_iter_mcsafe(const void *addr, size_t bytes,
664 struct pipe_inode_info *pipe = i->pipe;
665 unsigned int p_mask = pipe->ring_size - 1;
667 size_t n, off, xfer = 0;
672 bytes = n = push_pipe(i, bytes, &i_head, &off);
676 size_t chunk = min_t(size_t, n, PAGE_SIZE - off);
679 rem = memcpy_mcsafe_to_page(pipe->bufs[i_head & p_mask].page,
682 i->iov_offset = off + chunk - rem;
696 * _copy_to_iter_mcsafe - copy to user with source-read error exception handling
697 * @addr: source kernel address
698 * @bytes: total transfer length
699 * @iter: destination iterator
701 * The pmem driver arranges for filesystem-dax to use this facility via
702 * dax_copy_to_iter() for protecting read/write to persistent memory.
703 * Unless / until an architecture can guarantee identical performance
704 * between _copy_to_iter_mcsafe() and _copy_to_iter() it would be a
705 * performance regression to switch more users to the mcsafe version.
707 * Otherwise, the main differences between this and typical _copy_to_iter().
709 * * Typical tail/residue handling after a fault retries the copy
710 * byte-by-byte until the fault happens again. Re-triggering machine
711 * checks is potentially fatal so the implementation uses source
712 * alignment and poison alignment assumptions to avoid re-triggering
713 * hardware exceptions.
715 * * ITER_KVEC, ITER_PIPE, and ITER_BVEC can return short copies.
716 * Compare to copy_to_iter() where only ITER_IOVEC attempts might return
719 * See MCSAFE_TEST for self-test.
721 size_t _copy_to_iter_mcsafe(const void *addr, size_t bytes, struct iov_iter *i)
723 const char *from = addr;
724 unsigned long rem, curr_addr, s_addr = (unsigned long) addr;
726 if (unlikely(iov_iter_is_pipe(i)))
727 return copy_pipe_to_iter_mcsafe(addr, bytes, i);
728 if (iter_is_iovec(i))
730 iterate_and_advance(i, bytes, v,
731 copyout_mcsafe(v.iov_base, (from += v.iov_len) - v.iov_len, v.iov_len),
733 rem = memcpy_mcsafe_to_page(v.bv_page, v.bv_offset,
734 (from += v.bv_len) - v.bv_len, v.bv_len);
736 curr_addr = (unsigned long) from;
737 bytes = curr_addr - s_addr - rem;
742 rem = memcpy_mcsafe(v.iov_base, (from += v.iov_len) - v.iov_len,
745 curr_addr = (unsigned long) from;
746 bytes = curr_addr - s_addr - rem;
754 EXPORT_SYMBOL_GPL(_copy_to_iter_mcsafe);
755 #endif /* CONFIG_ARCH_HAS_UACCESS_MCSAFE */
757 size_t _copy_from_iter(void *addr, size_t bytes, struct iov_iter *i)
760 if (unlikely(iov_iter_is_pipe(i))) {
764 if (iter_is_iovec(i))
766 iterate_and_advance(i, bytes, v,
767 copyin((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len),
768 memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
769 v.bv_offset, v.bv_len),
770 memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
775 EXPORT_SYMBOL(_copy_from_iter);
777 bool _copy_from_iter_full(void *addr, size_t bytes, struct iov_iter *i)
780 if (unlikely(iov_iter_is_pipe(i))) {
784 if (unlikely(i->count < bytes))
787 if (iter_is_iovec(i))
789 iterate_all_kinds(i, bytes, v, ({
790 if (copyin((to += v.iov_len) - v.iov_len,
791 v.iov_base, v.iov_len))
794 memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
795 v.bv_offset, v.bv_len),
796 memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
799 iov_iter_advance(i, bytes);
802 EXPORT_SYMBOL(_copy_from_iter_full);
804 size_t _copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i)
807 if (unlikely(iov_iter_is_pipe(i))) {
811 iterate_and_advance(i, bytes, v,
812 __copy_from_user_inatomic_nocache((to += v.iov_len) - v.iov_len,
813 v.iov_base, v.iov_len),
814 memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
815 v.bv_offset, v.bv_len),
816 memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
821 EXPORT_SYMBOL(_copy_from_iter_nocache);
823 #ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE
825 * _copy_from_iter_flushcache - write destination through cpu cache
826 * @addr: destination kernel address
827 * @bytes: total transfer length
828 * @iter: source iterator
830 * The pmem driver arranges for filesystem-dax to use this facility via
831 * dax_copy_from_iter() for ensuring that writes to persistent memory
832 * are flushed through the CPU cache. It is differentiated from
833 * _copy_from_iter_nocache() in that guarantees all data is flushed for
834 * all iterator types. The _copy_from_iter_nocache() only attempts to
835 * bypass the cache for the ITER_IOVEC case, and on some archs may use
836 * instructions that strand dirty-data in the cache.
838 size_t _copy_from_iter_flushcache(void *addr, size_t bytes, struct iov_iter *i)
841 if (unlikely(iov_iter_is_pipe(i))) {
845 iterate_and_advance(i, bytes, v,
846 __copy_from_user_flushcache((to += v.iov_len) - v.iov_len,
847 v.iov_base, v.iov_len),
848 memcpy_page_flushcache((to += v.bv_len) - v.bv_len, v.bv_page,
849 v.bv_offset, v.bv_len),
850 memcpy_flushcache((to += v.iov_len) - v.iov_len, v.iov_base,
856 EXPORT_SYMBOL_GPL(_copy_from_iter_flushcache);
859 bool _copy_from_iter_full_nocache(void *addr, size_t bytes, struct iov_iter *i)
862 if (unlikely(iov_iter_is_pipe(i))) {
866 if (unlikely(i->count < bytes))
868 iterate_all_kinds(i, bytes, v, ({
869 if (__copy_from_user_inatomic_nocache((to += v.iov_len) - v.iov_len,
870 v.iov_base, v.iov_len))
873 memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
874 v.bv_offset, v.bv_len),
875 memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
878 iov_iter_advance(i, bytes);
881 EXPORT_SYMBOL(_copy_from_iter_full_nocache);
883 static inline bool page_copy_sane(struct page *page, size_t offset, size_t n)
886 size_t v = n + offset;
889 * The general case needs to access the page order in order
890 * to compute the page size.
891 * However, we mostly deal with order-0 pages and thus can
892 * avoid a possible cache line miss for requests that fit all
895 if (n <= v && v <= PAGE_SIZE)
898 head = compound_head(page);
899 v += (page - head) << PAGE_SHIFT;
901 if (likely(n <= v && v <= (page_size(head))))
907 size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes,
910 if (unlikely(!page_copy_sane(page, offset, bytes)))
912 if (i->type & (ITER_BVEC|ITER_KVEC)) {
913 void *kaddr = kmap_atomic(page);
914 size_t wanted = copy_to_iter(kaddr + offset, bytes, i);
915 kunmap_atomic(kaddr);
917 } else if (unlikely(iov_iter_is_discard(i)))
919 else if (likely(!iov_iter_is_pipe(i)))
920 return copy_page_to_iter_iovec(page, offset, bytes, i);
922 return copy_page_to_iter_pipe(page, offset, bytes, i);
924 EXPORT_SYMBOL(copy_page_to_iter);
926 size_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes,
929 if (unlikely(!page_copy_sane(page, offset, bytes)))
931 if (unlikely(iov_iter_is_pipe(i) || iov_iter_is_discard(i))) {
935 if (i->type & (ITER_BVEC|ITER_KVEC)) {
936 void *kaddr = kmap_atomic(page);
937 size_t wanted = _copy_from_iter(kaddr + offset, bytes, i);
938 kunmap_atomic(kaddr);
941 return copy_page_from_iter_iovec(page, offset, bytes, i);
943 EXPORT_SYMBOL(copy_page_from_iter);
945 static size_t pipe_zero(size_t bytes, struct iov_iter *i)
947 struct pipe_inode_info *pipe = i->pipe;
948 unsigned int p_mask = pipe->ring_size - 1;
955 bytes = n = push_pipe(i, bytes, &i_head, &off);
960 size_t chunk = min_t(size_t, n, PAGE_SIZE - off);
961 memzero_page(pipe->bufs[i_head & p_mask].page, off, chunk);
963 i->iov_offset = off + chunk;
972 size_t iov_iter_zero(size_t bytes, struct iov_iter *i)
974 if (unlikely(iov_iter_is_pipe(i)))
975 return pipe_zero(bytes, i);
976 iterate_and_advance(i, bytes, v,
977 clear_user(v.iov_base, v.iov_len),
978 memzero_page(v.bv_page, v.bv_offset, v.bv_len),
979 memset(v.iov_base, 0, v.iov_len)
984 EXPORT_SYMBOL(iov_iter_zero);
986 size_t iov_iter_copy_from_user_atomic(struct page *page,
987 struct iov_iter *i, unsigned long offset, size_t bytes)
989 char *kaddr = kmap_atomic(page), *p = kaddr + offset;
990 if (unlikely(!page_copy_sane(page, offset, bytes))) {
991 kunmap_atomic(kaddr);
994 if (unlikely(iov_iter_is_pipe(i) || iov_iter_is_discard(i))) {
995 kunmap_atomic(kaddr);
999 iterate_all_kinds(i, bytes, v,
1000 copyin((p += v.iov_len) - v.iov_len, v.iov_base, v.iov_len),
1001 memcpy_from_page((p += v.bv_len) - v.bv_len, v.bv_page,
1002 v.bv_offset, v.bv_len),
1003 memcpy((p += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
1005 kunmap_atomic(kaddr);
1008 EXPORT_SYMBOL(iov_iter_copy_from_user_atomic);
1010 static inline void pipe_truncate(struct iov_iter *i)
1012 struct pipe_inode_info *pipe = i->pipe;
1013 unsigned int p_tail = pipe->tail;
1014 unsigned int p_head = pipe->head;
1015 unsigned int p_mask = pipe->ring_size - 1;
1017 if (!pipe_empty(p_head, p_tail)) {
1018 struct pipe_buffer *buf;
1019 unsigned int i_head = i->head;
1020 size_t off = i->iov_offset;
1023 buf = &pipe->bufs[i_head & p_mask];
1024 buf->len = off - buf->offset;
1027 while (p_head != i_head) {
1029 pipe_buf_release(pipe, &pipe->bufs[p_head & p_mask]);
1032 pipe->head = p_head;
1036 static void pipe_advance(struct iov_iter *i, size_t size)
1038 struct pipe_inode_info *pipe = i->pipe;
1039 if (unlikely(i->count < size))
1042 struct pipe_buffer *buf;
1043 unsigned int p_mask = pipe->ring_size - 1;
1044 unsigned int i_head = i->head;
1045 size_t off = i->iov_offset, left = size;
1047 if (off) /* make it relative to the beginning of buffer */
1048 left += off - pipe->bufs[i_head & p_mask].offset;
1050 buf = &pipe->bufs[i_head & p_mask];
1051 if (left <= buf->len)
1057 i->iov_offset = buf->offset + left;
1060 /* ... and discard everything past that point */
1064 void iov_iter_advance(struct iov_iter *i, size_t size)
1066 if (unlikely(iov_iter_is_pipe(i))) {
1067 pipe_advance(i, size);
1070 if (unlikely(iov_iter_is_discard(i))) {
1074 iterate_and_advance(i, size, v, 0, 0, 0)
1076 EXPORT_SYMBOL(iov_iter_advance);
1078 void iov_iter_revert(struct iov_iter *i, size_t unroll)
1082 if (WARN_ON(unroll > MAX_RW_COUNT))
1085 if (unlikely(iov_iter_is_pipe(i))) {
1086 struct pipe_inode_info *pipe = i->pipe;
1087 unsigned int p_mask = pipe->ring_size - 1;
1088 unsigned int i_head = i->head;
1089 size_t off = i->iov_offset;
1091 struct pipe_buffer *b = &pipe->bufs[i_head & p_mask];
1092 size_t n = off - b->offset;
1098 if (!unroll && i_head == i->start_head) {
1103 b = &pipe->bufs[i_head & p_mask];
1104 off = b->offset + b->len;
1106 i->iov_offset = off;
1111 if (unlikely(iov_iter_is_discard(i)))
1113 if (unroll <= i->iov_offset) {
1114 i->iov_offset -= unroll;
1117 unroll -= i->iov_offset;
1118 if (iov_iter_is_bvec(i)) {
1119 const struct bio_vec *bvec = i->bvec;
1121 size_t n = (--bvec)->bv_len;
1125 i->iov_offset = n - unroll;
1130 } else { /* same logics for iovec and kvec */
1131 const struct iovec *iov = i->iov;
1133 size_t n = (--iov)->iov_len;
1137 i->iov_offset = n - unroll;
1144 EXPORT_SYMBOL(iov_iter_revert);
1147 * Return the count of just the current iov_iter segment.
1149 size_t iov_iter_single_seg_count(const struct iov_iter *i)
1151 if (unlikely(iov_iter_is_pipe(i)))
1152 return i->count; // it is a silly place, anyway
1153 if (i->nr_segs == 1)
1155 if (unlikely(iov_iter_is_discard(i)))
1157 else if (iov_iter_is_bvec(i))
1158 return min(i->count, i->bvec->bv_len - i->iov_offset);
1160 return min(i->count, i->iov->iov_len - i->iov_offset);
1162 EXPORT_SYMBOL(iov_iter_single_seg_count);
1164 void iov_iter_kvec(struct iov_iter *i, unsigned int direction,
1165 const struct kvec *kvec, unsigned long nr_segs,
1168 WARN_ON(direction & ~(READ | WRITE));
1169 i->type = ITER_KVEC | (direction & (READ | WRITE));
1171 i->nr_segs = nr_segs;
1175 EXPORT_SYMBOL(iov_iter_kvec);
1177 void iov_iter_bvec(struct iov_iter *i, unsigned int direction,
1178 const struct bio_vec *bvec, unsigned long nr_segs,
1181 WARN_ON(direction & ~(READ | WRITE));
1182 i->type = ITER_BVEC | (direction & (READ | WRITE));
1184 i->nr_segs = nr_segs;
1188 EXPORT_SYMBOL(iov_iter_bvec);
1190 void iov_iter_pipe(struct iov_iter *i, unsigned int direction,
1191 struct pipe_inode_info *pipe,
1194 BUG_ON(direction != READ);
1195 WARN_ON(pipe_full(pipe->head, pipe->tail, pipe->ring_size));
1196 i->type = ITER_PIPE | READ;
1198 i->head = pipe->head;
1201 i->start_head = i->head;
1203 EXPORT_SYMBOL(iov_iter_pipe);
1206 * iov_iter_discard - Initialise an I/O iterator that discards data
1207 * @i: The iterator to initialise.
1208 * @direction: The direction of the transfer.
1209 * @count: The size of the I/O buffer in bytes.
1211 * Set up an I/O iterator that just discards everything that's written to it.
1212 * It's only available as a READ iterator.
1214 void iov_iter_discard(struct iov_iter *i, unsigned int direction, size_t count)
1216 BUG_ON(direction != READ);
1217 i->type = ITER_DISCARD | READ;
1221 EXPORT_SYMBOL(iov_iter_discard);
1223 unsigned long iov_iter_alignment(const struct iov_iter *i)
1225 unsigned long res = 0;
1226 size_t size = i->count;
1228 if (unlikely(iov_iter_is_pipe(i))) {
1229 unsigned int p_mask = i->pipe->ring_size - 1;
1231 if (size && i->iov_offset && allocated(&i->pipe->bufs[i->head & p_mask]))
1232 return size | i->iov_offset;
1235 iterate_all_kinds(i, size, v,
1236 (res |= (unsigned long)v.iov_base | v.iov_len, 0),
1237 res |= v.bv_offset | v.bv_len,
1238 res |= (unsigned long)v.iov_base | v.iov_len
1242 EXPORT_SYMBOL(iov_iter_alignment);
1244 unsigned long iov_iter_gap_alignment(const struct iov_iter *i)
1246 unsigned long res = 0;
1247 size_t size = i->count;
1249 if (unlikely(iov_iter_is_pipe(i) || iov_iter_is_discard(i))) {
1254 iterate_all_kinds(i, size, v,
1255 (res |= (!res ? 0 : (unsigned long)v.iov_base) |
1256 (size != v.iov_len ? size : 0), 0),
1257 (res |= (!res ? 0 : (unsigned long)v.bv_offset) |
1258 (size != v.bv_len ? size : 0)),
1259 (res |= (!res ? 0 : (unsigned long)v.iov_base) |
1260 (size != v.iov_len ? size : 0))
1264 EXPORT_SYMBOL(iov_iter_gap_alignment);
1266 static inline ssize_t __pipe_get_pages(struct iov_iter *i,
1268 struct page **pages,
1272 struct pipe_inode_info *pipe = i->pipe;
1273 unsigned int p_mask = pipe->ring_size - 1;
1274 ssize_t n = push_pipe(i, maxsize, &iter_head, start);
1281 get_page(*pages++ = pipe->bufs[iter_head & p_mask].page);
1289 static ssize_t pipe_get_pages(struct iov_iter *i,
1290 struct page **pages, size_t maxsize, unsigned maxpages,
1293 unsigned int iter_head, npages;
1302 data_start(i, &iter_head, start);
1303 /* Amount of free space: some of this one + all after this one */
1304 npages = pipe_space_for_user(iter_head, i->pipe->tail, i->pipe);
1305 capacity = min(npages, maxpages) * PAGE_SIZE - *start;
1307 return __pipe_get_pages(i, min(maxsize, capacity), pages, iter_head, start);
1310 ssize_t iov_iter_get_pages(struct iov_iter *i,
1311 struct page **pages, size_t maxsize, unsigned maxpages,
1314 if (maxsize > i->count)
1317 if (unlikely(iov_iter_is_pipe(i)))
1318 return pipe_get_pages(i, pages, maxsize, maxpages, start);
1319 if (unlikely(iov_iter_is_discard(i)))
1322 iterate_all_kinds(i, maxsize, v, ({
1323 unsigned long addr = (unsigned long)v.iov_base;
1324 size_t len = v.iov_len + (*start = addr & (PAGE_SIZE - 1));
1328 if (len > maxpages * PAGE_SIZE)
1329 len = maxpages * PAGE_SIZE;
1330 addr &= ~(PAGE_SIZE - 1);
1331 n = DIV_ROUND_UP(len, PAGE_SIZE);
1332 res = get_user_pages_fast(addr, n,
1333 iov_iter_rw(i) != WRITE ? FOLL_WRITE : 0,
1335 if (unlikely(res < 0))
1337 return (res == n ? len : res * PAGE_SIZE) - *start;
1339 /* can't be more than PAGE_SIZE */
1340 *start = v.bv_offset;
1341 get_page(*pages = v.bv_page);
1349 EXPORT_SYMBOL(iov_iter_get_pages);
1351 static struct page **get_pages_array(size_t n)
1353 return kvmalloc_array(n, sizeof(struct page *), GFP_KERNEL);
1356 static ssize_t pipe_get_pages_alloc(struct iov_iter *i,
1357 struct page ***pages, size_t maxsize,
1361 unsigned int iter_head, npages;
1370 data_start(i, &iter_head, start);
1371 /* Amount of free space: some of this one + all after this one */
1372 npages = pipe_space_for_user(iter_head, i->pipe->tail, i->pipe);
1373 n = npages * PAGE_SIZE - *start;
1377 npages = DIV_ROUND_UP(maxsize + *start, PAGE_SIZE);
1378 p = get_pages_array(npages);
1381 n = __pipe_get_pages(i, maxsize, p, iter_head, start);
1389 ssize_t iov_iter_get_pages_alloc(struct iov_iter *i,
1390 struct page ***pages, size_t maxsize,
1395 if (maxsize > i->count)
1398 if (unlikely(iov_iter_is_pipe(i)))
1399 return pipe_get_pages_alloc(i, pages, maxsize, start);
1400 if (unlikely(iov_iter_is_discard(i)))
1403 iterate_all_kinds(i, maxsize, v, ({
1404 unsigned long addr = (unsigned long)v.iov_base;
1405 size_t len = v.iov_len + (*start = addr & (PAGE_SIZE - 1));
1409 addr &= ~(PAGE_SIZE - 1);
1410 n = DIV_ROUND_UP(len, PAGE_SIZE);
1411 p = get_pages_array(n);
1414 res = get_user_pages_fast(addr, n,
1415 iov_iter_rw(i) != WRITE ? FOLL_WRITE : 0, p);
1416 if (unlikely(res < 0)) {
1421 return (res == n ? len : res * PAGE_SIZE) - *start;
1423 /* can't be more than PAGE_SIZE */
1424 *start = v.bv_offset;
1425 *pages = p = get_pages_array(1);
1428 get_page(*p = v.bv_page);
1436 EXPORT_SYMBOL(iov_iter_get_pages_alloc);
1438 size_t csum_and_copy_from_iter(void *addr, size_t bytes, __wsum *csum,
1445 if (unlikely(iov_iter_is_pipe(i) || iov_iter_is_discard(i))) {
1449 iterate_and_advance(i, bytes, v, ({
1451 next = csum_and_copy_from_user(v.iov_base,
1452 (to += v.iov_len) - v.iov_len,
1453 v.iov_len, 0, &err);
1455 sum = csum_block_add(sum, next, off);
1458 err ? v.iov_len : 0;
1460 char *p = kmap_atomic(v.bv_page);
1461 sum = csum_and_memcpy((to += v.bv_len) - v.bv_len,
1462 p + v.bv_offset, v.bv_len,
1467 sum = csum_and_memcpy((to += v.iov_len) - v.iov_len,
1468 v.iov_base, v.iov_len,
1476 EXPORT_SYMBOL(csum_and_copy_from_iter);
1478 bool csum_and_copy_from_iter_full(void *addr, size_t bytes, __wsum *csum,
1485 if (unlikely(iov_iter_is_pipe(i) || iov_iter_is_discard(i))) {
1489 if (unlikely(i->count < bytes))
1491 iterate_all_kinds(i, bytes, v, ({
1493 next = csum_and_copy_from_user(v.iov_base,
1494 (to += v.iov_len) - v.iov_len,
1495 v.iov_len, 0, &err);
1498 sum = csum_block_add(sum, next, off);
1502 char *p = kmap_atomic(v.bv_page);
1503 sum = csum_and_memcpy((to += v.bv_len) - v.bv_len,
1504 p + v.bv_offset, v.bv_len,
1509 sum = csum_and_memcpy((to += v.iov_len) - v.iov_len,
1510 v.iov_base, v.iov_len,
1516 iov_iter_advance(i, bytes);
1519 EXPORT_SYMBOL(csum_and_copy_from_iter_full);
1521 size_t csum_and_copy_to_iter(const void *addr, size_t bytes, void *csump,
1524 const char *from = addr;
1525 __wsum *csum = csump;
1529 if (unlikely(iov_iter_is_pipe(i)))
1530 return csum_and_copy_to_pipe_iter(addr, bytes, csum, i);
1533 if (unlikely(iov_iter_is_discard(i))) {
1534 WARN_ON(1); /* for now */
1537 iterate_and_advance(i, bytes, v, ({
1539 next = csum_and_copy_to_user((from += v.iov_len) - v.iov_len,
1541 v.iov_len, 0, &err);
1543 sum = csum_block_add(sum, next, off);
1546 err ? v.iov_len : 0;
1548 char *p = kmap_atomic(v.bv_page);
1549 sum = csum_and_memcpy(p + v.bv_offset,
1550 (from += v.bv_len) - v.bv_len,
1551 v.bv_len, sum, off);
1555 sum = csum_and_memcpy(v.iov_base,
1556 (from += v.iov_len) - v.iov_len,
1557 v.iov_len, sum, off);
1564 EXPORT_SYMBOL(csum_and_copy_to_iter);
1566 size_t hash_and_copy_to_iter(const void *addr, size_t bytes, void *hashp,
1569 #ifdef CONFIG_CRYPTO
1570 struct ahash_request *hash = hashp;
1571 struct scatterlist sg;
1574 copied = copy_to_iter(addr, bytes, i);
1575 sg_init_one(&sg, addr, copied);
1576 ahash_request_set_crypt(hash, &sg, NULL, copied);
1577 crypto_ahash_update(hash);
1583 EXPORT_SYMBOL(hash_and_copy_to_iter);
1585 int iov_iter_npages(const struct iov_iter *i, int maxpages)
1587 size_t size = i->count;
1592 if (unlikely(iov_iter_is_discard(i)))
1595 if (unlikely(iov_iter_is_pipe(i))) {
1596 struct pipe_inode_info *pipe = i->pipe;
1597 unsigned int iter_head;
1603 data_start(i, &iter_head, &off);
1604 /* some of this one + all after this one */
1605 npages = pipe_space_for_user(iter_head, pipe->tail, pipe);
1606 if (npages >= maxpages)
1608 } else iterate_all_kinds(i, size, v, ({
1609 unsigned long p = (unsigned long)v.iov_base;
1610 npages += DIV_ROUND_UP(p + v.iov_len, PAGE_SIZE)
1612 if (npages >= maxpages)
1616 if (npages >= maxpages)
1619 unsigned long p = (unsigned long)v.iov_base;
1620 npages += DIV_ROUND_UP(p + v.iov_len, PAGE_SIZE)
1622 if (npages >= maxpages)
1628 EXPORT_SYMBOL(iov_iter_npages);
1630 const void *dup_iter(struct iov_iter *new, struct iov_iter *old, gfp_t flags)
1633 if (unlikely(iov_iter_is_pipe(new))) {
1637 if (unlikely(iov_iter_is_discard(new)))
1639 if (iov_iter_is_bvec(new))
1640 return new->bvec = kmemdup(new->bvec,
1641 new->nr_segs * sizeof(struct bio_vec),
1644 /* iovec and kvec have identical layout */
1645 return new->iov = kmemdup(new->iov,
1646 new->nr_segs * sizeof(struct iovec),
1649 EXPORT_SYMBOL(dup_iter);
1652 * import_iovec() - Copy an array of &struct iovec from userspace
1653 * into the kernel, check that it is valid, and initialize a new
1654 * &struct iov_iter iterator to access it.
1656 * @type: One of %READ or %WRITE.
1657 * @uvector: Pointer to the userspace array.
1658 * @nr_segs: Number of elements in userspace array.
1659 * @fast_segs: Number of elements in @iov.
1660 * @iov: (input and output parameter) Pointer to pointer to (usually small
1661 * on-stack) kernel array.
1662 * @i: Pointer to iterator that will be initialized on success.
1664 * If the array pointed to by *@iov is large enough to hold all @nr_segs,
1665 * then this function places %NULL in *@iov on return. Otherwise, a new
1666 * array will be allocated and the result placed in *@iov. This means that
1667 * the caller may call kfree() on *@iov regardless of whether the small
1668 * on-stack array was used or not (and regardless of whether this function
1669 * returns an error or not).
1671 * Return: Negative error code on error, bytes imported on success
1673 ssize_t import_iovec(int type, const struct iovec __user * uvector,
1674 unsigned nr_segs, unsigned fast_segs,
1675 struct iovec **iov, struct iov_iter *i)
1679 n = rw_copy_check_uvector(type, uvector, nr_segs, fast_segs,
1687 iov_iter_init(i, type, p, nr_segs, n);
1688 *iov = p == *iov ? NULL : p;
1691 EXPORT_SYMBOL(import_iovec);
1693 #ifdef CONFIG_COMPAT
1694 #include <linux/compat.h>
1696 ssize_t compat_import_iovec(int type,
1697 const struct compat_iovec __user * uvector,
1698 unsigned nr_segs, unsigned fast_segs,
1699 struct iovec **iov, struct iov_iter *i)
1703 n = compat_rw_copy_check_uvector(type, uvector, nr_segs, fast_segs,
1711 iov_iter_init(i, type, p, nr_segs, n);
1712 *iov = p == *iov ? NULL : p;
1715 EXPORT_SYMBOL(compat_import_iovec);
1718 int import_single_range(int rw, void __user *buf, size_t len,
1719 struct iovec *iov, struct iov_iter *i)
1721 if (len > MAX_RW_COUNT)
1723 if (unlikely(!access_ok(buf, len)))
1726 iov->iov_base = buf;
1728 iov_iter_init(i, rw, iov, 1, len);
1731 EXPORT_SYMBOL(import_single_range);
1733 int iov_iter_for_each_range(struct iov_iter *i, size_t bytes,
1734 int (*f)(struct kvec *vec, void *context),
1742 iterate_all_kinds(i, bytes, v, -EINVAL, ({
1743 w.iov_base = kmap(v.bv_page) + v.bv_offset;
1744 w.iov_len = v.bv_len;
1745 err = f(&w, context);
1749 err = f(&w, context);})
1753 EXPORT_SYMBOL(iov_iter_for_each_range);