^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * "splice": joining two ropes together by interweaving their strands.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * This is the "extended pipe" functionality, where a pipe is used as
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * an arbitrary in-memory buffer. Think of a pipe as a small kernel
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * buffer that you can use to transfer data from one end to the other.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) * The traditional unix read/write is extended with a "splice()" operation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) * that transfers data buffers to or from a pipe buffer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) * Named by Larry McVoy, original implementation from Linus, extended by
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) * Jens to support splicing to files, network, direct splicing, etc and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) * fixing lots of bugs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) * Copyright (C) 2005-2006 Jens Axboe <axboe@kernel.dk>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) * Copyright (C) 2005-2006 Linus Torvalds <torvalds@osdl.org>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) * Copyright (C) 2006 Ingo Molnar <mingo@elte.hu>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <linux/bvec.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include <linux/fs.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #include <linux/file.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #include <linux/pagemap.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #include <linux/splice.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) #include <linux/memcontrol.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) #include <linux/mm_inline.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) #include <linux/swap.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) #include <linux/writeback.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) #include <linux/export.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) #include <linux/syscalls.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) #include <linux/uio.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) #include <linux/security.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) #include <linux/gfp.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) #include <linux/socket.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) #include <linux/sched/signal.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) #include "internal.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) * Attempt to steal a page from a pipe buffer. This should perhaps go into
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) * a vm helper function, it's already simplified quite a bit by the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) * addition of remove_mapping(). If success is returned, the caller may
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) * attempt to reuse this page for another destination.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) static bool page_cache_pipe_buf_try_steal(struct pipe_inode_info *pipe,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) struct pipe_buffer *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) struct page *page = buf->page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) struct address_space *mapping;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) lock_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) mapping = page_mapping(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) if (mapping) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) WARN_ON(!PageUptodate(page));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) * At least for ext2 with nobh option, we need to wait on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) * writeback completing on this page, since we'll remove it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) * from the pagecache. Otherwise truncate wont wait on the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) * page, allowing the disk blocks to be reused by someone else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) * before we actually wrote our data to them. fs corruption
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) * ensues.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) wait_on_page_writeback(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) if (page_has_private(page) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) !try_to_release_page(page, GFP_KERNEL))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) goto out_unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) * If we succeeded in removing the mapping, set LRU flag
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) * and return good.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) if (remove_mapping(mapping, page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) buf->flags |= PIPE_BUF_FLAG_LRU;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) * Raced with truncate or failed to remove page from current
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) * address space, unlock and return failure.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) out_unlock:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) unlock_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) static void page_cache_pipe_buf_release(struct pipe_inode_info *pipe,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) struct pipe_buffer *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) put_page(buf->page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) buf->flags &= ~PIPE_BUF_FLAG_LRU;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) * Check whether the contents of buf is OK to access. Since the content
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) * is a page cache page, IO may be in flight.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) static int page_cache_pipe_buf_confirm(struct pipe_inode_info *pipe,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) struct pipe_buffer *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) struct page *page = buf->page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) if (!PageUptodate(page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) lock_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) * Page got truncated/unhashed. This will cause a 0-byte
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) * splice, if this is the first page.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) if (!page->mapping) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) err = -ENODATA;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) goto error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) * Uh oh, read-error from disk.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) if (!PageUptodate(page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) err = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) goto error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) * Page is ok afterall, we are done.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) unlock_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) error:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) unlock_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) const struct pipe_buf_operations page_cache_pipe_buf_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) .confirm = page_cache_pipe_buf_confirm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) .release = page_cache_pipe_buf_release,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) .try_steal = page_cache_pipe_buf_try_steal,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) .get = generic_pipe_buf_get,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) static bool user_page_pipe_buf_try_steal(struct pipe_inode_info *pipe,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) struct pipe_buffer *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) if (!(buf->flags & PIPE_BUF_FLAG_GIFT))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) buf->flags |= PIPE_BUF_FLAG_LRU;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) return generic_pipe_buf_try_steal(pipe, buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) static const struct pipe_buf_operations user_page_pipe_buf_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) .release = page_cache_pipe_buf_release,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) .try_steal = user_page_pipe_buf_try_steal,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) .get = generic_pipe_buf_get,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) static void wakeup_pipe_readers(struct pipe_inode_info *pipe)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) smp_mb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) if (waitqueue_active(&pipe->rd_wait))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) wake_up_interruptible(&pipe->rd_wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) * splice_to_pipe - fill passed data into a pipe
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) * @pipe: pipe to fill
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) * @spd: data to fill
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) * Description:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) * @spd contains a map of pages and len/offset tuples, along with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) * the struct pipe_buf_operations associated with these pages. This
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) * function will link that data to the pipe.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) ssize_t splice_to_pipe(struct pipe_inode_info *pipe,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) struct splice_pipe_desc *spd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) unsigned int spd_pages = spd->nr_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) unsigned int tail = pipe->tail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) unsigned int head = pipe->head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) unsigned int mask = pipe->ring_size - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) int ret = 0, page_nr = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) if (!spd_pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) if (unlikely(!pipe->readers)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) send_sig(SIGPIPE, current, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) ret = -EPIPE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) while (!pipe_full(head, tail, pipe->max_usage)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) struct pipe_buffer *buf = &pipe->bufs[head & mask];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) buf->page = spd->pages[page_nr];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) buf->offset = spd->partial[page_nr].offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) buf->len = spd->partial[page_nr].len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) buf->private = spd->partial[page_nr].private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) buf->ops = spd->ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) buf->flags = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) head++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) pipe->head = head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) page_nr++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) ret += buf->len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) if (!--spd->nr_pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) if (!ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) ret = -EAGAIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) while (page_nr < spd_pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) spd->spd_release(spd, page_nr++);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) EXPORT_SYMBOL_GPL(splice_to_pipe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) ssize_t add_to_pipe(struct pipe_inode_info *pipe, struct pipe_buffer *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) unsigned int head = pipe->head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) unsigned int tail = pipe->tail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) unsigned int mask = pipe->ring_size - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) if (unlikely(!pipe->readers)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) send_sig(SIGPIPE, current, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) ret = -EPIPE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) } else if (pipe_full(head, tail, pipe->max_usage)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) ret = -EAGAIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) pipe->bufs[head & mask] = *buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) pipe->head = head + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) return buf->len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) pipe_buf_release(pipe, buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) EXPORT_SYMBOL(add_to_pipe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) * Check if we need to grow the arrays holding pages and partial page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) * descriptions.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) int splice_grow_spd(const struct pipe_inode_info *pipe, struct splice_pipe_desc *spd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) unsigned int max_usage = READ_ONCE(pipe->max_usage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) spd->nr_pages_max = max_usage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) if (max_usage <= PIPE_DEF_BUFFERS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) spd->pages = kmalloc_array(max_usage, sizeof(struct page *), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) spd->partial = kmalloc_array(max_usage, sizeof(struct partial_page),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) if (spd->pages && spd->partial)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) kfree(spd->pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) kfree(spd->partial);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) void splice_shrink_spd(struct splice_pipe_desc *spd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) if (spd->nr_pages_max <= PIPE_DEF_BUFFERS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) kfree(spd->pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) kfree(spd->partial);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) * generic_file_splice_read - splice data from file to a pipe
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) * @in: file to splice from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) * @ppos: position in @in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) * @pipe: pipe to splice to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) * @len: number of bytes to splice
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) * @flags: splice modifier flags
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) * Description:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) * Will read pages from given file and fill them into a pipe. Can be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) * used as long as it has more or less sane ->read_iter().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) ssize_t generic_file_splice_read(struct file *in, loff_t *ppos,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) struct pipe_inode_info *pipe, size_t len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) unsigned int flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) struct iov_iter to;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) struct kiocb kiocb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) unsigned int i_head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) iov_iter_pipe(&to, READ, pipe, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) i_head = to.head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) init_sync_kiocb(&kiocb, in);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) kiocb.ki_pos = *ppos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) ret = call_read_iter(in, &kiocb, &to);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) if (ret > 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) *ppos = kiocb.ki_pos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) file_accessed(in);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) } else if (ret < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) to.head = i_head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) to.iov_offset = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) iov_iter_advance(&to, 0); /* to free what was emitted */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) * callers of ->splice_read() expect -EAGAIN on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) * "can't put anything in there", rather than -EFAULT.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) if (ret == -EFAULT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) ret = -EAGAIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) EXPORT_SYMBOL_NS(generic_file_splice_read, ANDROID_GKI_VFS_EXPORT_ONLY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) const struct pipe_buf_operations default_pipe_buf_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) .release = generic_pipe_buf_release,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) .try_steal = generic_pipe_buf_try_steal,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) .get = generic_pipe_buf_get,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) /* Pipe buffer operations for a socket and similar. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) const struct pipe_buf_operations nosteal_pipe_buf_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) .release = generic_pipe_buf_release,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) .get = generic_pipe_buf_get,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) EXPORT_SYMBOL(nosteal_pipe_buf_ops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) * Send 'sd->len' bytes to socket from 'sd->file' at position 'sd->pos'
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) * using sendpage(). Return the number of bytes sent.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) static int pipe_to_sendpage(struct pipe_inode_info *pipe,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) struct pipe_buffer *buf, struct splice_desc *sd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) struct file *file = sd->u.file;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) loff_t pos = sd->pos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) int more;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) if (!likely(file->f_op->sendpage))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) more = (sd->flags & SPLICE_F_MORE) ? MSG_MORE : 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) if (sd->len < sd->total_len &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) pipe_occupancy(pipe->head, pipe->tail) > 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) more |= MSG_SENDPAGE_NOTLAST;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) return file->f_op->sendpage(file, buf->page, buf->offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) sd->len, &pos, more);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) static void wakeup_pipe_writers(struct pipe_inode_info *pipe)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) smp_mb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) if (waitqueue_active(&pipe->wr_wait))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) wake_up_interruptible(&pipe->wr_wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) * splice_from_pipe_feed - feed available data from a pipe to a file
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) * @pipe: pipe to splice from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) * @sd: information to @actor
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) * @actor: handler that splices the data
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) * Description:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) * This function loops over the pipe and calls @actor to do the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) * actual moving of a single struct pipe_buffer to the desired
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) * destination. It returns when there's no more buffers left in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) * the pipe or if the requested number of bytes (@sd->total_len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) * have been copied. It returns a positive number (one) if the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) * pipe needs to be filled with more data, zero if the required
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) * number of bytes have been copied and -errno on error.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) * This, together with splice_from_pipe_{begin,end,next}, may be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) * used to implement the functionality of __splice_from_pipe() when
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) * locking is required around copying the pipe buffers to the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) * destination.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) static int splice_from_pipe_feed(struct pipe_inode_info *pipe, struct splice_desc *sd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) splice_actor *actor)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) unsigned int head = pipe->head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) unsigned int tail = pipe->tail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) unsigned int mask = pipe->ring_size - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) while (!pipe_empty(head, tail)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) struct pipe_buffer *buf = &pipe->bufs[tail & mask];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) sd->len = buf->len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) if (sd->len > sd->total_len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) sd->len = sd->total_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) ret = pipe_buf_confirm(pipe, buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) if (unlikely(ret)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) if (ret == -ENODATA)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) ret = actor(pipe, buf, sd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) if (ret <= 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) buf->offset += ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) buf->len -= ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) sd->num_spliced += ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) sd->len -= ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) sd->pos += ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) sd->total_len -= ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) if (!buf->len) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) pipe_buf_release(pipe, buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) tail++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) pipe->tail = tail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) if (pipe->files)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) sd->need_wakeup = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) if (!sd->total_len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) /* We know we have a pipe buffer, but maybe it's empty? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) static inline bool eat_empty_buffer(struct pipe_inode_info *pipe)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) unsigned int tail = pipe->tail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) unsigned int mask = pipe->ring_size - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) struct pipe_buffer *buf = &pipe->bufs[tail & mask];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) if (unlikely(!buf->len)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) pipe_buf_release(pipe, buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) pipe->tail = tail+1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) * splice_from_pipe_next - wait for some data to splice from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) * @pipe: pipe to splice from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) * @sd: information about the splice operation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) * Description:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) * This function will wait for some data and return a positive
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) * value (one) if pipe buffers are available. It will return zero
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) * or -errno if no more data needs to be spliced.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) static int splice_from_pipe_next(struct pipe_inode_info *pipe, struct splice_desc *sd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) * Check for signal early to make process killable when there are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) * always buffers available
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) if (signal_pending(current))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) return -ERESTARTSYS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) repeat:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) while (pipe_empty(pipe->head, pipe->tail)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) if (!pipe->writers)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) if (sd->num_spliced)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) if (sd->flags & SPLICE_F_NONBLOCK)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) return -EAGAIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) if (signal_pending(current))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) return -ERESTARTSYS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) if (sd->need_wakeup) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) wakeup_pipe_writers(pipe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) sd->need_wakeup = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) pipe_wait_readable(pipe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) if (eat_empty_buffer(pipe))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) goto repeat;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) * splice_from_pipe_begin - start splicing from pipe
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) * @sd: information about the splice operation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) * Description:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) * This function should be called before a loop containing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) * splice_from_pipe_next() and splice_from_pipe_feed() to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) * initialize the necessary fields of @sd.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) static void splice_from_pipe_begin(struct splice_desc *sd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) sd->num_spliced = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) sd->need_wakeup = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) * splice_from_pipe_end - finish splicing from pipe
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) * @pipe: pipe to splice from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) * @sd: information about the splice operation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) * Description:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) * This function will wake up pipe writers if necessary. It should
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) * be called after a loop containing splice_from_pipe_next() and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) * splice_from_pipe_feed().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) static void splice_from_pipe_end(struct pipe_inode_info *pipe, struct splice_desc *sd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) if (sd->need_wakeup)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) wakeup_pipe_writers(pipe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) * __splice_from_pipe - splice data from a pipe to given actor
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) * @pipe: pipe to splice from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) * @sd: information to @actor
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) * @actor: handler that splices the data
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) * Description:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) * This function does little more than loop over the pipe and call
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) * @actor to do the actual moving of a single struct pipe_buffer to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) * the desired destination. See pipe_to_file, pipe_to_sendpage, or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) * pipe_to_user.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) ssize_t __splice_from_pipe(struct pipe_inode_info *pipe, struct splice_desc *sd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) splice_actor *actor)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) splice_from_pipe_begin(sd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) cond_resched();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) ret = splice_from_pipe_next(pipe, sd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) if (ret > 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) ret = splice_from_pipe_feed(pipe, sd, actor);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) } while (ret > 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) splice_from_pipe_end(pipe, sd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) return sd->num_spliced ? sd->num_spliced : ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) EXPORT_SYMBOL(__splice_from_pipe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) * splice_from_pipe - splice data from a pipe to a file
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) * @pipe: pipe to splice from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) * @out: file to splice to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) * @ppos: position in @out
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) * @len: how many bytes to splice
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) * @flags: splice modifier flags
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) * @actor: handler that splices the data
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) * Description:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) * See __splice_from_pipe. This function locks the pipe inode,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) * otherwise it's identical to __splice_from_pipe().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) ssize_t splice_from_pipe(struct pipe_inode_info *pipe, struct file *out,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) loff_t *ppos, size_t len, unsigned int flags,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) splice_actor *actor)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) ssize_t ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) struct splice_desc sd = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) .total_len = len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) .flags = flags,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) .pos = *ppos,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) .u.file = out,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) pipe_lock(pipe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) ret = __splice_from_pipe(pipe, &sd, actor);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) pipe_unlock(pipe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) * iter_file_splice_write - splice data from a pipe to a file
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) * @pipe: pipe info
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) * @out: file to write to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) * @ppos: position in @out
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) * @len: number of bytes to splice
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) * @flags: splice modifier flags
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) * Description:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) * Will either move or copy pages (determined by @flags options) from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) * the given pipe inode to the given file.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) * This one is ->write_iter-based.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) ssize_t
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) iter_file_splice_write(struct pipe_inode_info *pipe, struct file *out,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) loff_t *ppos, size_t len, unsigned int flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) struct splice_desc sd = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) .total_len = len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) .flags = flags,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) .pos = *ppos,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) .u.file = out,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) int nbufs = pipe->max_usage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) struct bio_vec *array = kcalloc(nbufs, sizeof(struct bio_vec),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) ssize_t ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) if (unlikely(!array))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) pipe_lock(pipe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) splice_from_pipe_begin(&sd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) while (sd.total_len) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) struct iov_iter from;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) unsigned int head, tail, mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) size_t left;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) int n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) ret = splice_from_pipe_next(pipe, &sd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) if (ret <= 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) if (unlikely(nbufs < pipe->max_usage)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) kfree(array);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) nbufs = pipe->max_usage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) array = kcalloc(nbufs, sizeof(struct bio_vec),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) if (!array) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) ret = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) head = pipe->head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) tail = pipe->tail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) mask = pipe->ring_size - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) /* build the vector */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) left = sd.total_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) for (n = 0; !pipe_empty(head, tail) && left && n < nbufs; tail++, n++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) struct pipe_buffer *buf = &pipe->bufs[tail & mask];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) size_t this_len = buf->len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) if (this_len > left)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) this_len = left;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) ret = pipe_buf_confirm(pipe, buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) if (unlikely(ret)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) if (ret == -ENODATA)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) goto done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) array[n].bv_page = buf->page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) array[n].bv_len = this_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) array[n].bv_offset = buf->offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) left -= this_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) iov_iter_bvec(&from, WRITE, array, n, sd.total_len - left);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) ret = vfs_iter_write(out, &from, &sd.pos, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) if (ret <= 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) sd.num_spliced += ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) sd.total_len -= ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) *ppos = sd.pos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) /* dismiss the fully eaten buffers, adjust the partial one */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) tail = pipe->tail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) while (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) struct pipe_buffer *buf = &pipe->bufs[tail & mask];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) if (ret >= buf->len) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) ret -= buf->len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) buf->len = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) pipe_buf_release(pipe, buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) tail++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) pipe->tail = tail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) if (pipe->files)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) sd.need_wakeup = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) buf->offset += ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) buf->len -= ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) done:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) kfree(array);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) splice_from_pipe_end(pipe, &sd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) pipe_unlock(pipe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) if (sd.num_spliced)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) ret = sd.num_spliced;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) EXPORT_SYMBOL_NS(iter_file_splice_write, ANDROID_GKI_VFS_EXPORT_ONLY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) * generic_splice_sendpage - splice data from a pipe to a socket
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) * @pipe: pipe to splice from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) * @out: socket to write to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) * @ppos: position in @out
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) * @len: number of bytes to splice
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) * @flags: splice modifier flags
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) * Description:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) * Will send @len bytes from the pipe to a network socket. No data copying
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) * is involved.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) ssize_t generic_splice_sendpage(struct pipe_inode_info *pipe, struct file *out,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) loff_t *ppos, size_t len, unsigned int flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) return splice_from_pipe(pipe, out, ppos, len, flags, pipe_to_sendpage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) EXPORT_SYMBOL(generic_splice_sendpage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) static int warn_unsupported(struct file *file, const char *op)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) pr_debug_ratelimited(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) "splice %s not supported for file %pD4 (pid: %d comm: %.20s)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) op, file, current->pid, current->comm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) * Attempt to initiate a splice from pipe to file.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) static long do_splice_from(struct pipe_inode_info *pipe, struct file *out,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) loff_t *ppos, size_t len, unsigned int flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) if (unlikely(!out->f_op->splice_write))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) return warn_unsupported(out, "write");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) return out->f_op->splice_write(pipe, out, ppos, len, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) * Attempt to initiate a splice from a file to a pipe.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) static long do_splice_to(struct file *in, loff_t *ppos,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) struct pipe_inode_info *pipe, size_t len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) unsigned int flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) if (unlikely(!(in->f_mode & FMODE_READ)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) return -EBADF;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) ret = rw_verify_area(READ, in, ppos, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) if (unlikely(ret < 0))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) if (unlikely(len > MAX_RW_COUNT))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) len = MAX_RW_COUNT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) if (unlikely(!in->f_op->splice_read))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) return warn_unsupported(in, "read");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) return in->f_op->splice_read(in, ppos, pipe, len, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) * splice_direct_to_actor - splices data directly between two non-pipes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) * @in: file to splice from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) * @sd: actor information on where to splice to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) * @actor: handles the data splicing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) * Description:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) * This is a special case helper to splice directly between two
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) * points, without requiring an explicit pipe. Internally an allocated
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) * pipe is cached in the process, and reused during the lifetime of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) * that process.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) ssize_t splice_direct_to_actor(struct file *in, struct splice_desc *sd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) splice_direct_actor *actor)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) struct pipe_inode_info *pipe;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) long ret, bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) umode_t i_mode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) size_t len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) int i, flags, more;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) * We require the input being a regular file, as we don't want to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) * randomly drop data for eg socket -> socket splicing. Use the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) * piped splicing for that!
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) i_mode = file_inode(in)->i_mode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) if (unlikely(!S_ISREG(i_mode) && !S_ISBLK(i_mode)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) * neither in nor out is a pipe, setup an internal pipe attached to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) * 'out' and transfer the wanted data from 'in' to 'out' through that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) pipe = current->splice_pipe;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) if (unlikely(!pipe)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) pipe = alloc_pipe_info();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) if (!pipe)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) * We don't have an immediate reader, but we'll read the stuff
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) * out of the pipe right after the splice_to_pipe(). So set
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) * PIPE_READERS appropriately.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) pipe->readers = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) current->splice_pipe = pipe;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) * Do the splice.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) bytes = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) len = sd->total_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) flags = sd->flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) * Don't block on output, we have to drain the direct pipe.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) sd->flags &= ~SPLICE_F_NONBLOCK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) more = sd->flags & SPLICE_F_MORE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) WARN_ON_ONCE(!pipe_empty(pipe->head, pipe->tail));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) while (len) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) unsigned int p_space;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) size_t read_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) loff_t pos = sd->pos, prev_pos = pos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) /* Don't try to read more the pipe has space for. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) p_space = pipe->max_usage -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) pipe_occupancy(pipe->head, pipe->tail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) read_len = min_t(size_t, len, p_space << PAGE_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) ret = do_splice_to(in, &pos, pipe, read_len, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) if (unlikely(ret <= 0))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) goto out_release;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) read_len = ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) sd->total_len = read_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) * If more data is pending, set SPLICE_F_MORE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) * If this is the last data and SPLICE_F_MORE was not set
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) * initially, clears it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) if (read_len < len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) sd->flags |= SPLICE_F_MORE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) else if (!more)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) sd->flags &= ~SPLICE_F_MORE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) * NOTE: nonblocking mode only applies to the input. We
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) * must not do the output in nonblocking mode as then we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) * could get stuck data in the internal pipe:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888) ret = actor(pipe, sd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) if (unlikely(ret <= 0)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) sd->pos = prev_pos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) goto out_release;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) bytes += ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) len -= ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) sd->pos = pos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) if (ret < read_len) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) sd->pos = prev_pos + ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) goto out_release;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) done:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) pipe->tail = pipe->head = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) file_accessed(in);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907) return bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) out_release:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) * If we did an incomplete transfer we must release
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) * the pipe buffers in question:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) for (i = 0; i < pipe->ring_size; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) struct pipe_buffer *buf = &pipe->bufs[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) if (buf->ops)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) pipe_buf_release(pipe, buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) if (!bytes)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) bytes = ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924) goto done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926) EXPORT_SYMBOL(splice_direct_to_actor);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) static int direct_splice_actor(struct pipe_inode_info *pipe,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) struct splice_desc *sd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) struct file *file = sd->u.file;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) return do_splice_from(pipe, file, sd->opos, sd->total_len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) sd->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938) * do_splice_direct - splices data directly between two files
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) * @in: file to splice from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) * @ppos: input file offset
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) * @out: file to splice to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) * @opos: output file offset
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943) * @len: number of bytes to splice
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) * @flags: splice modifier flags
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946) * Description:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947) * For use by do_sendfile(). splice can easily emulate sendfile, but
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) * doing it in the application would incur an extra system call
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949) * (splice in + splice out, as compared to just sendfile()). So this helper
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950) * can splice directly through a process-private pipe.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) long do_splice_direct(struct file *in, loff_t *ppos, struct file *out,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) loff_t *opos, size_t len, unsigned int flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956) struct splice_desc sd = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957) .len = len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) .total_len = len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959) .flags = flags,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960) .pos = *ppos,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961) .u.file = out,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962) .opos = opos,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964) long ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966) if (unlikely(!(out->f_mode & FMODE_WRITE)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967) return -EBADF;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969) if (unlikely(out->f_flags & O_APPEND))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972) ret = rw_verify_area(WRITE, out, opos, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973) if (unlikely(ret < 0))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976) ret = splice_direct_to_actor(in, &sd, direct_splice_actor);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977) if (ret > 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978) *ppos = sd.pos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982) EXPORT_SYMBOL(do_splice_direct);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984) static int wait_for_space(struct pipe_inode_info *pipe, unsigned flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986) for (;;) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987) if (unlikely(!pipe->readers)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988) send_sig(SIGPIPE, current, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989) return -EPIPE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991) if (!pipe_full(pipe->head, pipe->tail, pipe->max_usage))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993) if (flags & SPLICE_F_NONBLOCK)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994) return -EAGAIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995) if (signal_pending(current))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996) return -ERESTARTSYS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997) pipe_wait_writable(pipe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001) static int splice_pipe_to_pipe(struct pipe_inode_info *ipipe,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002) struct pipe_inode_info *opipe,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003) size_t len, unsigned int flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006) * Determine where to splice to/from.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008) long do_splice(struct file *in, loff_t *off_in, struct file *out,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009) loff_t *off_out, size_t len, unsigned int flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011) struct pipe_inode_info *ipipe;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012) struct pipe_inode_info *opipe;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013) loff_t offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014) long ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016) if (unlikely(!(in->f_mode & FMODE_READ) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017) !(out->f_mode & FMODE_WRITE)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018) return -EBADF;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020) ipipe = get_pipe_info(in, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021) opipe = get_pipe_info(out, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023) if (ipipe && opipe) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024) if (off_in || off_out)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025) return -ESPIPE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027) /* Splicing to self would be fun, but... */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028) if (ipipe == opipe)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031) if ((in->f_flags | out->f_flags) & O_NONBLOCK)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032) flags |= SPLICE_F_NONBLOCK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034) return splice_pipe_to_pipe(ipipe, opipe, len, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037) if (ipipe) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038) if (off_in)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039) return -ESPIPE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040) if (off_out) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041) if (!(out->f_mode & FMODE_PWRITE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043) offset = *off_out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045) offset = out->f_pos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048) if (unlikely(out->f_flags & O_APPEND))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051) ret = rw_verify_area(WRITE, out, &offset, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052) if (unlikely(ret < 0))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055) if (in->f_flags & O_NONBLOCK)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056) flags |= SPLICE_F_NONBLOCK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058) file_start_write(out);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059) ret = do_splice_from(ipipe, out, &offset, len, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060) file_end_write(out);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1061)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1062) if (!off_out)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1063) out->f_pos = offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1064) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1065) *off_out = offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1066)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1067) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1068) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1069)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1070) if (opipe) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1071) if (off_out)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1072) return -ESPIPE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1073) if (off_in) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1074) if (!(in->f_mode & FMODE_PREAD))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1075) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1076) offset = *off_in;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1077) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1078) offset = in->f_pos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1079) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1080)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1081) if (out->f_flags & O_NONBLOCK)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1082) flags |= SPLICE_F_NONBLOCK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1083)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1084) pipe_lock(opipe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1085) ret = wait_for_space(opipe, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1086) if (!ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1087) unsigned int p_space;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1088)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1089) /* Don't try to read more the pipe has space for. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1090) p_space = opipe->max_usage - pipe_occupancy(opipe->head, opipe->tail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1091) len = min_t(size_t, len, p_space << PAGE_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1092)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1093) ret = do_splice_to(in, &offset, opipe, len, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1094) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1095) pipe_unlock(opipe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1096) if (ret > 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1097) wakeup_pipe_readers(opipe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1098) if (!off_in)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1099) in->f_pos = offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1100) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1101) *off_in = offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1102)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1103) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1104) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1105)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1106) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1107) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1108)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1109) static long __do_splice(struct file *in, loff_t __user *off_in,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1110) struct file *out, loff_t __user *off_out,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1111) size_t len, unsigned int flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1112) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1113) struct pipe_inode_info *ipipe;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1114) struct pipe_inode_info *opipe;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1115) loff_t offset, *__off_in = NULL, *__off_out = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1116) long ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1117)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1118) ipipe = get_pipe_info(in, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1119) opipe = get_pipe_info(out, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1120)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1121) if (ipipe && off_in)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1122) return -ESPIPE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1123) if (opipe && off_out)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1124) return -ESPIPE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1125)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1126) if (off_out) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1127) if (copy_from_user(&offset, off_out, sizeof(loff_t)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1128) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1129) __off_out = &offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1130) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1131) if (off_in) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1132) if (copy_from_user(&offset, off_in, sizeof(loff_t)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1133) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1134) __off_in = &offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1135) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1136)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1137) ret = do_splice(in, __off_in, out, __off_out, len, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1138) if (ret < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1139) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1140)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1141) if (__off_out && copy_to_user(off_out, __off_out, sizeof(loff_t)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1142) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1143) if (__off_in && copy_to_user(off_in, __off_in, sizeof(loff_t)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1144) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1145)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1146) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1147) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1148)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1149) static int iter_to_pipe(struct iov_iter *from,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1150) struct pipe_inode_info *pipe,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1151) unsigned flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1152) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1153) struct pipe_buffer buf = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1154) .ops = &user_page_pipe_buf_ops,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1155) .flags = flags
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1156) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1157) size_t total = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1158) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1159) bool failed = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1160)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1161) while (iov_iter_count(from) && !failed) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1162) struct page *pages[16];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1163) ssize_t copied;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1164) size_t start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1165) int n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1166)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1167) copied = iov_iter_get_pages(from, pages, ~0UL, 16, &start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1168) if (copied <= 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1169) ret = copied;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1170) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1171) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1172)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1173) for (n = 0; copied; n++, start = 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1174) int size = min_t(int, copied, PAGE_SIZE - start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1175) if (!failed) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1176) buf.page = pages[n];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1177) buf.offset = start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1178) buf.len = size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1179) ret = add_to_pipe(pipe, &buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1180) if (unlikely(ret < 0)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1181) failed = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1182) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1183) iov_iter_advance(from, ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1184) total += ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1185) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1186) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1187) put_page(pages[n]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1188) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1189) copied -= size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1190) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1191) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1192) return total ? total : ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1193) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1194)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1195) static int pipe_to_user(struct pipe_inode_info *pipe, struct pipe_buffer *buf,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1196) struct splice_desc *sd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1197) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1198) int n = copy_page_to_iter(buf->page, buf->offset, sd->len, sd->u.data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1199) return n == sd->len ? n : -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1200) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1201)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1202) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1203) * For lack of a better implementation, implement vmsplice() to userspace
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1204) * as a simple copy of the pipes pages to the user iov.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1205) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1206) static long vmsplice_to_user(struct file *file, struct iov_iter *iter,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1207) unsigned int flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1208) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1209) struct pipe_inode_info *pipe = get_pipe_info(file, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1210) struct splice_desc sd = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1211) .total_len = iov_iter_count(iter),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1212) .flags = flags,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1213) .u.data = iter
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1214) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1215) long ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1216)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1217) if (!pipe)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1218) return -EBADF;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1219)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1220) if (sd.total_len) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1221) pipe_lock(pipe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1222) ret = __splice_from_pipe(pipe, &sd, pipe_to_user);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1223) pipe_unlock(pipe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1224) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1225)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1226) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1227) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1228)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1229) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1230) * vmsplice splices a user address range into a pipe. It can be thought of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1231) * as splice-from-memory, where the regular splice is splice-from-file (or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1232) * to file). In both cases the output is a pipe, naturally.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1233) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1234) static long vmsplice_to_pipe(struct file *file, struct iov_iter *iter,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1235) unsigned int flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1236) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1237) struct pipe_inode_info *pipe;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1238) long ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1239) unsigned buf_flag = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1240)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1241) if (flags & SPLICE_F_GIFT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1242) buf_flag = PIPE_BUF_FLAG_GIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1243)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1244) pipe = get_pipe_info(file, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1245) if (!pipe)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1246) return -EBADF;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1247)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1248) pipe_lock(pipe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1249) ret = wait_for_space(pipe, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1250) if (!ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1251) ret = iter_to_pipe(iter, pipe, buf_flag);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1252) pipe_unlock(pipe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1253) if (ret > 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1254) wakeup_pipe_readers(pipe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1255) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1256) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1257)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1258) static int vmsplice_type(struct fd f, int *type)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1259) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1260) if (!f.file)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1261) return -EBADF;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1262) if (f.file->f_mode & FMODE_WRITE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1263) *type = WRITE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1264) } else if (f.file->f_mode & FMODE_READ) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1265) *type = READ;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1266) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1267) fdput(f);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1268) return -EBADF;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1269) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1270) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1271) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1272)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1273) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1274) * Note that vmsplice only really supports true splicing _from_ user memory
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1275) * to a pipe, not the other way around. Splicing from user memory is a simple
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1276) * operation that can be supported without any funky alignment restrictions
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1277) * or nasty vm tricks. We simply map in the user memory and fill them into
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1278) * a pipe. The reverse isn't quite as easy, though. There are two possible
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1279) * solutions for that:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1280) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1281) * - memcpy() the data internally, at which point we might as well just
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1282) * do a regular read() on the buffer anyway.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1283) * - Lots of nasty vm tricks, that are neither fast nor flexible (it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1284) * has restriction limitations on both ends of the pipe).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1285) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1286) * Currently we punt and implement it as a normal copy, see pipe_to_user().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1287) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1288) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1289) SYSCALL_DEFINE4(vmsplice, int, fd, const struct iovec __user *, uiov,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1290) unsigned long, nr_segs, unsigned int, flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1291) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1292) struct iovec iovstack[UIO_FASTIOV];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1293) struct iovec *iov = iovstack;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1294) struct iov_iter iter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1295) ssize_t error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1296) struct fd f;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1297) int type;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1298)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1299) if (unlikely(flags & ~SPLICE_F_ALL))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1300) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1301)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1302) f = fdget(fd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1303) error = vmsplice_type(f, &type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1304) if (error)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1305) return error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1306)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1307) error = import_iovec(type, uiov, nr_segs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1308) ARRAY_SIZE(iovstack), &iov, &iter);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1309) if (error < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1310) goto out_fdput;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1311)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1312) if (!iov_iter_count(&iter))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1313) error = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1314) else if (iov_iter_rw(&iter) == WRITE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1315) error = vmsplice_to_pipe(f.file, &iter, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1316) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1317) error = vmsplice_to_user(f.file, &iter, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1318)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1319) kfree(iov);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1320) out_fdput:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1321) fdput(f);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1322) return error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1323) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1324)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1325) SYSCALL_DEFINE6(splice, int, fd_in, loff_t __user *, off_in,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1326) int, fd_out, loff_t __user *, off_out,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1327) size_t, len, unsigned int, flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1328) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1329) struct fd in, out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1330) long error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1331)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1332) if (unlikely(!len))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1333) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1334)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1335) if (unlikely(flags & ~SPLICE_F_ALL))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1336) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1337)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1338) error = -EBADF;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1339) in = fdget(fd_in);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1340) if (in.file) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1341) out = fdget(fd_out);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1342) if (out.file) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1343) error = __do_splice(in.file, off_in, out.file, off_out,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1344) len, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1345) fdput(out);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1346) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1347) fdput(in);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1348) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1349) return error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1350) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1351)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1352) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1353) * Make sure there's data to read. Wait for input if we can, otherwise
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1354) * return an appropriate error.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1355) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1356) static int ipipe_prep(struct pipe_inode_info *pipe, unsigned int flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1357) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1358) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1359)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1360) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1361) * Check the pipe occupancy without the inode lock first. This function
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1362) * is speculative anyways, so missing one is ok.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1363) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1364) if (!pipe_empty(pipe->head, pipe->tail))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1365) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1366)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1367) ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1368) pipe_lock(pipe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1369)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1370) while (pipe_empty(pipe->head, pipe->tail)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1371) if (signal_pending(current)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1372) ret = -ERESTARTSYS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1373) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1374) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1375) if (!pipe->writers)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1376) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1377) if (flags & SPLICE_F_NONBLOCK) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1378) ret = -EAGAIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1379) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1380) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1381) pipe_wait_readable(pipe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1382) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1383)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1384) pipe_unlock(pipe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1385) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1386) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1387)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1388) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1389) * Make sure there's writeable room. Wait for room if we can, otherwise
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1390) * return an appropriate error.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1391) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1392) static int opipe_prep(struct pipe_inode_info *pipe, unsigned int flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1393) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1394) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1395)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1396) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1397) * Check pipe occupancy without the inode lock first. This function
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1398) * is speculative anyways, so missing one is ok.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1399) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1400) if (!pipe_full(pipe->head, pipe->tail, pipe->max_usage))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1401) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1402)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1403) ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1404) pipe_lock(pipe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1405)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1406) while (pipe_full(pipe->head, pipe->tail, pipe->max_usage)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1407) if (!pipe->readers) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1408) send_sig(SIGPIPE, current, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1409) ret = -EPIPE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1410) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1411) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1412) if (flags & SPLICE_F_NONBLOCK) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1413) ret = -EAGAIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1414) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1415) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1416) if (signal_pending(current)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1417) ret = -ERESTARTSYS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1418) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1419) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1420) pipe_wait_writable(pipe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1421) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1422)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1423) pipe_unlock(pipe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1424) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1425) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1426)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1427) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1428) * Splice contents of ipipe to opipe.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1429) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1430) static int splice_pipe_to_pipe(struct pipe_inode_info *ipipe,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1431) struct pipe_inode_info *opipe,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1432) size_t len, unsigned int flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1433) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1434) struct pipe_buffer *ibuf, *obuf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1435) unsigned int i_head, o_head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1436) unsigned int i_tail, o_tail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1437) unsigned int i_mask, o_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1438) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1439) bool input_wakeup = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1440)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1441)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1442) retry:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1443) ret = ipipe_prep(ipipe, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1444) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1445) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1446)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1447) ret = opipe_prep(opipe, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1448) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1449) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1450)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1451) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1452) * Potential ABBA deadlock, work around it by ordering lock
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1453) * grabbing by pipe info address. Otherwise two different processes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1454) * could deadlock (one doing tee from A -> B, the other from B -> A).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1455) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1456) pipe_double_lock(ipipe, opipe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1457)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1458) i_tail = ipipe->tail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1459) i_mask = ipipe->ring_size - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1460) o_head = opipe->head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1461) o_mask = opipe->ring_size - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1462)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1463) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1464) size_t o_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1465)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1466) if (!opipe->readers) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1467) send_sig(SIGPIPE, current, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1468) if (!ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1469) ret = -EPIPE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1470) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1471) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1472)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1473) i_head = ipipe->head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1474) o_tail = opipe->tail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1475)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1476) if (pipe_empty(i_head, i_tail) && !ipipe->writers)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1477) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1478)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1479) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1480) * Cannot make any progress, because either the input
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1481) * pipe is empty or the output pipe is full.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1482) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1483) if (pipe_empty(i_head, i_tail) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1484) pipe_full(o_head, o_tail, opipe->max_usage)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1485) /* Already processed some buffers, break */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1486) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1487) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1488)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1489) if (flags & SPLICE_F_NONBLOCK) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1490) ret = -EAGAIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1491) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1492) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1493)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1494) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1495) * We raced with another reader/writer and haven't
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1496) * managed to process any buffers. A zero return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1497) * value means EOF, so retry instead.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1498) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1499) pipe_unlock(ipipe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1500) pipe_unlock(opipe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1501) goto retry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1502) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1503)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1504) ibuf = &ipipe->bufs[i_tail & i_mask];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1505) obuf = &opipe->bufs[o_head & o_mask];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1506)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1507) if (len >= ibuf->len) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1508) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1509) * Simply move the whole buffer from ipipe to opipe
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1510) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1511) *obuf = *ibuf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1512) ibuf->ops = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1513) i_tail++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1514) ipipe->tail = i_tail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1515) input_wakeup = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1516) o_len = obuf->len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1517) o_head++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1518) opipe->head = o_head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1519) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1520) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1521) * Get a reference to this pipe buffer,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1522) * so we can copy the contents over.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1523) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1524) if (!pipe_buf_get(ipipe, ibuf)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1525) if (ret == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1526) ret = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1527) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1528) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1529) *obuf = *ibuf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1530)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1531) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1532) * Don't inherit the gift and merge flags, we need to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1533) * prevent multiple steals of this page.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1534) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1535) obuf->flags &= ~PIPE_BUF_FLAG_GIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1536) obuf->flags &= ~PIPE_BUF_FLAG_CAN_MERGE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1537)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1538) obuf->len = len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1539) ibuf->offset += len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1540) ibuf->len -= len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1541) o_len = len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1542) o_head++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1543) opipe->head = o_head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1544) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1545) ret += o_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1546) len -= o_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1547) } while (len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1548)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1549) pipe_unlock(ipipe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1550) pipe_unlock(opipe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1551)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1552) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1553) * If we put data in the output pipe, wakeup any potential readers.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1554) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1555) if (ret > 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1556) wakeup_pipe_readers(opipe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1557)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1558) if (input_wakeup)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1559) wakeup_pipe_writers(ipipe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1560)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1561) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1562) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1563)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1564) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1565) * Link contents of ipipe to opipe.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1566) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1567) static int link_pipe(struct pipe_inode_info *ipipe,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1568) struct pipe_inode_info *opipe,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1569) size_t len, unsigned int flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1570) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1571) struct pipe_buffer *ibuf, *obuf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1572) unsigned int i_head, o_head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1573) unsigned int i_tail, o_tail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1574) unsigned int i_mask, o_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1575) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1576)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1577) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1578) * Potential ABBA deadlock, work around it by ordering lock
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1579) * grabbing by pipe info address. Otherwise two different processes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1580) * could deadlock (one doing tee from A -> B, the other from B -> A).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1581) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1582) pipe_double_lock(ipipe, opipe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1583)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1584) i_tail = ipipe->tail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1585) i_mask = ipipe->ring_size - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1586) o_head = opipe->head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1587) o_mask = opipe->ring_size - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1588)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1589) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1590) if (!opipe->readers) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1591) send_sig(SIGPIPE, current, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1592) if (!ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1593) ret = -EPIPE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1594) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1595) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1596)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1597) i_head = ipipe->head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1598) o_tail = opipe->tail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1599)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1600) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1601) * If we have iterated all input buffers or run out of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1602) * output room, break.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1603) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1604) if (pipe_empty(i_head, i_tail) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1605) pipe_full(o_head, o_tail, opipe->max_usage))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1606) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1607)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1608) ibuf = &ipipe->bufs[i_tail & i_mask];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1609) obuf = &opipe->bufs[o_head & o_mask];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1610)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1611) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1612) * Get a reference to this pipe buffer,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1613) * so we can copy the contents over.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1614) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1615) if (!pipe_buf_get(ipipe, ibuf)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1616) if (ret == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1617) ret = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1618) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1619) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1620)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1621) *obuf = *ibuf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1622)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1623) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1624) * Don't inherit the gift and merge flag, we need to prevent
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1625) * multiple steals of this page.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1626) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1627) obuf->flags &= ~PIPE_BUF_FLAG_GIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1628) obuf->flags &= ~PIPE_BUF_FLAG_CAN_MERGE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1629)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1630) if (obuf->len > len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1631) obuf->len = len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1632) ret += obuf->len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1633) len -= obuf->len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1634)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1635) o_head++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1636) opipe->head = o_head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1637) i_tail++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1638) } while (len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1639)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1640) pipe_unlock(ipipe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1641) pipe_unlock(opipe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1642)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1643) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1644) * If we put data in the output pipe, wakeup any potential readers.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1645) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1646) if (ret > 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1647) wakeup_pipe_readers(opipe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1648)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1649) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1650) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1651)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1652) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1653) * This is a tee(1) implementation that works on pipes. It doesn't copy
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1654) * any data, it simply references the 'in' pages on the 'out' pipe.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1655) * The 'flags' used are the SPLICE_F_* variants, currently the only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1656) * applicable one is SPLICE_F_NONBLOCK.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1657) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1658) long do_tee(struct file *in, struct file *out, size_t len, unsigned int flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1659) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1660) struct pipe_inode_info *ipipe = get_pipe_info(in, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1661) struct pipe_inode_info *opipe = get_pipe_info(out, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1662) int ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1663)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1664) if (unlikely(!(in->f_mode & FMODE_READ) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1665) !(out->f_mode & FMODE_WRITE)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1666) return -EBADF;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1667)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1668) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1669) * Duplicate the contents of ipipe to opipe without actually
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1670) * copying the data.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1671) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1672) if (ipipe && opipe && ipipe != opipe) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1673) if ((in->f_flags | out->f_flags) & O_NONBLOCK)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1674) flags |= SPLICE_F_NONBLOCK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1675)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1676) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1677) * Keep going, unless we encounter an error. The ipipe/opipe
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1678) * ordering doesn't really matter.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1679) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1680) ret = ipipe_prep(ipipe, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1681) if (!ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1682) ret = opipe_prep(opipe, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1683) if (!ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1684) ret = link_pipe(ipipe, opipe, len, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1685) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1686) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1687)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1688) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1689) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1690)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1691) SYSCALL_DEFINE4(tee, int, fdin, int, fdout, size_t, len, unsigned int, flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1692) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1693) struct fd in, out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1694) int error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1695)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1696) if (unlikely(flags & ~SPLICE_F_ALL))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1697) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1698)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1699) if (unlikely(!len))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1700) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1701)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1702) error = -EBADF;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1703) in = fdget(fdin);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1704) if (in.file) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1705) out = fdget(fdout);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1706) if (out.file) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1707) error = do_tee(in.file, out.file, len, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1708) fdput(out);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1709) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1710) fdput(in);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1711) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1712)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1713) return error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1714) }
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags