^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * linux/fs/pipe.c
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Copyright (C) 1991, 1992, 1999 Linus Torvalds
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include <linux/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <linux/file.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/poll.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/init.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/fs.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/log2.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/mount.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/pseudo_fs.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <linux/magic.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <linux/pipe_fs_i.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <linux/uio.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <linux/highmem.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include <linux/pagemap.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #include <linux/audit.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #include <linux/syscalls.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #include <linux/fcntl.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) #include <linux/memcontrol.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) #include <linux/watch_queue.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) #include <linux/uaccess.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) #include <asm/ioctls.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) #include "internal.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) * New pipe buffers will be restricted to this size while the user is exceeding
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) * their pipe buffer quota. The general pipe use case needs at least two
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) * buffers: one for data yet to be read, and one for new data. If this is less
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) * than two, then a write to a non-empty pipe may block even if the pipe is not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) * full. This can occur with GNU make jobserver or similar uses of pipes as
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) * semaphores: multiple processes may be waiting to write tokens back to the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) * pipe before reading tokens: https://lore.kernel.org/lkml/1628086770.5rn8p04n6j.none@localhost/.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) * Users can reduce their pipe buffers with F_SETPIPE_SZ below this at their
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) * own risk, namely: pipe writes to non-full pipes may block until the pipe is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) * emptied.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) #define PIPE_MIN_DEF_BUFFERS 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) * The max size that a non-root user is allowed to grow the pipe. Can
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) * be set by root in /proc/sys/fs/pipe-max-size
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) unsigned int pipe_max_size = 1048576;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) /* Maximum allocatable pages per user. Hard limit is unset by default, soft
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) * matches default values.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) unsigned long pipe_user_pages_hard;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) unsigned long pipe_user_pages_soft = PIPE_DEF_BUFFERS * INR_OPEN_CUR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) * We use head and tail indices that aren't masked off, except at the point of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) * dereference, but rather they're allowed to wrap naturally. This means there
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) * isn't a dead spot in the buffer, but the ring has to be a power of two and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) * <= 2^31.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) * -- David Howells 2019-09-23.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) * Reads with count = 0 should always return 0.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) * -- Julian Bradfield 1999-06-07.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) * FIFOs and Pipes now generate SIGIO for both readers and writers.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) * -- Jeremy Elson <jelson@circlemud.org> 2001-08-16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) * pipe_read & write cleanup
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) * -- Manfred Spraul <manfred@colorfullife.com> 2002-05-09
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) static void pipe_lock_nested(struct pipe_inode_info *pipe, int subclass)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) if (pipe->files)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) mutex_lock_nested(&pipe->mutex, subclass);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) void pipe_lock(struct pipe_inode_info *pipe)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) * pipe_lock() nests non-pipe inode locks (for writing to a file)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) pipe_lock_nested(pipe, I_MUTEX_PARENT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) EXPORT_SYMBOL(pipe_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) void pipe_unlock(struct pipe_inode_info *pipe)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) if (pipe->files)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) mutex_unlock(&pipe->mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) EXPORT_SYMBOL(pipe_unlock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) static inline void __pipe_lock(struct pipe_inode_info *pipe)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) mutex_lock_nested(&pipe->mutex, I_MUTEX_PARENT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) static inline void __pipe_unlock(struct pipe_inode_info *pipe)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) mutex_unlock(&pipe->mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) void pipe_double_lock(struct pipe_inode_info *pipe1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) struct pipe_inode_info *pipe2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) BUG_ON(pipe1 == pipe2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) if (pipe1 < pipe2) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) pipe_lock_nested(pipe1, I_MUTEX_PARENT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) pipe_lock_nested(pipe2, I_MUTEX_CHILD);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) pipe_lock_nested(pipe2, I_MUTEX_PARENT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) pipe_lock_nested(pipe1, I_MUTEX_CHILD);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) static void anon_pipe_buf_release(struct pipe_inode_info *pipe,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) struct pipe_buffer *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) struct page *page = buf->page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) * If nobody else uses this page, and we don't already have a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) * temporary page, let's keep track of it as a one-deep
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) * allocation cache. (Otherwise just release our reference to it)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) if (page_count(page) == 1 && !pipe->tmp_page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) pipe->tmp_page = page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) put_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) static bool anon_pipe_buf_try_steal(struct pipe_inode_info *pipe,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) struct pipe_buffer *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) struct page *page = buf->page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) if (page_count(page) != 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) memcg_kmem_uncharge_page(page, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) __SetPageLocked(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) * generic_pipe_buf_try_steal - attempt to take ownership of a &pipe_buffer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) * @pipe: the pipe that the buffer belongs to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) * @buf: the buffer to attempt to steal
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) * Description:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) * This function attempts to steal the &struct page attached to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) * @buf. If successful, this function returns 0 and returns with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) * the page locked. The caller may then reuse the page for whatever
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) * he wishes; the typical use is insertion into a different file
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) * page cache.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) bool generic_pipe_buf_try_steal(struct pipe_inode_info *pipe,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) struct pipe_buffer *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) struct page *page = buf->page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) * A reference of one is golden, that means that the owner of this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) * page is the only one holding a reference to it. lock the page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) * and return OK.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) if (page_count(page) == 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) lock_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) EXPORT_SYMBOL(generic_pipe_buf_try_steal);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) * generic_pipe_buf_get - get a reference to a &struct pipe_buffer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) * @pipe: the pipe that the buffer belongs to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) * @buf: the buffer to get a reference to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) * Description:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) * This function grabs an extra reference to @buf. It's used in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) * in the tee() system call, when we duplicate the buffers in one
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) * pipe into another.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) bool generic_pipe_buf_get(struct pipe_inode_info *pipe, struct pipe_buffer *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) return try_get_page(buf->page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) EXPORT_SYMBOL(generic_pipe_buf_get);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) * generic_pipe_buf_release - put a reference to a &struct pipe_buffer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) * @pipe: the pipe that the buffer belongs to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) * @buf: the buffer to put a reference to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) * Description:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) * This function releases a reference to @buf.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) void generic_pipe_buf_release(struct pipe_inode_info *pipe,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) struct pipe_buffer *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) put_page(buf->page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) EXPORT_SYMBOL(generic_pipe_buf_release);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) static const struct pipe_buf_operations anon_pipe_buf_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) .release = anon_pipe_buf_release,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) .try_steal = anon_pipe_buf_try_steal,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) .get = generic_pipe_buf_get,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) /* Done while waiting without holding the pipe lock - thus the READ_ONCE() */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) static inline bool pipe_readable(const struct pipe_inode_info *pipe)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) unsigned int head = READ_ONCE(pipe->head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) unsigned int tail = READ_ONCE(pipe->tail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) unsigned int writers = READ_ONCE(pipe->writers);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) return !pipe_empty(head, tail) || !writers;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) static ssize_t
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) pipe_read(struct kiocb *iocb, struct iov_iter *to)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) size_t total_len = iov_iter_count(to);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) struct file *filp = iocb->ki_filp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) struct pipe_inode_info *pipe = filp->private_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) bool was_full, wake_next_reader = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) ssize_t ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) /* Null read succeeds. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) if (unlikely(total_len == 0))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) __pipe_lock(pipe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) * We only wake up writers if the pipe was full when we started
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) * reading in order to avoid unnecessary wakeups.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) * But when we do wake up writers, we do so using a sync wakeup
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) * (WF_SYNC), because we want them to get going and generate more
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) * data for us.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) was_full = pipe_full(pipe->head, pipe->tail, pipe->max_usage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) for (;;) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) /* Read ->head with a barrier vs post_one_notification() */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) unsigned int head = smp_load_acquire(&pipe->head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) unsigned int tail = pipe->tail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) unsigned int mask = pipe->ring_size - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) #ifdef CONFIG_WATCH_QUEUE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) if (pipe->note_loss) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) struct watch_notification n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) if (total_len < 8) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) if (ret == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) ret = -ENOBUFS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) n.type = WATCH_TYPE_META;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) n.subtype = WATCH_META_LOSS_NOTIFICATION;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) n.info = watch_sizeof(n);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) if (copy_to_iter(&n, sizeof(n), to) != sizeof(n)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) if (ret == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) ret = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) ret += sizeof(n);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) total_len -= sizeof(n);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) pipe->note_loss = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) if (!pipe_empty(head, tail)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) struct pipe_buffer *buf = &pipe->bufs[tail & mask];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) size_t chars = buf->len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) size_t written;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) int error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) if (chars > total_len) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) if (buf->flags & PIPE_BUF_FLAG_WHOLE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) if (ret == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) ret = -ENOBUFS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) chars = total_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) error = pipe_buf_confirm(pipe, buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) if (error) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) if (!ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) ret = error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) written = copy_page_to_iter(buf->page, buf->offset, chars, to);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) if (unlikely(written < chars)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) if (!ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) ret = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) ret += chars;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) buf->offset += chars;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) buf->len -= chars;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) /* Was it a packet buffer? Clean up and exit */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) if (buf->flags & PIPE_BUF_FLAG_PACKET) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) total_len = chars;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) buf->len = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) if (!buf->len) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) pipe_buf_release(pipe, buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) spin_lock_irq(&pipe->rd_wait.lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) #ifdef CONFIG_WATCH_QUEUE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) if (buf->flags & PIPE_BUF_FLAG_LOSS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) pipe->note_loss = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) tail++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) pipe->tail = tail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) spin_unlock_irq(&pipe->rd_wait.lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) total_len -= chars;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) if (!total_len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) break; /* common path: read succeeded */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) if (!pipe_empty(head, tail)) /* More to do? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) if (!pipe->writers)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) if (filp->f_flags & O_NONBLOCK) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) ret = -EAGAIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) __pipe_unlock(pipe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) * We only get here if we didn't actually read anything.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) * However, we could have seen (and removed) a zero-sized
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) * pipe buffer, and might have made space in the buffers
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) * that way.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) * You can't make zero-sized pipe buffers by doing an empty
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) * write (not even in packet mode), but they can happen if
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) * the writer gets an EFAULT when trying to fill a buffer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) * that already got allocated and inserted in the buffer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) * array.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) * So we still need to wake up any pending writers in the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) * _very_ unlikely case that the pipe was full, but we got
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) * no data.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) if (unlikely(was_full)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) wake_up_interruptible_sync_poll(&pipe->wr_wait, EPOLLOUT | EPOLLWRNORM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) * But because we didn't read anything, at this point we can
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) * just return directly with -ERESTARTSYS if we're interrupted,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) * since we've done any required wakeups and there's no need
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) * to mark anything accessed. And we've dropped the lock.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) if (wait_event_interruptible_exclusive(pipe->rd_wait, pipe_readable(pipe)) < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) return -ERESTARTSYS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) __pipe_lock(pipe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) was_full = pipe_full(pipe->head, pipe->tail, pipe->max_usage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) wake_next_reader = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) if (pipe_empty(pipe->head, pipe->tail))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) wake_next_reader = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) __pipe_unlock(pipe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) if (was_full) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) wake_up_interruptible_sync_poll(&pipe->wr_wait, EPOLLOUT | EPOLLWRNORM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) if (wake_next_reader)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) wake_up_interruptible_sync_poll(&pipe->rd_wait, EPOLLIN | EPOLLRDNORM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) if (ret > 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) file_accessed(filp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) static inline int is_packetized(struct file *file)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) return (file->f_flags & O_DIRECT) != 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) /* Done while waiting without holding the pipe lock - thus the READ_ONCE() */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) static inline bool pipe_writable(const struct pipe_inode_info *pipe)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) unsigned int head = READ_ONCE(pipe->head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) unsigned int tail = READ_ONCE(pipe->tail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) unsigned int max_usage = READ_ONCE(pipe->max_usage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) return !pipe_full(head, tail, max_usage) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) !READ_ONCE(pipe->readers);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) static ssize_t
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) pipe_write(struct kiocb *iocb, struct iov_iter *from)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) struct file *filp = iocb->ki_filp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) struct pipe_inode_info *pipe = filp->private_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) unsigned int head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) ssize_t ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) size_t total_len = iov_iter_count(from);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) ssize_t chars;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) bool was_empty = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) bool wake_next_writer = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) /* Null write succeeds. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) if (unlikely(total_len == 0))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) __pipe_lock(pipe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) if (!pipe->readers) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) send_sig(SIGPIPE, current, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) ret = -EPIPE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) #ifdef CONFIG_WATCH_QUEUE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) if (pipe->watch_queue) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) ret = -EXDEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) * Epoll nonsensically wants a wakeup whether the pipe
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) * was already empty or not.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) * If it wasn't empty we try to merge new data into
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) * the last buffer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) * That naturally merges small writes, but it also
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) * page-aligns the rest of the writes for large writes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) * spanning multiple pages.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) head = pipe->head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) was_empty = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) chars = total_len & (PAGE_SIZE-1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) if (chars && !pipe_empty(head, pipe->tail)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) unsigned int mask = pipe->ring_size - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) struct pipe_buffer *buf = &pipe->bufs[(head - 1) & mask];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) int offset = buf->offset + buf->len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) if ((buf->flags & PIPE_BUF_FLAG_CAN_MERGE) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) offset + chars <= PAGE_SIZE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) ret = pipe_buf_confirm(pipe, buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) ret = copy_page_from_iter(buf->page, offset, chars, from);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) if (unlikely(ret < chars)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) ret = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) buf->len += ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) if (!iov_iter_count(from))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) for (;;) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) if (!pipe->readers) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) send_sig(SIGPIPE, current, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) if (!ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) ret = -EPIPE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) head = pipe->head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) if (!pipe_full(head, pipe->tail, pipe->max_usage)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) unsigned int mask = pipe->ring_size - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) struct pipe_buffer *buf = &pipe->bufs[head & mask];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) struct page *page = pipe->tmp_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) int copied;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) if (!page) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) page = alloc_page(GFP_HIGHUSER | __GFP_ACCOUNT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) if (unlikely(!page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) ret = ret ? : -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) pipe->tmp_page = page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) /* Allocate a slot in the ring in advance and attach an
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) * empty buffer. If we fault or otherwise fail to use
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) * it, either the reader will consume it or it'll still
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) * be there for the next write.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) spin_lock_irq(&pipe->rd_wait.lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) head = pipe->head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) if (pipe_full(head, pipe->tail, pipe->max_usage)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) spin_unlock_irq(&pipe->rd_wait.lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) pipe->head = head + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) spin_unlock_irq(&pipe->rd_wait.lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) /* Insert it into the buffer array */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) buf = &pipe->bufs[head & mask];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) buf->page = page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) buf->ops = &anon_pipe_buf_ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) buf->offset = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) buf->len = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) if (is_packetized(filp))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) buf->flags = PIPE_BUF_FLAG_PACKET;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) buf->flags = PIPE_BUF_FLAG_CAN_MERGE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) pipe->tmp_page = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) copied = copy_page_from_iter(page, 0, PAGE_SIZE, from);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) if (unlikely(copied < PAGE_SIZE && iov_iter_count(from))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) if (!ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) ret = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) ret += copied;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) buf->offset = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) buf->len = copied;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) if (!iov_iter_count(from))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) if (!pipe_full(head, pipe->tail, pipe->max_usage))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) /* Wait for buffer space to become available. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) if (filp->f_flags & O_NONBLOCK) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) if (!ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) ret = -EAGAIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) if (signal_pending(current)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) if (!ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) ret = -ERESTARTSYS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) * We're going to release the pipe lock and wait for more
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) * space. We wake up any readers if necessary, and then
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) * after waiting we need to re-check whether the pipe
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) * become empty while we dropped the lock.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) __pipe_unlock(pipe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) if (was_empty) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) wake_up_interruptible_sync_poll(&pipe->rd_wait, EPOLLIN | EPOLLRDNORM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) wait_event_interruptible_exclusive(pipe->wr_wait, pipe_writable(pipe));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) __pipe_lock(pipe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) was_empty = pipe_empty(pipe->head, pipe->tail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) wake_next_writer = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) if (pipe_full(pipe->head, pipe->tail, pipe->max_usage))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) wake_next_writer = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) __pipe_unlock(pipe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) * If we do do a wakeup event, we do a 'sync' wakeup, because we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) * want the reader to start processing things asap, rather than
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) * leave the data pending.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) * This is particularly important for small writes, because of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) * how (for example) the GNU make jobserver uses small writes to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) * wake up pending jobs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) if (was_empty) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) wake_up_interruptible_sync_poll(&pipe->rd_wait, EPOLLIN | EPOLLRDNORM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) if (wake_next_writer)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) wake_up_interruptible_sync_poll(&pipe->wr_wait, EPOLLOUT | EPOLLWRNORM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) if (ret > 0 && sb_start_write_trylock(file_inode(filp)->i_sb)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) int err = file_update_time(filp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) ret = err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) sb_end_write(file_inode(filp)->i_sb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) static long pipe_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) struct pipe_inode_info *pipe = filp->private_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) int count, head, tail, mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) switch (cmd) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) case FIONREAD:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) __pipe_lock(pipe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) count = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) head = pipe->head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) tail = pipe->tail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) mask = pipe->ring_size - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) while (tail != head) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) count += pipe->bufs[tail & mask].len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) tail++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) __pipe_unlock(pipe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) return put_user(count, (int __user *)arg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) #ifdef CONFIG_WATCH_QUEUE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) case IOC_WATCH_QUEUE_SET_SIZE: {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) __pipe_lock(pipe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) ret = watch_queue_set_size(pipe, arg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) __pipe_unlock(pipe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) case IOC_WATCH_QUEUE_SET_FILTER:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) return watch_queue_set_filter(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) pipe, (struct watch_notification_filter __user *)arg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) return -ENOIOCTLCMD;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) /* No kernel lock held - fine */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) static __poll_t
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) pipe_poll(struct file *filp, poll_table *wait)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) __poll_t mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) struct pipe_inode_info *pipe = filp->private_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) unsigned int head, tail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) * Reading pipe state only -- no need for acquiring the semaphore.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) * But because this is racy, the code has to add the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) * entry to the poll table _first_ ..
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) if (filp->f_mode & FMODE_READ)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) poll_wait(filp, &pipe->rd_wait, wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) if (filp->f_mode & FMODE_WRITE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) poll_wait(filp, &pipe->wr_wait, wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) * .. and only then can you do the racy tests. That way,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) * if something changes and you got it wrong, the poll
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) * table entry will wake you up and fix it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) head = READ_ONCE(pipe->head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) tail = READ_ONCE(pipe->tail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) mask = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) if (filp->f_mode & FMODE_READ) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) if (!pipe_empty(head, tail))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) mask |= EPOLLIN | EPOLLRDNORM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) if (!pipe->writers && filp->f_version != pipe->w_counter)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) mask |= EPOLLHUP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) if (filp->f_mode & FMODE_WRITE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) if (!pipe_full(head, tail, pipe->max_usage))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) mask |= EPOLLOUT | EPOLLWRNORM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) * Most Unices do not set EPOLLERR for FIFOs but on Linux they
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) * behave exactly like pipes for poll().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) if (!pipe->readers)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) mask |= EPOLLERR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) return mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) static void put_pipe_info(struct inode *inode, struct pipe_inode_info *pipe)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) int kill = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) spin_lock(&inode->i_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) if (!--pipe->files) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) inode->i_pipe = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) kill = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) spin_unlock(&inode->i_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) if (kill)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) free_pipe_info(pipe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) static int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) pipe_release(struct inode *inode, struct file *file)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) struct pipe_inode_info *pipe = file->private_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) __pipe_lock(pipe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) if (file->f_mode & FMODE_READ)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) pipe->readers--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) if (file->f_mode & FMODE_WRITE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) pipe->writers--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) /* Was that the last reader or writer, but not the other side? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) if (!pipe->readers != !pipe->writers) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) wake_up_interruptible_all(&pipe->rd_wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) wake_up_interruptible_all(&pipe->wr_wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) __pipe_unlock(pipe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) put_pipe_info(inode, pipe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) static int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) pipe_fasync(int fd, struct file *filp, int on)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) struct pipe_inode_info *pipe = filp->private_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) int retval = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) __pipe_lock(pipe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) if (filp->f_mode & FMODE_READ)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) retval = fasync_helper(fd, filp, on, &pipe->fasync_readers);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) if ((filp->f_mode & FMODE_WRITE) && retval >= 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) retval = fasync_helper(fd, filp, on, &pipe->fasync_writers);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) if (retval < 0 && (filp->f_mode & FMODE_READ))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) /* this can happen only if on == T */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) fasync_helper(-1, filp, 0, &pipe->fasync_readers);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) __pipe_unlock(pipe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) return retval;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) unsigned long account_pipe_buffers(struct user_struct *user,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) unsigned long old, unsigned long new)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) return atomic_long_add_return(new - old, &user->pipe_bufs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) bool too_many_pipe_buffers_soft(unsigned long user_bufs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) unsigned long soft_limit = READ_ONCE(pipe_user_pages_soft);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) return soft_limit && user_bufs > soft_limit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) bool too_many_pipe_buffers_hard(unsigned long user_bufs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) unsigned long hard_limit = READ_ONCE(pipe_user_pages_hard);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) return hard_limit && user_bufs > hard_limit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) bool pipe_is_unprivileged_user(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) return !capable(CAP_SYS_RESOURCE) && !capable(CAP_SYS_ADMIN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) struct pipe_inode_info *alloc_pipe_info(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) struct pipe_inode_info *pipe;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) unsigned long pipe_bufs = PIPE_DEF_BUFFERS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) struct user_struct *user = get_current_user();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) unsigned long user_bufs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) unsigned int max_size = READ_ONCE(pipe_max_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) pipe = kzalloc(sizeof(struct pipe_inode_info), GFP_KERNEL_ACCOUNT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) if (pipe == NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) goto out_free_uid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) if (pipe_bufs * PAGE_SIZE > max_size && !capable(CAP_SYS_RESOURCE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) pipe_bufs = max_size >> PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) user_bufs = account_pipe_buffers(user, 0, pipe_bufs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) if (too_many_pipe_buffers_soft(user_bufs) && pipe_is_unprivileged_user()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) user_bufs = account_pipe_buffers(user, pipe_bufs, PIPE_MIN_DEF_BUFFERS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) pipe_bufs = PIPE_MIN_DEF_BUFFERS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) if (too_many_pipe_buffers_hard(user_bufs) && pipe_is_unprivileged_user())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) goto out_revert_acct;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) pipe->bufs = kcalloc(pipe_bufs, sizeof(struct pipe_buffer),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) GFP_KERNEL_ACCOUNT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) if (pipe->bufs) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) init_waitqueue_head(&pipe->rd_wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) init_waitqueue_head(&pipe->wr_wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) pipe->r_counter = pipe->w_counter = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) pipe->max_usage = pipe_bufs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) pipe->ring_size = pipe_bufs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) pipe->nr_accounted = pipe_bufs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) pipe->user = user;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) mutex_init(&pipe->mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) return pipe;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) out_revert_acct:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) (void) account_pipe_buffers(user, pipe_bufs, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) kfree(pipe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) out_free_uid:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) free_uid(user);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) void free_pipe_info(struct pipe_inode_info *pipe)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) #ifdef CONFIG_WATCH_QUEUE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) if (pipe->watch_queue)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) watch_queue_clear(pipe->watch_queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) (void) account_pipe_buffers(pipe->user, pipe->nr_accounted, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) free_uid(pipe->user);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) for (i = 0; i < pipe->ring_size; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) struct pipe_buffer *buf = pipe->bufs + i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) if (buf->ops)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) pipe_buf_release(pipe, buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) #ifdef CONFIG_WATCH_QUEUE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) if (pipe->watch_queue)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) put_watch_queue(pipe->watch_queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) if (pipe->tmp_page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) __free_page(pipe->tmp_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) kfree(pipe->bufs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) kfree(pipe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) static struct vfsmount *pipe_mnt __read_mostly;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) * pipefs_dname() is called from d_path().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) static char *pipefs_dname(struct dentry *dentry, char *buffer, int buflen)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) return dynamic_dname(dentry, buffer, buflen, "pipe:[%lu]",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) d_inode(dentry)->i_ino);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) static const struct dentry_operations pipefs_dentry_operations = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) .d_dname = pipefs_dname,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) static struct inode * get_pipe_inode(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) struct inode *inode = new_inode_pseudo(pipe_mnt->mnt_sb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) struct pipe_inode_info *pipe;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) if (!inode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) goto fail_inode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) inode->i_ino = get_next_ino();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) pipe = alloc_pipe_info();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) if (!pipe)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) goto fail_iput;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) inode->i_pipe = pipe;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) pipe->files = 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) pipe->readers = pipe->writers = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888) inode->i_fop = &pipefifo_fops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) * Mark the inode dirty from the very beginning,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) * that way it will never be moved to the dirty
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893) * list because "mark_inode_dirty()" will think
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) * that it already _is_ on the dirty list.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) inode->i_state = I_DIRTY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) inode->i_mode = S_IFIFO | S_IRUSR | S_IWUSR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) inode->i_uid = current_fsuid();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) inode->i_gid = current_fsgid();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) inode->i_atime = inode->i_mtime = inode->i_ctime = current_time(inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) return inode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) fail_iput:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) iput(inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907) fail_inode:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) int create_pipe_files(struct file **res, int flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) struct inode *inode = get_pipe_inode();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) struct file *f;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) int error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) if (!inode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) return -ENFILE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) if (flags & O_NOTIFICATION_PIPE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) error = watch_queue_init(inode->i_pipe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) if (error) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) free_pipe_info(inode->i_pipe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924) iput(inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) return error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) f = alloc_file_pseudo(inode, pipe_mnt, "",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) O_WRONLY | (flags & (O_NONBLOCK | O_DIRECT)),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) &pipefifo_fops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932) if (IS_ERR(f)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) free_pipe_info(inode->i_pipe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) iput(inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) return PTR_ERR(f);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938) f->private_data = inode->i_pipe;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) res[0] = alloc_file_clone(f, O_RDONLY | (flags & O_NONBLOCK),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) &pipefifo_fops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) if (IS_ERR(res[0])) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943) put_pipe_info(inode, inode->i_pipe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) fput(f);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) return PTR_ERR(res[0]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947) res[0]->private_data = inode->i_pipe;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) res[1] = f;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949) stream_open(inode, res[0]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950) stream_open(inode, res[1]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) static int __do_pipe_flags(int *fd, struct file **files, int flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956) int error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957) int fdw, fdr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959) if (flags & ~(O_CLOEXEC | O_NONBLOCK | O_DIRECT | O_NOTIFICATION_PIPE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962) error = create_pipe_files(files, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963) if (error)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964) return error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966) error = get_unused_fd_flags(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967) if (error < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968) goto err_read_pipe;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969) fdr = error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971) error = get_unused_fd_flags(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972) if (error < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973) goto err_fdr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974) fdw = error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976) audit_fd_pair(fdr, fdw);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977) fd[0] = fdr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978) fd[1] = fdw;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981) err_fdr:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982) put_unused_fd(fdr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983) err_read_pipe:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984) fput(files[0]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985) fput(files[1]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986) return error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989) int do_pipe_flags(int *fd, int flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991) struct file *files[2];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992) int error = __do_pipe_flags(fd, files, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993) if (!error) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994) fd_install(fd[0], files[0]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995) fd_install(fd[1], files[1]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997) return error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001) * sys_pipe() is the normal C calling standard for creating
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002) * a pipe. It's not the way Unix traditionally does this, though.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004) static int do_pipe2(int __user *fildes, int flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006) struct file *files[2];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007) int fd[2];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008) int error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010) error = __do_pipe_flags(fd, files, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011) if (!error) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012) if (unlikely(copy_to_user(fildes, fd, sizeof(fd)))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013) fput(files[0]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014) fput(files[1]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015) put_unused_fd(fd[0]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016) put_unused_fd(fd[1]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017) error = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019) fd_install(fd[0], files[0]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020) fd_install(fd[1], files[1]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023) return error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026) SYSCALL_DEFINE2(pipe2, int __user *, fildes, int, flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028) return do_pipe2(fildes, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031) SYSCALL_DEFINE1(pipe, int __user *, fildes)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033) return do_pipe2(fildes, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037) * This is the stupid "wait for pipe to be readable or writable"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038) * model.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040) * See pipe_read/write() for the proper kind of exclusive wait,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041) * but that requires that we wake up any other readers/writers
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042) * if we then do not end up reading everything (ie the whole
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043) * "wake_next_reader/writer" logic in pipe_read/write()).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045) void pipe_wait_readable(struct pipe_inode_info *pipe)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047) pipe_unlock(pipe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048) wait_event_interruptible(pipe->rd_wait, pipe_readable(pipe));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049) pipe_lock(pipe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052) void pipe_wait_writable(struct pipe_inode_info *pipe)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054) pipe_unlock(pipe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055) wait_event_interruptible(pipe->wr_wait, pipe_writable(pipe));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056) pipe_lock(pipe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060) * This depends on both the wait (here) and the wakeup (wake_up_partner)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1061) * holding the pipe lock, so "*cnt" is stable and we know a wakeup cannot
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1062) * race with the count check and waitqueue prep.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1063) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1064) * Normally in order to avoid races, you'd do the prepare_to_wait() first,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1065) * then check the condition you're waiting for, and only then sleep. But
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1066) * because of the pipe lock, we can check the condition before being on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1067) * the wait queue.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1068) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1069) * We use the 'rd_wait' waitqueue for pipe partner waiting.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1070) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1071) static int wait_for_partner(struct pipe_inode_info *pipe, unsigned int *cnt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1072) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1073) DEFINE_WAIT(rdwait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1074) int cur = *cnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1075)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1076) while (cur == *cnt) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1077) prepare_to_wait(&pipe->rd_wait, &rdwait, TASK_INTERRUPTIBLE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1078) pipe_unlock(pipe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1079) schedule();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1080) finish_wait(&pipe->rd_wait, &rdwait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1081) pipe_lock(pipe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1082) if (signal_pending(current))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1083) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1084) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1085) return cur == *cnt ? -ERESTARTSYS : 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1086) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1087)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1088) static void wake_up_partner(struct pipe_inode_info *pipe)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1089) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1090) wake_up_interruptible_all(&pipe->rd_wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1091) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1092)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1093) static int fifo_open(struct inode *inode, struct file *filp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1094) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1095) struct pipe_inode_info *pipe;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1096) bool is_pipe = inode->i_sb->s_magic == PIPEFS_MAGIC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1097) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1098)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1099) filp->f_version = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1101) spin_lock(&inode->i_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1102) if (inode->i_pipe) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1103) pipe = inode->i_pipe;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1104) pipe->files++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1105) spin_unlock(&inode->i_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1106) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1107) spin_unlock(&inode->i_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1108) pipe = alloc_pipe_info();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1109) if (!pipe)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1110) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1111) pipe->files = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1112) spin_lock(&inode->i_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1113) if (unlikely(inode->i_pipe)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1114) inode->i_pipe->files++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1115) spin_unlock(&inode->i_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1116) free_pipe_info(pipe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1117) pipe = inode->i_pipe;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1118) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1119) inode->i_pipe = pipe;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1120) spin_unlock(&inode->i_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1121) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1122) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1123) filp->private_data = pipe;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1124) /* OK, we have a pipe and it's pinned down */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1125)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1126) __pipe_lock(pipe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1127)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1128) /* We can only do regular read/write on fifos */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1129) stream_open(inode, filp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1130)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1131) switch (filp->f_mode & (FMODE_READ | FMODE_WRITE)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1132) case FMODE_READ:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1133) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1134) * O_RDONLY
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1135) * POSIX.1 says that O_NONBLOCK means return with the FIFO
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1136) * opened, even when there is no process writing the FIFO.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1137) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1138) pipe->r_counter++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1139) if (pipe->readers++ == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1140) wake_up_partner(pipe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1141)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1142) if (!is_pipe && !pipe->writers) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1143) if ((filp->f_flags & O_NONBLOCK)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1144) /* suppress EPOLLHUP until we have
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1145) * seen a writer */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1146) filp->f_version = pipe->w_counter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1147) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1148) if (wait_for_partner(pipe, &pipe->w_counter))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1149) goto err_rd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1150) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1151) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1152) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1153)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1154) case FMODE_WRITE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1155) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1156) * O_WRONLY
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1157) * POSIX.1 says that O_NONBLOCK means return -1 with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1158) * errno=ENXIO when there is no process reading the FIFO.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1159) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1160) ret = -ENXIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1161) if (!is_pipe && (filp->f_flags & O_NONBLOCK) && !pipe->readers)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1162) goto err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1163)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1164) pipe->w_counter++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1165) if (!pipe->writers++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1166) wake_up_partner(pipe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1167)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1168) if (!is_pipe && !pipe->readers) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1169) if (wait_for_partner(pipe, &pipe->r_counter))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1170) goto err_wr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1171) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1172) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1173)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1174) case FMODE_READ | FMODE_WRITE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1175) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1176) * O_RDWR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1177) * POSIX.1 leaves this case "undefined" when O_NONBLOCK is set.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1178) * This implementation will NEVER block on a O_RDWR open, since
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1179) * the process can at least talk to itself.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1180) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1181)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1182) pipe->readers++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1183) pipe->writers++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1184) pipe->r_counter++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1185) pipe->w_counter++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1186) if (pipe->readers == 1 || pipe->writers == 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1187) wake_up_partner(pipe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1188) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1189)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1190) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1191) ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1192) goto err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1193) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1194)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1195) /* Ok! */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1196) __pipe_unlock(pipe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1197) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1198)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1199) err_rd:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1200) if (!--pipe->readers)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1201) wake_up_interruptible(&pipe->wr_wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1202) ret = -ERESTARTSYS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1203) goto err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1204)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1205) err_wr:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1206) if (!--pipe->writers)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1207) wake_up_interruptible_all(&pipe->rd_wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1208) ret = -ERESTARTSYS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1209) goto err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1210)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1211) err:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1212) __pipe_unlock(pipe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1213)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1214) put_pipe_info(inode, pipe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1215) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1216) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1217)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1218) const struct file_operations pipefifo_fops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1219) .open = fifo_open,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1220) .llseek = no_llseek,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1221) .read_iter = pipe_read,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1222) .write_iter = pipe_write,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1223) .poll = pipe_poll,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1224) .unlocked_ioctl = pipe_ioctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1225) .release = pipe_release,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1226) .fasync = pipe_fasync,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1227) .splice_write = iter_file_splice_write,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1228) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1229)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1230) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1231) * Currently we rely on the pipe array holding a power-of-2 number
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1232) * of pages. Returns 0 on error.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1233) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1234) unsigned int round_pipe_size(unsigned long size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1235) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1236) if (size > (1U << 31))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1237) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1238)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1239) /* Minimum pipe size, as required by POSIX */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1240) if (size < PAGE_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1241) return PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1242)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1243) return roundup_pow_of_two(size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1244) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1245)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1246) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1247) * Resize the pipe ring to a number of slots.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1248) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1249) int pipe_resize_ring(struct pipe_inode_info *pipe, unsigned int nr_slots)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1250) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1251) struct pipe_buffer *bufs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1252) unsigned int head, tail, mask, n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1253)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1254) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1255) * We can shrink the pipe, if arg is greater than the ring occupancy.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1256) * Since we don't expect a lot of shrink+grow operations, just free and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1257) * allocate again like we would do for growing. If the pipe currently
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1258) * contains more buffers than arg, then return busy.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1259) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1260) mask = pipe->ring_size - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1261) head = pipe->head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1262) tail = pipe->tail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1263) n = pipe_occupancy(pipe->head, pipe->tail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1264) if (nr_slots < n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1265) return -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1266)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1267) bufs = kcalloc(nr_slots, sizeof(*bufs),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1268) GFP_KERNEL_ACCOUNT | __GFP_NOWARN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1269) if (unlikely(!bufs))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1270) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1271)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1272) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1273) * The pipe array wraps around, so just start the new one at zero
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1274) * and adjust the indices.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1275) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1276) if (n > 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1277) unsigned int h = head & mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1278) unsigned int t = tail & mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1279) if (h > t) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1280) memcpy(bufs, pipe->bufs + t,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1281) n * sizeof(struct pipe_buffer));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1282) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1283) unsigned int tsize = pipe->ring_size - t;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1284) if (h > 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1285) memcpy(bufs + tsize, pipe->bufs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1286) h * sizeof(struct pipe_buffer));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1287) memcpy(bufs, pipe->bufs + t,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1288) tsize * sizeof(struct pipe_buffer));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1289) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1290) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1291)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1292) head = n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1293) tail = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1294)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1295) kfree(pipe->bufs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1296) pipe->bufs = bufs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1297) pipe->ring_size = nr_slots;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1298) if (pipe->max_usage > nr_slots)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1299) pipe->max_usage = nr_slots;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1300) pipe->tail = tail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1301) pipe->head = head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1302)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1303) /* This might have made more room for writers */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1304) wake_up_interruptible(&pipe->wr_wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1305) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1306) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1307)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1308) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1309) * Allocate a new array of pipe buffers and copy the info over. Returns the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1310) * pipe size if successful, or return -ERROR on error.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1311) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1312) static long pipe_set_size(struct pipe_inode_info *pipe, unsigned long arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1313) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1314) unsigned long user_bufs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1315) unsigned int nr_slots, size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1316) long ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1317)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1318) #ifdef CONFIG_WATCH_QUEUE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1319) if (pipe->watch_queue)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1320) return -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1321) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1322)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1323) size = round_pipe_size(arg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1324) nr_slots = size >> PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1325)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1326) if (!nr_slots)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1327) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1328)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1329) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1330) * If trying to increase the pipe capacity, check that an
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1331) * unprivileged user is not trying to exceed various limits
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1332) * (soft limit check here, hard limit check just below).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1333) * Decreasing the pipe capacity is always permitted, even
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1334) * if the user is currently over a limit.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1335) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1336) if (nr_slots > pipe->max_usage &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1337) size > pipe_max_size && !capable(CAP_SYS_RESOURCE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1338) return -EPERM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1339)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1340) user_bufs = account_pipe_buffers(pipe->user, pipe->nr_accounted, nr_slots);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1341)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1342) if (nr_slots > pipe->max_usage &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1343) (too_many_pipe_buffers_hard(user_bufs) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1344) too_many_pipe_buffers_soft(user_bufs)) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1345) pipe_is_unprivileged_user()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1346) ret = -EPERM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1347) goto out_revert_acct;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1348) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1349)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1350) ret = pipe_resize_ring(pipe, nr_slots);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1351) if (ret < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1352) goto out_revert_acct;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1353)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1354) pipe->max_usage = nr_slots;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1355) pipe->nr_accounted = nr_slots;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1356) return pipe->max_usage * PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1357)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1358) out_revert_acct:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1359) (void) account_pipe_buffers(pipe->user, nr_slots, pipe->nr_accounted);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1360) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1361) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1362)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1363) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1364) * After the inode slimming patch, i_pipe/i_bdev/i_cdev share the same
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1365) * location, so checking ->i_pipe is not enough to verify that this is a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1366) * pipe.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1367) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1368) struct pipe_inode_info *get_pipe_info(struct file *file, bool for_splice)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1369) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1370) struct pipe_inode_info *pipe = file->private_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1371)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1372) if (file->f_op != &pipefifo_fops || !pipe)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1373) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1374) #ifdef CONFIG_WATCH_QUEUE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1375) if (for_splice && pipe->watch_queue)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1376) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1377) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1378) return pipe;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1379) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1380)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1381) long pipe_fcntl(struct file *file, unsigned int cmd, unsigned long arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1382) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1383) struct pipe_inode_info *pipe;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1384) long ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1385)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1386) pipe = get_pipe_info(file, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1387) if (!pipe)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1388) return -EBADF;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1389)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1390) __pipe_lock(pipe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1391)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1392) switch (cmd) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1393) case F_SETPIPE_SZ:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1394) ret = pipe_set_size(pipe, arg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1395) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1396) case F_GETPIPE_SZ:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1397) ret = pipe->max_usage * PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1398) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1399) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1400) ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1401) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1402) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1403)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1404) __pipe_unlock(pipe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1405) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1406) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1407)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1408) static const struct super_operations pipefs_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1409) .destroy_inode = free_inode_nonrcu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1410) .statfs = simple_statfs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1411) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1412)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1413) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1414) * pipefs should _never_ be mounted by userland - too much of security hassle,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1415) * no real gain from having the whole whorehouse mounted. So we don't need
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1416) * any operations on the root directory. However, we need a non-trivial
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1417) * d_name - pipe: will go nicely and kill the special-casing in procfs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1418) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1419)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1420) static int pipefs_init_fs_context(struct fs_context *fc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1421) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1422) struct pseudo_fs_context *ctx = init_pseudo(fc, PIPEFS_MAGIC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1423) if (!ctx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1424) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1425) ctx->ops = &pipefs_ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1426) ctx->dops = &pipefs_dentry_operations;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1427) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1428) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1429)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1430) static struct file_system_type pipe_fs_type = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1431) .name = "pipefs",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1432) .init_fs_context = pipefs_init_fs_context,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1433) .kill_sb = kill_anon_super,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1434) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1435)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1436) static int __init init_pipe_fs(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1437) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1438) int err = register_filesystem(&pipe_fs_type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1439)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1440) if (!err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1441) pipe_mnt = kern_mount(&pipe_fs_type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1442) if (IS_ERR(pipe_mnt)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1443) err = PTR_ERR(pipe_mnt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1444) unregister_filesystem(&pipe_fs_type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1445) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1446) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1447) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1448) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1449)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1450) fs_initcall(init_pipe_fs);
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags