^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * linux/fs/nfs/direct.c
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Copyright (C) 2003 by Chuck Lever <cel@netapp.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * High-performance uncached I/O for the Linux NFS client
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) * There are important applications whose performance or correctness
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) * depends on uncached access to file data. Database clusters
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) * (multiple copies of the same instance running on separate hosts)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) * implement their own cache coherency protocol that subsumes file
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) * system cache protocols. Applications that process datasets
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) * considerably larger than the client's memory do not always benefit
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) * from a local cache. A streaming video server, for instance, has no
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) * need to cache the contents of a file.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) * When an application requests uncached I/O, all read and write requests
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) * are made directly to the server; data stored or fetched via these
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) * requests is not cached in the Linux page cache. The client does not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) * correct unaligned requests from applications. All requested bytes are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) * held on permanent storage before a direct write system call returns to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) * an application.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) * Solaris implements an uncached I/O facility called directio() that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) * is used for backups and sequential I/O to very large files. Solaris
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) * also supports uncaching whole NFS partitions with "-o forcedirectio,"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) * an undocumented mount option.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) * Designed by Jeff Kimmel, Chuck Lever, and Trond Myklebust, with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) * help from Andrew Morton.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) * 18 Dec 2001 Initial implementation for 2.4 --cel
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) * 08 Jul 2002 Version for 2.4.19, with bug fixes --trondmy
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) * 08 Jun 2003 Port to 2.5 APIs --cel
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) * 31 Mar 2004 Handle direct I/O without VFS support --cel
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) * 15 Sep 2004 Parallel async reads --cel
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) * 04 May 2005 support O_DIRECT with aio --cel
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) #include <linux/errno.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) #include <linux/sched.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) #include <linux/file.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) #include <linux/pagemap.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) #include <linux/kref.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) #include <linux/task_io_accounting_ops.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) #include <linux/nfs_fs.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) #include <linux/nfs_page.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) #include <linux/sunrpc/clnt.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) #include <linux/uaccess.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) #include <linux/atomic.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) #include "internal.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) #include "iostat.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) #include "pnfs.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) #define NFSDBG_FACILITY NFSDBG_VFS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) static struct kmem_cache *nfs_direct_cachep;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) struct nfs_direct_req {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) struct kref kref; /* release manager */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) /* I/O parameters */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) struct nfs_open_context *ctx; /* file open context info */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) struct nfs_lock_context *l_ctx; /* Lock context info */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) struct kiocb * iocb; /* controlling i/o request */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) struct inode * inode; /* target file of i/o */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) /* completion state */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) atomic_t io_count; /* i/os we're waiting for */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) spinlock_t lock; /* protect completion state */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) loff_t io_start; /* Start offset for I/O */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) ssize_t count, /* bytes actually processed */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) max_count, /* max expected count */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) bytes_left, /* bytes left to be sent */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) error; /* any reported error */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) struct completion completion; /* wait for i/o completion */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) /* commit state */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) struct nfs_mds_commit_info mds_cinfo; /* Storage for cinfo */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) struct pnfs_ds_commit_info ds_cinfo; /* Storage for cinfo */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) struct work_struct work;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) int flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) /* for write */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) #define NFS_ODIRECT_DO_COMMIT (1) /* an unstable reply was received */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) #define NFS_ODIRECT_RESCHED_WRITES (2) /* write verification failed */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) /* for read */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) #define NFS_ODIRECT_SHOULD_DIRTY (3) /* dirty user-space page after read */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) #define NFS_ODIRECT_DONE INT_MAX /* write verification failed */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) static const struct nfs_pgio_completion_ops nfs_direct_write_completion_ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) static const struct nfs_commit_completion_ops nfs_direct_commit_completion_ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) static void nfs_direct_write_complete(struct nfs_direct_req *dreq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) static void nfs_direct_write_schedule_work(struct work_struct *work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) static inline void get_dreq(struct nfs_direct_req *dreq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) atomic_inc(&dreq->io_count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) static inline int put_dreq(struct nfs_direct_req *dreq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) return atomic_dec_and_test(&dreq->io_count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) nfs_direct_handle_truncated(struct nfs_direct_req *dreq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) const struct nfs_pgio_header *hdr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) ssize_t dreq_len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) if (!(test_bit(NFS_IOHDR_ERROR, &hdr->flags) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) test_bit(NFS_IOHDR_EOF, &hdr->flags)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) if (dreq->max_count >= dreq_len) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) dreq->max_count = dreq_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) if (dreq->count > dreq_len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) dreq->count = dreq_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) if (test_bit(NFS_IOHDR_ERROR, &hdr->flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) dreq->error = hdr->error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) else /* Clear outstanding error if this is EOF */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) dreq->error = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) nfs_direct_count_bytes(struct nfs_direct_req *dreq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) const struct nfs_pgio_header *hdr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) loff_t hdr_end = hdr->io_start + hdr->good_bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) ssize_t dreq_len = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) if (hdr_end > dreq->io_start)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) dreq_len = hdr_end - dreq->io_start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) nfs_direct_handle_truncated(dreq, hdr, dreq_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) if (dreq_len > dreq->max_count)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) dreq_len = dreq->max_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) if (dreq->count < dreq_len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) dreq->count = dreq_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) * nfs_direct_IO - NFS address space operation for direct I/O
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) * @iocb: target I/O control block
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) * @iter: I/O buffer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) * The presence of this routine in the address space ops vector means
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) * the NFS client supports direct I/O. However, for most direct IO, we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) * shunt off direct read and write requests before the VFS gets them,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) * so this method is only ever called for swap.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) ssize_t nfs_direct_IO(struct kiocb *iocb, struct iov_iter *iter)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) struct inode *inode = iocb->ki_filp->f_mapping->host;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) /* we only support swap file calling nfs_direct_IO */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) if (!IS_SWAPFILE(inode))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) VM_BUG_ON(iov_iter_count(iter) != PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) if (iov_iter_rw(iter) == READ)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) return nfs_file_direct_read(iocb, iter);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) return nfs_file_direct_write(iocb, iter);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) static void nfs_direct_release_pages(struct page **pages, unsigned int npages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) unsigned int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) for (i = 0; i < npages; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) put_page(pages[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) void nfs_init_cinfo_from_dreq(struct nfs_commit_info *cinfo,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) struct nfs_direct_req *dreq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) cinfo->inode = dreq->inode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) cinfo->mds = &dreq->mds_cinfo;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) cinfo->ds = &dreq->ds_cinfo;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) cinfo->dreq = dreq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) cinfo->completion_ops = &nfs_direct_commit_completion_ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) static inline struct nfs_direct_req *nfs_direct_req_alloc(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) struct nfs_direct_req *dreq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) dreq = kmem_cache_zalloc(nfs_direct_cachep, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) if (!dreq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) kref_init(&dreq->kref);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) kref_get(&dreq->kref);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) init_completion(&dreq->completion);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) INIT_LIST_HEAD(&dreq->mds_cinfo.list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) pnfs_init_ds_commit_info(&dreq->ds_cinfo);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) INIT_WORK(&dreq->work, nfs_direct_write_schedule_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) spin_lock_init(&dreq->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) return dreq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) static void nfs_direct_req_free(struct kref *kref)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) struct nfs_direct_req *dreq = container_of(kref, struct nfs_direct_req, kref);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) pnfs_release_ds_info(&dreq->ds_cinfo, dreq->inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) if (dreq->l_ctx != NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) nfs_put_lock_context(dreq->l_ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) if (dreq->ctx != NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) put_nfs_open_context(dreq->ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) kmem_cache_free(nfs_direct_cachep, dreq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) static void nfs_direct_req_release(struct nfs_direct_req *dreq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) kref_put(&dreq->kref, nfs_direct_req_free);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) ssize_t nfs_dreq_bytes_left(struct nfs_direct_req *dreq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) return dreq->bytes_left;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) EXPORT_SYMBOL_GPL(nfs_dreq_bytes_left);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) * Collects and returns the final error value/byte-count.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) static ssize_t nfs_direct_wait(struct nfs_direct_req *dreq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) ssize_t result = -EIOCBQUEUED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) /* Async requests don't wait here */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) if (dreq->iocb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) result = wait_for_completion_killable(&dreq->completion);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) if (!result) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) result = dreq->count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) WARN_ON_ONCE(dreq->count < 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) if (!result)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) result = dreq->error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) return (ssize_t) result;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) * Synchronous I/O uses a stack-allocated iocb. Thus we can't trust
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) * the iocb is still valid here if this is a synchronous request.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) static void nfs_direct_complete(struct nfs_direct_req *dreq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) struct inode *inode = dreq->inode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) inode_dio_end(inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) if (dreq->iocb) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) long res = (long) dreq->error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) if (dreq->count != 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) res = (long) dreq->count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) WARN_ON_ONCE(dreq->count < 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) dreq->iocb->ki_complete(dreq->iocb, res, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) complete(&dreq->completion);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) nfs_direct_req_release(dreq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) static void nfs_direct_read_completion(struct nfs_pgio_header *hdr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) unsigned long bytes = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) struct nfs_direct_req *dreq = hdr->dreq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) spin_lock(&dreq->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) if (test_bit(NFS_IOHDR_REDO, &hdr->flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) spin_unlock(&dreq->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) goto out_put;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) nfs_direct_count_bytes(dreq, hdr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) spin_unlock(&dreq->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) while (!list_empty(&hdr->pages)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) struct nfs_page *req = nfs_list_entry(hdr->pages.next);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) struct page *page = req->wb_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) if (!PageCompound(page) && bytes < hdr->good_bytes &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) (dreq->flags == NFS_ODIRECT_SHOULD_DIRTY))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) set_page_dirty(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) bytes += req->wb_bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) nfs_list_remove_request(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) nfs_release_request(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) out_put:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) if (put_dreq(dreq))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) nfs_direct_complete(dreq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) hdr->release(hdr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) static void nfs_read_sync_pgio_error(struct list_head *head, int error)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) struct nfs_page *req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) while (!list_empty(head)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) req = nfs_list_entry(head->next);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) nfs_list_remove_request(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) nfs_release_request(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) static void nfs_direct_pgio_init(struct nfs_pgio_header *hdr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) get_dreq(hdr->dreq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) static const struct nfs_pgio_completion_ops nfs_direct_read_completion_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) .error_cleanup = nfs_read_sync_pgio_error,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) .init_hdr = nfs_direct_pgio_init,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) .completion = nfs_direct_read_completion,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) * For each rsize'd chunk of the user's buffer, dispatch an NFS READ
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) * operation. If nfs_readdata_alloc() or get_user_pages() fails,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) * bail and stop sending more reads. Read length accounting is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) * handled automatically by nfs_direct_read_result(). Otherwise, if
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) * no requests have been sent, just return an error.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) static ssize_t nfs_direct_read_schedule_iovec(struct nfs_direct_req *dreq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) struct iov_iter *iter,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) loff_t pos)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) struct nfs_pageio_descriptor desc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) struct inode *inode = dreq->inode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) ssize_t result = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) size_t requested_bytes = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) size_t rsize = max_t(size_t, NFS_SERVER(inode)->rsize, PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) nfs_pageio_init_read(&desc, dreq->inode, false,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) &nfs_direct_read_completion_ops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) get_dreq(dreq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) desc.pg_dreq = dreq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) inode_dio_begin(inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) while (iov_iter_count(iter)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) struct page **pagevec;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) size_t bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) size_t pgbase;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) unsigned npages, i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) result = iov_iter_get_pages_alloc(iter, &pagevec,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) rsize, &pgbase);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) if (result < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) bytes = result;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) iov_iter_advance(iter, bytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) npages = (result + pgbase + PAGE_SIZE - 1) / PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) for (i = 0; i < npages; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) struct nfs_page *req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) unsigned int req_len = min_t(size_t, bytes, PAGE_SIZE - pgbase);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) /* XXX do we need to do the eof zeroing found in async_filler? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) req = nfs_create_request(dreq->ctx, pagevec[i],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) pgbase, req_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) if (IS_ERR(req)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) result = PTR_ERR(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) req->wb_index = pos >> PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) req->wb_offset = pos & ~PAGE_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) if (!nfs_pageio_add_request(&desc, req)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) result = desc.pg_error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) nfs_release_request(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) pgbase = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) bytes -= req_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) requested_bytes += req_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) pos += req_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) dreq->bytes_left -= req_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) nfs_direct_release_pages(pagevec, npages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) kvfree(pagevec);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) if (result < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) nfs_pageio_complete(&desc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) * If no bytes were started, return the error, and let the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) * generic layer handle the completion.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) if (requested_bytes == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) inode_dio_end(inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) nfs_direct_req_release(dreq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) return result < 0 ? result : -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) if (put_dreq(dreq))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) nfs_direct_complete(dreq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) return requested_bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) * nfs_file_direct_read - file direct read operation for NFS files
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) * @iocb: target I/O control block
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) * @iter: vector of user buffers into which to read data
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) * We use this function for direct reads instead of calling
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) * generic_file_aio_read() in order to avoid gfar's check to see if
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) * the request starts before the end of the file. For that check
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) * to work, we must generate a GETATTR before each direct read, and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) * even then there is a window between the GETATTR and the subsequent
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) * READ where the file size could change. Our preference is simply
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) * to do all reads the application wants, and the server will take
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) * care of managing the end of file boundary.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) * This function also eliminates unnecessarily updating the file's
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) * atime locally, as the NFS server sets the file's atime, and this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) * client must read the updated atime from the server back into its
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) * cache.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) ssize_t nfs_file_direct_read(struct kiocb *iocb, struct iov_iter *iter)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) struct file *file = iocb->ki_filp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) struct address_space *mapping = file->f_mapping;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) struct inode *inode = mapping->host;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) struct nfs_direct_req *dreq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) struct nfs_lock_context *l_ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) ssize_t result, requested;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) size_t count = iov_iter_count(iter);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) nfs_add_stats(mapping->host, NFSIOS_DIRECTREADBYTES, count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) dfprintk(FILE, "NFS: direct read(%pD2, %zd@%Ld)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) file, count, (long long) iocb->ki_pos);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) result = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) if (!count)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) task_io_account_read(count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) result = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) dreq = nfs_direct_req_alloc();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) if (dreq == NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) dreq->inode = inode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) dreq->bytes_left = dreq->max_count = count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) dreq->io_start = iocb->ki_pos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) dreq->ctx = get_nfs_open_context(nfs_file_open_context(iocb->ki_filp));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) l_ctx = nfs_get_lock_context(dreq->ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) if (IS_ERR(l_ctx)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) result = PTR_ERR(l_ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) nfs_direct_req_release(dreq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) goto out_release;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) dreq->l_ctx = l_ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) if (!is_sync_kiocb(iocb))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) dreq->iocb = iocb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) if (iter_is_iovec(iter))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) dreq->flags = NFS_ODIRECT_SHOULD_DIRTY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) nfs_start_io_direct(inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) NFS_I(inode)->read_io += count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) requested = nfs_direct_read_schedule_iovec(dreq, iter, iocb->ki_pos);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) nfs_end_io_direct(inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) if (requested > 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) result = nfs_direct_wait(dreq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) if (result > 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) requested -= result;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) iocb->ki_pos += result;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) iov_iter_revert(iter, requested);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) result = requested;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) out_release:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) nfs_direct_req_release(dreq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) return result;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) nfs_direct_join_group(struct list_head *list, struct inode *inode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) struct nfs_page *req, *next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) list_for_each_entry(req, list, wb_list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) if (req->wb_head != req || req->wb_this_page == req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) for (next = req->wb_this_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) next != req->wb_head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) next = next->wb_this_page) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) nfs_list_remove_request(next);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) nfs_release_request(next);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) nfs_join_page_group(req, inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) nfs_direct_write_scan_commit_list(struct inode *inode,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) struct list_head *list,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) struct nfs_commit_info *cinfo)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) mutex_lock(&NFS_I(cinfo->inode)->commit_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) pnfs_recover_commit_reqs(list, cinfo);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) nfs_scan_commit_list(&cinfo->mds->list, list, cinfo, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) mutex_unlock(&NFS_I(cinfo->inode)->commit_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) static void nfs_direct_write_reschedule(struct nfs_direct_req *dreq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) struct nfs_pageio_descriptor desc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) struct nfs_page *req, *tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) LIST_HEAD(reqs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) struct nfs_commit_info cinfo;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) LIST_HEAD(failed);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) nfs_init_cinfo_from_dreq(&cinfo, dreq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) nfs_direct_write_scan_commit_list(dreq->inode, &reqs, &cinfo);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) nfs_direct_join_group(&reqs, dreq->inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) dreq->count = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) dreq->max_count = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) list_for_each_entry(req, &reqs, wb_list)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) dreq->max_count += req->wb_bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) nfs_clear_pnfs_ds_commit_verifiers(&dreq->ds_cinfo);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) get_dreq(dreq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) nfs_pageio_init_write(&desc, dreq->inode, FLUSH_STABLE, false,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) &nfs_direct_write_completion_ops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) desc.pg_dreq = dreq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) list_for_each_entry_safe(req, tmp, &reqs, wb_list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) /* Bump the transmission count */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) req->wb_nio++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) if (!nfs_pageio_add_request(&desc, req)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) nfs_list_move_request(req, &failed);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) spin_lock(&cinfo.inode->i_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) dreq->flags = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) if (desc.pg_error < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) dreq->error = desc.pg_error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) dreq->error = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) spin_unlock(&cinfo.inode->i_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) nfs_release_request(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) nfs_pageio_complete(&desc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) while (!list_empty(&failed)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) req = nfs_list_entry(failed.next);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) nfs_list_remove_request(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) nfs_unlock_and_release_request(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) if (put_dreq(dreq))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) nfs_direct_write_complete(dreq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) static void nfs_direct_commit_complete(struct nfs_commit_data *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) const struct nfs_writeverf *verf = data->res.verf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) struct nfs_direct_req *dreq = data->dreq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) struct nfs_commit_info cinfo;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) struct nfs_page *req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) int status = data->task.tk_status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) if (status < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) /* Errors in commit are fatal */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) dreq->error = status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) dreq->max_count = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) dreq->count = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) dreq->flags = NFS_ODIRECT_DONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) } else if (dreq->flags == NFS_ODIRECT_DONE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) status = dreq->error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) nfs_init_cinfo_from_dreq(&cinfo, dreq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) while (!list_empty(&data->pages)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) req = nfs_list_entry(data->pages.next);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) nfs_list_remove_request(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) if (status >= 0 && !nfs_write_match_verf(verf, req)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) dreq->flags = NFS_ODIRECT_RESCHED_WRITES;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) * Despite the reboot, the write was successful,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) * so reset wb_nio.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) req->wb_nio = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) nfs_mark_request_commit(req, NULL, &cinfo, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) } else /* Error or match */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) nfs_release_request(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) nfs_unlock_and_release_request(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) if (nfs_commit_end(cinfo.mds))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) nfs_direct_write_complete(dreq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) static void nfs_direct_resched_write(struct nfs_commit_info *cinfo,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) struct nfs_page *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) struct nfs_direct_req *dreq = cinfo->dreq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) spin_lock(&dreq->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) if (dreq->flags != NFS_ODIRECT_DONE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) dreq->flags = NFS_ODIRECT_RESCHED_WRITES;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) spin_unlock(&dreq->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) nfs_mark_request_commit(req, NULL, cinfo, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) static const struct nfs_commit_completion_ops nfs_direct_commit_completion_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) .completion = nfs_direct_commit_complete,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) .resched_write = nfs_direct_resched_write,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) static void nfs_direct_commit_schedule(struct nfs_direct_req *dreq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) int res;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) struct nfs_commit_info cinfo;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) LIST_HEAD(mds_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) nfs_init_cinfo_from_dreq(&cinfo, dreq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) nfs_scan_commit(dreq->inode, &mds_list, &cinfo);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) res = nfs_generic_commit_list(dreq->inode, &mds_list, 0, &cinfo);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) if (res < 0) /* res == -ENOMEM */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) nfs_direct_write_reschedule(dreq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) static void nfs_direct_write_clear_reqs(struct nfs_direct_req *dreq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) struct nfs_commit_info cinfo;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) struct nfs_page *req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) LIST_HEAD(reqs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) nfs_init_cinfo_from_dreq(&cinfo, dreq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) nfs_direct_write_scan_commit_list(dreq->inode, &reqs, &cinfo);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) while (!list_empty(&reqs)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) req = nfs_list_entry(reqs.next);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) nfs_list_remove_request(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) nfs_release_request(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) nfs_unlock_and_release_request(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) static void nfs_direct_write_schedule_work(struct work_struct *work)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) struct nfs_direct_req *dreq = container_of(work, struct nfs_direct_req, work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) int flags = dreq->flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) dreq->flags = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) switch (flags) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) case NFS_ODIRECT_DO_COMMIT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) nfs_direct_commit_schedule(dreq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) case NFS_ODIRECT_RESCHED_WRITES:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) nfs_direct_write_reschedule(dreq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) nfs_direct_write_clear_reqs(dreq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) nfs_zap_mapping(dreq->inode, dreq->inode->i_mapping);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) nfs_direct_complete(dreq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) static void nfs_direct_write_complete(struct nfs_direct_req *dreq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) queue_work(nfsiod_workqueue, &dreq->work); /* Calls nfs_direct_write_schedule_work */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) static void nfs_direct_write_completion(struct nfs_pgio_header *hdr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) struct nfs_direct_req *dreq = hdr->dreq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) struct nfs_commit_info cinfo;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) struct nfs_page *req = nfs_list_entry(hdr->pages.next);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) int flags = NFS_ODIRECT_DONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) nfs_init_cinfo_from_dreq(&cinfo, dreq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) spin_lock(&dreq->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) if (test_bit(NFS_IOHDR_REDO, &hdr->flags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) spin_unlock(&dreq->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) goto out_put;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) nfs_direct_count_bytes(dreq, hdr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) if (hdr->good_bytes != 0 && nfs_write_need_commit(hdr)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) if (!dreq->flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) dreq->flags = NFS_ODIRECT_DO_COMMIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) flags = dreq->flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) spin_unlock(&dreq->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) while (!list_empty(&hdr->pages)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) req = nfs_list_entry(hdr->pages.next);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) nfs_list_remove_request(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) if (flags == NFS_ODIRECT_DO_COMMIT) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) kref_get(&req->wb_kref);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) memcpy(&req->wb_verf, &hdr->verf.verifier,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) sizeof(req->wb_verf));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) nfs_mark_request_commit(req, hdr->lseg, &cinfo,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) hdr->ds_commit_idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) } else if (flags == NFS_ODIRECT_RESCHED_WRITES) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) kref_get(&req->wb_kref);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) nfs_mark_request_commit(req, NULL, &cinfo, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) nfs_unlock_and_release_request(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) out_put:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) if (put_dreq(dreq))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) nfs_direct_write_complete(dreq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) hdr->release(hdr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) static void nfs_write_sync_pgio_error(struct list_head *head, int error)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) struct nfs_page *req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) while (!list_empty(head)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) req = nfs_list_entry(head->next);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) nfs_list_remove_request(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) nfs_unlock_and_release_request(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) static void nfs_direct_write_reschedule_io(struct nfs_pgio_header *hdr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) struct nfs_direct_req *dreq = hdr->dreq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) spin_lock(&dreq->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) if (dreq->error == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) dreq->flags = NFS_ODIRECT_RESCHED_WRITES;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) /* fake unstable write to let common nfs resend pages */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) hdr->verf.committed = NFS_UNSTABLE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) hdr->good_bytes = hdr->args.offset + hdr->args.count -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) hdr->io_start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) spin_unlock(&dreq->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) static const struct nfs_pgio_completion_ops nfs_direct_write_completion_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) .error_cleanup = nfs_write_sync_pgio_error,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) .init_hdr = nfs_direct_pgio_init,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) .completion = nfs_direct_write_completion,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) .reschedule_io = nfs_direct_write_reschedule_io,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) * NB: Return the value of the first error return code. Subsequent
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) * errors after the first one are ignored.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) * For each wsize'd chunk of the user's buffer, dispatch an NFS WRITE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) * operation. If nfs_writedata_alloc() or get_user_pages() fails,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) * bail and stop sending more writes. Write length accounting is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) * handled automatically by nfs_direct_write_result(). Otherwise, if
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) * no requests have been sent, just return an error.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) static ssize_t nfs_direct_write_schedule_iovec(struct nfs_direct_req *dreq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) struct iov_iter *iter,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) loff_t pos)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) struct nfs_pageio_descriptor desc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) struct inode *inode = dreq->inode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) ssize_t result = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) size_t requested_bytes = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) size_t wsize = max_t(size_t, NFS_SERVER(inode)->wsize, PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) nfs_pageio_init_write(&desc, inode, FLUSH_COND_STABLE, false,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) &nfs_direct_write_completion_ops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) desc.pg_dreq = dreq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) get_dreq(dreq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) inode_dio_begin(inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) NFS_I(inode)->write_io += iov_iter_count(iter);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) while (iov_iter_count(iter)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) struct page **pagevec;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) size_t bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) size_t pgbase;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) unsigned npages, i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) result = iov_iter_get_pages_alloc(iter, &pagevec,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) wsize, &pgbase);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) if (result < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) bytes = result;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) iov_iter_advance(iter, bytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) npages = (result + pgbase + PAGE_SIZE - 1) / PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) for (i = 0; i < npages; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) struct nfs_page *req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) unsigned int req_len = min_t(size_t, bytes, PAGE_SIZE - pgbase);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) req = nfs_create_request(dreq->ctx, pagevec[i],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) pgbase, req_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) if (IS_ERR(req)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) result = PTR_ERR(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) if (desc.pg_error < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) nfs_free_request(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) result = desc.pg_error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) nfs_lock_request(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) req->wb_index = pos >> PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) req->wb_offset = pos & ~PAGE_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) if (!nfs_pageio_add_request(&desc, req)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) result = desc.pg_error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) nfs_unlock_and_release_request(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) pgbase = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) bytes -= req_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) requested_bytes += req_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) pos += req_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) dreq->bytes_left -= req_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) nfs_direct_release_pages(pagevec, npages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) kvfree(pagevec);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) if (result < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) nfs_pageio_complete(&desc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) * If no bytes were started, return the error, and let the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) * generic layer handle the completion.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) if (requested_bytes == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) inode_dio_end(inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) nfs_direct_req_release(dreq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) return result < 0 ? result : -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) if (put_dreq(dreq))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) nfs_direct_write_complete(dreq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) return requested_bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) * nfs_file_direct_write - file direct write operation for NFS files
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) * @iocb: target I/O control block
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) * @iter: vector of user buffers from which to write data
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) * We use this function for direct writes instead of calling
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) * generic_file_aio_write() in order to avoid taking the inode
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) * semaphore and updating the i_size. The NFS server will set
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) * the new i_size and this client must read the updated size
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) * back into its cache. We let the server do generic write
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) * parameter checking and report problems.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) * We eliminate local atime updates, see direct read above.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888) * We avoid unnecessary page cache invalidations for normal cached
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) * readers of this file.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) * Note that O_APPEND is not supported for NFS direct writes, as there
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) * is no atomic O_APPEND write facility in the NFS protocol.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) ssize_t nfs_file_direct_write(struct kiocb *iocb, struct iov_iter *iter)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) ssize_t result, requested;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) size_t count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) struct file *file = iocb->ki_filp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) struct address_space *mapping = file->f_mapping;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) struct inode *inode = mapping->host;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) struct nfs_direct_req *dreq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) struct nfs_lock_context *l_ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903) loff_t pos, end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) dfprintk(FILE, "NFS: direct write(%pD2, %zd@%Ld)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) file, iov_iter_count(iter), (long long) iocb->ki_pos);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) result = generic_write_checks(iocb, iter);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) if (result <= 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) return result;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) count = result;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) nfs_add_stats(mapping->host, NFSIOS_DIRECTWRITTENBYTES, count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) pos = iocb->ki_pos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) end = (pos + iov_iter_count(iter) - 1) >> PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) task_io_account_write(count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) result = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) dreq = nfs_direct_req_alloc();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) if (!dreq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924) dreq->inode = inode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) dreq->bytes_left = dreq->max_count = count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926) dreq->io_start = pos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927) dreq->ctx = get_nfs_open_context(nfs_file_open_context(iocb->ki_filp));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) l_ctx = nfs_get_lock_context(dreq->ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) if (IS_ERR(l_ctx)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) result = PTR_ERR(l_ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) nfs_direct_req_release(dreq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932) goto out_release;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) dreq->l_ctx = l_ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) if (!is_sync_kiocb(iocb))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) dreq->iocb = iocb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) pnfs_init_ds_commit_info_ops(&dreq->ds_cinfo, inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) nfs_start_io_direct(inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) requested = nfs_direct_write_schedule_iovec(dreq, iter, pos);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943) if (mapping->nrpages) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) invalidate_inode_pages2_range(mapping,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) pos >> PAGE_SHIFT, end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) nfs_end_io_direct(inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950) if (requested > 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) result = nfs_direct_wait(dreq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952) if (result > 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) requested -= result;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) iocb->ki_pos = pos + result;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) /* XXX: should check the generic_write_sync retval */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956) generic_write_sync(iocb, result);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) iov_iter_revert(iter, requested);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960) result = requested;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962) out_release:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963) nfs_direct_req_release(dreq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965) return result;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969) * nfs_init_directcache - create a slab cache for nfs_direct_req structures
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972) int __init nfs_init_directcache(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974) nfs_direct_cachep = kmem_cache_create("nfs_direct_cache",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975) sizeof(struct nfs_direct_req),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976) 0, (SLAB_RECLAIM_ACCOUNT|
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977) SLAB_MEM_SPREAD),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978) NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979) if (nfs_direct_cachep == NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986) * nfs_destroy_directcache - destroy the slab cache for nfs_direct_req structures
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989) void nfs_destroy_directcache(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991) kmem_cache_destroy(nfs_direct_cachep);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992) }
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags