^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Copyright (c) 2000-2005 Silicon Graphics, Inc.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * Copyright (c) 2016-2018 Christoph Hellwig.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * All Rights Reserved.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) #include "xfs.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include "xfs_shared.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include "xfs_format.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include "xfs_log_format.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include "xfs_trans_resv.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include "xfs_mount.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include "xfs_inode.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include "xfs_trans.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include "xfs_iomap.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include "xfs_trace.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include "xfs_bmap.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include "xfs_bmap_util.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include "xfs_reflink.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) struct xfs_writepage_ctx {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) struct iomap_writepage_ctx ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) unsigned int data_seq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) unsigned int cow_seq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) static inline struct xfs_writepage_ctx *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) XFS_WPC(struct iomap_writepage_ctx *ctx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) return container_of(ctx, struct xfs_writepage_ctx, ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) * Fast and loose check if this write could update the on-disk inode size.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) static inline bool xfs_ioend_is_append(struct iomap_ioend *ioend)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) return ioend->io_offset + ioend->io_size >
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) XFS_I(ioend->io_inode)->i_d.di_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) STATIC int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) xfs_setfilesize_trans_alloc(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) struct iomap_ioend *ioend)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) struct xfs_mount *mp = XFS_I(ioend->io_inode)->i_mount;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) struct xfs_trans *tp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) int error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) error = xfs_trans_alloc(mp, &M_RES(mp)->tr_fsyncts, 0, 0, 0, &tp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) if (error)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) return error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) ioend->io_private = tp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) * We may pass freeze protection with a transaction. So tell lockdep
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) * we released it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) __sb_writers_release(ioend->io_inode->i_sb, SB_FREEZE_FS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) * We hand off the transaction to the completion thread now, so
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) * clear the flag here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) current_restore_flags_nested(&tp->t_pflags, PF_MEMALLOC_NOFS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) * Update on-disk file size now that data has been written to disk.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) STATIC int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) __xfs_setfilesize(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) struct xfs_inode *ip,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) struct xfs_trans *tp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) xfs_off_t offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) size_t size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) xfs_fsize_t isize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) xfs_ilock(ip, XFS_ILOCK_EXCL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) isize = xfs_new_eof(ip, offset + size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) if (!isize) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) xfs_iunlock(ip, XFS_ILOCK_EXCL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) xfs_trans_cancel(tp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) trace_xfs_setfilesize(ip, offset, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) ip->i_d.di_size = isize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) xfs_trans_ijoin(tp, ip, XFS_ILOCK_EXCL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) return xfs_trans_commit(tp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) xfs_setfilesize(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) struct xfs_inode *ip,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) xfs_off_t offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) size_t size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) struct xfs_mount *mp = ip->i_mount;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) struct xfs_trans *tp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) int error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) error = xfs_trans_alloc(mp, &M_RES(mp)->tr_fsyncts, 0, 0, 0, &tp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) if (error)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) return error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) return __xfs_setfilesize(ip, tp, offset, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) STATIC int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) xfs_setfilesize_ioend(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) struct iomap_ioend *ioend,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) int error)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) struct xfs_inode *ip = XFS_I(ioend->io_inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) struct xfs_trans *tp = ioend->io_private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) * The transaction may have been allocated in the I/O submission thread,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) * thus we need to mark ourselves as being in a transaction manually.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) * Similarly for freeze protection.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) current_set_flags_nested(&tp->t_pflags, PF_MEMALLOC_NOFS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) __sb_writers_acquired(VFS_I(ip)->i_sb, SB_FREEZE_FS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) /* we abort the update if there was an IO error */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) if (error) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) xfs_trans_cancel(tp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) return error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) return __xfs_setfilesize(ip, tp, ioend->io_offset, ioend->io_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) * IO write completion.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) STATIC void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) xfs_end_ioend(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) struct iomap_ioend *ioend)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) struct xfs_inode *ip = XFS_I(ioend->io_inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) xfs_off_t offset = ioend->io_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) size_t size = ioend->io_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) unsigned int nofs_flag;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) int error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) * We can allocate memory here while doing writeback on behalf of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) * memory reclaim. To avoid memory allocation deadlocks set the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) * task-wide nofs context for the following operations.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) nofs_flag = memalloc_nofs_save();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) * Just clean up the in-memory strutures if the fs has been shut down.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) if (XFS_FORCED_SHUTDOWN(ip->i_mount)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) error = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) goto done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) * Clean up any COW blocks on an I/O error.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) error = blk_status_to_errno(ioend->io_bio->bi_status);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) if (unlikely(error)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) if (ioend->io_flags & IOMAP_F_SHARED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) xfs_reflink_cancel_cow_range(ip, offset, size, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) goto done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) * Success: commit the COW or unwritten blocks if needed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) if (ioend->io_flags & IOMAP_F_SHARED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) error = xfs_reflink_end_cow(ip, offset, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) else if (ioend->io_type == IOMAP_UNWRITTEN)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) error = xfs_iomap_write_unwritten(ip, offset, size, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) ASSERT(!xfs_ioend_is_append(ioend) || ioend->io_private);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) done:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) if (ioend->io_private)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) error = xfs_setfilesize_ioend(ioend, error);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) iomap_finish_ioends(ioend, error);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) memalloc_nofs_restore(nofs_flag);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) * If the to be merged ioend has a preallocated transaction for file
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) * size updates we need to ensure the ioend it is merged into also
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) * has one. If it already has one we can simply cancel the transaction
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) * as it is guaranteed to be clean.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) xfs_ioend_merge_private(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) struct iomap_ioend *ioend,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) struct iomap_ioend *next)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) if (!ioend->io_private) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) ioend->io_private = next->io_private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) next->io_private = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) xfs_setfilesize_ioend(next, -ECANCELED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) /* Finish all pending io completions. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) xfs_end_io(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) struct work_struct *work)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) struct xfs_inode *ip =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) container_of(work, struct xfs_inode, i_ioend_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) struct iomap_ioend *ioend;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) struct list_head tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) spin_lock_irqsave(&ip->i_ioend_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) list_replace_init(&ip->i_ioend_list, &tmp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) spin_unlock_irqrestore(&ip->i_ioend_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) iomap_sort_ioends(&tmp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) while ((ioend = list_first_entry_or_null(&tmp, struct iomap_ioend,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) io_list))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) list_del_init(&ioend->io_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) iomap_ioend_try_merge(ioend, &tmp, xfs_ioend_merge_private);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) xfs_end_ioend(ioend);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) static inline bool xfs_ioend_needs_workqueue(struct iomap_ioend *ioend)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) return ioend->io_private ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) ioend->io_type == IOMAP_UNWRITTEN ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) (ioend->io_flags & IOMAP_F_SHARED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) STATIC void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) xfs_end_bio(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) struct bio *bio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) struct iomap_ioend *ioend = bio->bi_private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) struct xfs_inode *ip = XFS_I(ioend->io_inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) ASSERT(xfs_ioend_needs_workqueue(ioend));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) spin_lock_irqsave(&ip->i_ioend_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) if (list_empty(&ip->i_ioend_list))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) WARN_ON_ONCE(!queue_work(ip->i_mount->m_unwritten_workqueue,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) &ip->i_ioend_work));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) list_add_tail(&ioend->io_list, &ip->i_ioend_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) spin_unlock_irqrestore(&ip->i_ioend_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) * Fast revalidation of the cached writeback mapping. Return true if the current
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) * mapping is valid, false otherwise.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) static bool
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) xfs_imap_valid(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) struct iomap_writepage_ctx *wpc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) struct xfs_inode *ip,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) loff_t offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) if (offset < wpc->iomap.offset ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) offset >= wpc->iomap.offset + wpc->iomap.length)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) * If this is a COW mapping, it is sufficient to check that the mapping
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) * covers the offset. Be careful to check this first because the caller
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) * can revalidate a COW mapping without updating the data seqno.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) if (wpc->iomap.flags & IOMAP_F_SHARED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) * This is not a COW mapping. Check the sequence number of the data fork
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) * because concurrent changes could have invalidated the extent. Check
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) * the COW fork because concurrent changes since the last time we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) * checked (and found nothing at this offset) could have added
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) * overlapping blocks.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) if (XFS_WPC(wpc)->data_seq != READ_ONCE(ip->i_df.if_seq))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) if (xfs_inode_has_cow_data(ip) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) XFS_WPC(wpc)->cow_seq != READ_ONCE(ip->i_cowfp->if_seq))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) * Pass in a dellalloc extent and convert it to real extents, return the real
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) * extent that maps offset_fsb in wpc->iomap.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) * The current page is held locked so nothing could have removed the block
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) * backing offset_fsb, although it could have moved from the COW to the data
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) * fork by another thread.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) static int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) xfs_convert_blocks(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) struct iomap_writepage_ctx *wpc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) struct xfs_inode *ip,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) int whichfork,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) loff_t offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) int error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) unsigned *seq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) if (whichfork == XFS_COW_FORK)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) seq = &XFS_WPC(wpc)->cow_seq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) seq = &XFS_WPC(wpc)->data_seq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) * Attempt to allocate whatever delalloc extent currently backs offset
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) * and put the result into wpc->iomap. Allocate in a loop because it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) * may take several attempts to allocate real blocks for a contiguous
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) * delalloc extent if free space is sufficiently fragmented.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) error = xfs_bmapi_convert_delalloc(ip, whichfork, offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) &wpc->iomap, seq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) if (error)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) return error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) } while (wpc->iomap.offset + wpc->iomap.length <= offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) static int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) xfs_map_blocks(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) struct iomap_writepage_ctx *wpc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) struct inode *inode,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) loff_t offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) struct xfs_inode *ip = XFS_I(inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) struct xfs_mount *mp = ip->i_mount;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) ssize_t count = i_blocksize(inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) xfs_fileoff_t offset_fsb = XFS_B_TO_FSBT(mp, offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) xfs_fileoff_t end_fsb = XFS_B_TO_FSB(mp, offset + count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) xfs_fileoff_t cow_fsb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) int whichfork;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) struct xfs_bmbt_irec imap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) struct xfs_iext_cursor icur;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) int retries = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) int error = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) if (XFS_FORCED_SHUTDOWN(mp))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) * COW fork blocks can overlap data fork blocks even if the blocks
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) * aren't shared. COW I/O always takes precedent, so we must always
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) * check for overlap on reflink inodes unless the mapping is already a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) * COW one, or the COW fork hasn't changed from the last time we looked
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) * at it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) * It's safe to check the COW fork if_seq here without the ILOCK because
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) * we've indirectly protected against concurrent updates: writeback has
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) * the page locked, which prevents concurrent invalidations by reflink
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) * and directio and prevents concurrent buffered writes to the same
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) * page. Changes to if_seq always happen under i_lock, which protects
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) * against concurrent updates and provides a memory barrier on the way
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) * out that ensures that we always see the current value.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) if (xfs_imap_valid(wpc, ip, offset))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) * If we don't have a valid map, now it's time to get a new one for this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) * offset. This will convert delayed allocations (including COW ones)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) * into real extents. If we return without a valid map, it means we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) * landed in a hole and we skip the block.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) retry:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) cow_fsb = NULLFILEOFF;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) whichfork = XFS_DATA_FORK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) xfs_ilock(ip, XFS_ILOCK_SHARED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) ASSERT(ip->i_df.if_format != XFS_DINODE_FMT_BTREE ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) (ip->i_df.if_flags & XFS_IFEXTENTS));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) * Check if this is offset is covered by a COW extents, and if yes use
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) * it directly instead of looking up anything in the data fork.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) if (xfs_inode_has_cow_data(ip) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) xfs_iext_lookup_extent(ip, ip->i_cowfp, offset_fsb, &icur, &imap))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) cow_fsb = imap.br_startoff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) if (cow_fsb != NULLFILEOFF && cow_fsb <= offset_fsb) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) XFS_WPC(wpc)->cow_seq = READ_ONCE(ip->i_cowfp->if_seq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) xfs_iunlock(ip, XFS_ILOCK_SHARED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) whichfork = XFS_COW_FORK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) goto allocate_blocks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) * No COW extent overlap. Revalidate now that we may have updated
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) * ->cow_seq. If the data mapping is still valid, we're done.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) if (xfs_imap_valid(wpc, ip, offset)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) xfs_iunlock(ip, XFS_ILOCK_SHARED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) * If we don't have a valid map, now it's time to get a new one for this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) * offset. This will convert delayed allocations (including COW ones)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) * into real extents.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) if (!xfs_iext_lookup_extent(ip, &ip->i_df, offset_fsb, &icur, &imap))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) imap.br_startoff = end_fsb; /* fake a hole past EOF */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) XFS_WPC(wpc)->data_seq = READ_ONCE(ip->i_df.if_seq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) xfs_iunlock(ip, XFS_ILOCK_SHARED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) /* landed in a hole or beyond EOF? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) if (imap.br_startoff > offset_fsb) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) imap.br_blockcount = imap.br_startoff - offset_fsb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) imap.br_startoff = offset_fsb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) imap.br_startblock = HOLESTARTBLOCK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) imap.br_state = XFS_EXT_NORM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) * Truncate to the next COW extent if there is one. This is the only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) * opportunity to do this because we can skip COW fork lookups for the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) * subsequent blocks in the mapping; however, the requirement to treat
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) * the COW range separately remains.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) if (cow_fsb != NULLFILEOFF &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) cow_fsb < imap.br_startoff + imap.br_blockcount)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) imap.br_blockcount = cow_fsb - imap.br_startoff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) /* got a delalloc extent? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) if (imap.br_startblock != HOLESTARTBLOCK &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) isnullstartblock(imap.br_startblock))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) goto allocate_blocks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) xfs_bmbt_to_iomap(ip, &wpc->iomap, &imap, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) trace_xfs_map_blocks_found(ip, offset, count, whichfork, &imap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) allocate_blocks:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) error = xfs_convert_blocks(wpc, ip, whichfork, offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) if (error) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) * If we failed to find the extent in the COW fork we might have
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) * raced with a COW to data fork conversion or truncate.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) * Restart the lookup to catch the extent in the data fork for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) * the former case, but prevent additional retries to avoid
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) * looping forever for the latter case.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) if (error == -EAGAIN && whichfork == XFS_COW_FORK && !retries++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) goto retry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) ASSERT(error != -EAGAIN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) return error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) * Due to merging the return real extent might be larger than the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) * original delalloc one. Trim the return extent to the next COW
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) * boundary again to force a re-lookup.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) if (whichfork != XFS_COW_FORK && cow_fsb != NULLFILEOFF) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) loff_t cow_offset = XFS_FSB_TO_B(mp, cow_fsb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) if (cow_offset < wpc->iomap.offset + wpc->iomap.length)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) wpc->iomap.length = cow_offset - wpc->iomap.offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) ASSERT(wpc->iomap.offset <= offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) ASSERT(wpc->iomap.offset + wpc->iomap.length > offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) trace_xfs_map_blocks_alloc(ip, offset, count, whichfork, &imap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) static int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) xfs_prepare_ioend(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) struct iomap_ioend *ioend,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) int status)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) unsigned int nofs_flag;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) * We can allocate memory here while doing writeback on behalf of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) * memory reclaim. To avoid memory allocation deadlocks set the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) * task-wide nofs context for the following operations.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) nofs_flag = memalloc_nofs_save();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) /* Convert CoW extents to regular */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) if (!status && (ioend->io_flags & IOMAP_F_SHARED)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) status = xfs_reflink_convert_cow(XFS_I(ioend->io_inode),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) ioend->io_offset, ioend->io_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) /* Reserve log space if we might write beyond the on-disk inode size. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) if (!status &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) ((ioend->io_flags & IOMAP_F_SHARED) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) ioend->io_type != IOMAP_UNWRITTEN) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) xfs_ioend_is_append(ioend) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) !ioend->io_private)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) status = xfs_setfilesize_trans_alloc(ioend);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) memalloc_nofs_restore(nofs_flag);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) if (xfs_ioend_needs_workqueue(ioend))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) ioend->io_bio->bi_end_io = xfs_end_bio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) return status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) * If the page has delalloc blocks on it, we need to punch them out before we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) * invalidate the page. If we don't, we leave a stale delalloc mapping on the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) * inode that can trip up a later direct I/O read operation on the same region.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) * We prevent this by truncating away the delalloc regions on the page. Because
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) * they are delalloc, we can do this without needing a transaction. Indeed - if
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) * we get ENOSPC errors, we have to be able to do this truncation without a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) * transaction as there is no space left for block reservation (typically why we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) * see a ENOSPC in writeback).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) xfs_discard_page(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) struct page *page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) loff_t fileoff)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) struct inode *inode = page->mapping->host;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) struct xfs_inode *ip = XFS_I(inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) struct xfs_mount *mp = ip->i_mount;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) unsigned int pageoff = offset_in_page(fileoff);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) xfs_fileoff_t start_fsb = XFS_B_TO_FSBT(mp, fileoff);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) xfs_fileoff_t pageoff_fsb = XFS_B_TO_FSBT(mp, pageoff);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) int error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) if (XFS_FORCED_SHUTDOWN(mp))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) goto out_invalidate;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) xfs_alert_ratelimited(mp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) "page discard on page "PTR_FMT", inode 0x%llx, offset %llu.",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) page, ip->i_ino, fileoff);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) error = xfs_bmap_punch_delalloc_range(ip, start_fsb,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) i_blocks_per_page(inode, page) - pageoff_fsb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) if (error && !XFS_FORCED_SHUTDOWN(mp))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) xfs_alert(mp, "page discard unable to remove delalloc mapping.");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) out_invalidate:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) iomap_invalidatepage(page, pageoff, PAGE_SIZE - pageoff);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) static const struct iomap_writeback_ops xfs_writeback_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) .map_blocks = xfs_map_blocks,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) .prepare_ioend = xfs_prepare_ioend,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) .discard_page = xfs_discard_page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) STATIC int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) xfs_vm_writepage(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) struct page *page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) struct writeback_control *wbc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) struct xfs_writepage_ctx wpc = { };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) return iomap_writepage(page, wbc, &wpc.ctx, &xfs_writeback_ops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) STATIC int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) xfs_vm_writepages(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) struct address_space *mapping,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) struct writeback_control *wbc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) struct xfs_writepage_ctx wpc = { };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) xfs_iflags_clear(XFS_I(mapping->host), XFS_ITRUNCATED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) return iomap_writepages(mapping, wbc, &wpc.ctx, &xfs_writeback_ops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) STATIC int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) xfs_dax_writepages(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) struct address_space *mapping,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) struct writeback_control *wbc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) struct xfs_inode *ip = XFS_I(mapping->host);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) xfs_iflags_clear(ip, XFS_ITRUNCATED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) return dax_writeback_mapping_range(mapping,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) xfs_inode_buftarg(ip)->bt_daxdev, wbc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) STATIC sector_t
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) xfs_vm_bmap(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) struct address_space *mapping,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) sector_t block)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) struct xfs_inode *ip = XFS_I(mapping->host);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) trace_xfs_vm_bmap(ip);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) * The swap code (ab-)uses ->bmap to get a block mapping and then
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) * bypasses the file system for actual I/O. We really can't allow
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) * that on reflinks inodes, so we have to skip out here. And yes,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) * 0 is the magic code for a bmap error.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) * Since we don't pass back blockdev info, we can't return bmap
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) * information for rt files either.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) if (xfs_is_cow_inode(ip) || XFS_IS_REALTIME_INODE(ip))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) return iomap_bmap(mapping, block, &xfs_read_iomap_ops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) STATIC int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) xfs_vm_readpage(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) struct file *unused,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) return iomap_readpage(page, &xfs_read_iomap_ops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) STATIC void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) xfs_vm_readahead(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) struct readahead_control *rac)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) iomap_readahead(rac, &xfs_read_iomap_ops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) static int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) xfs_iomap_swapfile_activate(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) struct swap_info_struct *sis,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) struct file *swap_file,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) sector_t *span)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) sis->bdev = xfs_inode_buftarg(XFS_I(file_inode(swap_file)))->bt_bdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) return iomap_swapfile_activate(sis, swap_file, span,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) &xfs_read_iomap_ops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) const struct address_space_operations xfs_address_space_operations = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) .readpage = xfs_vm_readpage,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) .readahead = xfs_vm_readahead,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) .writepage = xfs_vm_writepage,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) .writepages = xfs_vm_writepages,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) .set_page_dirty = iomap_set_page_dirty,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) .releasepage = iomap_releasepage,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) .invalidatepage = iomap_invalidatepage,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) .bmap = xfs_vm_bmap,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) .direct_IO = noop_direct_IO,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) .migratepage = iomap_migrate_page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) .is_partially_uptodate = iomap_is_partially_uptodate,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) .error_remove_page = generic_error_remove_page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) .swap_activate = xfs_iomap_swapfile_activate,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) const struct address_space_operations xfs_dax_aops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) .writepages = xfs_dax_writepages,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) .direct_IO = noop_direct_IO,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) .set_page_dirty = noop_set_page_dirty,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) .invalidatepage = noop_invalidatepage,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) .swap_activate = xfs_iomap_swapfile_activate,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) };