^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * mm/truncate.c - code for taking down pages from address_spaces
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Copyright (C) 2002, Linus Torvalds
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * 10Sep2002 Andrew Morton
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) * Initial version.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/backing-dev.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/dax.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/gfp.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/swap.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/export.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <linux/pagemap.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <linux/highmem.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <linux/pagevec.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <linux/task_io_accounting_ops.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include <linux/buffer_head.h> /* grr. try_to_release_page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) do_invalidatepage */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #include <linux/shmem_fs.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #include <linux/cleancache.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) #include <linux/rmap.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) #include "internal.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) * Regular page slots are stabilized by the page lock even without the tree
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) * itself locked. These unlocked entries need verification under the tree
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) * lock.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) static inline void __clear_shadow_entry(struct address_space *mapping,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) pgoff_t index, void *entry)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) XA_STATE(xas, &mapping->i_pages, index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) xas_set_update(&xas, workingset_update_node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) if (xas_load(&xas) != entry)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) xas_store(&xas, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) mapping->nrexceptional--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) static void clear_shadow_entry(struct address_space *mapping, pgoff_t index,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) void *entry)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) xa_lock_irq(&mapping->i_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) __clear_shadow_entry(mapping, index, entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) xa_unlock_irq(&mapping->i_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) * Unconditionally remove exceptional entries. Usually called from truncate
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) * path. Note that the pagevec may be altered by this function by removing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) * exceptional entries similar to what pagevec_remove_exceptionals does.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) static void truncate_exceptional_pvec_entries(struct address_space *mapping,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) struct pagevec *pvec, pgoff_t *indices,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) pgoff_t end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) int i, j;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) bool dax, lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) /* Handled by shmem itself */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) if (shmem_mapping(mapping))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) for (j = 0; j < pagevec_count(pvec); j++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) if (xa_is_value(pvec->pages[j]))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) if (j == pagevec_count(pvec))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) dax = dax_mapping(mapping);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) lock = !dax && indices[j] < end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) if (lock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) xa_lock_irq(&mapping->i_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) for (i = j; i < pagevec_count(pvec); i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) struct page *page = pvec->pages[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) pgoff_t index = indices[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) if (!xa_is_value(page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) pvec->pages[j++] = page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) if (index >= end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) if (unlikely(dax)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) dax_delete_mapping_entry(mapping, index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) __clear_shadow_entry(mapping, index, page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) if (lock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) xa_unlock_irq(&mapping->i_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) pvec->nr = j;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) * Invalidate exceptional entry if easily possible. This handles exceptional
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) * entries for invalidate_inode_pages().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) static int invalidate_exceptional_entry(struct address_space *mapping,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) pgoff_t index, void *entry)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) /* Handled by shmem itself, or for DAX we do nothing. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) if (shmem_mapping(mapping) || dax_mapping(mapping))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) clear_shadow_entry(mapping, index, entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) * Invalidate exceptional entry if clean. This handles exceptional entries for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) * invalidate_inode_pages2() so for DAX it evicts only clean entries.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) static int invalidate_exceptional_entry2(struct address_space *mapping,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) pgoff_t index, void *entry)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) /* Handled by shmem itself */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) if (shmem_mapping(mapping))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) if (dax_mapping(mapping))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) return dax_invalidate_mapping_entry_sync(mapping, index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) clear_shadow_entry(mapping, index, entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) * do_invalidatepage - invalidate part or all of a page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) * @page: the page which is affected
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) * @offset: start of the range to invalidate
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) * @length: length of the range to invalidate
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) * do_invalidatepage() is called when all or part of the page has become
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) * invalidated by a truncate operation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) * do_invalidatepage() does not have to release all buffers, but it must
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) * ensure that no dirty buffer is left outside @offset and that no I/O
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) * is underway against any of the blocks which are outside the truncation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) * point. Because the caller is about to free (and possibly reuse) those
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) * blocks on-disk.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) void do_invalidatepage(struct page *page, unsigned int offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) unsigned int length)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) void (*invalidatepage)(struct page *, unsigned int, unsigned int);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) invalidatepage = page->mapping->a_ops->invalidatepage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) #ifdef CONFIG_BLOCK
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) if (!invalidatepage)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) invalidatepage = block_invalidatepage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) if (invalidatepage)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) (*invalidatepage)(page, offset, length);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) * If truncate cannot remove the fs-private metadata from the page, the page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) * becomes orphaned. It will be left on the LRU and may even be mapped into
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) * user pagetables if we're racing with filemap_fault().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) * We need to bail out if page->mapping is no longer equal to the original
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) * mapping. This happens a) when the VM reclaimed the page while we waited on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) * its lock, b) when a concurrent invalidate_mapping_pages got there first and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) * c) when tmpfs swizzles a page between a tmpfs inode and swapper_space.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) static void truncate_cleanup_page(struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) if (page_mapped(page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) unmap_mapping_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) if (page_has_private(page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) do_invalidatepage(page, 0, thp_size(page));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) * Some filesystems seem to re-dirty the page even after
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) * the VM has canceled the dirty bit (eg ext3 journaling).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) * Hence dirty accounting check is placed after invalidation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) cancel_dirty_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) ClearPageMappedToDisk(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) * This is for invalidate_mapping_pages(). That function can be called at
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) * any time, and is not supposed to throw away dirty pages. But pages can
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) * be marked dirty at any time too, so use remove_mapping which safely
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) * discards clean, unused pages.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) * Returns non-zero if the page was successfully invalidated.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) static int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) invalidate_complete_page(struct address_space *mapping, struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) if (page->mapping != mapping)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) if (page_has_private(page) && !try_to_release_page(page, 0))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) ret = remove_mapping(mapping, page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) int truncate_inode_page(struct address_space *mapping, struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) VM_BUG_ON_PAGE(PageTail(page), page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) if (page->mapping != mapping)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) truncate_cleanup_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) delete_from_page_cache(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) * Used to get rid of pages on hardware memory corruption.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) int generic_error_remove_page(struct address_space *mapping, struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) if (!mapping)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) * Only punch for normal data pages for now.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) * Handling other types like directories would need more auditing.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) if (!S_ISREG(mapping->host->i_mode))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) return truncate_inode_page(mapping, page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) EXPORT_SYMBOL(generic_error_remove_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) * Safely invalidate one page from its pagecache mapping.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) * It only drops clean, unused pages. The page must be locked.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) * Returns 1 if the page is successfully invalidated, otherwise 0.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) int invalidate_inode_page(struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) struct address_space *mapping = page_mapping(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) if (!mapping)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) if (PageDirty(page) || PageWriteback(page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) if (page_mapped(page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) return invalidate_complete_page(mapping, page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) * truncate_inode_pages_range - truncate range of pages specified by start & end byte offsets
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) * @mapping: mapping to truncate
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) * @lstart: offset from which to truncate
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) * @lend: offset to which to truncate (inclusive)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) * Truncate the page cache, removing the pages that are between
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) * specified offsets (and zeroing out partial pages
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) * if lstart or lend + 1 is not page aligned).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) * Truncate takes two passes - the first pass is nonblocking. It will not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) * block on page locks and it will not block on writeback. The second pass
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) * will wait. This is to prevent as much IO as possible in the affected region.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) * The first pass will remove most pages, so the search cost of the second pass
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) * is low.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) * We pass down the cache-hot hint to the page freeing code. Even if the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) * mapping is large, it is probably the case that the final pages are the most
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) * recently touched, and freeing happens in ascending file offset order.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) * Note that since ->invalidatepage() accepts range to invalidate
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) * truncate_inode_pages_range is able to handle cases where lend + 1 is not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) * page aligned properly.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) void truncate_inode_pages_range(struct address_space *mapping,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) loff_t lstart, loff_t lend)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) pgoff_t start; /* inclusive */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) pgoff_t end; /* exclusive */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) unsigned int partial_start; /* inclusive */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) unsigned int partial_end; /* exclusive */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) struct pagevec pvec;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) pgoff_t indices[PAGEVEC_SIZE];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) pgoff_t index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) if (mapping->nrpages == 0 && mapping->nrexceptional == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) /* Offsets within partial pages */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) partial_start = lstart & (PAGE_SIZE - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) partial_end = (lend + 1) & (PAGE_SIZE - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) * 'start' and 'end' always covers the range of pages to be fully
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) * truncated. Partial pages are covered with 'partial_start' at the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) * start of the range and 'partial_end' at the end of the range.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) * Note that 'end' is exclusive while 'lend' is inclusive.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) start = (lstart + PAGE_SIZE - 1) >> PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) if (lend == -1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) * lend == -1 indicates end-of-file so we have to set 'end'
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) * to the highest possible pgoff_t and since the type is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) * unsigned we're using -1.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) end = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) end = (lend + 1) >> PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) pagevec_init(&pvec);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) index = start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) while (index < end && pagevec_lookup_entries(&pvec, mapping, index,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) min(end - index, (pgoff_t)PAGEVEC_SIZE),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) indices)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) * Pagevec array has exceptional entries and we may also fail
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) * to lock some pages. So we store pages that can be deleted
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) * in a new pagevec.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) struct pagevec locked_pvec;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) pagevec_init(&locked_pvec);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) for (i = 0; i < pagevec_count(&pvec); i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) struct page *page = pvec.pages[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) /* We rely upon deletion not changing page->index */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) index = indices[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) if (index >= end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) if (xa_is_value(page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) if (!trylock_page(page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) WARN_ON(page_to_index(page) != index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) if (PageWriteback(page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) unlock_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) if (page->mapping != mapping) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) unlock_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) pagevec_add(&locked_pvec, page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) for (i = 0; i < pagevec_count(&locked_pvec); i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) truncate_cleanup_page(locked_pvec.pages[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) delete_from_page_cache_batch(mapping, &locked_pvec);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) for (i = 0; i < pagevec_count(&locked_pvec); i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) unlock_page(locked_pvec.pages[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) truncate_exceptional_pvec_entries(mapping, &pvec, indices, end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) pagevec_release(&pvec);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) cond_resched();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) index++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) if (partial_start) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) struct page *page = find_lock_page(mapping, start - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) if (page) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) unsigned int top = PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) if (start > end) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) /* Truncation within a single page */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) top = partial_end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) partial_end = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) wait_on_page_writeback(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) zero_user_segment(page, partial_start, top);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) cleancache_invalidate_page(mapping, page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) if (page_has_private(page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) do_invalidatepage(page, partial_start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) top - partial_start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) unlock_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) put_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) if (partial_end) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) struct page *page = find_lock_page(mapping, end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) if (page) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) wait_on_page_writeback(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) zero_user_segment(page, 0, partial_end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) cleancache_invalidate_page(mapping, page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) if (page_has_private(page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) do_invalidatepage(page, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) partial_end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) unlock_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) put_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) * If the truncation happened within a single page no pages
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) * will be released, just zeroed, so we can bail out now.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) if (start >= end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) index = start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) for ( ; ; ) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) cond_resched();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) if (!pagevec_lookup_entries(&pvec, mapping, index,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) min(end - index, (pgoff_t)PAGEVEC_SIZE), indices)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) /* If all gone from start onwards, we're done */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) if (index == start)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) /* Otherwise restart to make sure all gone */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) index = start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) if (index == start && indices[0] >= end) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) /* All gone out of hole to be punched, we're done */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) pagevec_remove_exceptionals(&pvec);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) pagevec_release(&pvec);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) for (i = 0; i < pagevec_count(&pvec); i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) struct page *page = pvec.pages[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) /* We rely upon deletion not changing page->index */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) index = indices[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) if (index >= end) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) /* Restart punch to make sure all gone */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) index = start - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) if (xa_is_value(page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) lock_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) WARN_ON(page_to_index(page) != index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) wait_on_page_writeback(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) truncate_inode_page(mapping, page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) unlock_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) truncate_exceptional_pvec_entries(mapping, &pvec, indices, end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) pagevec_release(&pvec);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) index++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) cleancache_invalidate_inode(mapping);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) EXPORT_SYMBOL(truncate_inode_pages_range);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) * truncate_inode_pages - truncate *all* the pages from an offset
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) * @mapping: mapping to truncate
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) * @lstart: offset from which to truncate
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) * Called under (and serialised by) inode->i_mutex.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) * Note: When this function returns, there can be a page in the process of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) * deletion (inside __delete_from_page_cache()) in the specified range. Thus
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) * mapping->nrpages can be non-zero when this function returns even after
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) * truncation of the whole mapping.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) void truncate_inode_pages(struct address_space *mapping, loff_t lstart)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) truncate_inode_pages_range(mapping, lstart, (loff_t)-1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) EXPORT_SYMBOL(truncate_inode_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) * truncate_inode_pages_final - truncate *all* pages before inode dies
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) * @mapping: mapping to truncate
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) * Called under (and serialized by) inode->i_mutex.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) * Filesystems have to use this in the .evict_inode path to inform the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) * VM that this is the final truncate and the inode is going away.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) void truncate_inode_pages_final(struct address_space *mapping)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) unsigned long nrexceptional;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) unsigned long nrpages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) * Page reclaim can not participate in regular inode lifetime
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) * management (can't call iput()) and thus can race with the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) * inode teardown. Tell it when the address space is exiting,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) * so that it does not install eviction information after the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) * final truncate has begun.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) mapping_set_exiting(mapping);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) * When reclaim installs eviction entries, it increases
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) * nrexceptional first, then decreases nrpages. Make sure we see
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) * this in the right order or we might miss an entry.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) nrpages = mapping->nrpages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) smp_rmb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) nrexceptional = mapping->nrexceptional;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) if (nrpages || nrexceptional) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) * As truncation uses a lockless tree lookup, cycle
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) * the tree lock to make sure any ongoing tree
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) * modification that does not see AS_EXITING is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) * completed before starting the final truncate.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) xa_lock_irq(&mapping->i_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) xa_unlock_irq(&mapping->i_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) * Cleancache needs notification even if there are no pages or shadow
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) * entries.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) truncate_inode_pages(mapping, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) EXPORT_SYMBOL(truncate_inode_pages_final);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) static unsigned long __invalidate_mapping_pages(struct address_space *mapping,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) pgoff_t start, pgoff_t end, unsigned long *nr_pagevec)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) pgoff_t indices[PAGEVEC_SIZE];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) struct pagevec pvec;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) pgoff_t index = start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) unsigned long ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) unsigned long count = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) pagevec_init(&pvec);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) while (index <= end && pagevec_lookup_entries(&pvec, mapping, index,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) min(end - index, (pgoff_t)PAGEVEC_SIZE - 1) + 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) indices)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) for (i = 0; i < pagevec_count(&pvec); i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) struct page *page = pvec.pages[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) /* We rely upon deletion not changing page->index */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) index = indices[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) if (index > end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) if (xa_is_value(page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) invalidate_exceptional_entry(mapping, index,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) if (!trylock_page(page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) WARN_ON(page_to_index(page) != index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) /* Middle of THP: skip */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) if (PageTransTail(page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) unlock_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) } else if (PageTransHuge(page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) index += HPAGE_PMD_NR - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) i += HPAGE_PMD_NR - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) * 'end' is in the middle of THP. Don't
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) * invalidate the page as the part outside of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) * 'end' could be still useful.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) if (index > end) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) unlock_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) /* Take a pin outside pagevec */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) get_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) * Drop extra pins before trying to invalidate
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) * the huge page.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) pagevec_remove_exceptionals(&pvec);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) pagevec_release(&pvec);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) ret = invalidate_inode_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) unlock_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) * Invalidation is a hint that the page is no longer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) * of interest and try to speed up its reclaim.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) if (!ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) deactivate_file_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) /* It is likely on the pagevec of a remote CPU */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) if (nr_pagevec)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) (*nr_pagevec)++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) if (PageTransHuge(page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) put_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) count += ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) pagevec_remove_exceptionals(&pvec);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) pagevec_release(&pvec);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) cond_resched();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) index++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) return count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) * invalidate_mapping_pages - Invalidate all the unlocked pages of one inode
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) * @mapping: the address_space which holds the pages to invalidate
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) * @start: the offset 'from' which to invalidate
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) * @end: the offset 'to' which to invalidate (inclusive)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) * This function only removes the unlocked pages, if you want to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) * remove all the pages of one inode, you must call truncate_inode_pages.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) * invalidate_mapping_pages() will not block on IO activity. It will not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) * invalidate pages which are dirty, locked, under writeback or mapped into
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) * pagetables.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) * Return: the number of the pages that were invalidated
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) unsigned long invalidate_mapping_pages(struct address_space *mapping,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) pgoff_t start, pgoff_t end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) return __invalidate_mapping_pages(mapping, start, end, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) EXPORT_SYMBOL(invalidate_mapping_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) * This helper is similar with the above one, except that it accounts for pages
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) * that are likely on a pagevec and count them in @nr_pagevec, which will used by
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) * the caller.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) void invalidate_mapping_pagevec(struct address_space *mapping,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) pgoff_t start, pgoff_t end, unsigned long *nr_pagevec)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) __invalidate_mapping_pages(mapping, start, end, nr_pagevec);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) * This is like invalidate_complete_page(), except it ignores the page's
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) * refcount. We do this because invalidate_inode_pages2() needs stronger
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) * invalidation guarantees, and cannot afford to leave pages behind because
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) * shrink_page_list() has a temp ref on them, or because they're transiently
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) * sitting in the lru_cache_add() pagevecs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) static int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) invalidate_complete_page2(struct address_space *mapping, struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) if (page->mapping != mapping)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) if (page_has_private(page) && !try_to_release_page(page, GFP_KERNEL))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) xa_lock_irqsave(&mapping->i_pages, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) if (PageDirty(page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) goto failed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) BUG_ON(page_has_private(page));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) __delete_from_page_cache(page, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) xa_unlock_irqrestore(&mapping->i_pages, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) if (mapping->a_ops->freepage)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) mapping->a_ops->freepage(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) put_page(page); /* pagecache ref */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) failed:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) xa_unlock_irqrestore(&mapping->i_pages, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) static int do_launder_page(struct address_space *mapping, struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) if (!PageDirty(page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) if (page->mapping != mapping || mapping->a_ops->launder_page == NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) return mapping->a_ops->launder_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) * invalidate_inode_pages2_range - remove range of pages from an address_space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) * @mapping: the address_space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) * @start: the page offset 'from' which to invalidate
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) * @end: the page offset 'to' which to invalidate (inclusive)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) * Any pages which are found to be mapped into pagetables are unmapped prior to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) * invalidation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) * Return: -EBUSY if any pages could not be invalidated.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) int invalidate_inode_pages2_range(struct address_space *mapping,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) pgoff_t start, pgoff_t end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) pgoff_t indices[PAGEVEC_SIZE];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) struct pagevec pvec;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) pgoff_t index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) int ret2 = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) int did_range_unmap = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) if (mapping->nrpages == 0 && mapping->nrexceptional == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) pagevec_init(&pvec);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) index = start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) while (index <= end && pagevec_lookup_entries(&pvec, mapping, index,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) min(end - index, (pgoff_t)PAGEVEC_SIZE - 1) + 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) indices)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) for (i = 0; i < pagevec_count(&pvec); i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) struct page *page = pvec.pages[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) /* We rely upon deletion not changing page->index */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) index = indices[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) if (index > end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) if (xa_is_value(page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) if (!invalidate_exceptional_entry2(mapping,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) index, page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) ret = -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) if (!did_range_unmap && page_mapped(page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) * If page is mapped, before taking its lock,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) * zap the rest of the file in one hit.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) unmap_mapping_pages(mapping, index,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) (1 + end - index), false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) did_range_unmap = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) lock_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) WARN_ON(page_to_index(page) != index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) if (page->mapping != mapping) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) unlock_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) wait_on_page_writeback(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) if (page_mapped(page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) unmap_mapping_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) BUG_ON(page_mapped(page));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) ret2 = do_launder_page(mapping, page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) if (ret2 == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) if (!invalidate_complete_page2(mapping, page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) ret2 = -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) if (ret2 < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) ret = ret2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) unlock_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) pagevec_remove_exceptionals(&pvec);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) pagevec_release(&pvec);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) cond_resched();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) index++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) * For DAX we invalidate page tables after invalidating page cache. We
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) * could invalidate page tables while invalidating each entry however
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) * that would be expensive. And doing range unmapping before doesn't
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) * work as we have no cheap way to find whether page cache entry didn't
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) * get remapped later.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) if (dax_mapping(mapping)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) unmap_mapping_pages(mapping, start, end - start + 1, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) cleancache_invalidate_inode(mapping);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) EXPORT_SYMBOL_GPL(invalidate_inode_pages2_range);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) * invalidate_inode_pages2 - remove all pages from an address_space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) * @mapping: the address_space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) * Any pages which are found to be mapped into pagetables are unmapped prior to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) * invalidation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) * Return: -EBUSY if any pages could not be invalidated.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) int invalidate_inode_pages2(struct address_space *mapping)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) return invalidate_inode_pages2_range(mapping, 0, -1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) EXPORT_SYMBOL_GPL(invalidate_inode_pages2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) * truncate_pagecache - unmap and remove pagecache that has been truncated
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) * @inode: inode
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) * @newsize: new file size
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) * inode's new i_size must already be written before truncate_pagecache
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) * is called.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) * This function should typically be called before the filesystem
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) * releases resources associated with the freed range (eg. deallocates
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) * blocks). This way, pagecache will always stay logically coherent
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) * with on-disk format, and the filesystem would not have to deal with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) * situations such as writepage being called for a page that has already
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) * had its underlying blocks deallocated.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) void truncate_pagecache(struct inode *inode, loff_t newsize)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) struct address_space *mapping = inode->i_mapping;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) loff_t holebegin = round_up(newsize, PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) * unmap_mapping_range is called twice, first simply for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) * efficiency so that truncate_inode_pages does fewer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) * single-page unmaps. However after this first call, and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) * before truncate_inode_pages finishes, it is possible for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) * private pages to be COWed, which remain after
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) * truncate_inode_pages finishes, hence the second
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) * unmap_mapping_range call must be made for correctness.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) unmap_mapping_range(mapping, holebegin, 0, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) truncate_inode_pages(mapping, newsize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) unmap_mapping_range(mapping, holebegin, 0, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) EXPORT_SYMBOL(truncate_pagecache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) * truncate_setsize - update inode and pagecache for a new file size
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) * @inode: inode
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) * @newsize: new file size
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) * truncate_setsize updates i_size and performs pagecache truncation (if
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) * necessary) to @newsize. It will be typically be called from the filesystem's
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) * setattr function when ATTR_SIZE is passed in.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) * Must be called with a lock serializing truncates and writes (generally
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) * i_mutex but e.g. xfs uses a different lock) and before all filesystem
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) * specific block truncation has been performed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) void truncate_setsize(struct inode *inode, loff_t newsize)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) loff_t oldsize = inode->i_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) i_size_write(inode, newsize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) if (newsize > oldsize)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) pagecache_isize_extended(inode, oldsize, newsize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) truncate_pagecache(inode, newsize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) EXPORT_SYMBOL(truncate_setsize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) * pagecache_isize_extended - update pagecache after extension of i_size
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) * @inode: inode for which i_size was extended
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) * @from: original inode size
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) * @to: new inode size
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) * Handle extension of inode size either caused by extending truncate or by
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) * write starting after current i_size. We mark the page straddling current
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) * i_size RO so that page_mkwrite() is called on the nearest write access to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) * the page. This way filesystem can be sure that page_mkwrite() is called on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) * the page before user writes to the page via mmap after the i_size has been
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) * changed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) * The function must be called after i_size is updated so that page fault
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) * coming after we unlock the page will already see the new i_size.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) * The function must be called while we still hold i_mutex - this not only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) * makes sure i_size is stable but also that userspace cannot observe new
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) * i_size value before we are prepared to store mmap writes at new inode size.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) void pagecache_isize_extended(struct inode *inode, loff_t from, loff_t to)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) int bsize = i_blocksize(inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) loff_t rounded_from;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) pgoff_t index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) WARN_ON(to > inode->i_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) if (from >= to || bsize == PAGE_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893) /* Page straddling @from will not have any hole block created? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) rounded_from = round_up(from, bsize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) if (to <= rounded_from || !(rounded_from & (PAGE_SIZE - 1)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) index = from >> PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) page = find_lock_page(inode->i_mapping, index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) /* Page not cached? Nothing to do */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) if (!page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) * See clear_page_dirty_for_io() for details why set_page_dirty()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) * is needed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907) if (page_mkclean(page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) set_page_dirty(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) unlock_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) put_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) EXPORT_SYMBOL(pagecache_isize_extended);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) * truncate_pagecache_range - unmap and remove pagecache that is hole-punched
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) * @inode: inode
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) * @lstart: offset of beginning of hole
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) * @lend: offset of last byte of hole
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) * This function should typically be called before the filesystem
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) * releases resources associated with the freed range (eg. deallocates
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) * blocks). This way, pagecache will always stay logically coherent
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) * with on-disk format, and the filesystem would not have to deal with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924) * situations such as writepage being called for a page that has already
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) * had its underlying blocks deallocated.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927) void truncate_pagecache_range(struct inode *inode, loff_t lstart, loff_t lend)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) struct address_space *mapping = inode->i_mapping;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) loff_t unmap_start = round_up(lstart, PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) loff_t unmap_end = round_down(1 + lend, PAGE_SIZE) - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) * This rounding is currently just for example: unmap_mapping_range
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) * expands its hole outwards, whereas we want it to contract the hole
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) * inwards. However, existing callers of truncate_pagecache_range are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) * doing their own page rounding first. Note that unmap_mapping_range
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) * allows holelen 0 for all, and we allow lend -1 for end of file.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) * Unlike in truncate_pagecache, unmap_mapping_range is called only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) * once (before truncating pagecache), and without "even_cows" flag:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943) * hole-punching should not remove private COWed pages from the hole.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) if ((u64)unmap_end > (u64)unmap_start)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946) unmap_mapping_range(mapping, unmap_start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947) 1 + unmap_end - unmap_start, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) truncate_inode_pages_range(mapping, lstart, lend);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950) EXPORT_SYMBOL(truncate_pagecache_range);
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags