Orange Pi5 kernel

^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  2) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  3) #include <linux/pagemap.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  4) #include <linux/xarray.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  5) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  6) #include <linux/swap.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  7) #include <linux/swapops.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  8) #include <asm/mte.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  9) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) static DEFINE_XARRAY(mte_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) void *mte_allocate_tag_storage(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) 	/* tags granule is 16 bytes, 2 tags stored per byte */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) 	return kmalloc(PAGE_SIZE / 16 / 2, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) void mte_free_tag_storage(char *storage)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) 	kfree(storage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) int mte_save_tags(struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) 	void *tag_storage, *ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) 	if (!test_bit(PG_mte_tagged, &page->flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) 		return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) 	tag_storage = mte_allocate_tag_storage();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) 	if (!tag_storage)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) 		return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) 	mte_save_page_tags(page_address(page), tag_storage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) 	/* page_private contains the swap entry.val set in do_swap_page */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) 	ret = xa_store(&mte_pages, page_private(page), tag_storage, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) 	if (WARN(xa_is_err(ret), "Failed to store MTE tags")) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) 		mte_free_tag_storage(tag_storage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) 		return xa_err(ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) 	} else if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) 		/* Entry is being replaced, free the old entry */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) 		mte_free_tag_storage(ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) bool mte_restore_tags(swp_entry_t entry, struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) 	void *tags = xa_load(&mte_pages, entry.val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) 	if (!tags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) 		return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) 	page_kasan_tag_reset(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) 	 * We need smp_wmb() in between setting the flags and clearing the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) 	 * tags because if another thread reads page->flags and builds a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) 	 * tagged address out of it, there is an actual dependency to the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) 	 * memory access, but on the current thread we do not guarantee that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) 	 * the new page->flags are visible before the tags were updated.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) 	smp_wmb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) 	mte_restore_page_tags(page_address(page), tags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) 	return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) void mte_invalidate_tags(int type, pgoff_t offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) 	swp_entry_t entry = swp_entry(type, offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) 	void *tags = xa_erase(&mte_pages, entry.val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) 	mte_free_tag_storage(tags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) void mte_invalidate_tags_area(int type)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) 	swp_entry_t entry = swp_entry(type, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) 	swp_entry_t last_entry = swp_entry(type + 1, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) 	void *tags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) 	XA_STATE(xa_state, &mte_pages, entry.val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) 	xa_lock(&mte_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) 	xas_for_each(&xa_state, tags, last_entry.val - 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) 		__xa_erase(&mte_pages, xa_state.xa_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) 		mte_free_tag_storage(tags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) 	xa_unlock(&mte_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) }