^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * mm/mprotect.c
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * (C) Copyright 1994 Linus Torvalds
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * (C) Copyright 2002 Christoph Hellwig
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) * Address space accounting code <alan@lxorguk.ukuu.org.uk>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) * (C) Copyright 2002 Red Hat Inc, All Rights Reserved
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/pagewalk.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/hugetlb.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/shm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/mman.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/fs.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/highmem.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <linux/security.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <linux/mempolicy.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <linux/personality.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <linux/syscalls.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include <linux/swap.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #include <linux/swapops.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #include <linux/mmu_notifier.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #include <linux/migrate.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) #include <linux/perf_event.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) #include <linux/pkeys.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) #include <linux/ksm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) #include <linux/uaccess.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) #include <linux/mm_inline.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) #include <linux/pgtable.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) #include <asm/cacheflush.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) #include <asm/mmu_context.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) #include <asm/tlbflush.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) #include "internal.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) static unsigned long change_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) unsigned long addr, unsigned long end, pgprot_t newprot,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) unsigned long cp_flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) pte_t *pte, oldpte;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) spinlock_t *ptl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) unsigned long pages = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) int target_node = NUMA_NO_NODE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) bool dirty_accountable = cp_flags & MM_CP_DIRTY_ACCT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) bool prot_numa = cp_flags & MM_CP_PROT_NUMA;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) bool uffd_wp = cp_flags & MM_CP_UFFD_WP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) bool uffd_wp_resolve = cp_flags & MM_CP_UFFD_WP_RESOLVE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) * Can be called with only the mmap_lock for reading by
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) * prot_numa so we must check the pmd isn't constantly
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) * changing from under us from pmd_none to pmd_trans_huge
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) * and/or the other way around.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) if (pmd_trans_unstable(pmd))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) * The pmd points to a regular pte so the pmd can't change
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) * from under us even if the mmap_lock is only hold for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) * reading.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) /* Get target node for single threaded private VMAs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) if (prot_numa && !(vma->vm_flags & VM_SHARED) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) atomic_read(&vma->vm_mm->mm_users) == 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) target_node = numa_node_id();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) flush_tlb_batched_pending(vma->vm_mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) arch_enter_lazy_mmu_mode();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) oldpte = *pte;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) if (pte_present(oldpte)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) pte_t ptent;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) bool preserve_write = prot_numa && pte_write(oldpte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) * Avoid trapping faults against the zero or KSM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) * pages. See similar comment in change_huge_pmd.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) if (prot_numa) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) /* Avoid TLB flush if possible */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) if (pte_protnone(oldpte))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) page = vm_normal_page(vma, addr, oldpte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) if (!page || PageKsm(page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) /* Also skip shared copy-on-write pages */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) if (is_cow_mapping(vma->vm_flags) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) page_count(page) != 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) * While migration can move some dirty pages,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) * it cannot move them all from MIGRATE_ASYNC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) * context.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) if (page_is_file_lru(page) && PageDirty(page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) * Don't mess with PTEs if page is already on the node
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) * a single-threaded process is running on.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) if (target_node == page_to_nid(page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) oldpte = ptep_modify_prot_start(vma, addr, pte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) ptent = pte_modify(oldpte, newprot);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) if (preserve_write)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) ptent = pte_mk_savedwrite(ptent);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) if (uffd_wp) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) ptent = pte_wrprotect(ptent);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) ptent = pte_mkuffd_wp(ptent);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) } else if (uffd_wp_resolve) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) * Leave the write bit to be handled
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) * by PF interrupt handler, then
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) * things like COW could be properly
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) * handled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) ptent = pte_clear_uffd_wp(ptent);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) /* Avoid taking write faults for known dirty pages */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) if (dirty_accountable && pte_dirty(ptent) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) (pte_soft_dirty(ptent) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) !(vma->vm_flags & VM_SOFTDIRTY))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) ptent = pte_mkwrite(ptent);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) ptep_modify_prot_commit(vma, addr, pte, oldpte, ptent);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) pages++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) } else if (is_swap_pte(oldpte)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) swp_entry_t entry = pte_to_swp_entry(oldpte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) pte_t newpte;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) if (is_write_migration_entry(entry)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) * A protection check is difficult so
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) * just be safe and disable write
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) make_migration_entry_read(&entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) newpte = swp_entry_to_pte(entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) if (pte_swp_soft_dirty(oldpte))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) newpte = pte_swp_mksoft_dirty(newpte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) if (pte_swp_uffd_wp(oldpte))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) newpte = pte_swp_mkuffd_wp(newpte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) } else if (is_write_device_private_entry(entry)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) * We do not preserve soft-dirtiness. See
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) * copy_one_pte() for explanation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) make_device_private_entry_read(&entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) newpte = swp_entry_to_pte(entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) if (pte_swp_uffd_wp(oldpte))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) newpte = pte_swp_mkuffd_wp(newpte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) newpte = oldpte;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) if (uffd_wp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) newpte = pte_swp_mkuffd_wp(newpte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) else if (uffd_wp_resolve)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) newpte = pte_swp_clear_uffd_wp(newpte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) if (!pte_same(oldpte, newpte)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) set_pte_at(vma->vm_mm, addr, pte, newpte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) pages++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) } while (pte++, addr += PAGE_SIZE, addr != end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) arch_leave_lazy_mmu_mode();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) pte_unmap_unlock(pte - 1, ptl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) return pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) * Used when setting automatic NUMA hinting protection where it is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) * critical that a numa hinting PMD is not confused with a bad PMD.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) static inline int pmd_none_or_clear_bad_unless_trans_huge(pmd_t *pmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) pmd_t pmdval = pmd_read_atomic(pmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) /* See pmd_none_or_trans_huge_or_clear_bad for info on barrier */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) #ifdef CONFIG_TRANSPARENT_HUGEPAGE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) barrier();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) if (pmd_none(pmdval))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) if (pmd_trans_huge(pmdval))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) if (unlikely(pmd_bad(pmdval))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) pmd_clear_bad(pmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) static inline unsigned long change_pmd_range(struct vm_area_struct *vma,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) pud_t *pud, unsigned long addr, unsigned long end,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) pgprot_t newprot, unsigned long cp_flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) pmd_t *pmd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) unsigned long next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) unsigned long pages = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) unsigned long nr_huge_updates = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) struct mmu_notifier_range range;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) range.start = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) pmd = pmd_offset(pud, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) unsigned long this_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) next = pmd_addr_end(addr, end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) * Automatic NUMA balancing walks the tables with mmap_lock
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) * held for read. It's possible a parallel update to occur
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) * between pmd_trans_huge() and a pmd_none_or_clear_bad()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) * check leading to a false positive and clearing.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) * Hence, it's necessary to atomically read the PMD value
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) * for all the checks.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) if (!is_swap_pmd(*pmd) && !pmd_devmap(*pmd) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) pmd_none_or_clear_bad_unless_trans_huge(pmd))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) goto next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) /* invoke the mmu notifier if the pmd is populated */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) if (!range.start) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) mmu_notifier_range_init(&range,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) MMU_NOTIFY_PROTECTION_VMA, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) vma, vma->vm_mm, addr, end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) mmu_notifier_invalidate_range_start(&range);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) if (is_swap_pmd(*pmd) || pmd_trans_huge(*pmd) || pmd_devmap(*pmd)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) if (next - addr != HPAGE_PMD_SIZE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) __split_huge_pmd(vma, pmd, addr, false, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) int nr_ptes = change_huge_pmd(vma, pmd, addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) newprot, cp_flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) if (nr_ptes) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) if (nr_ptes == HPAGE_PMD_NR) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) pages += HPAGE_PMD_NR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) nr_huge_updates++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) /* huge pmd was handled */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) goto next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) /* fall through, the trans huge pmd just split */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) this_pages = change_pte_range(vma, pmd, addr, next, newprot,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) cp_flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) pages += this_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) next:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) cond_resched();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) } while (pmd++, addr = next, addr != end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) if (range.start)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) mmu_notifier_invalidate_range_end(&range);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) if (nr_huge_updates)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) count_vm_numa_events(NUMA_HUGE_PTE_UPDATES, nr_huge_updates);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) return pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) static inline unsigned long change_pud_range(struct vm_area_struct *vma,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) p4d_t *p4d, unsigned long addr, unsigned long end,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) pgprot_t newprot, unsigned long cp_flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) pud_t *pud;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) unsigned long next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) unsigned long pages = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) pud = pud_offset(p4d, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) next = pud_addr_end(addr, end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) if (pud_none_or_clear_bad(pud))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) pages += change_pmd_range(vma, pud, addr, next, newprot,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) cp_flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) } while (pud++, addr = next, addr != end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) return pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) static inline unsigned long change_p4d_range(struct vm_area_struct *vma,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) pgd_t *pgd, unsigned long addr, unsigned long end,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) pgprot_t newprot, unsigned long cp_flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) p4d_t *p4d;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) unsigned long next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) unsigned long pages = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) p4d = p4d_offset(pgd, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) next = p4d_addr_end(addr, end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) if (p4d_none_or_clear_bad(p4d))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) pages += change_pud_range(vma, p4d, addr, next, newprot,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) cp_flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) } while (p4d++, addr = next, addr != end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) return pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) static unsigned long change_protection_range(struct vm_area_struct *vma,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) unsigned long addr, unsigned long end, pgprot_t newprot,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) unsigned long cp_flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) struct mm_struct *mm = vma->vm_mm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) pgd_t *pgd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) unsigned long next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) unsigned long start = addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) unsigned long pages = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) BUG_ON(addr >= end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) pgd = pgd_offset(mm, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) flush_cache_range(vma, addr, end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) inc_tlb_flush_pending(mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) next = pgd_addr_end(addr, end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) if (pgd_none_or_clear_bad(pgd))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) pages += change_p4d_range(vma, pgd, addr, next, newprot,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) cp_flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) } while (pgd++, addr = next, addr != end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) /* Only flush the TLB if we actually modified any entries: */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) if (pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) flush_tlb_range(vma, start, end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) dec_tlb_flush_pending(mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) return pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) unsigned long change_protection(struct vm_area_struct *vma, unsigned long start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) unsigned long end, pgprot_t newprot,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) unsigned long cp_flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) unsigned long pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) BUG_ON((cp_flags & MM_CP_UFFD_WP_ALL) == MM_CP_UFFD_WP_ALL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) if (is_vm_hugetlb_page(vma))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) pages = hugetlb_change_protection(vma, start, end, newprot);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) pages = change_protection_range(vma, start, end, newprot,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) cp_flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) return pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) static int prot_none_pte_entry(pte_t *pte, unsigned long addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) unsigned long next, struct mm_walk *walk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) return pfn_modify_allowed(pte_pfn(*pte), *(pgprot_t *)(walk->private)) ?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) 0 : -EACCES;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) static int prot_none_hugetlb_entry(pte_t *pte, unsigned long hmask,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) unsigned long addr, unsigned long next,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) struct mm_walk *walk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) return pfn_modify_allowed(pte_pfn(*pte), *(pgprot_t *)(walk->private)) ?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) 0 : -EACCES;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) static int prot_none_test(unsigned long addr, unsigned long next,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) struct mm_walk *walk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) static const struct mm_walk_ops prot_none_walk_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) .pte_entry = prot_none_pte_entry,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) .hugetlb_entry = prot_none_hugetlb_entry,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) .test_walk = prot_none_test,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) mprotect_fixup(struct vm_area_struct *vma, struct vm_area_struct **pprev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) unsigned long start, unsigned long end, unsigned long newflags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) struct mm_struct *mm = vma->vm_mm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) unsigned long oldflags = vma->vm_flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) long nrpages = (end - start) >> PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) unsigned long charged = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) pgoff_t pgoff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) int error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) int dirty_accountable = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) if (newflags == oldflags) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) *pprev = vma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) * Do PROT_NONE PFN permission checks here when we can still
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) * bail out without undoing a lot of state. This is a rather
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) * uncommon case, so doesn't need to be very optimized.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) if (arch_has_pfn_modify_check() &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) (vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) (newflags & VM_ACCESS_FLAGS) == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) pgprot_t new_pgprot = vm_get_page_prot(newflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) error = walk_page_range(current->mm, start, end,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) &prot_none_walk_ops, &new_pgprot);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) if (error)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) return error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) * If we make a private mapping writable we increase our commit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) * but (without finer accounting) cannot reduce our commit if we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) * make it unwritable again. hugetlb mapping were accounted for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) * even if read-only so there is no need to account for them here
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) if (newflags & VM_WRITE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) /* Check space limits when area turns into data. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) if (!may_expand_vm(mm, newflags, nrpages) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) may_expand_vm(mm, oldflags, nrpages))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) if (!(oldflags & (VM_ACCOUNT|VM_WRITE|VM_HUGETLB|
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) VM_SHARED|VM_NORESERVE))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) charged = nrpages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) if (security_vm_enough_memory_mm(mm, charged))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) newflags |= VM_ACCOUNT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) * First try to merge with previous and/or next vma.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) *pprev = vma_merge(mm, *pprev, start, end, newflags,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) vma->vm_userfaultfd_ctx, vma_get_anon_name(vma));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) if (*pprev) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) vma = *pprev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) VM_WARN_ON((vma->vm_flags ^ newflags) & ~VM_SOFTDIRTY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) goto success;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) *pprev = vma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) if (start != vma->vm_start) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) error = split_vma(mm, vma, start, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) if (error)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) goto fail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) if (end != vma->vm_end) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) error = split_vma(mm, vma, end, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) if (error)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) goto fail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) success:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) * vm_flags and vm_page_prot are protected by the mmap_lock
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) * held in write mode.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) vm_write_begin(vma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) WRITE_ONCE(vma->vm_flags, newflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) dirty_accountable = vma_wants_writenotify(vma, vma->vm_page_prot);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) vma_set_page_prot(vma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) change_protection(vma, start, end, vma->vm_page_prot,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) dirty_accountable ? MM_CP_DIRTY_ACCT : 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) vm_write_end(vma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) * Private VM_LOCKED VMA becoming writable: trigger COW to avoid major
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) * fault on access.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) if ((oldflags & (VM_WRITE | VM_SHARED | VM_LOCKED)) == VM_LOCKED &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) (newflags & VM_WRITE)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) populate_vma_page_range(vma, start, end, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) vm_stat_account(mm, oldflags, -nrpages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) vm_stat_account(mm, newflags, nrpages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) perf_event_mmap(vma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) fail:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) vm_unacct_memory(charged);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) return error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) * pkey==-1 when doing a legacy mprotect()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) static int do_mprotect_pkey(unsigned long start, size_t len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) unsigned long prot, int pkey)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) unsigned long nstart, end, tmp, reqprot;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) struct vm_area_struct *vma, *prev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) int error = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) const int grows = prot & (PROT_GROWSDOWN|PROT_GROWSUP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) const bool rier = (current->personality & READ_IMPLIES_EXEC) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) (prot & PROT_READ);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) start = untagged_addr(start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) prot &= ~(PROT_GROWSDOWN|PROT_GROWSUP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) if (grows == (PROT_GROWSDOWN|PROT_GROWSUP)) /* can't be both */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) if (start & ~PAGE_MASK)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) if (!len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) len = PAGE_ALIGN(len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) end = start + len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) if (end <= start)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) if (!arch_validate_prot(prot, start))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) reqprot = prot;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) if (mmap_write_lock_killable(current->mm))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) return -EINTR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) * If userspace did not allocate the pkey, do not let
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) * them use it here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) error = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) if ((pkey != -1) && !mm_pkey_is_allocated(current->mm, pkey))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) vma = find_vma(current->mm, start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) error = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) if (!vma)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) prev = vma->vm_prev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) if (unlikely(grows & PROT_GROWSDOWN)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) if (vma->vm_start >= end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) start = vma->vm_start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) error = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) if (!(vma->vm_flags & VM_GROWSDOWN))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) if (vma->vm_start > start)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) if (unlikely(grows & PROT_GROWSUP)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) end = vma->vm_end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) error = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) if (!(vma->vm_flags & VM_GROWSUP))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) if (start > vma->vm_start)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) prev = vma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) for (nstart = start ; ; ) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) unsigned long mask_off_old_flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) unsigned long newflags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) int new_vma_pkey;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) /* Here we know that vma->vm_start <= nstart < vma->vm_end. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) /* Does the application expect PROT_READ to imply PROT_EXEC */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) if (rier && (vma->vm_flags & VM_MAYEXEC))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) prot |= PROT_EXEC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) * Each mprotect() call explicitly passes r/w/x permissions.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) * If a permission is not passed to mprotect(), it must be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) * cleared from the VMA.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) mask_off_old_flags = VM_READ | VM_WRITE | VM_EXEC |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) VM_FLAGS_CLEAR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) new_vma_pkey = arch_override_mprotect_pkey(vma, prot, pkey);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) newflags = calc_vm_prot_bits(prot, new_vma_pkey);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) newflags |= (vma->vm_flags & ~mask_off_old_flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) /* newflags >> 4 shift VM_MAY% in place of VM_% */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) if ((newflags & ~(newflags >> 4)) & VM_ACCESS_FLAGS) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) error = -EACCES;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) /* Allow architectures to sanity-check the new flags */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) if (!arch_validate_flags(newflags)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) error = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) error = security_file_mprotect(vma, reqprot, prot);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) if (error)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) tmp = vma->vm_end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) if (tmp > end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) tmp = end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) error = mprotect_fixup(vma, &prev, nstart, tmp, newflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) if (error)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) nstart = tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) if (nstart < prev->vm_end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) nstart = prev->vm_end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) if (nstart >= end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) vma = prev->vm_next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) if (!vma || vma->vm_start != nstart) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) error = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) prot = reqprot;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) mmap_write_unlock(current->mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) return error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) SYSCALL_DEFINE3(mprotect, unsigned long, start, size_t, len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) unsigned long, prot)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) return do_mprotect_pkey(start, len, prot, -1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) #ifdef CONFIG_ARCH_HAS_PKEYS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) SYSCALL_DEFINE4(pkey_mprotect, unsigned long, start, size_t, len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) unsigned long, prot, int, pkey)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) return do_mprotect_pkey(start, len, prot, pkey);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) SYSCALL_DEFINE2(pkey_alloc, unsigned long, flags, unsigned long, init_val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) int pkey;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) /* No flags supported yet. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) if (flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) /* check for unsupported init values */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) if (init_val & ~PKEY_ACCESS_MASK)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) mmap_write_lock(current->mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) pkey = mm_pkey_alloc(current->mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) ret = -ENOSPC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) if (pkey == -1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) ret = arch_set_user_pkey_access(current, pkey, init_val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) mm_pkey_free(current->mm, pkey);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) ret = pkey;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) mmap_write_unlock(current->mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) SYSCALL_DEFINE1(pkey_free, int, pkey)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) mmap_write_lock(current->mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) ret = mm_pkey_free(current->mm, pkey);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) mmap_write_unlock(current->mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) * We could provie warnings or errors if any VMA still
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) * has the pkey set here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) #endif /* CONFIG_ARCH_HAS_PKEYS */
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags