^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) /* SPDX-License-Identifier: GPL-2.0 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) #ifndef _LINUX_PGALLLC_TRACK_H
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) #define _LINUX_PGALLLC_TRACK_H
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) #if defined(CONFIG_MMU)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) static inline p4d_t *p4d_alloc_track(struct mm_struct *mm, pgd_t *pgd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) unsigned long address,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) pgtbl_mod_mask *mod_mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) if (unlikely(pgd_none(*pgd))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) if (__p4d_alloc(mm, pgd, address))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) *mod_mask |= PGTBL_PGD_MODIFIED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) return p4d_offset(pgd, address);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) static inline pud_t *pud_alloc_track(struct mm_struct *mm, p4d_t *p4d,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) unsigned long address,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) pgtbl_mod_mask *mod_mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) if (unlikely(p4d_none(*p4d))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) if (__pud_alloc(mm, p4d, address))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) *mod_mask |= PGTBL_P4D_MODIFIED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) return pud_offset(p4d, address);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) static inline pmd_t *pmd_alloc_track(struct mm_struct *mm, pud_t *pud,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) unsigned long address,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) pgtbl_mod_mask *mod_mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) if (unlikely(pud_none(*pud))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) if (__pmd_alloc(mm, pud, address))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) *mod_mask |= PGTBL_PUD_MODIFIED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) return pmd_offset(pud, address);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) #endif /* CONFIG_MMU */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) #define pte_alloc_kernel_track(pmd, address, mask) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) ((unlikely(pmd_none(*(pmd))) && \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) (__pte_alloc_kernel(pmd) || ({*(mask)|=PGTBL_PMD_MODIFIED;0;})))?\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) NULL: pte_offset_kernel(pmd, address))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) #endif /* _LINUX_PGALLLC_TRACK_H */