^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) #include <linux/sched.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) #include <linux/errno.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) #include <linux/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) #include <linux/nmi.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) #include <linux/swap.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include <linux/smp.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <linux/highmem.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/pagemap.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/spinlock.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <asm/cpu_entry_area.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <asm/fixmap.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <asm/e820/api.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <asm/tlb.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <asm/tlbflush.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <asm/io.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <linux/vmalloc.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) unsigned int __VMALLOC_RESERVE = 128 << 20;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) * Associate a virtual page frame with a given physical page frame
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) * and protection flags for that frame.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) void set_pte_vaddr(unsigned long vaddr, pte_t pteval)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) pgd_t *pgd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) p4d_t *p4d;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) pud_t *pud;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) pmd_t *pmd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) pte_t *pte;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) pgd = swapper_pg_dir + pgd_index(vaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) if (pgd_none(*pgd)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) BUG();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) p4d = p4d_offset(pgd, vaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) if (p4d_none(*p4d)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) BUG();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) pud = pud_offset(p4d, vaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) if (pud_none(*pud)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) BUG();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) pmd = pmd_offset(pud, vaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) if (pmd_none(*pmd)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) BUG();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) pte = pte_offset_kernel(pmd, vaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) if (!pte_none(pteval))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) set_pte_at(&init_mm, vaddr, pte, pteval);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) pte_clear(&init_mm, vaddr, pte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) * It's enough to flush this one mapping.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) * (PGE mappings get flushed as well)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) flush_tlb_one_kernel(vaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) unsigned long __FIXADDR_TOP = 0xfffff000;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) EXPORT_SYMBOL(__FIXADDR_TOP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) * vmalloc=size forces the vmalloc area to be exactly 'size'
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) * bytes. This can be used to increase (or decrease) the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) * vmalloc area - the default is 128m.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) static int __init parse_vmalloc(char *arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) if (!arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) /* Add VMALLOC_OFFSET to the parsed value due to vm area guard hole*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) __VMALLOC_RESERVE = memparse(arg, &arg) + VMALLOC_OFFSET;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) early_param("vmalloc", parse_vmalloc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) * reservetop=size reserves a hole at the top of the kernel address space which
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) * a hypervisor can load into later. Needed for dynamically loaded hypervisors,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) * so relocating the fixmap can be done before paging initialization.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) static int __init parse_reservetop(char *arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) unsigned long address;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) if (!arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) address = memparse(arg, &arg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) reserve_top_address(address);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) early_ioremap_init();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) early_param("reservetop", parse_reservetop);