Orange Pi5 kernel

^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   3)  * alternative runtime patching
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   4)  * inspired by the x86 version
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   5)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   6)  * Copyright (C) 2014 ARM Ltd.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   7)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   8) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   9) #define pr_fmt(fmt) "alternatives: " fmt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  10) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  11) #include <linux/init.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  12) #include <linux/cpu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  13) #include <asm/cacheflush.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  14) #include <asm/alternative.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  15) #include <asm/cpufeature.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  16) #include <asm/insn.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  17) #include <asm/sections.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  18) #include <linux/stop_machine.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  19) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  20) #define __ALT_PTR(a,f)		((void *)&(a)->f + (a)->f)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  21) #define ALT_ORIG_PTR(a)		__ALT_PTR(a, orig_offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  22) #define ALT_REPL_PTR(a)		__ALT_PTR(a, alt_offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  23) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  24) /* Volatile, as we may be patching the guts of READ_ONCE() */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  25) static volatile int all_alternatives_applied;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  26) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  27) static DECLARE_BITMAP(applied_alternatives, ARM64_NCAPS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  28) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  29) struct alt_region {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  30) 	struct alt_instr *begin;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  31) 	struct alt_instr *end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  32) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  33) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  34) bool alternative_is_applied(u16 cpufeature)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  35) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  36) 	if (WARN_ON(cpufeature >= ARM64_NCAPS))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  37) 		return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  38) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  39) 	return test_bit(cpufeature, applied_alternatives);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  40) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  41) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  42) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  43)  * Check if the target PC is within an alternative block.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  44)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  45) static bool branch_insn_requires_update(struct alt_instr *alt, unsigned long pc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  46) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  47) 	unsigned long replptr = (unsigned long)ALT_REPL_PTR(alt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  48) 	return !(pc >= replptr && pc <= (replptr + alt->alt_len));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  49) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  50) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  51) #define align_down(x, a)	((unsigned long)(x) & ~(((unsigned long)(a)) - 1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  52) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  53) static u32 get_alt_insn(struct alt_instr *alt, __le32 *insnptr, __le32 *altinsnptr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  54) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  55) 	u32 insn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  56) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  57) 	insn = le32_to_cpu(*altinsnptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  58) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  59) 	if (aarch64_insn_is_branch_imm(insn)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  60) 		s32 offset = aarch64_get_branch_offset(insn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  61) 		unsigned long target;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  62) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  63) 		target = (unsigned long)altinsnptr + offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  64) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  65) 		/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  66) 		 * If we're branching inside the alternate sequence,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  67) 		 * do not rewrite the instruction, as it is already
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  68) 		 * correct. Otherwise, generate the new instruction.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  69) 		 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  70) 		if (branch_insn_requires_update(alt, target)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  71) 			offset = target - (unsigned long)insnptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  72) 			insn = aarch64_set_branch_offset(insn, offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  73) 		}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  74) 	} else if (aarch64_insn_is_adrp(insn)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  75) 		s32 orig_offset, new_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  76) 		unsigned long target;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  77) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  78) 		/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  79) 		 * If we're replacing an adrp instruction, which uses PC-relative
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  80) 		 * immediate addressing, adjust the offset to reflect the new
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  81) 		 * PC. adrp operates on 4K aligned addresses.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  82) 		 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  83) 		orig_offset  = aarch64_insn_adrp_get_offset(insn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  84) 		target = align_down(altinsnptr, SZ_4K) + orig_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  85) 		new_offset = target - align_down(insnptr, SZ_4K);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  86) 		insn = aarch64_insn_adrp_set_offset(insn, new_offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  87) 	} else if (aarch64_insn_uses_literal(insn)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  88) 		/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  89) 		 * Disallow patching unhandled instructions using PC relative
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  90) 		 * literal addresses
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  91) 		 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  92) 		BUG();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  93) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  94) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  95) 	return insn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  96) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  97) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  98) static void patch_alternative(struct alt_instr *alt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  99) 			      __le32 *origptr, __le32 *updptr, int nr_inst)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) 	__le32 *replptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) 	int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) 	replptr = ALT_REPL_PTR(alt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) 	for (i = 0; i < nr_inst; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) 		u32 insn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) 		insn = get_alt_insn(alt, origptr + i, replptr + i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) 		updptr[i] = cpu_to_le32(insn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114)  * We provide our own, private D-cache cleaning function so that we don't
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115)  * accidentally call into the cache.S code, which is patched by us at
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116)  * runtime.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) static void clean_dcache_range_nopatch(u64 start, u64 end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) 	u64 cur, d_size, ctr_el0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) 	ctr_el0 = read_sanitised_ftr_reg(SYS_CTR_EL0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) 	d_size = 4 << cpuid_feature_extract_unsigned_field(ctr_el0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) 							   CTR_DMINLINE_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) 	cur = start & ~(d_size - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) 	do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) 		/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) 		 * We must clean+invalidate to the PoC in order to avoid
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) 		 * Cortex-A53 errata 826319, 827319, 824069 and 819472
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) 		 * (this corresponds to ARM64_WORKAROUND_CLEAN_CACHE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) 		 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) 		asm volatile("dc civac, %0" : : "r" (cur) : "memory");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) 	} while (cur += d_size, cur < end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) static void __nocfi __apply_alternatives(void *alt_region,  bool is_module,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) 					 unsigned long *feature_mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) 	struct alt_instr *alt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) 	struct alt_region *region = alt_region;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) 	__le32 *origptr, *updptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) 	alternative_cb_t alt_cb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) 	for (alt = region->begin; alt < region->end; alt++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) 		int nr_inst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) 		if (!test_bit(alt->cpufeature, feature_mask))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) 			continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) 		/* Use ARM64_CB_PATCH as an unconditional patch */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) 		if (alt->cpufeature < ARM64_CB_PATCH &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) 		    !cpus_have_cap(alt->cpufeature))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) 			continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) 		if (alt->cpufeature == ARM64_CB_PATCH)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) 			BUG_ON(alt->alt_len != 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) 		else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) 			BUG_ON(alt->alt_len != alt->orig_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) 		pr_info_once("patching kernel code\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) 		origptr = ALT_ORIG_PTR(alt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) 		updptr = is_module ? origptr : lm_alias(origptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) 		nr_inst = alt->orig_len / AARCH64_INSN_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) 		if (alt->cpufeature < ARM64_CB_PATCH)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) 			alt_cb = patch_alternative;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) 		else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) 			alt_cb  = ALT_REPL_PTR(alt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) 		alt_cb(alt, origptr, updptr, nr_inst);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) 		if (!is_module) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) 			clean_dcache_range_nopatch((u64)origptr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) 						   (u64)(origptr + nr_inst));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) 		}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) 	 * The core module code takes care of cache maintenance in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) 	 * flush_module_icache().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) 	if (!is_module) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) 		dsb(ish);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) 		__flush_icache_all();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) 		isb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) 		/* Ignore ARM64_CB bit from feature mask */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) 		bitmap_or(applied_alternatives, applied_alternatives,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) 			  feature_mask, ARM64_NCAPS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) 		bitmap_and(applied_alternatives, applied_alternatives,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) 			   cpu_hwcaps, ARM64_NCAPS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197)  * We might be patching the stop_machine state machine, so implement a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198)  * really simple polling protocol here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) static int __apply_alternatives_multi_stop(void *unused)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) 	struct alt_region region = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) 		.begin	= (struct alt_instr *)__alt_instructions,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) 		.end	= (struct alt_instr *)__alt_instructions_end,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) 	};
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) 	/* We always have a CPU 0 at this point (__init) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) 	if (smp_processor_id()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) 		while (!all_alternatives_applied)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) 			cpu_relax();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) 		isb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) 	} else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) 		DECLARE_BITMAP(remaining_capabilities, ARM64_NPATCHABLE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) 		bitmap_complement(remaining_capabilities, boot_capabilities,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) 				  ARM64_NPATCHABLE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) 		BUG_ON(all_alternatives_applied);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) 		__apply_alternatives(&region, false, remaining_capabilities);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) 		/* Barriers provided by the cache flushing */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) 		all_alternatives_applied = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) void __init apply_alternatives_all(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) 	/* better not try code patching on a live SMP system */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) 	stop_machine(__apply_alternatives_multi_stop, NULL, cpu_online_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234)  * This is called very early in the boot process (directly after we run
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235)  * a feature detect on the boot CPU). No need to worry about other CPUs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236)  * here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) void __init apply_boot_alternatives(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) 	struct alt_region region = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) 		.begin	= (struct alt_instr *)__alt_instructions,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) 		.end	= (struct alt_instr *)__alt_instructions_end,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) 	};
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) 	/* If called on non-boot cpu things could go wrong */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) 	WARN_ON(smp_processor_id() != 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) 	__apply_alternatives(&region, false, &boot_capabilities[0]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) #ifdef CONFIG_MODULES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) void apply_alternatives_module(void *start, size_t length)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) 	struct alt_region region = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) 		.begin	= start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) 		.end	= start + length,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) 	};
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) 	DECLARE_BITMAP(all_capabilities, ARM64_NPATCHABLE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) 	bitmap_fill(all_capabilities, ARM64_NPATCHABLE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) 	__apply_alternatives(&region, true, &all_capabilities[0]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) #endif