Orange Pi5 kernel

^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   1) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   2)  * Coherency fabric (Aurora) support for Armada 370, 375, 38x and XP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   3)  * platforms.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   4)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   5)  * Copyright (C) 2012 Marvell
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   6)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   7)  * Yehuda Yitschak <yehuday@marvell.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   8)  * Gregory Clement <gregory.clement@free-electrons.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   9)  * Thomas Petazzoni <thomas.petazzoni@free-electrons.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  10)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  11)  * This file is licensed under the terms of the GNU General Public
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  12)  * License version 2.  This program is licensed "as is" without any
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  13)  * warranty of any kind, whether express or implied.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  14)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  15)  * The Armada 370, 375, 38x and XP SOCs have a coherency fabric which is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  16)  * responsible for ensuring hardware coherency between all CPUs and between
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  17)  * CPUs and I/O masters. This file initializes the coherency fabric and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  18)  * supplies basic routines for configuring and controlling hardware coherency
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  19)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  20) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  21) #define pr_fmt(fmt) "mvebu-coherency: " fmt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  22) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  23) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  24) #include <linux/init.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  25) #include <linux/of_address.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  26) #include <linux/io.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  27) #include <linux/smp.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  28) #include <linux/dma-map-ops.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  29) #include <linux/platform_device.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  30) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  31) #include <linux/mbus.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  32) #include <linux/pci.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  33) #include <asm/smp_plat.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  34) #include <asm/cacheflush.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  35) #include <asm/mach/map.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  36) #include <asm/dma-mapping.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  37) #include "coherency.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  38) #include "mvebu-soc-id.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  39) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  40) unsigned long coherency_phys_base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  41) void __iomem *coherency_base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  42) static void __iomem *coherency_cpu_base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  43) static void __iomem *cpu_config_base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  44) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  45) /* Coherency fabric registers */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  46) #define IO_SYNC_BARRIER_CTL_OFFSET		   0x0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  47) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  48) enum {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  49) 	COHERENCY_FABRIC_TYPE_NONE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  50) 	COHERENCY_FABRIC_TYPE_ARMADA_370_XP,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  51) 	COHERENCY_FABRIC_TYPE_ARMADA_375,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  52) 	COHERENCY_FABRIC_TYPE_ARMADA_380,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  53) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  54) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  55) static const struct of_device_id of_coherency_table[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  56) 	{.compatible = "marvell,coherency-fabric",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  57) 	 .data = (void *) COHERENCY_FABRIC_TYPE_ARMADA_370_XP },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  58) 	{.compatible = "marvell,armada-375-coherency-fabric",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  59) 	 .data = (void *) COHERENCY_FABRIC_TYPE_ARMADA_375 },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  60) 	{.compatible = "marvell,armada-380-coherency-fabric",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  61) 	 .data = (void *) COHERENCY_FABRIC_TYPE_ARMADA_380 },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  62) 	{ /* end of list */ },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  63) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  64) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  65) /* Functions defined in coherency_ll.S */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  66) int ll_enable_coherency(void);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  67) void ll_add_cpu_to_smp_group(void);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  68) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  69) #define CPU_CONFIG_SHARED_L2 BIT(16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  70) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  71) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  72)  * Disable the "Shared L2 Present" bit in CPU Configuration register
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  73)  * on Armada XP.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  74)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  75)  * The "Shared L2 Present" bit affects the "level of coherence" value
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  76)  * in the clidr CP15 register.  Cache operation functions such as
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  77)  * "flush all" and "invalidate all" operate on all the cache levels
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  78)  * that included in the defined level of coherence. When HW I/O
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  79)  * coherency is used, this bit causes unnecessary flushes of the L2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  80)  * cache.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  81)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  82) static void armada_xp_clear_shared_l2(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  83) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  84) 	u32 reg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  85) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  86) 	if (!cpu_config_base)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  87) 		return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  88) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  89) 	reg = readl(cpu_config_base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  90) 	reg &= ~CPU_CONFIG_SHARED_L2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  91) 	writel(reg, cpu_config_base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  92) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  93) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  94) static int mvebu_hwcc_notifier(struct notifier_block *nb,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  95) 			       unsigned long event, void *__dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  96) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  97) 	struct device *dev = __dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  98) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  99) 	if (event != BUS_NOTIFY_ADD_DEVICE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) 		return NOTIFY_DONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) 	set_dma_ops(dev, &arm_coherent_dma_ops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) 	return NOTIFY_OK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) static struct notifier_block mvebu_hwcc_nb = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) 	.notifier_call = mvebu_hwcc_notifier,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) static struct notifier_block mvebu_hwcc_pci_nb __maybe_unused = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) 	.notifier_call = mvebu_hwcc_notifier,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) static int armada_xp_clear_l2_starting(unsigned int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) 	armada_xp_clear_shared_l2();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) static void __init armada_370_coherency_init(struct device_node *np)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) 	struct resource res;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) 	struct device_node *cpu_config_np;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) 	of_address_to_resource(np, 0, &res);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) 	coherency_phys_base = res.start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) 	 * Ensure secondary CPUs will see the updated value,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) 	 * which they read before they join the coherency
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) 	 * fabric, and therefore before they are coherent with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) 	 * the boot CPU cache.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) 	sync_cache_w(&coherency_phys_base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) 	coherency_base = of_iomap(np, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) 	coherency_cpu_base = of_iomap(np, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) 	cpu_config_np = of_find_compatible_node(NULL, NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) 						"marvell,armada-xp-cpu-config");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) 	if (!cpu_config_np)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) 		goto exit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) 	cpu_config_base = of_iomap(cpu_config_np, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) 	if (!cpu_config_base) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) 		of_node_put(cpu_config_np);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) 		goto exit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) 	of_node_put(cpu_config_np);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) 	cpuhp_setup_state_nocalls(CPUHP_AP_ARM_MVEBU_COHERENCY,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) 				  "arm/mvebu/coherency:starting",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) 				  armada_xp_clear_l2_starting, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) exit:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) 	set_cpu_coherent();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158)  * This ioremap hook is used on Armada 375/38x to ensure that all MMIO
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159)  * areas are mapped as MT_UNCACHED instead of MT_DEVICE. This is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160)  * needed for the HW I/O coherency mechanism to work properly without
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161)  * deadlock.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) static void __iomem *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) armada_wa_ioremap_caller(phys_addr_t phys_addr, size_t size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) 			 unsigned int mtype, void *caller)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) 	mtype = MT_UNCACHED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) 	return __arm_ioremap_caller(phys_addr, size, mtype, caller);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) static void __init armada_375_380_coherency_init(struct device_node *np)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) 	struct device_node *cache_dn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) 	coherency_cpu_base = of_iomap(np, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) 	arch_ioremap_caller = armada_wa_ioremap_caller;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) 	pci_ioremap_set_mem_type(MT_UNCACHED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) 	 * We should switch the PL310 to I/O coherency mode only if
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) 	 * I/O coherency is actually enabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) 	if (!coherency_available())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) 		return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) 	 * Add the PL310 property "arm,io-coherent". This makes sure the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) 	 * outer sync operation is not used, which allows to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) 	 * workaround the system erratum that causes deadlocks when
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) 	 * doing PCIe in an SMP situation on Armada 375 and Armada
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) 	 * 38x.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) 	for_each_compatible_node(cache_dn, NULL, "arm,pl310-cache") {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) 		struct property *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) 		p = kzalloc(sizeof(*p), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) 		p->name = kstrdup("arm,io-coherent", GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) 		of_add_property(cache_dn, p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) static int coherency_type(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) 	struct device_node *np;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) 	const struct of_device_id *match;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) 	int type;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) 	 * The coherency fabric is needed:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) 	 * - For coherency between processors on Armada XP, so only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) 	 *   when SMP is enabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) 	 * - For coherency between the processor and I/O devices, but
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) 	 *   this coherency requires many pre-requisites (write
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) 	 *   allocate cache policy, shareable pages, SMP bit set) that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) 	 *   are only meant in SMP situations.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) 	 *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) 	 * Note that this means that on Armada 370, there is currently
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) 	 * no way to use hardware I/O coherency, because even when
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) 	 * CONFIG_SMP is enabled, is_smp() returns false due to the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) 	 * Armada 370 being a single-core processor. To lift this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) 	 * limitation, we would have to find a way to make the cache
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) 	 * policy set to write-allocate (on all Armada SoCs), and to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) 	 * set the shareable attribute in page tables (on all Armada
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) 	 * SoCs except the Armada 370). Unfortunately, such decisions
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) 	 * are taken very early in the kernel boot process, at a point
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) 	 * where we don't know yet on which SoC we are running.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) 	if (!is_smp())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) 		return COHERENCY_FABRIC_TYPE_NONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) 	np = of_find_matching_node_and_match(NULL, of_coherency_table, &match);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) 	if (!np)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) 		return COHERENCY_FABRIC_TYPE_NONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) 	type = (int) match->data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) 	of_node_put(np);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) 	return type;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) int set_cpu_coherent(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) 	int type = coherency_type();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) 	if (type == COHERENCY_FABRIC_TYPE_ARMADA_370_XP) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) 		if (!coherency_base) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) 			pr_warn("Can't make current CPU cache coherent.\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) 			pr_warn("Coherency fabric is not initialized\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) 			return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) 		}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) 		armada_xp_clear_shared_l2();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) 		ll_add_cpu_to_smp_group();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) 		return ll_enable_coherency();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) int coherency_available(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) 	return coherency_type() != COHERENCY_FABRIC_TYPE_NONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) int __init coherency_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) 	int type = coherency_type();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) 	struct device_node *np;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) 	np = of_find_matching_node(NULL, of_coherency_table);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) 	if (type == COHERENCY_FABRIC_TYPE_ARMADA_370_XP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) 		armada_370_coherency_init(np);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) 	else if (type == COHERENCY_FABRIC_TYPE_ARMADA_375 ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) 		 type == COHERENCY_FABRIC_TYPE_ARMADA_380)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) 		armada_375_380_coherency_init(np);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) 	of_node_put(np);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) static int __init coherency_late_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) 	if (coherency_available())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) 		bus_register_notifier(&platform_bus_type,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) 				      &mvebu_hwcc_nb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) postcore_initcall(coherency_late_init);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) #if IS_ENABLED(CONFIG_PCI)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) static int __init coherency_pci_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) 	if (coherency_available())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) 		bus_register_notifier(&pci_bus_type,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) 				       &mvebu_hwcc_pci_nb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) arch_initcall(coherency_pci_init);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) #endif