^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * CPU complex suspend & resume functions for Tegra SoCs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Copyright (c) 2009-2012, NVIDIA Corporation. All rights reserved.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include <linux/clk/tegra.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <linux/cpumask.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/cpu_pm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/delay.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/err.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/io.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/spinlock.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/suspend.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <linux/firmware/trusted_foundations.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <soc/tegra/flowctrl.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include <soc/tegra/fuse.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #include <soc/tegra/pm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #include <soc/tegra/pmc.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) #include <asm/cacheflush.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) #include <asm/firmware.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) #include <asm/idmap.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) #include <asm/proc-fns.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) #include <asm/smp_plat.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) #include <asm/suspend.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) #include <asm/tlbflush.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) #include "iomap.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) #include "pm.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) #include "reset.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) #include "sleep.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) #ifdef CONFIG_PM_SLEEP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) static DEFINE_SPINLOCK(tegra_lp2_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) static u32 iram_save_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) static void *iram_save_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) struct tegra_lp1_iram tegra_lp1_iram;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) void (*tegra_tear_down_cpu)(void);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) void (*tegra_sleep_core_finish)(unsigned long v2p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) static int (*tegra_sleep_func)(unsigned long v2p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) static void tegra_tear_down_cpu_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) switch (tegra_get_chip_id()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) case TEGRA20:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) if (IS_ENABLED(CONFIG_ARCH_TEGRA_2x_SOC))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) tegra_tear_down_cpu = tegra20_tear_down_cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) case TEGRA30:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) case TEGRA114:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) case TEGRA124:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) if (IS_ENABLED(CONFIG_ARCH_TEGRA_3x_SOC) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) IS_ENABLED(CONFIG_ARCH_TEGRA_114_SOC) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) IS_ENABLED(CONFIG_ARCH_TEGRA_124_SOC))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) tegra_tear_down_cpu = tegra30_tear_down_cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) * restore_cpu_complex
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) * restores cpu clock setting, clears flow controller
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) * Always called on CPU 0.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) static void restore_cpu_complex(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) int cpu = smp_processor_id();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) BUG_ON(cpu != 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) #ifdef CONFIG_SMP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) cpu = cpu_logical_map(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) /* Restore the CPU clock settings */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) tegra_cpu_clock_resume();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) flowctrl_cpu_suspend_exit(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) * suspend_cpu_complex
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) * saves pll state for use by restart_plls, prepares flow controller for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) * transition to suspend state
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) * Must always be called on cpu 0.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) static void suspend_cpu_complex(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) int cpu = smp_processor_id();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) BUG_ON(cpu != 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) #ifdef CONFIG_SMP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) cpu = cpu_logical_map(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) /* Save the CPU clock settings */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) tegra_cpu_clock_suspend();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) flowctrl_cpu_suspend_enter(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) void tegra_pm_clear_cpu_in_lp2(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) int phy_cpu_id = cpu_logical_map(smp_processor_id());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) u32 *cpu_in_lp2 = tegra_cpu_lp2_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) spin_lock(&tegra_lp2_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) BUG_ON(!(*cpu_in_lp2 & BIT(phy_cpu_id)));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) *cpu_in_lp2 &= ~BIT(phy_cpu_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) spin_unlock(&tegra_lp2_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) void tegra_pm_set_cpu_in_lp2(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) int phy_cpu_id = cpu_logical_map(smp_processor_id());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) u32 *cpu_in_lp2 = tegra_cpu_lp2_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) spin_lock(&tegra_lp2_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) BUG_ON((*cpu_in_lp2 & BIT(phy_cpu_id)));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) *cpu_in_lp2 |= BIT(phy_cpu_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) spin_unlock(&tegra_lp2_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) static int tegra_sleep_cpu(unsigned long v2p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) if (tegra_cpu_car_ops->rail_off_ready &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) WARN_ON(!tegra_cpu_rail_off_ready()))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) return -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) * L2 cache disabling using kernel API only allowed when all
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) * secondary CPU's are offline. Cache have to be disabled with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) * MMU-on if cache maintenance is done via Trusted Foundations
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) * firmware. Note that CPUIDLE won't ever enter powergate on Tegra30
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) * if any of secondary CPU's is online and this is the LP2-idle
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) * code-path only for Tegra20/30.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) #ifdef CONFIG_OUTER_CACHE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) if (trusted_foundations_registered() && outer_cache.disable)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) outer_cache.disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) * Note that besides of setting up CPU reset vector this firmware
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) * call may also do the following, depending on the FW version:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) * 1) Disable L2. But this doesn't matter since we already
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) * disabled the L2.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) * 2) Disable D-cache. This need to be taken into account in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) * particular by the tegra_disable_clean_inv_dcache() which
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) * shall avoid the re-disable.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) call_firmware_op(prepare_idle, TF_PM_MODE_LP2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) setup_mm_for_reboot();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) tegra_sleep_cpu_finish(v2p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) /* should never here */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) BUG();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) static void tegra_pm_set(enum tegra_suspend_mode mode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) u32 value;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) switch (tegra_get_chip_id()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) case TEGRA20:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) case TEGRA30:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) /* Turn off CRAIL */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) value = flowctrl_read_cpu_csr(0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) value &= ~FLOW_CTRL_CSR_ENABLE_EXT_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) value |= FLOW_CTRL_CSR_ENABLE_EXT_CRAIL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) flowctrl_write_cpu_csr(0, value);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) tegra_pmc_enter_suspend_mode(mode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) int tegra_pm_enter_lp2(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) tegra_pm_set(TEGRA_SUSPEND_LP2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) cpu_cluster_pm_enter();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) suspend_cpu_complex();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) err = cpu_suspend(PHYS_OFFSET - PAGE_OFFSET, &tegra_sleep_cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) * Resume L2 cache if it wasn't re-enabled early during resume,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) * which is the case for Tegra30 that has to re-enable the cache
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) * via firmware call. In other cases cache is already enabled and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) * hence re-enabling is a no-op. This is always a no-op on Tegra114+.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) outer_resume();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) restore_cpu_complex();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) cpu_cluster_pm_exit();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) call_firmware_op(prepare_idle, TF_PM_MODE_NONE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) enum tegra_suspend_mode tegra_pm_validate_suspend_mode(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) enum tegra_suspend_mode mode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) * The Tegra devices support suspending to LP1 or lower currently.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) if (mode > TEGRA_SUSPEND_LP1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) return TEGRA_SUSPEND_LP1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) return mode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) static int tegra_sleep_core(unsigned long v2p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) * Cache have to be disabled with MMU-on if cache maintenance is done
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) * via Trusted Foundations firmware. This is a no-op on Tegra114+.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) if (trusted_foundations_registered())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) outer_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) call_firmware_op(prepare_idle, TF_PM_MODE_LP1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) setup_mm_for_reboot();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) tegra_sleep_core_finish(v2p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) /* should never here */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) BUG();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) * tegra_lp1_iram_hook
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) * Hooking the address of LP1 reset vector and SDRAM self-refresh code in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) * SDRAM. These codes not be copied to IRAM in this fuction. We need to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) * copy these code to IRAM before LP0/LP1 suspend and restore the content
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) * of IRAM after resume.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) static bool tegra_lp1_iram_hook(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) switch (tegra_get_chip_id()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) case TEGRA20:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) if (IS_ENABLED(CONFIG_ARCH_TEGRA_2x_SOC))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) tegra20_lp1_iram_hook();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) case TEGRA30:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) case TEGRA114:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) case TEGRA124:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) if (IS_ENABLED(CONFIG_ARCH_TEGRA_3x_SOC) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) IS_ENABLED(CONFIG_ARCH_TEGRA_114_SOC) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) IS_ENABLED(CONFIG_ARCH_TEGRA_124_SOC))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) tegra30_lp1_iram_hook();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) if (!tegra_lp1_iram.start_addr || !tegra_lp1_iram.end_addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) iram_save_size = tegra_lp1_iram.end_addr - tegra_lp1_iram.start_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) iram_save_addr = kmalloc(iram_save_size, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) if (!iram_save_addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) static bool tegra_sleep_core_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) switch (tegra_get_chip_id()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) case TEGRA20:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) if (IS_ENABLED(CONFIG_ARCH_TEGRA_2x_SOC))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) tegra20_sleep_core_init();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) case TEGRA30:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) case TEGRA114:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) case TEGRA124:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) if (IS_ENABLED(CONFIG_ARCH_TEGRA_3x_SOC) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) IS_ENABLED(CONFIG_ARCH_TEGRA_114_SOC) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) IS_ENABLED(CONFIG_ARCH_TEGRA_124_SOC))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) tegra30_sleep_core_init();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) if (!tegra_sleep_core_finish)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) static void tegra_suspend_enter_lp1(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) /* copy the reset vector & SDRAM shutdown code into IRAM */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) memcpy(iram_save_addr, IO_ADDRESS(TEGRA_IRAM_LPx_RESUME_AREA),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) iram_save_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) memcpy(IO_ADDRESS(TEGRA_IRAM_LPx_RESUME_AREA),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) tegra_lp1_iram.start_addr, iram_save_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) *((u32 *)tegra_cpu_lp1_mask) = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) static void tegra_suspend_exit_lp1(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) /* restore IRAM */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) memcpy(IO_ADDRESS(TEGRA_IRAM_LPx_RESUME_AREA), iram_save_addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) iram_save_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) *(u32 *)tegra_cpu_lp1_mask = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) static const char *lp_state[TEGRA_MAX_SUSPEND_MODE] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) [TEGRA_SUSPEND_NONE] = "none",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) [TEGRA_SUSPEND_LP2] = "LP2",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) [TEGRA_SUSPEND_LP1] = "LP1",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) [TEGRA_SUSPEND_LP0] = "LP0",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) static int tegra_suspend_enter(suspend_state_t state)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) enum tegra_suspend_mode mode = tegra_pmc_get_suspend_mode();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) if (WARN_ON(mode < TEGRA_SUSPEND_NONE ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) mode >= TEGRA_MAX_SUSPEND_MODE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) pr_info("Entering suspend state %s\n", lp_state[mode]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) tegra_pm_set(mode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) local_fiq_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) suspend_cpu_complex();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) switch (mode) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) case TEGRA_SUSPEND_LP1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) tegra_suspend_enter_lp1();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) case TEGRA_SUSPEND_LP2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) tegra_pm_set_cpu_in_lp2();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) cpu_suspend(PHYS_OFFSET - PAGE_OFFSET, tegra_sleep_func);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) * Resume L2 cache if it wasn't re-enabled early during resume,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) * which is the case for Tegra30 that has to re-enable the cache
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) * via firmware call. In other cases cache is already enabled and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) * hence re-enabling is a no-op.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) outer_resume();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) switch (mode) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) case TEGRA_SUSPEND_LP1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) tegra_suspend_exit_lp1();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) case TEGRA_SUSPEND_LP2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) tegra_pm_clear_cpu_in_lp2();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) restore_cpu_complex();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) local_fiq_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) call_firmware_op(prepare_idle, TF_PM_MODE_NONE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) static const struct platform_suspend_ops tegra_suspend_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) .valid = suspend_valid_only_mem,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) .enter = tegra_suspend_enter,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) void __init tegra_init_suspend(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) enum tegra_suspend_mode mode = tegra_pmc_get_suspend_mode();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) if (mode == TEGRA_SUSPEND_NONE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) tegra_tear_down_cpu_init();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) if (mode >= TEGRA_SUSPEND_LP1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) if (!tegra_lp1_iram_hook() || !tegra_sleep_core_init()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) pr_err("%s: unable to allocate memory for SDRAM"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) "self-refresh -- LP0/LP1 unavailable\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) __func__);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) tegra_pmc_set_suspend_mode(TEGRA_SUSPEND_LP2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) mode = TEGRA_SUSPEND_LP2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) /* set up sleep function for cpu_suspend */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) switch (mode) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) case TEGRA_SUSPEND_LP1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) tegra_sleep_func = tegra_sleep_core;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) case TEGRA_SUSPEND_LP2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) tegra_sleep_func = tegra_sleep_cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) suspend_set_ops(&tegra_suspend_ops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) int tegra_pm_park_secondary_cpu(unsigned long cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) if (cpu > 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) tegra_disable_clean_inv_dcache(TEGRA_FLUSH_CACHE_LOUIS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) if (tegra_get_chip_id() == TEGRA20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) tegra20_hotplug_shutdown();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) tegra30_hotplug_shutdown();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) #endif