^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Copyright (c) 2014-2020, NVIDIA CORPORATION. All rights reserved.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) #include <linux/io.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include <linux/clk.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <linux/delay.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/of.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <soc/tegra/mc.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include "tegra210-emc.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include "tegra210-mc.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) * Enable flags for specifying verbosity.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #define INFO (1 << 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #define STEPS (1 << 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #define SUB_STEPS (1 << 2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #define PRELOCK (1 << 3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #define PRELOCK_STEPS (1 << 4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #define ACTIVE_EN (1 << 5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) #define PRAMP_UP (1 << 6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) #define PRAMP_DN (1 << 7)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) #define EMA_WRITES (1 << 10)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) #define EMA_UPDATES (1 << 11)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) #define PER_TRAIN (1 << 16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) #define CC_PRINT (1 << 17)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) #define CCFIFO (1 << 29)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) #define REGS (1 << 30)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) #define REG_LISTS (1 << 31)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) #define emc_dbg(emc, flags, ...) dev_dbg(emc->dev, __VA_ARGS__)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) #define DVFS_CLOCK_CHANGE_VERSION 21021
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) #define EMC_PRELOCK_VERSION 2101
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) enum {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) DVFS_SEQUENCE = 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) WRITE_TRAINING_SEQUENCE = 2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) PERIODIC_TRAINING_SEQUENCE = 3,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) DVFS_PT1 = 10,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) DVFS_UPDATE = 11,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) TRAINING_PT1 = 12,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) TRAINING_UPDATE = 13,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) PERIODIC_TRAINING_UPDATE = 14
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) * PTFV defines - basically just indexes into the per table PTFV array.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) #define PTFV_DQSOSC_MOVAVG_C0D0U0_INDEX 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) #define PTFV_DQSOSC_MOVAVG_C0D0U1_INDEX 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) #define PTFV_DQSOSC_MOVAVG_C0D1U0_INDEX 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) #define PTFV_DQSOSC_MOVAVG_C0D1U1_INDEX 3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) #define PTFV_DQSOSC_MOVAVG_C1D0U0_INDEX 4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) #define PTFV_DQSOSC_MOVAVG_C1D0U1_INDEX 5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) #define PTFV_DQSOSC_MOVAVG_C1D1U0_INDEX 6
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) #define PTFV_DQSOSC_MOVAVG_C1D1U1_INDEX 7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) #define PTFV_DVFS_SAMPLES_INDEX 9
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) #define PTFV_MOVAVG_WEIGHT_INDEX 10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) #define PTFV_CONFIG_CTRL_INDEX 11
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) #define PTFV_CONFIG_CTRL_USE_PREVIOUS_EMA (1 << 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) * Do arithmetic in fixed point.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) #define MOVAVG_PRECISION_FACTOR 100
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) * The division portion of the average operation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) #define __AVERAGE_PTFV(dev) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) ({ next->ptfv_list[PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX] = \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) next->ptfv_list[PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX] / \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) next->ptfv_list[PTFV_DVFS_SAMPLES_INDEX]; })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) * Convert val to fixed point and add it to the temporary average.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) #define __INCREMENT_PTFV(dev, val) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) ({ next->ptfv_list[PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX] += \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) ((val) * MOVAVG_PRECISION_FACTOR); })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) * Convert a moving average back to integral form and return the value.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) #define __MOVAVG_AC(timing, dev) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) ((timing)->ptfv_list[PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX] / \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) MOVAVG_PRECISION_FACTOR)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) /* Weighted update. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) #define __WEIGHTED_UPDATE_PTFV(dev, nval) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) do { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) int w = PTFV_MOVAVG_WEIGHT_INDEX; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) int dqs = PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) next->ptfv_list[dqs] = \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) ((nval * MOVAVG_PRECISION_FACTOR) + \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) (next->ptfv_list[dqs] * \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) next->ptfv_list[w])) / \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) (next->ptfv_list[w] + 1); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) emc_dbg(emc, EMA_UPDATES, "%s: (s=%lu) EMA: %u\n", \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) __stringify(dev), nval, next->ptfv_list[dqs]); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) } while (0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) /* Access a particular average. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) #define __MOVAVG(timing, dev) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) ((timing)->ptfv_list[PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) static u32 update_clock_tree_delay(struct tegra210_emc *emc, int type)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) bool periodic_training_update = type == PERIODIC_TRAINING_UPDATE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) struct tegra210_emc_timing *last = emc->last;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) struct tegra210_emc_timing *next = emc->next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) u32 last_timing_rate_mhz = last->rate / 1000;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) u32 next_timing_rate_mhz = next->rate / 1000;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) bool dvfs_update = type == DVFS_UPDATE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) s32 tdel = 0, tmdel = 0, adel = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) bool dvfs_pt1 = type == DVFS_PT1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) unsigned long cval = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) u32 temp[2][2], value;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) unsigned int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) * Dev0 MSB.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) if (dvfs_pt1 || periodic_training_update) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) value = tegra210_emc_mrr_read(emc, 2, 19);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) for (i = 0; i < emc->num_channels; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) temp[i][0] = (value & 0x00ff) << 8;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) temp[i][1] = (value & 0xff00) << 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) value >>= 16;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) * Dev0 LSB.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) value = tegra210_emc_mrr_read(emc, 2, 18);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) for (i = 0; i < emc->num_channels; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) temp[i][0] |= (value & 0x00ff) >> 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) temp[i][1] |= (value & 0xff00) >> 8;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) value >>= 16;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) if (dvfs_pt1 || periodic_training_update) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) cval = tegra210_emc_actual_osc_clocks(last->run_clocks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) cval *= 1000000;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) cval /= last_timing_rate_mhz * 2 * temp[0][0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) if (dvfs_pt1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) __INCREMENT_PTFV(C0D0U0, cval);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) else if (dvfs_update)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) __AVERAGE_PTFV(C0D0U0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) else if (periodic_training_update)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) __WEIGHTED_UPDATE_PTFV(C0D0U0, cval);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) if (dvfs_update || periodic_training_update) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) tdel = next->current_dram_clktree[C0D0U0] -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) __MOVAVG_AC(next, C0D0U0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) tmdel = (tdel < 0) ? -1 * tdel : tdel;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) adel = tmdel;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) if (tmdel * 128 * next_timing_rate_mhz / 1000000 >
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) next->tree_margin)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) next->current_dram_clktree[C0D0U0] =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) __MOVAVG_AC(next, C0D0U0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) if (dvfs_pt1 || periodic_training_update) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) cval = tegra210_emc_actual_osc_clocks(last->run_clocks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) cval *= 1000000;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) cval /= last_timing_rate_mhz * 2 * temp[0][1];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) if (dvfs_pt1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) __INCREMENT_PTFV(C0D0U1, cval);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) else if (dvfs_update)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) __AVERAGE_PTFV(C0D0U1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) else if (periodic_training_update)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) __WEIGHTED_UPDATE_PTFV(C0D0U1, cval);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) if (dvfs_update || periodic_training_update) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) tdel = next->current_dram_clktree[C0D0U1] -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) __MOVAVG_AC(next, C0D0U1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) tmdel = (tdel < 0) ? -1 * tdel : tdel;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) if (tmdel > adel)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) adel = tmdel;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) if (tmdel * 128 * next_timing_rate_mhz / 1000000 >
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) next->tree_margin)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) next->current_dram_clktree[C0D0U1] =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) __MOVAVG_AC(next, C0D0U1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) if (emc->num_channels > 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) if (dvfs_pt1 || periodic_training_update) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) cval = tegra210_emc_actual_osc_clocks(last->run_clocks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) cval *= 1000000;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) cval /= last_timing_rate_mhz * 2 * temp[1][0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) if (dvfs_pt1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) __INCREMENT_PTFV(C1D0U0, cval);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) else if (dvfs_update)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) __AVERAGE_PTFV(C1D0U0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) else if (periodic_training_update)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) __WEIGHTED_UPDATE_PTFV(C1D0U0, cval);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) if (dvfs_update || periodic_training_update) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) tdel = next->current_dram_clktree[C1D0U0] -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) __MOVAVG_AC(next, C1D0U0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) tmdel = (tdel < 0) ? -1 * tdel : tdel;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) if (tmdel > adel)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) adel = tmdel;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) if (tmdel * 128 * next_timing_rate_mhz / 1000000 >
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) next->tree_margin)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) next->current_dram_clktree[C1D0U0] =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) __MOVAVG_AC(next, C1D0U0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) if (dvfs_pt1 || periodic_training_update) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) cval = tegra210_emc_actual_osc_clocks(last->run_clocks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) cval *= 1000000;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) cval /= last_timing_rate_mhz * 2 * temp[1][1];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) if (dvfs_pt1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) __INCREMENT_PTFV(C1D0U1, cval);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) else if (dvfs_update)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) __AVERAGE_PTFV(C1D0U1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) else if (periodic_training_update)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) __WEIGHTED_UPDATE_PTFV(C1D0U1, cval);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) if (dvfs_update || periodic_training_update) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) tdel = next->current_dram_clktree[C1D0U1] -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) __MOVAVG_AC(next, C1D0U1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) tmdel = (tdel < 0) ? -1 * tdel : tdel;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) if (tmdel > adel)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) adel = tmdel;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) if (tmdel * 128 * next_timing_rate_mhz / 1000000 >
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) next->tree_margin)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) next->current_dram_clktree[C1D0U1] =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) __MOVAVG_AC(next, C1D0U1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) if (emc->num_devices < 2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) goto done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) * Dev1 MSB.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) if (dvfs_pt1 || periodic_training_update) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) value = tegra210_emc_mrr_read(emc, 1, 19);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) for (i = 0; i < emc->num_channels; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) temp[i][0] = (value & 0x00ff) << 8;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) temp[i][1] = (value & 0xff00) << 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) value >>= 16;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) * Dev1 LSB.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) value = tegra210_emc_mrr_read(emc, 2, 18);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) for (i = 0; i < emc->num_channels; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) temp[i][0] |= (value & 0x00ff) >> 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) temp[i][1] |= (value & 0xff00) >> 8;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) value >>= 16;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) if (dvfs_pt1 || periodic_training_update) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) cval = tegra210_emc_actual_osc_clocks(last->run_clocks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) cval *= 1000000;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) cval /= last_timing_rate_mhz * 2 * temp[0][0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) if (dvfs_pt1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) __INCREMENT_PTFV(C0D1U0, cval);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) else if (dvfs_update)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) __AVERAGE_PTFV(C0D1U0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) else if (periodic_training_update)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) __WEIGHTED_UPDATE_PTFV(C0D1U0, cval);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) if (dvfs_update || periodic_training_update) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) tdel = next->current_dram_clktree[C0D1U0] -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) __MOVAVG_AC(next, C0D1U0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) tmdel = (tdel < 0) ? -1 * tdel : tdel;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) if (tmdel > adel)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) adel = tmdel;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) if (tmdel * 128 * next_timing_rate_mhz / 1000000 >
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) next->tree_margin)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) next->current_dram_clktree[C0D1U0] =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) __MOVAVG_AC(next, C0D1U0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) if (dvfs_pt1 || periodic_training_update) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) cval = tegra210_emc_actual_osc_clocks(last->run_clocks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) cval *= 1000000;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) cval /= last_timing_rate_mhz * 2 * temp[0][1];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) if (dvfs_pt1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) __INCREMENT_PTFV(C0D1U1, cval);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) else if (dvfs_update)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) __AVERAGE_PTFV(C0D1U1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) else if (periodic_training_update)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) __WEIGHTED_UPDATE_PTFV(C0D1U1, cval);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) if (dvfs_update || periodic_training_update) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) tdel = next->current_dram_clktree[C0D1U1] -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) __MOVAVG_AC(next, C0D1U1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) tmdel = (tdel < 0) ? -1 * tdel : tdel;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) if (tmdel > adel)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) adel = tmdel;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) if (tmdel * 128 * next_timing_rate_mhz / 1000000 >
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) next->tree_margin)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) next->current_dram_clktree[C0D1U1] =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) __MOVAVG_AC(next, C0D1U1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) if (emc->num_channels > 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) if (dvfs_pt1 || periodic_training_update) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) cval = tegra210_emc_actual_osc_clocks(last->run_clocks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) cval *= 1000000;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) cval /= last_timing_rate_mhz * 2 * temp[1][0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) if (dvfs_pt1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) __INCREMENT_PTFV(C1D1U0, cval);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) else if (dvfs_update)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) __AVERAGE_PTFV(C1D1U0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) else if (periodic_training_update)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) __WEIGHTED_UPDATE_PTFV(C1D1U0, cval);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) if (dvfs_update || periodic_training_update) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) tdel = next->current_dram_clktree[C1D1U0] -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) __MOVAVG_AC(next, C1D1U0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) tmdel = (tdel < 0) ? -1 * tdel : tdel;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) if (tmdel > adel)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) adel = tmdel;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) if (tmdel * 128 * next_timing_rate_mhz / 1000000 >
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) next->tree_margin)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) next->current_dram_clktree[C1D1U0] =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) __MOVAVG_AC(next, C1D1U0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) if (dvfs_pt1 || periodic_training_update) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) cval = tegra210_emc_actual_osc_clocks(last->run_clocks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) cval *= 1000000;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) cval /= last_timing_rate_mhz * 2 * temp[1][1];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) if (dvfs_pt1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) __INCREMENT_PTFV(C1D1U1, cval);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) else if (dvfs_update)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) __AVERAGE_PTFV(C1D1U1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) else if (periodic_training_update)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) __WEIGHTED_UPDATE_PTFV(C1D1U1, cval);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) if (dvfs_update || periodic_training_update) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) tdel = next->current_dram_clktree[C1D1U1] -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) __MOVAVG_AC(next, C1D1U1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) tmdel = (tdel < 0) ? -1 * tdel : tdel;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) if (tmdel > adel)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) adel = tmdel;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) if (tmdel * 128 * next_timing_rate_mhz / 1000000 >
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) next->tree_margin)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) next->current_dram_clktree[C1D1U1] =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) __MOVAVG_AC(next, C1D1U1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) done:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) return adel;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) static u32 periodic_compensation_handler(struct tegra210_emc *emc, u32 type,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) struct tegra210_emc_timing *last,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) struct tegra210_emc_timing *next)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) #define __COPY_EMA(nt, lt, dev) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) ({ __MOVAVG(nt, dev) = __MOVAVG(lt, dev) * \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) (nt)->ptfv_list[PTFV_DVFS_SAMPLES_INDEX]; })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) u32 i, adel = 0, samples = next->ptfv_list[PTFV_DVFS_SAMPLES_INDEX];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) u32 delay;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) delay = tegra210_emc_actual_osc_clocks(last->run_clocks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) delay *= 1000;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) delay = 2 + (delay / last->rate);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) if (!next->periodic_training)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) if (type == DVFS_SEQUENCE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) if (last->periodic_training &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) (next->ptfv_list[PTFV_CONFIG_CTRL_INDEX] &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) PTFV_CONFIG_CTRL_USE_PREVIOUS_EMA)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) * If the previous frequency was using periodic
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) * calibration then we can reuse the previous
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) * frequencies EMA data.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) __COPY_EMA(next, last, C0D0U0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) __COPY_EMA(next, last, C0D0U1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) __COPY_EMA(next, last, C1D0U0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) __COPY_EMA(next, last, C1D0U1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) __COPY_EMA(next, last, C0D1U0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) __COPY_EMA(next, last, C0D1U1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) __COPY_EMA(next, last, C1D1U0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) __COPY_EMA(next, last, C1D1U1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) /* Reset the EMA.*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) __MOVAVG(next, C0D0U0) = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) __MOVAVG(next, C0D0U1) = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) __MOVAVG(next, C1D0U0) = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) __MOVAVG(next, C1D0U1) = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) __MOVAVG(next, C0D1U0) = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) __MOVAVG(next, C0D1U1) = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) __MOVAVG(next, C1D1U0) = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) __MOVAVG(next, C1D1U1) = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) for (i = 0; i < samples; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) tegra210_emc_start_periodic_compensation(emc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) udelay(delay);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) * Generate next sample of data.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) adel = update_clock_tree_delay(emc, DVFS_PT1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) * Seems like it should be part of the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) * 'if (last_timing->periodic_training)' conditional
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) * since is already done for the else clause.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) adel = update_clock_tree_delay(emc, DVFS_UPDATE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) if (type == PERIODIC_TRAINING_SEQUENCE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) tegra210_emc_start_periodic_compensation(emc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) udelay(delay);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) adel = update_clock_tree_delay(emc, PERIODIC_TRAINING_UPDATE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) return adel;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) static u32 tegra210_emc_r21021_periodic_compensation(struct tegra210_emc *emc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) u32 emc_cfg, emc_cfg_o, emc_cfg_update, del, value;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) u32 list[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) EMC_DATA_BRLSHFT_0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) EMC_DATA_BRLSHFT_1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) struct tegra210_emc_timing *last = emc->last;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) unsigned int items = ARRAY_SIZE(list), i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) unsigned long delay;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) if (last->periodic_training) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) emc_dbg(emc, PER_TRAIN, "Periodic training starting\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) value = emc_readl(emc, EMC_DBG);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) emc_cfg_o = emc_readl(emc, EMC_CFG);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) emc_cfg = emc_cfg_o & ~(EMC_CFG_DYN_SELF_REF |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) EMC_CFG_DRAM_ACPD |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) EMC_CFG_DRAM_CLKSTOP_PD);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) * 1. Power optimizations should be off.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) emc_writel(emc, emc_cfg, EMC_CFG);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) /* Does emc_timing_update() for above changes. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) tegra210_emc_dll_disable(emc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) for (i = 0; i < emc->num_channels; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) tegra210_emc_wait_for_update(emc, i, EMC_EMC_STATUS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) EMC_EMC_STATUS_DRAM_IN_POWERDOWN_MASK,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) for (i = 0; i < emc->num_channels; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) tegra210_emc_wait_for_update(emc, i, EMC_EMC_STATUS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) EMC_EMC_STATUS_DRAM_IN_SELF_REFRESH_MASK,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) emc_cfg_update = value = emc_readl(emc, EMC_CFG_UPDATE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) value &= ~EMC_CFG_UPDATE_UPDATE_DLL_IN_UPDATE_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) value |= (2 << EMC_CFG_UPDATE_UPDATE_DLL_IN_UPDATE_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) emc_writel(emc, value, EMC_CFG_UPDATE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) * 2. osc kick off - this assumes training and dvfs have set
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) * correct MR23.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) tegra210_emc_start_periodic_compensation(emc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) * 3. Let dram capture its clock tree delays.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) delay = tegra210_emc_actual_osc_clocks(last->run_clocks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) delay *= 1000;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) delay /= last->rate + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) udelay(delay);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) * 4. Check delta wrt previous values (save value if margin
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) * exceeds what is set in table).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) del = periodic_compensation_handler(emc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) PERIODIC_TRAINING_SEQUENCE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) last, last);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) * 5. Apply compensation w.r.t. trained values (if clock tree
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) * has drifted more than the set margin).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) if (last->tree_margin < ((del * 128 * (last->rate / 1000)) / 1000000)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) for (i = 0; i < items; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) value = tegra210_emc_compensate(last, list[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) emc_dbg(emc, EMA_WRITES, "0x%08x <= 0x%08x\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) list[i], value);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) emc_writel(emc, value, list[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) emc_writel(emc, emc_cfg_o, EMC_CFG);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) * 6. Timing update actally applies the new trimmers.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) tegra210_emc_timing_update(emc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) /* 6.1. Restore the UPDATE_DLL_IN_UPDATE field. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) emc_writel(emc, emc_cfg_update, EMC_CFG_UPDATE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) /* 6.2. Restore the DLL. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) tegra210_emc_dll_enable(emc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) * Do the clock change sequence.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) static void tegra210_emc_r21021_set_clock(struct tegra210_emc *emc, u32 clksrc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) /* state variables */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) static bool fsp_for_next_freq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) /* constant configuration parameters */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) const bool save_restore_clkstop_pd = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) const u32 zqcal_before_cc_cutoff = 2400;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) const bool cya_allow_ref_cc = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) const bool cya_issue_pc_ref = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) const bool opt_cc_short_zcal = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) const bool ref_b4_sref_en = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) const u32 tZQCAL_lpddr4 = 1000000;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) const bool opt_short_zcal = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) const bool opt_do_sw_qrst = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) const u32 opt_dvfs_mode = MAN_SR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) * This is the timing table for the source frequency. It does _not_
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) * necessarily correspond to the actual timing values in the EMC at the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) * moment. If the boot BCT differs from the table then this can happen.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) * However, we need it for accessing the dram_timings (which are not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) * really registers) array for the current frequency.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) struct tegra210_emc_timing *fake, *last = emc->last, *next = emc->next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) u32 tRTM, RP_war, R2P_war, TRPab_war, deltaTWATM, W2P_war, tRPST;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) u32 mr13_flip_fspwr, mr13_flip_fspop, ramp_up_wait, ramp_down_wait;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) u32 zq_wait_long, zq_latch_dvfs_wait_time, tZQCAL_lpddr4_fc_adj;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) u32 emc_auto_cal_config, auto_cal_en, emc_cfg, emc_sel_dpd_ctrl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) u32 tFC_lpddr4 = 1000 * next->dram_timings[T_FC_LPDDR4];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) u32 bg_reg_mode_change, enable_bglp_reg, enable_bg_reg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) bool opt_zcal_en_cc = false, is_lpddr3 = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) bool compensate_trimmer_applicable = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) u32 emc_dbg, emc_cfg_pipe_clk, emc_pin;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) u32 src_clk_period, dst_clk_period; /* in picoseconds */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) bool shared_zq_resistor = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) u32 value, dram_type;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) u32 opt_dll_mode = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) unsigned long delay;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) unsigned int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) emc_dbg(emc, INFO, "Running clock change.\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) /* XXX fake == last */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) fake = tegra210_emc_find_timing(emc, last->rate * 1000UL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) fsp_for_next_freq = !fsp_for_next_freq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) value = emc_readl(emc, EMC_FBIO_CFG5) & EMC_FBIO_CFG5_DRAM_TYPE_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) dram_type = value >> EMC_FBIO_CFG5_DRAM_TYPE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) if (last->burst_regs[EMC_ZCAL_WAIT_CNT_INDEX] & BIT(31))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) shared_zq_resistor = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) if ((next->burst_regs[EMC_ZCAL_INTERVAL_INDEX] != 0 &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) last->burst_regs[EMC_ZCAL_INTERVAL_INDEX] == 0) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) dram_type == DRAM_TYPE_LPDDR4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) opt_zcal_en_cc = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) if (dram_type == DRAM_TYPE_DDR3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) opt_dll_mode = tegra210_emc_get_dll_state(next);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) if ((next->burst_regs[EMC_FBIO_CFG5_INDEX] & BIT(25)) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) (dram_type == DRAM_TYPE_LPDDR2))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) is_lpddr3 = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) emc_readl(emc, EMC_CFG);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) emc_readl(emc, EMC_AUTO_CAL_CONFIG);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) src_clk_period = 1000000000 / last->rate;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) dst_clk_period = 1000000000 / next->rate;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) if (dst_clk_period <= zqcal_before_cc_cutoff)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) tZQCAL_lpddr4_fc_adj = tZQCAL_lpddr4 - tFC_lpddr4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) tZQCAL_lpddr4_fc_adj = tZQCAL_lpddr4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) tZQCAL_lpddr4_fc_adj /= dst_clk_period;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) emc_dbg = emc_readl(emc, EMC_DBG);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) emc_pin = emc_readl(emc, EMC_PIN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) emc_cfg_pipe_clk = emc_readl(emc, EMC_CFG_PIPE_CLK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) emc_cfg = next->burst_regs[EMC_CFG_INDEX];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) emc_cfg &= ~(EMC_CFG_DYN_SELF_REF | EMC_CFG_DRAM_ACPD |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) EMC_CFG_DRAM_CLKSTOP_SR | EMC_CFG_DRAM_CLKSTOP_PD);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) emc_sel_dpd_ctrl = next->emc_sel_dpd_ctrl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) emc_sel_dpd_ctrl &= ~(EMC_SEL_DPD_CTRL_CLK_SEL_DPD_EN |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) EMC_SEL_DPD_CTRL_CA_SEL_DPD_EN |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) EMC_SEL_DPD_CTRL_RESET_SEL_DPD_EN |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) EMC_SEL_DPD_CTRL_ODT_SEL_DPD_EN |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) EMC_SEL_DPD_CTRL_DATA_SEL_DPD_EN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) emc_dbg(emc, INFO, "Clock change version: %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) DVFS_CLOCK_CHANGE_VERSION);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) emc_dbg(emc, INFO, "DRAM type = %d\n", dram_type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) emc_dbg(emc, INFO, "DRAM dev #: %u\n", emc->num_devices);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) emc_dbg(emc, INFO, "Next EMC clksrc: 0x%08x\n", clksrc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) emc_dbg(emc, INFO, "DLL clksrc: 0x%08x\n", next->dll_clk_src);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) emc_dbg(emc, INFO, "last rate: %u, next rate %u\n", last->rate,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) next->rate);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) emc_dbg(emc, INFO, "last period: %u, next period: %u\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) src_clk_period, dst_clk_period);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) emc_dbg(emc, INFO, " shared_zq_resistor: %d\n", !!shared_zq_resistor);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) emc_dbg(emc, INFO, " num_channels: %u\n", emc->num_channels);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) emc_dbg(emc, INFO, " opt_dll_mode: %d\n", opt_dll_mode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) * Step 1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) * Pre DVFS SW sequence.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) emc_dbg(emc, STEPS, "Step 1\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) emc_dbg(emc, STEPS, "Step 1.1: Disable DLL temporarily.\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) value = emc_readl(emc, EMC_CFG_DIG_DLL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) value &= ~EMC_CFG_DIG_DLL_CFG_DLL_EN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) emc_writel(emc, value, EMC_CFG_DIG_DLL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) tegra210_emc_timing_update(emc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) for (i = 0; i < emc->num_channels; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) tegra210_emc_wait_for_update(emc, i, EMC_CFG_DIG_DLL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) EMC_CFG_DIG_DLL_CFG_DLL_EN, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) emc_dbg(emc, STEPS, "Step 1.2: Disable AUTOCAL temporarily.\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) emc_auto_cal_config = next->emc_auto_cal_config;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) auto_cal_en = emc_auto_cal_config & EMC_AUTO_CAL_CONFIG_AUTO_CAL_ENABLE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) emc_auto_cal_config &= ~EMC_AUTO_CAL_CONFIG_AUTO_CAL_START;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) emc_auto_cal_config |= EMC_AUTO_CAL_CONFIG_AUTO_CAL_MEASURE_STALL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) emc_auto_cal_config |= EMC_AUTO_CAL_CONFIG_AUTO_CAL_UPDATE_STALL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) emc_auto_cal_config |= auto_cal_en;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) emc_writel(emc, emc_auto_cal_config, EMC_AUTO_CAL_CONFIG);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) emc_readl(emc, EMC_AUTO_CAL_CONFIG); /* Flush write. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) emc_dbg(emc, STEPS, "Step 1.3: Disable other power features.\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) tegra210_emc_set_shadow_bypass(emc, ACTIVE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) emc_writel(emc, emc_cfg, EMC_CFG);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) emc_writel(emc, emc_sel_dpd_ctrl, EMC_SEL_DPD_CTRL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) tegra210_emc_set_shadow_bypass(emc, ASSEMBLY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) if (next->periodic_training) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) tegra210_emc_reset_dram_clktree_values(next);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) for (i = 0; i < emc->num_channels; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) tegra210_emc_wait_for_update(emc, i, EMC_EMC_STATUS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) EMC_EMC_STATUS_DRAM_IN_POWERDOWN_MASK,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) for (i = 0; i < emc->num_channels; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) tegra210_emc_wait_for_update(emc, i, EMC_EMC_STATUS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) EMC_EMC_STATUS_DRAM_IN_SELF_REFRESH_MASK,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) tegra210_emc_start_periodic_compensation(emc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) delay = 1000 * tegra210_emc_actual_osc_clocks(last->run_clocks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) udelay((delay / last->rate) + 2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) value = periodic_compensation_handler(emc, DVFS_SEQUENCE, fake,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) next);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) value = (value * 128 * next->rate / 1000) / 1000000;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) if (next->periodic_training && value > next->tree_margin)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) compensate_trimmer_applicable = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) emc_writel(emc, EMC_INTSTATUS_CLKCHANGE_COMPLETE, EMC_INTSTATUS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) tegra210_emc_set_shadow_bypass(emc, ACTIVE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) emc_writel(emc, emc_cfg, EMC_CFG);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) emc_writel(emc, emc_sel_dpd_ctrl, EMC_SEL_DPD_CTRL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) emc_writel(emc, emc_cfg_pipe_clk | EMC_CFG_PIPE_CLK_CLK_ALWAYS_ON,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) EMC_CFG_PIPE_CLK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) emc_writel(emc, next->emc_fdpd_ctrl_cmd_no_ramp &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) ~EMC_FDPD_CTRL_CMD_NO_RAMP_CMD_DPD_NO_RAMP_ENABLE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) EMC_FDPD_CTRL_CMD_NO_RAMP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) bg_reg_mode_change =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) ((next->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) EMC_PMACRO_BG_BIAS_CTRL_0_BGLP_E_PWRD) ^
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) (last->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) EMC_PMACRO_BG_BIAS_CTRL_0_BGLP_E_PWRD)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) ((next->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) EMC_PMACRO_BG_BIAS_CTRL_0_BG_E_PWRD) ^
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) (last->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) EMC_PMACRO_BG_BIAS_CTRL_0_BG_E_PWRD));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) enable_bglp_reg =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) (next->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) EMC_PMACRO_BG_BIAS_CTRL_0_BGLP_E_PWRD) == 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) enable_bg_reg =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) (next->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) EMC_PMACRO_BG_BIAS_CTRL_0_BG_E_PWRD) == 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) if (bg_reg_mode_change) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) if (enable_bg_reg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) emc_writel(emc, last->burst_regs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) [EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) ~EMC_PMACRO_BG_BIAS_CTRL_0_BG_E_PWRD,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) EMC_PMACRO_BG_BIAS_CTRL_0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) if (enable_bglp_reg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) emc_writel(emc, last->burst_regs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) [EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) ~EMC_PMACRO_BG_BIAS_CTRL_0_BGLP_E_PWRD,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) EMC_PMACRO_BG_BIAS_CTRL_0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) /* Check if we need to turn on VREF generator. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) if ((((last->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX] &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_E_IVREF) == 0) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) ((next->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX] &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_E_IVREF) == 1)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) (((last->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX] &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQS_E_IVREF) == 0) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) ((next->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX] &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQS_E_IVREF) != 0))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) u32 pad_tx_ctrl =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) next->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) u32 last_pad_tx_ctrl =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) last->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) u32 next_dq_e_ivref, next_dqs_e_ivref;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) next_dqs_e_ivref = pad_tx_ctrl &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQS_E_IVREF;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) next_dq_e_ivref = pad_tx_ctrl &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_E_IVREF;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) value = (last_pad_tx_ctrl &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) ~EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_E_IVREF &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) ~EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQS_E_IVREF) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) next_dq_e_ivref | next_dqs_e_ivref;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) emc_writel(emc, value, EMC_PMACRO_DATA_PAD_TX_CTRL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) udelay(1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) } else if (bg_reg_mode_change) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) udelay(1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) tegra210_emc_set_shadow_bypass(emc, ASSEMBLY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) * Step 2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) * Prelock the DLL.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) emc_dbg(emc, STEPS, "Step 2\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) if (next->burst_regs[EMC_CFG_DIG_DLL_INDEX] &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) EMC_CFG_DIG_DLL_CFG_DLL_EN) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) emc_dbg(emc, INFO, "Prelock enabled for target frequency.\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) value = tegra210_emc_dll_prelock(emc, clksrc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) emc_dbg(emc, INFO, "DLL out: 0x%03x\n", value);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) emc_dbg(emc, INFO, "Disabling DLL for target frequency.\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) tegra210_emc_dll_disable(emc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) * Step 3:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) * Prepare autocal for the clock change.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) emc_dbg(emc, STEPS, "Step 3\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) tegra210_emc_set_shadow_bypass(emc, ACTIVE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) emc_writel(emc, next->emc_auto_cal_config2, EMC_AUTO_CAL_CONFIG2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) emc_writel(emc, next->emc_auto_cal_config3, EMC_AUTO_CAL_CONFIG3);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) emc_writel(emc, next->emc_auto_cal_config4, EMC_AUTO_CAL_CONFIG4);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) emc_writel(emc, next->emc_auto_cal_config5, EMC_AUTO_CAL_CONFIG5);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) emc_writel(emc, next->emc_auto_cal_config6, EMC_AUTO_CAL_CONFIG6);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) emc_writel(emc, next->emc_auto_cal_config7, EMC_AUTO_CAL_CONFIG7);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) emc_writel(emc, next->emc_auto_cal_config8, EMC_AUTO_CAL_CONFIG8);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) tegra210_emc_set_shadow_bypass(emc, ASSEMBLY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) emc_auto_cal_config |= (EMC_AUTO_CAL_CONFIG_AUTO_CAL_COMPUTE_START |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) auto_cal_en);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) emc_writel(emc, emc_auto_cal_config, EMC_AUTO_CAL_CONFIG);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) * Step 4:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) * Update EMC_CFG. (??)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) emc_dbg(emc, STEPS, "Step 4\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) if (src_clk_period > 50000 && dram_type == DRAM_TYPE_LPDDR4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) ccfifo_writel(emc, 1, EMC_SELF_REF, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) emc_writel(emc, next->emc_cfg_2, EMC_CFG_2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) * Step 5:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) * Prepare reference variables for ZQCAL regs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) emc_dbg(emc, STEPS, "Step 5\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) if (dram_type == DRAM_TYPE_LPDDR4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) zq_wait_long = max((u32)1, div_o3(1000000, dst_clk_period));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) else if (dram_type == DRAM_TYPE_LPDDR2 || is_lpddr3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) zq_wait_long = max(next->min_mrs_wait,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) div_o3(360000, dst_clk_period)) + 4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) else if (dram_type == DRAM_TYPE_DDR3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) zq_wait_long = max((u32)256,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) div_o3(320000, dst_clk_period) + 2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) zq_wait_long = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) * Step 6:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) * Training code - removed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) emc_dbg(emc, STEPS, "Step 6\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) * Step 7:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893) * Program FSP reference registers and send MRWs to new FSPWR.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) emc_dbg(emc, STEPS, "Step 7\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) emc_dbg(emc, SUB_STEPS, "Step 7.1: Bug 200024907 - Patch RP R2P");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) /* WAR 200024907 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) if (dram_type == DRAM_TYPE_LPDDR4) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) u32 nRTP = 16;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) if (src_clk_period >= 1000000 / 1866) /* 535.91 ps */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903) nRTP = 14;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) if (src_clk_period >= 1000000 / 1600) /* 625.00 ps */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) nRTP = 12;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) if (src_clk_period >= 1000000 / 1333) /* 750.19 ps */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) nRTP = 10;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) if (src_clk_period >= 1000000 / 1066) /* 938.09 ps */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) nRTP = 8;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) deltaTWATM = max_t(u32, div_o3(7500, src_clk_period), 8);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) * Originally there was a + .5 in the tRPST calculation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) * However since we can't do FP in the kernel and the tRTM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) * computation was in a floating point ceiling function, adding
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) * one to tRTP should be ok. There is no other source of non
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) * integer values, so the result was always going to be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) * something for the form: f_ceil(N + .5) = N + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924) tRPST = (last->emc_mrw & 0x80) >> 7;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) tRTM = fake->dram_timings[RL] + div_o3(3600, src_clk_period) +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926) max_t(u32, div_o3(7500, src_clk_period), 8) + tRPST +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927) 1 + nRTP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) emc_dbg(emc, INFO, "tRTM = %u, EMC_RP = %u\n", tRTM,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) next->burst_regs[EMC_RP_INDEX]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932) if (last->burst_regs[EMC_RP_INDEX] < tRTM) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) if (tRTM > (last->burst_regs[EMC_R2P_INDEX] +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) last->burst_regs[EMC_RP_INDEX])) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) R2P_war = tRTM - last->burst_regs[EMC_RP_INDEX];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) RP_war = last->burst_regs[EMC_RP_INDEX];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) TRPab_war = last->burst_regs[EMC_TRPAB_INDEX];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) if (R2P_war > 63) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) RP_war = R2P_war +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) last->burst_regs[EMC_RP_INDEX] - 63;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943) if (TRPab_war < RP_war)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) TRPab_war = RP_war;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946) R2P_war = 63;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949) R2P_war = last->burst_regs[EMC_R2P_INDEX];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950) RP_war = last->burst_regs[EMC_RP_INDEX];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) TRPab_war = last->burst_regs[EMC_TRPAB_INDEX];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) if (RP_war < deltaTWATM) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) W2P_war = last->burst_regs[EMC_W2P_INDEX]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956) + deltaTWATM - RP_war;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957) if (W2P_war > 63) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) RP_war = RP_war + W2P_war - 63;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959) if (TRPab_war < RP_war)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960) TRPab_war = RP_war;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961) W2P_war = 63;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964) W2P_war = last->burst_regs[
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965) EMC_W2P_INDEX];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968) if ((last->burst_regs[EMC_W2P_INDEX] ^ W2P_war) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969) (last->burst_regs[EMC_R2P_INDEX] ^ R2P_war) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970) (last->burst_regs[EMC_RP_INDEX] ^ RP_war) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971) (last->burst_regs[EMC_TRPAB_INDEX] ^ TRPab_war)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972) emc_writel(emc, RP_war, EMC_RP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973) emc_writel(emc, R2P_war, EMC_R2P);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974) emc_writel(emc, W2P_war, EMC_W2P);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975) emc_writel(emc, TRPab_war, EMC_TRPAB);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978) tegra210_emc_timing_update(emc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980) emc_dbg(emc, INFO, "Skipped WAR\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984) if (!fsp_for_next_freq) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985) mr13_flip_fspwr = (next->emc_mrw3 & 0xffffff3f) | 0x80;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986) mr13_flip_fspop = (next->emc_mrw3 & 0xffffff3f) | 0x00;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988) mr13_flip_fspwr = (next->emc_mrw3 & 0xffffff3f) | 0x40;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989) mr13_flip_fspop = (next->emc_mrw3 & 0xffffff3f) | 0xc0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992) if (dram_type == DRAM_TYPE_LPDDR4) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993) emc_writel(emc, mr13_flip_fspwr, EMC_MRW3);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994) emc_writel(emc, next->emc_mrw, EMC_MRW);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995) emc_writel(emc, next->emc_mrw2, EMC_MRW2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999) * Step 8:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000) * Program the shadow registers.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002) emc_dbg(emc, STEPS, "Step 8\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003) emc_dbg(emc, SUB_STEPS, "Writing burst_regs\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005) for (i = 0; i < next->num_burst; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006) const u16 *offsets = emc->offsets->burst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007) u16 offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009) if (!offsets[i])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012) value = next->burst_regs[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013) offset = offsets[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015) if (dram_type != DRAM_TYPE_LPDDR4 &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016) (offset == EMC_MRW6 || offset == EMC_MRW7 ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017) offset == EMC_MRW8 || offset == EMC_MRW9 ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018) offset == EMC_MRW10 || offset == EMC_MRW11 ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019) offset == EMC_MRW12 || offset == EMC_MRW13 ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020) offset == EMC_MRW14 || offset == EMC_MRW15 ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021) offset == EMC_TRAINING_CTRL))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024) /* Pain... And suffering. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025) if (offset == EMC_CFG) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026) value &= ~EMC_CFG_DRAM_ACPD;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027) value &= ~EMC_CFG_DYN_SELF_REF;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029) if (dram_type == DRAM_TYPE_LPDDR4) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030) value &= ~EMC_CFG_DRAM_CLKSTOP_SR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031) value &= ~EMC_CFG_DRAM_CLKSTOP_PD;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033) } else if (offset == EMC_MRS_WAIT_CNT &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034) dram_type == DRAM_TYPE_LPDDR2 &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035) opt_zcal_en_cc && !opt_cc_short_zcal &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036) opt_short_zcal) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037) value = (value & ~(EMC_MRS_WAIT_CNT_SHORT_WAIT_MASK <<
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038) EMC_MRS_WAIT_CNT_SHORT_WAIT_SHIFT)) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039) ((zq_wait_long & EMC_MRS_WAIT_CNT_SHORT_WAIT_MASK) <<
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040) EMC_MRS_WAIT_CNT_SHORT_WAIT_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041) } else if (offset == EMC_ZCAL_WAIT_CNT &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042) dram_type == DRAM_TYPE_DDR3 && opt_zcal_en_cc &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043) !opt_cc_short_zcal && opt_short_zcal) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044) value = (value & ~(EMC_ZCAL_WAIT_CNT_ZCAL_WAIT_CNT_MASK <<
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045) EMC_ZCAL_WAIT_CNT_ZCAL_WAIT_CNT_SHIFT)) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046) ((zq_wait_long & EMC_ZCAL_WAIT_CNT_ZCAL_WAIT_CNT_MASK) <<
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047) EMC_MRS_WAIT_CNT_SHORT_WAIT_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048) } else if (offset == EMC_ZCAL_INTERVAL && opt_zcal_en_cc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049) value = 0; /* EMC_ZCAL_INTERVAL reset value. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050) } else if (offset == EMC_PMACRO_AUTOCAL_CFG_COMMON) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051) value |= EMC_PMACRO_AUTOCAL_CFG_COMMON_E_CAL_BYPASS_DVFS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052) } else if (offset == EMC_PMACRO_DATA_PAD_TX_CTRL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053) value &= ~(EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSP_TX_E_DCC |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054) EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSN_TX_E_DCC |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055) EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_TX_E_DCC |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056) EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_CMD_TX_E_DCC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057) } else if (offset == EMC_PMACRO_CMD_PAD_TX_CTRL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058) value |= EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_DRVFORCEON;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059) value &= ~(EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSP_TX_E_DCC |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060) EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSN_TX_E_DCC |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1061) EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_E_DCC |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1062) EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_CMD_TX_E_DCC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1063) } else if (offset == EMC_PMACRO_BRICK_CTRL_RFU1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1064) value &= 0xf800f800;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1065) } else if (offset == EMC_PMACRO_COMMON_PAD_TX_CTRL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1066) value &= 0xfffffff0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1067) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1068)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1069) emc_writel(emc, value, offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1070) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1071)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1072) /* SW addition: do EMC refresh adjustment here. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1073) tegra210_emc_adjust_timing(emc, next);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1074)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1075) if (dram_type == DRAM_TYPE_LPDDR4) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1076) value = (23 << EMC_MRW_MRW_MA_SHIFT) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1077) (next->run_clocks & EMC_MRW_MRW_OP_MASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1078) emc_writel(emc, value, EMC_MRW);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1079) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1080)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1081) /* Per channel burst registers. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1082) emc_dbg(emc, SUB_STEPS, "Writing burst_regs_per_ch\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1083)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1084) for (i = 0; i < next->num_burst_per_ch; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1085) const struct tegra210_emc_per_channel_regs *burst =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1086) emc->offsets->burst_per_channel;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1087)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1088) if (!burst[i].offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1089) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1090)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1091) if (dram_type != DRAM_TYPE_LPDDR4 &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1092) (burst[i].offset == EMC_MRW6 ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1093) burst[i].offset == EMC_MRW7 ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1094) burst[i].offset == EMC_MRW8 ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1095) burst[i].offset == EMC_MRW9 ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1096) burst[i].offset == EMC_MRW10 ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1097) burst[i].offset == EMC_MRW11 ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1098) burst[i].offset == EMC_MRW12 ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1099) burst[i].offset == EMC_MRW13 ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1100) burst[i].offset == EMC_MRW14 ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1101) burst[i].offset == EMC_MRW15))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1102) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1103)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1104) /* Filter out second channel if not in DUAL_CHANNEL mode. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1105) if (emc->num_channels < 2 && burst[i].bank >= 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1106) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1107)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1108) emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1109) next->burst_reg_per_ch[i], burst[i].offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1110) emc_channel_writel(emc, burst[i].bank,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1111) next->burst_reg_per_ch[i],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1112) burst[i].offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1113) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1114)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1115) /* Vref regs. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1116) emc_dbg(emc, SUB_STEPS, "Writing vref_regs\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1117)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1118) for (i = 0; i < next->vref_num; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1119) const struct tegra210_emc_per_channel_regs *vref =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1120) emc->offsets->vref_per_channel;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1121)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1122) if (!vref[i].offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1123) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1124)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1125) if (emc->num_channels < 2 && vref[i].bank >= 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1126) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1127)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1128) emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1129) next->vref_perch_regs[i], vref[i].offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1130) emc_channel_writel(emc, vref[i].bank, next->vref_perch_regs[i],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1131) vref[i].offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1132) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1133)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1134) /* Trimmers. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1135) emc_dbg(emc, SUB_STEPS, "Writing trim_regs\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1136)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1137) for (i = 0; i < next->num_trim; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1138) const u16 *offsets = emc->offsets->trim;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1139)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1140) if (!offsets[i])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1141) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1142)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1143) if (compensate_trimmer_applicable &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1144) (offsets[i] == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0 ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1145) offsets[i] == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1 ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1146) offsets[i] == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2 ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1147) offsets[i] == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3 ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1148) offsets[i] == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0 ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1149) offsets[i] == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1 ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1150) offsets[i] == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2 ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1151) offsets[i] == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3 ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1152) offsets[i] == EMC_DATA_BRLSHFT_0 ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1153) offsets[i] == EMC_DATA_BRLSHFT_1)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1154) value = tegra210_emc_compensate(next, offsets[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1155) emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1156) value, offsets[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1157) emc_dbg(emc, EMA_WRITES, "0x%08x <= 0x%08x\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1158) (u32)(u64)offsets[i], value);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1159) emc_writel(emc, value, offsets[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1160) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1161) emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1162) next->trim_regs[i], offsets[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1163) emc_writel(emc, next->trim_regs[i], offsets[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1164) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1165) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1166)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1167) /* Per channel trimmers. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1168) emc_dbg(emc, SUB_STEPS, "Writing trim_regs_per_ch\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1169)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1170) for (i = 0; i < next->num_trim_per_ch; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1171) const struct tegra210_emc_per_channel_regs *trim =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1172) &emc->offsets->trim_per_channel[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1173) unsigned int offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1174)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1175) if (!trim[i].offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1176) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1177)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1178) if (emc->num_channels < 2 && trim[i].bank >= 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1179) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1180)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1181) offset = trim[i].offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1182)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1183) if (compensate_trimmer_applicable &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1184) (offset == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0 ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1185) offset == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1 ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1186) offset == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2 ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1187) offset == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3 ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1188) offset == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0 ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1189) offset == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1 ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1190) offset == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2 ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1191) offset == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3 ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1192) offset == EMC_DATA_BRLSHFT_0 ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1193) offset == EMC_DATA_BRLSHFT_1)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1194) value = tegra210_emc_compensate(next, offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1195) emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1196) value, offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1197) emc_dbg(emc, EMA_WRITES, "0x%08x <= 0x%08x\n", offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1198) value);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1199) emc_channel_writel(emc, trim[i].bank, value, offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1200) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1201) emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1202) next->trim_perch_regs[i], offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1203) emc_channel_writel(emc, trim[i].bank,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1204) next->trim_perch_regs[i], offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1205) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1206) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1207)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1208) emc_dbg(emc, SUB_STEPS, "Writing burst_mc_regs\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1209)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1210) for (i = 0; i < next->num_mc_regs; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1211) const u16 *offsets = emc->offsets->burst_mc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1212) u32 *values = next->burst_mc_regs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1213)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1214) emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1215) values[i], offsets[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1216) mc_writel(emc->mc, values[i], offsets[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1217) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1218)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1219) /* Registers to be programmed on the faster clock. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1220) if (next->rate < last->rate) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1221) const u16 *la = emc->offsets->la_scale;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1222)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1223) emc_dbg(emc, SUB_STEPS, "Writing la_scale_regs\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1224)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1225) for (i = 0; i < next->num_up_down; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1226) emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1227) next->la_scale_regs[i], la[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1228) mc_writel(emc->mc, next->la_scale_regs[i], la[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1229) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1230) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1231)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1232) /* Flush all the burst register writes. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1233) mc_readl(emc->mc, MC_EMEM_ADR_CFG);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1234)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1235) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1236) * Step 9:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1237) * LPDDR4 section A.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1238) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1239) emc_dbg(emc, STEPS, "Step 9\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1240)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1241) value = next->burst_regs[EMC_ZCAL_WAIT_CNT_INDEX];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1242) value &= ~EMC_ZCAL_WAIT_CNT_ZCAL_WAIT_CNT_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1243)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1244) if (dram_type == DRAM_TYPE_LPDDR4) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1245) emc_writel(emc, 0, EMC_ZCAL_INTERVAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1246) emc_writel(emc, value, EMC_ZCAL_WAIT_CNT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1247)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1248) value = emc_dbg | (EMC_DBG_WRITE_MUX_ACTIVE |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1249) EMC_DBG_WRITE_ACTIVE_ONLY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1250)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1251) emc_writel(emc, value, EMC_DBG);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1252) emc_writel(emc, 0, EMC_ZCAL_INTERVAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1253) emc_writel(emc, emc_dbg, EMC_DBG);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1254) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1255)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1256) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1257) * Step 10:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1258) * LPDDR4 and DDR3 common section.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1259) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1260) emc_dbg(emc, STEPS, "Step 10\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1261)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1262) if (opt_dvfs_mode == MAN_SR || dram_type == DRAM_TYPE_LPDDR4) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1263) if (dram_type == DRAM_TYPE_LPDDR4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1264) ccfifo_writel(emc, 0x101, EMC_SELF_REF, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1265) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1266) ccfifo_writel(emc, 0x1, EMC_SELF_REF, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1267)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1268) if (dram_type == DRAM_TYPE_LPDDR4 &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1269) dst_clk_period <= zqcal_before_cc_cutoff) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1270) ccfifo_writel(emc, mr13_flip_fspwr ^ 0x40, EMC_MRW3, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1271) ccfifo_writel(emc, (next->burst_regs[EMC_MRW6_INDEX] &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1272) 0xFFFF3F3F) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1273) (last->burst_regs[EMC_MRW6_INDEX] &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1274) 0x0000C0C0), EMC_MRW6, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1275) ccfifo_writel(emc, (next->burst_regs[EMC_MRW14_INDEX] &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1276) 0xFFFF0707) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1277) (last->burst_regs[EMC_MRW14_INDEX] &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1278) 0x00003838), EMC_MRW14, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1279)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1280) if (emc->num_devices > 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1281) ccfifo_writel(emc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1282) (next->burst_regs[EMC_MRW7_INDEX] &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1283) 0xFFFF3F3F) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1284) (last->burst_regs[EMC_MRW7_INDEX] &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1285) 0x0000C0C0), EMC_MRW7, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1286) ccfifo_writel(emc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1287) (next->burst_regs[EMC_MRW15_INDEX] &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1288) 0xFFFF0707) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1289) (last->burst_regs[EMC_MRW15_INDEX] &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1290) 0x00003838), EMC_MRW15, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1291) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1292)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1293) if (opt_zcal_en_cc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1294) if (emc->num_devices < 2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1295) ccfifo_writel(emc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1296) 2UL << EMC_ZQ_CAL_DEV_SEL_SHIFT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1297) | EMC_ZQ_CAL_ZQ_CAL_CMD,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1298) EMC_ZQ_CAL, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1299) else if (shared_zq_resistor)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1300) ccfifo_writel(emc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1301) 2UL << EMC_ZQ_CAL_DEV_SEL_SHIFT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1302) | EMC_ZQ_CAL_ZQ_CAL_CMD,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1303) EMC_ZQ_CAL, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1304) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1305) ccfifo_writel(emc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1306) EMC_ZQ_CAL_ZQ_CAL_CMD,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1307) EMC_ZQ_CAL, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1308) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1309) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1310) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1311)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1312) if (dram_type == DRAM_TYPE_LPDDR4) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1313) value = (1000 * fake->dram_timings[T_RP]) / src_clk_period;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1314) ccfifo_writel(emc, mr13_flip_fspop | 0x8, EMC_MRW3, value);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1315) ccfifo_writel(emc, 0, 0, tFC_lpddr4 / src_clk_period);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1316) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1317)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1318) if (dram_type == DRAM_TYPE_LPDDR4 || opt_dvfs_mode != MAN_SR) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1319) delay = 30;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1320)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1321) if (cya_allow_ref_cc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1322) delay += (1000 * fake->dram_timings[T_RP]) /
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1323) src_clk_period;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1324) delay += 4000 * fake->dram_timings[T_RFC];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1325) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1326)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1327) ccfifo_writel(emc, emc_pin & ~(EMC_PIN_PIN_CKE_PER_DEV |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1328) EMC_PIN_PIN_CKEB |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1329) EMC_PIN_PIN_CKE),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1330) EMC_PIN, delay);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1331) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1332)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1333) /* calculate reference delay multiplier */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1334) value = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1335)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1336) if (ref_b4_sref_en)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1337) value++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1338)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1339) if (cya_allow_ref_cc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1340) value++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1341)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1342) if (cya_issue_pc_ref)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1343) value++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1344)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1345) if (dram_type != DRAM_TYPE_LPDDR4) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1346) delay = ((1000 * fake->dram_timings[T_RP] / src_clk_period) +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1347) (1000 * fake->dram_timings[T_RFC] / src_clk_period));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1348) delay = value * delay + 20;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1349) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1350) delay = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1351) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1352)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1353) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1354) * Step 11:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1355) * Ramp down.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1356) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1357) emc_dbg(emc, STEPS, "Step 11\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1358)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1359) ccfifo_writel(emc, 0x0, EMC_CFG_SYNC, delay);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1360)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1361) value = emc_dbg | EMC_DBG_WRITE_MUX_ACTIVE | EMC_DBG_WRITE_ACTIVE_ONLY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1362) ccfifo_writel(emc, value, EMC_DBG, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1363)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1364) ramp_down_wait = tegra210_emc_dvfs_power_ramp_down(emc, src_clk_period,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1365) 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1366)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1367) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1368) * Step 12:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1369) * And finally - trigger the clock change.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1370) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1371) emc_dbg(emc, STEPS, "Step 12\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1372)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1373) ccfifo_writel(emc, 1, EMC_STALL_THEN_EXE_AFTER_CLKCHANGE, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1374) value &= ~EMC_DBG_WRITE_ACTIVE_ONLY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1375) ccfifo_writel(emc, value, EMC_DBG, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1376)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1377) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1378) * Step 13:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1379) * Ramp up.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1380) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1381) emc_dbg(emc, STEPS, "Step 13\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1382)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1383) ramp_up_wait = tegra210_emc_dvfs_power_ramp_up(emc, dst_clk_period, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1384) ccfifo_writel(emc, emc_dbg, EMC_DBG, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1385)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1386) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1387) * Step 14:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1388) * Bringup CKE pins.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1389) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1390) emc_dbg(emc, STEPS, "Step 14\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1391)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1392) if (dram_type == DRAM_TYPE_LPDDR4) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1393) value = emc_pin | EMC_PIN_PIN_CKE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1394)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1395) if (emc->num_devices <= 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1396) value &= ~(EMC_PIN_PIN_CKEB | EMC_PIN_PIN_CKE_PER_DEV);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1397) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1398) value |= EMC_PIN_PIN_CKEB | EMC_PIN_PIN_CKE_PER_DEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1399)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1400) ccfifo_writel(emc, value, EMC_PIN, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1401) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1402)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1403) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1404) * Step 15: (two step 15s ??)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1405) * Calculate zqlatch wait time; has dependency on ramping times.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1406) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1407) emc_dbg(emc, STEPS, "Step 15\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1408)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1409) if (dst_clk_period <= zqcal_before_cc_cutoff) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1410) s32 t = (s32)(ramp_up_wait + ramp_down_wait) /
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1411) (s32)dst_clk_period;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1412) zq_latch_dvfs_wait_time = (s32)tZQCAL_lpddr4_fc_adj - t;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1413) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1414) zq_latch_dvfs_wait_time = tZQCAL_lpddr4_fc_adj -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1415) div_o3(1000 * next->dram_timings[T_PDEX],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1416) dst_clk_period);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1417) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1418)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1419) emc_dbg(emc, INFO, "tZQCAL_lpddr4_fc_adj = %u\n", tZQCAL_lpddr4_fc_adj);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1420) emc_dbg(emc, INFO, "dst_clk_period = %u\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1421) dst_clk_period);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1422) emc_dbg(emc, INFO, "next->dram_timings[T_PDEX] = %u\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1423) next->dram_timings[T_PDEX]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1424) emc_dbg(emc, INFO, "zq_latch_dvfs_wait_time = %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1425) max_t(s32, 0, zq_latch_dvfs_wait_time));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1426)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1427) if (dram_type == DRAM_TYPE_LPDDR4 && opt_zcal_en_cc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1428) delay = div_o3(1000 * next->dram_timings[T_PDEX],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1429) dst_clk_period);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1430)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1431) if (emc->num_devices < 2) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1432) if (dst_clk_period > zqcal_before_cc_cutoff)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1433) ccfifo_writel(emc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1434) 2UL << EMC_ZQ_CAL_DEV_SEL_SHIFT |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1435) EMC_ZQ_CAL_ZQ_CAL_CMD, EMC_ZQ_CAL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1436) delay);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1437)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1438) value = (mr13_flip_fspop & 0xfffffff7) | 0x0c000000;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1439) ccfifo_writel(emc, value, EMC_MRW3, delay);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1440) ccfifo_writel(emc, 0, EMC_SELF_REF, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1441) ccfifo_writel(emc, 0, EMC_REF, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1442) ccfifo_writel(emc, 2UL << EMC_ZQ_CAL_DEV_SEL_SHIFT |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1443) EMC_ZQ_CAL_ZQ_LATCH_CMD,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1444) EMC_ZQ_CAL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1445) max_t(s32, 0, zq_latch_dvfs_wait_time));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1446) } else if (shared_zq_resistor) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1447) if (dst_clk_period > zqcal_before_cc_cutoff)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1448) ccfifo_writel(emc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1449) 2UL << EMC_ZQ_CAL_DEV_SEL_SHIFT |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1450) EMC_ZQ_CAL_ZQ_CAL_CMD, EMC_ZQ_CAL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1451) delay);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1452)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1453) ccfifo_writel(emc, 2UL << EMC_ZQ_CAL_DEV_SEL_SHIFT |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1454) EMC_ZQ_CAL_ZQ_LATCH_CMD, EMC_ZQ_CAL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1455) max_t(s32, 0, zq_latch_dvfs_wait_time) +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1456) delay);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1457) ccfifo_writel(emc, 1UL << EMC_ZQ_CAL_DEV_SEL_SHIFT |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1458) EMC_ZQ_CAL_ZQ_LATCH_CMD,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1459) EMC_ZQ_CAL, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1460)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1461) value = (mr13_flip_fspop & 0xfffffff7) | 0x0c000000;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1462) ccfifo_writel(emc, value, EMC_MRW3, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1463) ccfifo_writel(emc, 0, EMC_SELF_REF, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1464) ccfifo_writel(emc, 0, EMC_REF, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1465)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1466) ccfifo_writel(emc, 1UL << EMC_ZQ_CAL_DEV_SEL_SHIFT |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1467) EMC_ZQ_CAL_ZQ_LATCH_CMD, EMC_ZQ_CAL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1468) tZQCAL_lpddr4 / dst_clk_period);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1469) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1470) if (dst_clk_period > zqcal_before_cc_cutoff)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1471) ccfifo_writel(emc, EMC_ZQ_CAL_ZQ_CAL_CMD,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1472) EMC_ZQ_CAL, delay);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1473)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1474) value = (mr13_flip_fspop & 0xfffffff7) | 0x0c000000;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1475) ccfifo_writel(emc, value, EMC_MRW3, delay);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1476) ccfifo_writel(emc, 0, EMC_SELF_REF, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1477) ccfifo_writel(emc, 0, EMC_REF, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1478)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1479) ccfifo_writel(emc, EMC_ZQ_CAL_ZQ_LATCH_CMD, EMC_ZQ_CAL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1480) max_t(s32, 0, zq_latch_dvfs_wait_time));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1481) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1482) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1483)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1484) /* WAR: delay for zqlatch */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1485) ccfifo_writel(emc, 0, 0, 10);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1486)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1487) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1488) * Step 16:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1489) * LPDDR4 Conditional Training Kickoff. Removed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1490) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1491)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1492) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1493) * Step 17:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1494) * MANSR exit self refresh.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1495) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1496) emc_dbg(emc, STEPS, "Step 17\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1497)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1498) if (opt_dvfs_mode == MAN_SR && dram_type != DRAM_TYPE_LPDDR4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1499) ccfifo_writel(emc, 0, EMC_SELF_REF, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1500)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1501) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1502) * Step 18:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1503) * Send MRWs to LPDDR3/DDR3.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1504) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1505) emc_dbg(emc, STEPS, "Step 18\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1506)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1507) if (dram_type == DRAM_TYPE_LPDDR2) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1508) ccfifo_writel(emc, next->emc_mrw2, EMC_MRW2, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1509) ccfifo_writel(emc, next->emc_mrw, EMC_MRW, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1510) if (is_lpddr3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1511) ccfifo_writel(emc, next->emc_mrw4, EMC_MRW4, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1512) } else if (dram_type == DRAM_TYPE_DDR3) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1513) if (opt_dll_mode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1514) ccfifo_writel(emc, next->emc_emrs &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1515) ~EMC_EMRS_USE_EMRS_LONG_CNT, EMC_EMRS, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1516) ccfifo_writel(emc, next->emc_emrs2 &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1517) ~EMC_EMRS2_USE_EMRS2_LONG_CNT, EMC_EMRS2, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1518) ccfifo_writel(emc, next->emc_mrs |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1519) EMC_EMRS_USE_EMRS_LONG_CNT, EMC_MRS, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1520) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1521)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1522) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1523) * Step 19:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1524) * ZQCAL for LPDDR3/DDR3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1525) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1526) emc_dbg(emc, STEPS, "Step 19\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1527)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1528) if (opt_zcal_en_cc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1529) if (dram_type == DRAM_TYPE_LPDDR2) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1530) value = opt_cc_short_zcal ? 90000 : 360000;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1531) value = div_o3(value, dst_clk_period);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1532) value = value <<
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1533) EMC_MRS_WAIT_CNT2_MRS_EXT2_WAIT_CNT_SHIFT |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1534) value <<
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1535) EMC_MRS_WAIT_CNT2_MRS_EXT1_WAIT_CNT_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1536) ccfifo_writel(emc, value, EMC_MRS_WAIT_CNT2, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1537)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1538) value = opt_cc_short_zcal ? 0x56 : 0xab;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1539) ccfifo_writel(emc, 2 << EMC_MRW_MRW_DEV_SELECTN_SHIFT |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1540) EMC_MRW_USE_MRW_EXT_CNT |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1541) 10 << EMC_MRW_MRW_MA_SHIFT |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1542) value << EMC_MRW_MRW_OP_SHIFT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1543) EMC_MRW, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1544)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1545) if (emc->num_devices > 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1546) value = 1 << EMC_MRW_MRW_DEV_SELECTN_SHIFT |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1547) EMC_MRW_USE_MRW_EXT_CNT |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1548) 10 << EMC_MRW_MRW_MA_SHIFT |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1549) value << EMC_MRW_MRW_OP_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1550) ccfifo_writel(emc, value, EMC_MRW, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1551) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1552) } else if (dram_type == DRAM_TYPE_DDR3) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1553) value = opt_cc_short_zcal ? 0 : EMC_ZQ_CAL_LONG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1554)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1555) ccfifo_writel(emc, value |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1556) 2 << EMC_ZQ_CAL_DEV_SEL_SHIFT |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1557) EMC_ZQ_CAL_ZQ_CAL_CMD, EMC_ZQ_CAL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1558) 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1559)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1560) if (emc->num_devices > 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1561) value = value | 1 << EMC_ZQ_CAL_DEV_SEL_SHIFT |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1562) EMC_ZQ_CAL_ZQ_CAL_CMD;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1563) ccfifo_writel(emc, value, EMC_ZQ_CAL, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1564) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1565) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1566) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1567)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1568) if (bg_reg_mode_change) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1569) tegra210_emc_set_shadow_bypass(emc, ACTIVE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1570)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1571) if (ramp_up_wait <= 1250000)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1572) delay = (1250000 - ramp_up_wait) / dst_clk_period;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1573) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1574) delay = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1575)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1576) ccfifo_writel(emc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1577) next->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1578) EMC_PMACRO_BG_BIAS_CTRL_0, delay);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1579) tegra210_emc_set_shadow_bypass(emc, ASSEMBLY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1580) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1581)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1582) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1583) * Step 20:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1584) * Issue ref and optional QRST.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1585) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1586) emc_dbg(emc, STEPS, "Step 20\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1587)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1588) if (dram_type != DRAM_TYPE_LPDDR4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1589) ccfifo_writel(emc, 0, EMC_REF, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1590)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1591) if (opt_do_sw_qrst) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1592) ccfifo_writel(emc, 1, EMC_ISSUE_QRST, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1593) ccfifo_writel(emc, 0, EMC_ISSUE_QRST, 2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1594) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1595)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1596) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1597) * Step 21:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1598) * Restore ZCAL and ZCAL interval.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1599) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1600) emc_dbg(emc, STEPS, "Step 21\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1601)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1602) if (save_restore_clkstop_pd || opt_zcal_en_cc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1603) ccfifo_writel(emc, emc_dbg | EMC_DBG_WRITE_MUX_ACTIVE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1604) EMC_DBG, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1605) if (opt_zcal_en_cc && dram_type != DRAM_TYPE_LPDDR4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1606) ccfifo_writel(emc, next->burst_regs[EMC_ZCAL_INTERVAL_INDEX],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1607) EMC_ZCAL_INTERVAL, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1608)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1609) if (save_restore_clkstop_pd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1610) ccfifo_writel(emc, next->burst_regs[EMC_CFG_INDEX] &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1611) ~EMC_CFG_DYN_SELF_REF,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1612) EMC_CFG, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1613) ccfifo_writel(emc, emc_dbg, EMC_DBG, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1614) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1615)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1616) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1617) * Step 22:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1618) * Restore EMC_CFG_PIPE_CLK.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1619) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1620) emc_dbg(emc, STEPS, "Step 22\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1621)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1622) ccfifo_writel(emc, emc_cfg_pipe_clk, EMC_CFG_PIPE_CLK, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1623)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1624) if (bg_reg_mode_change) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1625) if (enable_bg_reg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1626) emc_writel(emc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1627) next->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1628) ~EMC_PMACRO_BG_BIAS_CTRL_0_BGLP_E_PWRD,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1629) EMC_PMACRO_BG_BIAS_CTRL_0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1630) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1631) emc_writel(emc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1632) next->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1633) ~EMC_PMACRO_BG_BIAS_CTRL_0_BG_E_PWRD,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1634) EMC_PMACRO_BG_BIAS_CTRL_0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1635) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1636)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1637) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1638) * Step 23:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1639) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1640) emc_dbg(emc, STEPS, "Step 23\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1641)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1642) value = emc_readl(emc, EMC_CFG_DIG_DLL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1643) value |= EMC_CFG_DIG_DLL_CFG_DLL_STALL_ALL_TRAFFIC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1644) value &= ~EMC_CFG_DIG_DLL_CFG_DLL_STALL_RW_UNTIL_LOCK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1645) value &= ~EMC_CFG_DIG_DLL_CFG_DLL_STALL_ALL_UNTIL_LOCK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1646) value &= ~EMC_CFG_DIG_DLL_CFG_DLL_EN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1647) value = (value & ~EMC_CFG_DIG_DLL_CFG_DLL_MODE_MASK) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1648) (2 << EMC_CFG_DIG_DLL_CFG_DLL_MODE_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1649) emc_writel(emc, value, EMC_CFG_DIG_DLL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1650)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1651) tegra210_emc_do_clock_change(emc, clksrc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1652)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1653) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1654) * Step 24:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1655) * Save training results. Removed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1656) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1657)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1658) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1659) * Step 25:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1660) * Program MC updown registers.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1661) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1662) emc_dbg(emc, STEPS, "Step 25\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1663)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1664) if (next->rate > last->rate) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1665) for (i = 0; i < next->num_up_down; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1666) mc_writel(emc->mc, next->la_scale_regs[i],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1667) emc->offsets->la_scale[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1668)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1669) tegra210_emc_timing_update(emc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1670) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1671)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1672) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1673) * Step 26:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1674) * Restore ZCAL registers.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1675) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1676) emc_dbg(emc, STEPS, "Step 26\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1677)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1678) if (dram_type == DRAM_TYPE_LPDDR4) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1679) tegra210_emc_set_shadow_bypass(emc, ACTIVE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1680) emc_writel(emc, next->burst_regs[EMC_ZCAL_WAIT_CNT_INDEX],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1681) EMC_ZCAL_WAIT_CNT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1682) emc_writel(emc, next->burst_regs[EMC_ZCAL_INTERVAL_INDEX],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1683) EMC_ZCAL_INTERVAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1684) tegra210_emc_set_shadow_bypass(emc, ASSEMBLY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1685) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1686)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1687) if (dram_type != DRAM_TYPE_LPDDR4 && opt_zcal_en_cc &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1688) !opt_short_zcal && opt_cc_short_zcal) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1689) udelay(2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1690)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1691) tegra210_emc_set_shadow_bypass(emc, ACTIVE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1692) if (dram_type == DRAM_TYPE_LPDDR2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1693) emc_writel(emc, next->burst_regs[EMC_MRS_WAIT_CNT_INDEX],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1694) EMC_MRS_WAIT_CNT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1695) else if (dram_type == DRAM_TYPE_DDR3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1696) emc_writel(emc, next->burst_regs[EMC_ZCAL_WAIT_CNT_INDEX],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1697) EMC_ZCAL_WAIT_CNT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1698) tegra210_emc_set_shadow_bypass(emc, ASSEMBLY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1699) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1700)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1701) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1702) * Step 27:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1703) * Restore EMC_CFG, FDPD registers.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1704) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1705) emc_dbg(emc, STEPS, "Step 27\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1706)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1707) tegra210_emc_set_shadow_bypass(emc, ACTIVE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1708) emc_writel(emc, next->burst_regs[EMC_CFG_INDEX], EMC_CFG);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1709) tegra210_emc_set_shadow_bypass(emc, ASSEMBLY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1710) emc_writel(emc, next->emc_fdpd_ctrl_cmd_no_ramp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1711) EMC_FDPD_CTRL_CMD_NO_RAMP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1712) emc_writel(emc, next->emc_sel_dpd_ctrl, EMC_SEL_DPD_CTRL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1713)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1714) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1715) * Step 28:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1716) * Training recover. Removed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1717) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1718) emc_dbg(emc, STEPS, "Step 28\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1719)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1720) tegra210_emc_set_shadow_bypass(emc, ACTIVE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1721) emc_writel(emc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1722) next->burst_regs[EMC_PMACRO_AUTOCAL_CFG_COMMON_INDEX],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1723) EMC_PMACRO_AUTOCAL_CFG_COMMON);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1724) tegra210_emc_set_shadow_bypass(emc, ASSEMBLY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1725)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1726) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1727) * Step 29:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1728) * Power fix WAR.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1729) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1730) emc_dbg(emc, STEPS, "Step 29\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1731)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1732) emc_writel(emc, EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE0 |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1733) EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE1 |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1734) EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE2 |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1735) EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE3 |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1736) EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE4 |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1737) EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE5 |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1738) EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE6 |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1739) EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE7,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1740) EMC_PMACRO_CFG_PM_GLOBAL_0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1741) emc_writel(emc, EMC_PMACRO_TRAINING_CTRL_0_CH0_TRAINING_E_WRPTR,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1742) EMC_PMACRO_TRAINING_CTRL_0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1743) emc_writel(emc, EMC_PMACRO_TRAINING_CTRL_1_CH1_TRAINING_E_WRPTR,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1744) EMC_PMACRO_TRAINING_CTRL_1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1745) emc_writel(emc, 0, EMC_PMACRO_CFG_PM_GLOBAL_0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1746)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1747) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1748) * Step 30:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1749) * Re-enable autocal.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1750) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1751) emc_dbg(emc, STEPS, "Step 30: Re-enable DLL and AUTOCAL\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1752)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1753) if (next->burst_regs[EMC_CFG_DIG_DLL_INDEX] & EMC_CFG_DIG_DLL_CFG_DLL_EN) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1754) value = emc_readl(emc, EMC_CFG_DIG_DLL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1755) value |= EMC_CFG_DIG_DLL_CFG_DLL_STALL_ALL_TRAFFIC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1756) value |= EMC_CFG_DIG_DLL_CFG_DLL_EN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1757) value &= ~EMC_CFG_DIG_DLL_CFG_DLL_STALL_RW_UNTIL_LOCK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1758) value &= ~EMC_CFG_DIG_DLL_CFG_DLL_STALL_ALL_UNTIL_LOCK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1759) value = (value & ~EMC_CFG_DIG_DLL_CFG_DLL_MODE_MASK) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1760) (2 << EMC_CFG_DIG_DLL_CFG_DLL_MODE_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1761) emc_writel(emc, value, EMC_CFG_DIG_DLL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1762) tegra210_emc_timing_update(emc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1763) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1764)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1765) emc_writel(emc, next->emc_auto_cal_config, EMC_AUTO_CAL_CONFIG);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1766)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1767) /* Done! Yay. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1768) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1769)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1770) const struct tegra210_emc_sequence tegra210_emc_r21021 = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1771) .revision = 0x7,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1772) .set_clock = tegra210_emc_r21021_set_clock,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1773) .periodic_compensation = tegra210_emc_r21021_periodic_compensation,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1774) };
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags