^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Enable PCIe link L0s/L1 state and Clock Power Management
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Copyright (C) 2007 Intel
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * Copyright (C) Zhang Yanmin (yanmin.zhang@intel.com)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * Copyright (C) Shaohua Li (shaohua.li@intel.com)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/moduleparam.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/pci.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/pci_regs.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/errno.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/pm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/init.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <linux/jiffies.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <linux/delay.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include "../pci.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #ifdef MODULE_PARAM_PREFIX
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #undef MODULE_PARAM_PREFIX
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) #define MODULE_PARAM_PREFIX "pcie_aspm."
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) /* Note: those are not register definitions */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) #define ASPM_STATE_L0S_UP (1) /* Upstream direction L0s state */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) #define ASPM_STATE_L0S_DW (2) /* Downstream direction L0s state */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) #define ASPM_STATE_L1 (4) /* L1 state */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) #define ASPM_STATE_L1_1 (8) /* ASPM L1.1 state */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) #define ASPM_STATE_L1_2 (0x10) /* ASPM L1.2 state */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) #define ASPM_STATE_L1_1_PCIPM (0x20) /* PCI PM L1.1 state */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) #define ASPM_STATE_L1_2_PCIPM (0x40) /* PCI PM L1.2 state */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) #define ASPM_STATE_L1_SS_PCIPM (ASPM_STATE_L1_1_PCIPM | ASPM_STATE_L1_2_PCIPM)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) #define ASPM_STATE_L1_2_MASK (ASPM_STATE_L1_2 | ASPM_STATE_L1_2_PCIPM)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) #define ASPM_STATE_L1SS (ASPM_STATE_L1_1 | ASPM_STATE_L1_1_PCIPM |\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) ASPM_STATE_L1_2_MASK)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) #define ASPM_STATE_L0S (ASPM_STATE_L0S_UP | ASPM_STATE_L0S_DW)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) #define ASPM_STATE_ALL (ASPM_STATE_L0S | ASPM_STATE_L1 | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) ASPM_STATE_L1SS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) struct aspm_latency {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) u32 l0s; /* L0s latency (nsec) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) u32 l1; /* L1 latency (nsec) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) struct pcie_link_state {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) struct pci_dev *pdev; /* Upstream component of the Link */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) struct pci_dev *downstream; /* Downstream component, function 0 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) struct pcie_link_state *root; /* pointer to the root port link */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) struct pcie_link_state *parent; /* pointer to the parent Link state */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) struct list_head sibling; /* node in link_list */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) /* ASPM state */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) u32 aspm_support:7; /* Supported ASPM state */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) u32 aspm_enabled:7; /* Enabled ASPM state */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) u32 aspm_capable:7; /* Capable ASPM state with latency */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) u32 aspm_default:7; /* Default ASPM state by BIOS */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) u32 aspm_disable:7; /* Disabled ASPM state */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) /* Clock PM state */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) u32 clkpm_capable:1; /* Clock PM capable? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) u32 clkpm_enabled:1; /* Current Clock PM state */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) u32 clkpm_default:1; /* Default Clock PM state by BIOS */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) u32 clkpm_disable:1; /* Clock PM disabled */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) /* Exit latencies */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) struct aspm_latency latency_up; /* Upstream direction exit latency */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) struct aspm_latency latency_dw; /* Downstream direction exit latency */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) * Endpoint acceptable latencies. A pcie downstream port only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) * has one slot under it, so at most there are 8 functions.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) struct aspm_latency acceptable[8];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) static int aspm_disabled, aspm_force;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) static bool aspm_support_enabled = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) static DEFINE_MUTEX(aspm_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) static LIST_HEAD(link_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) #define POLICY_DEFAULT 0 /* BIOS default setting */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) #define POLICY_PERFORMANCE 1 /* high performance */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) #define POLICY_POWERSAVE 2 /* high power saving */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) #define POLICY_POWER_SUPERSAVE 3 /* possibly even more power saving */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) #ifdef CONFIG_PCIEASPM_PERFORMANCE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) static int aspm_policy = POLICY_PERFORMANCE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) #elif defined CONFIG_PCIEASPM_POWERSAVE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) static int aspm_policy = POLICY_POWERSAVE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) #elif defined CONFIG_PCIEASPM_POWER_SUPERSAVE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) static int aspm_policy = POLICY_POWER_SUPERSAVE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) static int aspm_policy;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) static const char *policy_str[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) [POLICY_DEFAULT] = "default",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) [POLICY_PERFORMANCE] = "performance",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) [POLICY_POWERSAVE] = "powersave",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) [POLICY_POWER_SUPERSAVE] = "powersupersave"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) #define LINK_RETRAIN_TIMEOUT HZ
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) static int policy_to_aspm_state(struct pcie_link_state *link)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) switch (aspm_policy) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) case POLICY_PERFORMANCE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) /* Disable ASPM and Clock PM */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) case POLICY_POWERSAVE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) /* Enable ASPM L0s/L1 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) return (ASPM_STATE_L0S | ASPM_STATE_L1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) case POLICY_POWER_SUPERSAVE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) /* Enable Everything */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) return ASPM_STATE_ALL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) case POLICY_DEFAULT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) return link->aspm_default;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) static int policy_to_clkpm_state(struct pcie_link_state *link)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) switch (aspm_policy) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) case POLICY_PERFORMANCE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) /* Disable ASPM and Clock PM */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) case POLICY_POWERSAVE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) case POLICY_POWER_SUPERSAVE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) /* Enable Clock PM */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) case POLICY_DEFAULT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) return link->clkpm_default;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) static void pcie_set_clkpm_nocheck(struct pcie_link_state *link, int enable)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) struct pci_dev *child;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) struct pci_bus *linkbus = link->pdev->subordinate;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) u32 val = enable ? PCI_EXP_LNKCTL_CLKREQ_EN : 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) list_for_each_entry(child, &linkbus->devices, bus_list)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) pcie_capability_clear_and_set_word(child, PCI_EXP_LNKCTL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) PCI_EXP_LNKCTL_CLKREQ_EN,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) link->clkpm_enabled = !!enable;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) static void pcie_set_clkpm(struct pcie_link_state *link, int enable)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) * Don't enable Clock PM if the link is not Clock PM capable
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) * or Clock PM is disabled
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) if (!link->clkpm_capable || link->clkpm_disable)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) enable = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) /* Need nothing if the specified equals to current state */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) if (link->clkpm_enabled == enable)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) pcie_set_clkpm_nocheck(link, enable);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) static void pcie_clkpm_cap_init(struct pcie_link_state *link, int blacklist)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) int capable = 1, enabled = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) u32 reg32;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) u16 reg16;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) struct pci_dev *child;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) struct pci_bus *linkbus = link->pdev->subordinate;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) /* All functions should have the same cap and state, take the worst */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) list_for_each_entry(child, &linkbus->devices, bus_list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) pcie_capability_read_dword(child, PCI_EXP_LNKCAP, ®32);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) if (!(reg32 & PCI_EXP_LNKCAP_CLKPM)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) capable = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) enabled = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) pcie_capability_read_word(child, PCI_EXP_LNKCTL, ®16);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) if (!(reg16 & PCI_EXP_LNKCTL_CLKREQ_EN))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) enabled = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) link->clkpm_enabled = enabled;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) link->clkpm_default = enabled;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) link->clkpm_capable = capable;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) link->clkpm_disable = blacklist ? 1 : 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) static bool pcie_retrain_link(struct pcie_link_state *link)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) struct pci_dev *parent = link->pdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) unsigned long end_jiffies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) u16 reg16;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) pcie_capability_read_word(parent, PCI_EXP_LNKCTL, ®16);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) reg16 |= PCI_EXP_LNKCTL_RL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) pcie_capability_write_word(parent, PCI_EXP_LNKCTL, reg16);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) if (parent->clear_retrain_link) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) * Due to an erratum in some devices the Retrain Link bit
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) * needs to be cleared again manually to allow the link
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) * training to succeed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) reg16 &= ~PCI_EXP_LNKCTL_RL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) pcie_capability_write_word(parent, PCI_EXP_LNKCTL, reg16);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) /* Wait for link training end. Break out after waiting for timeout */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) end_jiffies = jiffies + LINK_RETRAIN_TIMEOUT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) pcie_capability_read_word(parent, PCI_EXP_LNKSTA, ®16);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) if (!(reg16 & PCI_EXP_LNKSTA_LT))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) msleep(1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) } while (time_before(jiffies, end_jiffies));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) return !(reg16 & PCI_EXP_LNKSTA_LT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) * pcie_aspm_configure_common_clock: check if the 2 ends of a link
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) * could use common clock. If they are, configure them to use the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) * common clock. That will reduce the ASPM state exit latency.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) static void pcie_aspm_configure_common_clock(struct pcie_link_state *link)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) int same_clock = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) u16 reg16, parent_reg, child_reg[8];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) struct pci_dev *child, *parent = link->pdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) struct pci_bus *linkbus = parent->subordinate;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) * All functions of a slot should have the same Slot Clock
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) * Configuration, so just check one function
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) child = list_entry(linkbus->devices.next, struct pci_dev, bus_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) BUG_ON(!pci_is_pcie(child));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) /* Check downstream component if bit Slot Clock Configuration is 1 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) pcie_capability_read_word(child, PCI_EXP_LNKSTA, ®16);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) if (!(reg16 & PCI_EXP_LNKSTA_SLC))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) same_clock = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) /* Check upstream component if bit Slot Clock Configuration is 1 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) pcie_capability_read_word(parent, PCI_EXP_LNKSTA, ®16);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) if (!(reg16 & PCI_EXP_LNKSTA_SLC))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) same_clock = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) /* Port might be already in common clock mode */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) pcie_capability_read_word(parent, PCI_EXP_LNKCTL, ®16);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) if (same_clock && (reg16 & PCI_EXP_LNKCTL_CCC)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) bool consistent = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) list_for_each_entry(child, &linkbus->devices, bus_list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) pcie_capability_read_word(child, PCI_EXP_LNKCTL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) ®16);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) if (!(reg16 & PCI_EXP_LNKCTL_CCC)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) consistent = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) if (consistent)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) pci_info(parent, "ASPM: current common clock configuration is inconsistent, reconfiguring\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) /* Configure downstream component, all functions */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) list_for_each_entry(child, &linkbus->devices, bus_list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) pcie_capability_read_word(child, PCI_EXP_LNKCTL, ®16);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) child_reg[PCI_FUNC(child->devfn)] = reg16;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) if (same_clock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) reg16 |= PCI_EXP_LNKCTL_CCC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) reg16 &= ~PCI_EXP_LNKCTL_CCC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) pcie_capability_write_word(child, PCI_EXP_LNKCTL, reg16);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) /* Configure upstream component */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) pcie_capability_read_word(parent, PCI_EXP_LNKCTL, ®16);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) parent_reg = reg16;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) if (same_clock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) reg16 |= PCI_EXP_LNKCTL_CCC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) reg16 &= ~PCI_EXP_LNKCTL_CCC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) pcie_capability_write_word(parent, PCI_EXP_LNKCTL, reg16);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) if (pcie_retrain_link(link))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) /* Training failed. Restore common clock configurations */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) pci_err(parent, "ASPM: Could not configure common clock\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) list_for_each_entry(child, &linkbus->devices, bus_list)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) pcie_capability_write_word(child, PCI_EXP_LNKCTL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) child_reg[PCI_FUNC(child->devfn)]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) pcie_capability_write_word(parent, PCI_EXP_LNKCTL, parent_reg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) /* Convert L0s latency encoding to ns */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) static u32 calc_l0s_latency(u32 lnkcap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) u32 encoding = (lnkcap & PCI_EXP_LNKCAP_L0SEL) >> 12;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) if (encoding == 0x7)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) return (5 * 1000); /* > 4us */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) return (64 << encoding);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) /* Convert L0s acceptable latency encoding to ns */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) static u32 calc_l0s_acceptable(u32 encoding)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) if (encoding == 0x7)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) return -1U;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) return (64 << encoding);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) /* Convert L1 latency encoding to ns */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) static u32 calc_l1_latency(u32 lnkcap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) u32 encoding = (lnkcap & PCI_EXP_LNKCAP_L1EL) >> 15;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) if (encoding == 0x7)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) return (65 * 1000); /* > 64us */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) return (1000 << encoding);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) /* Convert L1 acceptable latency encoding to ns */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) static u32 calc_l1_acceptable(u32 encoding)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) if (encoding == 0x7)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) return -1U;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) return (1000 << encoding);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) /* Convert L1SS T_pwr encoding to usec */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) static u32 calc_l1ss_pwron(struct pci_dev *pdev, u32 scale, u32 val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) switch (scale) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) case 0:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) return val * 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) case 1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) return val * 10;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) case 2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) return val * 100;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) pci_err(pdev, "%s: Invalid T_PwrOn scale: %u\n", __func__, scale);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) static void encode_l12_threshold(u32 threshold_us, u32 *scale, u32 *value)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) u32 threshold_ns = threshold_us * 1000;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) /* See PCIe r3.1, sec 7.33.3 and sec 6.18 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) if (threshold_ns < 32) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) *scale = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) *value = threshold_ns;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) } else if (threshold_ns < 1024) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) *scale = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) *value = threshold_ns >> 5;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) } else if (threshold_ns < 32768) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) *scale = 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) *value = threshold_ns >> 10;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) } else if (threshold_ns < 1048576) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) *scale = 3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) *value = threshold_ns >> 15;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) } else if (threshold_ns < 33554432) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) *scale = 4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) *value = threshold_ns >> 20;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) *scale = 5;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) *value = threshold_ns >> 25;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) static void pcie_aspm_check_latency(struct pci_dev *endpoint)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) u32 latency, l1_switch_latency = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) struct aspm_latency *acceptable;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) struct pcie_link_state *link;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) /* Device not in D0 doesn't need latency check */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) if ((endpoint->current_state != PCI_D0) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) (endpoint->current_state != PCI_UNKNOWN))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) link = endpoint->bus->self->link_state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) acceptable = &link->acceptable[PCI_FUNC(endpoint->devfn)];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) while (link) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) /* Check upstream direction L0s latency */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) if ((link->aspm_capable & ASPM_STATE_L0S_UP) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) (link->latency_up.l0s > acceptable->l0s))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) link->aspm_capable &= ~ASPM_STATE_L0S_UP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) /* Check downstream direction L0s latency */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) if ((link->aspm_capable & ASPM_STATE_L0S_DW) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) (link->latency_dw.l0s > acceptable->l0s))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) link->aspm_capable &= ~ASPM_STATE_L0S_DW;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) * Check L1 latency.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) * Every switch on the path to root complex need 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) * more microsecond for L1. Spec doesn't mention L0s.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) * The exit latencies for L1 substates are not advertised
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) * by a device. Since the spec also doesn't mention a way
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) * to determine max latencies introduced by enabling L1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) * substates on the components, it is not clear how to do
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) * a L1 substate exit latency check. We assume that the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) * L1 exit latencies advertised by a device include L1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) * substate latencies (and hence do not do any check).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) latency = max_t(u32, link->latency_up.l1, link->latency_dw.l1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) if ((link->aspm_capable & ASPM_STATE_L1) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) (latency + l1_switch_latency > acceptable->l1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) link->aspm_capable &= ~ASPM_STATE_L1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) l1_switch_latency += 1000;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) link = link->parent;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) * The L1 PM substate capability is only implemented in function 0 in a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) * multi function device.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) static struct pci_dev *pci_function_0(struct pci_bus *linkbus)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) struct pci_dev *child;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) list_for_each_entry(child, &linkbus->devices, bus_list)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) if (PCI_FUNC(child->devfn) == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) return child;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) static void pci_clear_and_set_dword(struct pci_dev *pdev, int pos,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) u32 clear, u32 set)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) u32 val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) pci_read_config_dword(pdev, pos, &val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) val &= ~clear;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) val |= set;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) pci_write_config_dword(pdev, pos, val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) /* Calculate L1.2 PM substate timing parameters */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) static void aspm_calc_l1ss_info(struct pcie_link_state *link,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) u32 parent_l1ss_cap, u32 child_l1ss_cap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) struct pci_dev *child = link->downstream, *parent = link->pdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) u32 val1, val2, scale1, scale2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) u32 t_common_mode, t_power_on, l1_2_threshold, scale, value;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) u32 ctl1 = 0, ctl2 = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) u32 pctl1, pctl2, cctl1, cctl2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) u32 pl1_2_enables, cl1_2_enables;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) if (!(link->aspm_support & ASPM_STATE_L1_2_MASK))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) /* Choose the greater of the two Port Common_Mode_Restore_Times */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) val1 = (parent_l1ss_cap & PCI_L1SS_CAP_CM_RESTORE_TIME) >> 8;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) val2 = (child_l1ss_cap & PCI_L1SS_CAP_CM_RESTORE_TIME) >> 8;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) t_common_mode = max(val1, val2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) /* Choose the greater of the two Port T_POWER_ON times */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) val1 = (parent_l1ss_cap & PCI_L1SS_CAP_P_PWR_ON_VALUE) >> 19;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) scale1 = (parent_l1ss_cap & PCI_L1SS_CAP_P_PWR_ON_SCALE) >> 16;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) val2 = (child_l1ss_cap & PCI_L1SS_CAP_P_PWR_ON_VALUE) >> 19;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) scale2 = (child_l1ss_cap & PCI_L1SS_CAP_P_PWR_ON_SCALE) >> 16;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) if (calc_l1ss_pwron(parent, scale1, val1) >
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) calc_l1ss_pwron(child, scale2, val2)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) ctl2 |= scale1 | (val1 << 3);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) t_power_on = calc_l1ss_pwron(parent, scale1, val1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) ctl2 |= scale2 | (val2 << 3);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) t_power_on = calc_l1ss_pwron(child, scale2, val2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) * Set LTR_L1.2_THRESHOLD to the time required to transition the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) * Link from L0 to L1.2 and back to L0 so we enter L1.2 only if
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) * downstream devices report (via LTR) that they can tolerate at
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) * least that much latency.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) * Based on PCIe r3.1, sec 5.5.3.3.1, Figures 5-16 and 5-17, and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) * Table 5-11. T(POWER_OFF) is at most 2us and T(L1.2) is at
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) * least 4us.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) l1_2_threshold = 2 + 4 + t_common_mode + t_power_on;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) encode_l12_threshold(l1_2_threshold, &scale, &value);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) ctl1 |= t_common_mode << 8 | scale << 29 | value << 16;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) pci_read_config_dword(parent, parent->l1ss + PCI_L1SS_CTL1, &pctl1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) pci_read_config_dword(parent, parent->l1ss + PCI_L1SS_CTL2, &pctl2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) pci_read_config_dword(child, child->l1ss + PCI_L1SS_CTL1, &cctl1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) pci_read_config_dword(child, child->l1ss + PCI_L1SS_CTL2, &cctl2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) if (ctl1 == pctl1 && ctl1 == cctl1 &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) ctl2 == pctl2 && ctl2 == cctl2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) /* Disable L1.2 while updating. See PCIe r5.0, sec 5.5.4, 7.8.3.3 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) pl1_2_enables = pctl1 & PCI_L1SS_CTL1_L1_2_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) cl1_2_enables = cctl1 & PCI_L1SS_CTL1_L1_2_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) if (pl1_2_enables || cl1_2_enables) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) pci_clear_and_set_dword(child, child->l1ss + PCI_L1SS_CTL1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) PCI_L1SS_CTL1_L1_2_MASK, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) pci_clear_and_set_dword(parent, parent->l1ss + PCI_L1SS_CTL1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) PCI_L1SS_CTL1_L1_2_MASK, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) /* Program T_POWER_ON times in both ports */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) pci_write_config_dword(parent, parent->l1ss + PCI_L1SS_CTL2, ctl2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) pci_write_config_dword(child, child->l1ss + PCI_L1SS_CTL2, ctl2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) /* Program Common_Mode_Restore_Time in upstream device */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) pci_clear_and_set_dword(parent, parent->l1ss + PCI_L1SS_CTL1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) PCI_L1SS_CTL1_CM_RESTORE_TIME, ctl1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) /* Program LTR_L1.2_THRESHOLD time in both ports */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) pci_clear_and_set_dword(parent, parent->l1ss + PCI_L1SS_CTL1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) PCI_L1SS_CTL1_LTR_L12_TH_VALUE |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) PCI_L1SS_CTL1_LTR_L12_TH_SCALE, ctl1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) pci_clear_and_set_dword(child, child->l1ss + PCI_L1SS_CTL1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) PCI_L1SS_CTL1_LTR_L12_TH_VALUE |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) PCI_L1SS_CTL1_LTR_L12_TH_SCALE, ctl1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) if (pl1_2_enables || cl1_2_enables) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) pci_clear_and_set_dword(parent, parent->l1ss + PCI_L1SS_CTL1, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) pl1_2_enables);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) pci_clear_and_set_dword(child, child->l1ss + PCI_L1SS_CTL1, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) cl1_2_enables);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) struct pci_dev *child = link->downstream, *parent = link->pdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) u32 parent_lnkcap, child_lnkcap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) u16 parent_lnkctl, child_lnkctl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) u32 parent_l1ss_cap, child_l1ss_cap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) u32 parent_l1ss_ctl1 = 0, child_l1ss_ctl1 = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) struct pci_bus *linkbus = parent->subordinate;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) if (blacklist) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) /* Set enabled/disable so that we will disable ASPM later */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) link->aspm_enabled = ASPM_STATE_ALL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) link->aspm_disable = ASPM_STATE_ALL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) * If ASPM not supported, don't mess with the clocks and link,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) * bail out now.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) pcie_capability_read_dword(parent, PCI_EXP_LNKCAP, &parent_lnkcap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) pcie_capability_read_dword(child, PCI_EXP_LNKCAP, &child_lnkcap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) if (!(parent_lnkcap & child_lnkcap & PCI_EXP_LNKCAP_ASPMS))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) /* Configure common clock before checking latencies */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) pcie_aspm_configure_common_clock(link);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) * Re-read upstream/downstream components' register state after
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) * clock configuration. L0s & L1 exit latencies in the otherwise
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) * read-only Link Capabilities may change depending on common clock
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) * configuration (PCIe r5.0, sec 7.5.3.6).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) pcie_capability_read_dword(parent, PCI_EXP_LNKCAP, &parent_lnkcap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) pcie_capability_read_dword(child, PCI_EXP_LNKCAP, &child_lnkcap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) pcie_capability_read_word(parent, PCI_EXP_LNKCTL, &parent_lnkctl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) pcie_capability_read_word(child, PCI_EXP_LNKCTL, &child_lnkctl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) * Setup L0s state
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) * Note that we must not enable L0s in either direction on a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) * given link unless components on both sides of the link each
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) * support L0s.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) if (parent_lnkcap & child_lnkcap & PCI_EXP_LNKCAP_ASPM_L0S)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) link->aspm_support |= ASPM_STATE_L0S;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) if (child_lnkctl & PCI_EXP_LNKCTL_ASPM_L0S)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) link->aspm_enabled |= ASPM_STATE_L0S_UP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) if (parent_lnkctl & PCI_EXP_LNKCTL_ASPM_L0S)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) link->aspm_enabled |= ASPM_STATE_L0S_DW;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) link->latency_up.l0s = calc_l0s_latency(parent_lnkcap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) link->latency_dw.l0s = calc_l0s_latency(child_lnkcap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) /* Setup L1 state */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) if (parent_lnkcap & child_lnkcap & PCI_EXP_LNKCAP_ASPM_L1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) link->aspm_support |= ASPM_STATE_L1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) if (parent_lnkctl & child_lnkctl & PCI_EXP_LNKCTL_ASPM_L1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) link->aspm_enabled |= ASPM_STATE_L1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) link->latency_up.l1 = calc_l1_latency(parent_lnkcap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) link->latency_dw.l1 = calc_l1_latency(child_lnkcap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) /* Setup L1 substate */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) pci_read_config_dword(parent, parent->l1ss + PCI_L1SS_CAP,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) &parent_l1ss_cap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) pci_read_config_dword(child, child->l1ss + PCI_L1SS_CAP,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) &child_l1ss_cap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) if (!(parent_l1ss_cap & PCI_L1SS_CAP_L1_PM_SS))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) parent_l1ss_cap = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) if (!(child_l1ss_cap & PCI_L1SS_CAP_L1_PM_SS))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) child_l1ss_cap = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) * If we don't have LTR for the entire path from the Root Complex
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) * to this device, we can't use ASPM L1.2 because it relies on the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) * LTR_L1.2_THRESHOLD. See PCIe r4.0, secs 5.5.4, 6.18.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) if (!child->ltr_path)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) child_l1ss_cap &= ~PCI_L1SS_CAP_ASPM_L1_2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_ASPM_L1_1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) link->aspm_support |= ASPM_STATE_L1_1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_ASPM_L1_2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) link->aspm_support |= ASPM_STATE_L1_2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_PCIPM_L1_1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) link->aspm_support |= ASPM_STATE_L1_1_PCIPM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_PCIPM_L1_2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) link->aspm_support |= ASPM_STATE_L1_2_PCIPM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) if (parent_l1ss_cap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) pci_read_config_dword(parent, parent->l1ss + PCI_L1SS_CTL1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) &parent_l1ss_ctl1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) if (child_l1ss_cap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) pci_read_config_dword(child, child->l1ss + PCI_L1SS_CTL1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) &child_l1ss_ctl1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_ASPM_L1_1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) link->aspm_enabled |= ASPM_STATE_L1_1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_ASPM_L1_2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) link->aspm_enabled |= ASPM_STATE_L1_2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_PCIPM_L1_1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) link->aspm_enabled |= ASPM_STATE_L1_1_PCIPM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_PCIPM_L1_2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) link->aspm_enabled |= ASPM_STATE_L1_2_PCIPM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) if (link->aspm_support & ASPM_STATE_L1SS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) aspm_calc_l1ss_info(link, parent_l1ss_cap, child_l1ss_cap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) /* Save default state */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) link->aspm_default = link->aspm_enabled;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) /* Setup initial capable state. Will be updated later */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) link->aspm_capable = link->aspm_support;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) /* Get and check endpoint acceptable latencies */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) list_for_each_entry(child, &linkbus->devices, bus_list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) u32 reg32, encoding;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) struct aspm_latency *acceptable =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) &link->acceptable[PCI_FUNC(child->devfn)];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) if (pci_pcie_type(child) != PCI_EXP_TYPE_ENDPOINT &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) pci_pcie_type(child) != PCI_EXP_TYPE_LEG_END)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) pcie_capability_read_dword(child, PCI_EXP_DEVCAP, ®32);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) /* Calculate endpoint L0s acceptable latency */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) encoding = (reg32 & PCI_EXP_DEVCAP_L0S) >> 6;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) acceptable->l0s = calc_l0s_acceptable(encoding);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) /* Calculate endpoint L1 acceptable latency */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) encoding = (reg32 & PCI_EXP_DEVCAP_L1) >> 9;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) acceptable->l1 = calc_l1_acceptable(encoding);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) pcie_aspm_check_latency(child);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) /* Configure the ASPM L1 substates */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) static void pcie_config_aspm_l1ss(struct pcie_link_state *link, u32 state)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) u32 val, enable_req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) struct pci_dev *child = link->downstream, *parent = link->pdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) enable_req = (link->aspm_enabled ^ state) & state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) * Here are the rules specified in the PCIe spec for enabling L1SS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) * - When enabling L1.x, enable bit at parent first, then at child
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) * - When disabling L1.x, disable bit at child first, then at parent
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) * - When enabling ASPM L1.x, need to disable L1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) * (at child followed by parent).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) * - The ASPM/PCIPM L1.2 must be disabled while programming timing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) * parameters
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) * To keep it simple, disable all L1SS bits first, and later enable
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) * what is needed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) /* Disable all L1 substates */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) pci_clear_and_set_dword(child, child->l1ss + PCI_L1SS_CTL1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) PCI_L1SS_CTL1_L1SS_MASK, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) pci_clear_and_set_dword(parent, parent->l1ss + PCI_L1SS_CTL1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) PCI_L1SS_CTL1_L1SS_MASK, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) * If needed, disable L1, and it gets enabled later
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) * in pcie_config_aspm_link().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) if (enable_req & (ASPM_STATE_L1_1 | ASPM_STATE_L1_2)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) pcie_capability_clear_and_set_word(child, PCI_EXP_LNKCTL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) PCI_EXP_LNKCTL_ASPM_L1, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) pcie_capability_clear_and_set_word(parent, PCI_EXP_LNKCTL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) PCI_EXP_LNKCTL_ASPM_L1, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) val = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) if (state & ASPM_STATE_L1_1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) val |= PCI_L1SS_CTL1_ASPM_L1_1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) if (state & ASPM_STATE_L1_2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) val |= PCI_L1SS_CTL1_ASPM_L1_2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) if (state & ASPM_STATE_L1_1_PCIPM)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) val |= PCI_L1SS_CTL1_PCIPM_L1_1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) if (state & ASPM_STATE_L1_2_PCIPM)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) val |= PCI_L1SS_CTL1_PCIPM_L1_2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) /* Enable what we need to enable */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) pci_clear_and_set_dword(parent, parent->l1ss + PCI_L1SS_CTL1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) PCI_L1SS_CTL1_L1SS_MASK, val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) pci_clear_and_set_dword(child, child->l1ss + PCI_L1SS_CTL1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) PCI_L1SS_CTL1_L1SS_MASK, val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) static void pcie_config_aspm_dev(struct pci_dev *pdev, u32 val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) pcie_capability_clear_and_set_word(pdev, PCI_EXP_LNKCTL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) PCI_EXP_LNKCTL_ASPMC, val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) static void pcie_config_aspm_link(struct pcie_link_state *link, u32 state)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) u32 upstream = 0, dwstream = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) struct pci_dev *child = link->downstream, *parent = link->pdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) struct pci_bus *linkbus = parent->subordinate;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) /* Enable only the states that were not explicitly disabled */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) state &= (link->aspm_capable & ~link->aspm_disable);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) /* Can't enable any substates if L1 is not enabled */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) if (!(state & ASPM_STATE_L1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) state &= ~ASPM_STATE_L1SS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) /* Spec says both ports must be in D0 before enabling PCI PM substates*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) if (parent->current_state != PCI_D0 || child->current_state != PCI_D0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) state &= ~ASPM_STATE_L1_SS_PCIPM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) state |= (link->aspm_enabled & ASPM_STATE_L1_SS_PCIPM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) /* Nothing to do if the link is already in the requested state */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) if (link->aspm_enabled == state)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) /* Convert ASPM state to upstream/downstream ASPM register state */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) if (state & ASPM_STATE_L0S_UP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) dwstream |= PCI_EXP_LNKCTL_ASPM_L0S;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) if (state & ASPM_STATE_L0S_DW)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) upstream |= PCI_EXP_LNKCTL_ASPM_L0S;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) if (state & ASPM_STATE_L1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) upstream |= PCI_EXP_LNKCTL_ASPM_L1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) dwstream |= PCI_EXP_LNKCTL_ASPM_L1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) if (link->aspm_capable & ASPM_STATE_L1SS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) pcie_config_aspm_l1ss(link, state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) * Spec 2.0 suggests all functions should be configured the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) * same setting for ASPM. Enabling ASPM L1 should be done in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) * upstream component first and then downstream, and vice
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) * versa for disabling ASPM L1. Spec doesn't mention L0S.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) if (state & ASPM_STATE_L1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) pcie_config_aspm_dev(parent, upstream);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) list_for_each_entry(child, &linkbus->devices, bus_list)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) pcie_config_aspm_dev(child, dwstream);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) if (!(state & ASPM_STATE_L1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) pcie_config_aspm_dev(parent, upstream);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) link->aspm_enabled = state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) static void pcie_config_aspm_path(struct pcie_link_state *link)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) while (link) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) pcie_config_aspm_link(link, policy_to_aspm_state(link));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) link = link->parent;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) static void free_link_state(struct pcie_link_state *link)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) link->pdev->link_state = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) kfree(link);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) static int pcie_aspm_sanity_check(struct pci_dev *pdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) struct pci_dev *child;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) u32 reg32;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) * Some functions in a slot might not all be PCIe functions,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) * very strange. Disable ASPM for the whole slot
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) list_for_each_entry(child, &pdev->subordinate->devices, bus_list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) if (!pci_is_pcie(child))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) * If ASPM is disabled then we're not going to change
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) * the BIOS state. It's safe to continue even if it's a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) * pre-1.1 device
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) if (aspm_disabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) * Disable ASPM for pre-1.1 PCIe device, we follow MS to use
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) * RBER bit to determine if a function is 1.1 version device
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) pcie_capability_read_dword(child, PCI_EXP_DEVCAP, ®32);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) if (!(reg32 & PCI_EXP_DEVCAP_RBER) && !aspm_force) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) pci_info(child, "disabling ASPM on pre-1.1 PCIe device. You can enable it with 'pcie_aspm=force'\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) static struct pcie_link_state *alloc_pcie_link_state(struct pci_dev *pdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) struct pcie_link_state *link;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) link = kzalloc(sizeof(*link), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) if (!link)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) INIT_LIST_HEAD(&link->sibling);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) link->pdev = pdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) link->downstream = pci_function_0(pdev->subordinate);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) * Root Ports and PCI/PCI-X to PCIe Bridges are roots of PCIe
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) * hierarchies. Note that some PCIe host implementations omit
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) * the root ports entirely, in which case a downstream port on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) * a switch may become the root of the link state chain for all
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) * its subordinate endpoints.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862) if (pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) pci_pcie_type(pdev) == PCI_EXP_TYPE_PCIE_BRIDGE ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) !pdev->bus->parent->self) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) link->root = link;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) struct pcie_link_state *parent;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) parent = pdev->bus->parent->self->link_state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) if (!parent) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) kfree(link);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) link->parent = parent;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) link->root = link->parent->root;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) list_add(&link->sibling, &link_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) pdev->link_state = link;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) return link;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) static void pcie_aspm_update_sysfs_visibility(struct pci_dev *pdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) struct pci_dev *child;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888) list_for_each_entry(child, &pdev->subordinate->devices, bus_list)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) sysfs_update_group(&child->dev.kobj, &aspm_ctrl_attr_group);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893) * pcie_aspm_init_link_state: Initiate PCI express link state.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) * It is called after the pcie and its children devices are scanned.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) * @pdev: the root port or switch downstream port
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) void pcie_aspm_init_link_state(struct pci_dev *pdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) struct pcie_link_state *link;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) int blacklist = !!pcie_aspm_sanity_check(pdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) if (!aspm_support_enabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) if (pdev->link_state)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) * We allocate pcie_link_state for the component on the upstream
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) * end of a Link, so there's nothing to do unless this device is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) * downstream port.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) if (!pcie_downstream_port(pdev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) /* VIA has a strange chipset, root port is under a bridge */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) if (pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) pdev->bus->self)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) down_read(&pci_bus_sem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) if (list_empty(&pdev->subordinate->devices))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) mutex_lock(&aspm_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926) link = alloc_pcie_link_state(pdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927) if (!link)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) goto unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) * Setup initial ASPM state. Note that we need to configure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) * upstream links also because capable state of them can be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932) * update through pcie_aspm_cap_init().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) pcie_aspm_cap_init(link, blacklist);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) /* Setup initial Clock PM state */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) pcie_clkpm_cap_init(link, blacklist);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) * At this stage drivers haven't had an opportunity to change the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) * link policy setting. Enabling ASPM on broken hardware can cripple
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) * it even before the driver has had a chance to disable ASPM, so
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943) * default to a safe level right now. If we're enabling ASPM beyond
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) * the BIOS's expectation, we'll do so once pci_enable_device() is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) * called.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947) if (aspm_policy != POLICY_POWERSAVE &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) aspm_policy != POLICY_POWER_SUPERSAVE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949) pcie_config_aspm_path(link);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950) pcie_set_clkpm(link, policy_to_clkpm_state(link));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) pcie_aspm_update_sysfs_visibility(pdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) unlock:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956) mutex_unlock(&aspm_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) up_read(&pci_bus_sem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961) /* Recheck latencies and update aspm_capable for links under the root */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962) static void pcie_update_aspm_capable(struct pcie_link_state *root)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964) struct pcie_link_state *link;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965) BUG_ON(root->parent);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966) list_for_each_entry(link, &link_list, sibling) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967) if (link->root != root)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969) link->aspm_capable = link->aspm_support;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971) list_for_each_entry(link, &link_list, sibling) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972) struct pci_dev *child;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973) struct pci_bus *linkbus = link->pdev->subordinate;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974) if (link->root != root)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976) list_for_each_entry(child, &linkbus->devices, bus_list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977) if ((pci_pcie_type(child) != PCI_EXP_TYPE_ENDPOINT) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978) (pci_pcie_type(child) != PCI_EXP_TYPE_LEG_END))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980) pcie_aspm_check_latency(child);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985) /* @pdev: the endpoint device */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986) void pcie_aspm_exit_link_state(struct pci_dev *pdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988) struct pci_dev *parent = pdev->bus->self;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989) struct pcie_link_state *link, *root, *parent_link;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991) if (!parent || !parent->link_state)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994) down_read(&pci_bus_sem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995) mutex_lock(&aspm_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997) * All PCIe functions are in one slot, remove one function will remove
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998) * the whole slot, so just wait until we are the last function left.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000) if (!list_empty(&parent->subordinate->devices))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003) link = parent->link_state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004) root = link->root;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005) parent_link = link->parent;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007) /* All functions are removed, so just disable ASPM for the link */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008) pcie_config_aspm_link(link, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009) list_del(&link->sibling);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010) /* Clock PM is for endpoint device */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011) free_link_state(link);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013) /* Recheck latencies and configure upstream links */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014) if (parent_link) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015) pcie_update_aspm_capable(root);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016) pcie_config_aspm_path(parent_link);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019) mutex_unlock(&aspm_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020) up_read(&pci_bus_sem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023) /* @pdev: the root port or switch downstream port */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024) void pcie_aspm_pm_state_change(struct pci_dev *pdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026) struct pcie_link_state *link = pdev->link_state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028) if (aspm_disabled || !link)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031) * Devices changed PM state, we should recheck if latency
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032) * meets all functions' requirement
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034) down_read(&pci_bus_sem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035) mutex_lock(&aspm_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036) pcie_update_aspm_capable(link->root);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037) pcie_config_aspm_path(link);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038) mutex_unlock(&aspm_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039) up_read(&pci_bus_sem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042) void pcie_aspm_powersave_config_link(struct pci_dev *pdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044) struct pcie_link_state *link = pdev->link_state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046) if (aspm_disabled || !link)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049) if (aspm_policy != POLICY_POWERSAVE &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050) aspm_policy != POLICY_POWER_SUPERSAVE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053) down_read(&pci_bus_sem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054) mutex_lock(&aspm_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055) pcie_config_aspm_path(link);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056) pcie_set_clkpm(link, policy_to_clkpm_state(link));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057) mutex_unlock(&aspm_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058) up_read(&pci_bus_sem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1061) static struct pcie_link_state *pcie_aspm_get_link(struct pci_dev *pdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1062) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1063) struct pci_dev *bridge;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1064)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1065) if (!pci_is_pcie(pdev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1066) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1067)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1068) bridge = pci_upstream_bridge(pdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1069) if (!bridge || !pci_is_pcie(bridge))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1070) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1071)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1072) return bridge->link_state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1073) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1074)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1075) static int __pci_disable_link_state(struct pci_dev *pdev, int state, bool sem)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1076) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1077) struct pcie_link_state *link = pcie_aspm_get_link(pdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1078)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1079) if (!link)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1080) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1081) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1082) * A driver requested that ASPM be disabled on this device, but
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1083) * if we don't have permission to manage ASPM (e.g., on ACPI
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1084) * systems we have to observe the FADT ACPI_FADT_NO_ASPM bit and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1085) * the _OSC method), we can't honor that request. Windows has
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1086) * a similar mechanism using "PciASPMOptOut", which is also
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1087) * ignored in this situation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1088) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1089) if (aspm_disabled) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1090) pci_warn(pdev, "can't disable ASPM; OS doesn't have ASPM control\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1091) return -EPERM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1092) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1093)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1094) if (sem)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1095) down_read(&pci_bus_sem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1096) mutex_lock(&aspm_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1097) if (state & PCIE_LINK_STATE_L0S)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1098) link->aspm_disable |= ASPM_STATE_L0S;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1099) if (state & PCIE_LINK_STATE_L1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1100) /* L1 PM substates require L1 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1101) link->aspm_disable |= ASPM_STATE_L1 | ASPM_STATE_L1SS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1102) if (state & PCIE_LINK_STATE_L1_1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1103) link->aspm_disable |= ASPM_STATE_L1_1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1104) if (state & PCIE_LINK_STATE_L1_2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1105) link->aspm_disable |= ASPM_STATE_L1_2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1106) if (state & PCIE_LINK_STATE_L1_1_PCIPM)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1107) link->aspm_disable |= ASPM_STATE_L1_1_PCIPM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1108) if (state & PCIE_LINK_STATE_L1_2_PCIPM)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1109) link->aspm_disable |= ASPM_STATE_L1_2_PCIPM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1110) pcie_config_aspm_link(link, policy_to_aspm_state(link));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1111)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1112) if (state & PCIE_LINK_STATE_CLKPM)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1113) link->clkpm_disable = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1114) pcie_set_clkpm(link, policy_to_clkpm_state(link));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1115) mutex_unlock(&aspm_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1116) if (sem)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1117) up_read(&pci_bus_sem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1118)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1119) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1120) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1121)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1122) int pci_disable_link_state_locked(struct pci_dev *pdev, int state)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1123) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1124) return __pci_disable_link_state(pdev, state, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1125) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1126) EXPORT_SYMBOL(pci_disable_link_state_locked);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1127)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1128) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1129) * pci_disable_link_state - Disable device's link state, so the link will
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1130) * never enter specific states. Note that if the BIOS didn't grant ASPM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1131) * control to the OS, this does nothing because we can't touch the LNKCTL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1132) * register. Returns 0 or a negative errno.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1133) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1134) * @pdev: PCI device
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1135) * @state: ASPM link state to disable
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1136) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1137) int pci_disable_link_state(struct pci_dev *pdev, int state)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1138) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1139) return __pci_disable_link_state(pdev, state, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1140) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1141) EXPORT_SYMBOL(pci_disable_link_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1142)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1143) static int pcie_aspm_set_policy(const char *val,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1144) const struct kernel_param *kp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1145) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1146) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1147) struct pcie_link_state *link;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1148)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1149) if (aspm_disabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1150) return -EPERM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1151) i = sysfs_match_string(policy_str, val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1152) if (i < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1153) return i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1154) if (i == aspm_policy)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1155) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1156)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1157) down_read(&pci_bus_sem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1158) mutex_lock(&aspm_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1159) aspm_policy = i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1160) list_for_each_entry(link, &link_list, sibling) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1161) pcie_config_aspm_link(link, policy_to_aspm_state(link));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1162) pcie_set_clkpm(link, policy_to_clkpm_state(link));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1163) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1164) mutex_unlock(&aspm_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1165) up_read(&pci_bus_sem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1166) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1167) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1168)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1169) static int pcie_aspm_get_policy(char *buffer, const struct kernel_param *kp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1170) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1171) int i, cnt = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1172) for (i = 0; i < ARRAY_SIZE(policy_str); i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1173) if (i == aspm_policy)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1174) cnt += sprintf(buffer + cnt, "[%s] ", policy_str[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1175) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1176) cnt += sprintf(buffer + cnt, "%s ", policy_str[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1177) cnt += sprintf(buffer + cnt, "\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1178) return cnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1179) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1180)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1181) module_param_call(policy, pcie_aspm_set_policy, pcie_aspm_get_policy,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1182) NULL, 0644);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1183)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1184) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1185) * pcie_aspm_enabled - Check if PCIe ASPM has been enabled for a device.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1186) * @pdev: Target device.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1187) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1188) * Relies on the upstream bridge's link_state being valid. The link_state
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1189) * is deallocated only when the last child of the bridge (i.e., @pdev or a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1190) * sibling) is removed, and the caller should be holding a reference to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1191) * @pdev, so this should be safe.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1192) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1193) bool pcie_aspm_enabled(struct pci_dev *pdev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1194) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1195) struct pcie_link_state *link = pcie_aspm_get_link(pdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1196)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1197) if (!link)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1198) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1199)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1200) return link->aspm_enabled;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1201) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1202) EXPORT_SYMBOL_GPL(pcie_aspm_enabled);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1203)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1204) static ssize_t aspm_attr_show_common(struct device *dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1205) struct device_attribute *attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1206) char *buf, u8 state)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1207) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1208) struct pci_dev *pdev = to_pci_dev(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1209) struct pcie_link_state *link = pcie_aspm_get_link(pdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1210)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1211) return sprintf(buf, "%d\n", (link->aspm_enabled & state) ? 1 : 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1212) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1213)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1214) static ssize_t aspm_attr_store_common(struct device *dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1215) struct device_attribute *attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1216) const char *buf, size_t len, u8 state)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1217) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1218) struct pci_dev *pdev = to_pci_dev(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1219) struct pcie_link_state *link = pcie_aspm_get_link(pdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1220) bool state_enable;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1221)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1222) if (strtobool(buf, &state_enable) < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1223) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1224)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1225) down_read(&pci_bus_sem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1226) mutex_lock(&aspm_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1227)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1228) if (state_enable) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1229) link->aspm_disable &= ~state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1230) /* need to enable L1 for substates */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1231) if (state & ASPM_STATE_L1SS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1232) link->aspm_disable &= ~ASPM_STATE_L1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1233) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1234) link->aspm_disable |= state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1235) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1236)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1237) pcie_config_aspm_link(link, policy_to_aspm_state(link));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1238)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1239) mutex_unlock(&aspm_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1240) up_read(&pci_bus_sem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1241)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1242) return len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1243) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1244)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1245) #define ASPM_ATTR(_f, _s) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1246) static ssize_t _f##_show(struct device *dev, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1247) struct device_attribute *attr, char *buf) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1248) { return aspm_attr_show_common(dev, attr, buf, ASPM_STATE_##_s); } \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1249) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1250) static ssize_t _f##_store(struct device *dev, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1251) struct device_attribute *attr, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1252) const char *buf, size_t len) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1253) { return aspm_attr_store_common(dev, attr, buf, len, ASPM_STATE_##_s); }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1254)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1255) ASPM_ATTR(l0s_aspm, L0S)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1256) ASPM_ATTR(l1_aspm, L1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1257) ASPM_ATTR(l1_1_aspm, L1_1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1258) ASPM_ATTR(l1_2_aspm, L1_2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1259) ASPM_ATTR(l1_1_pcipm, L1_1_PCIPM)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1260) ASPM_ATTR(l1_2_pcipm, L1_2_PCIPM)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1261)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1262) static ssize_t clkpm_show(struct device *dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1263) struct device_attribute *attr, char *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1264) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1265) struct pci_dev *pdev = to_pci_dev(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1266) struct pcie_link_state *link = pcie_aspm_get_link(pdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1267)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1268) return sprintf(buf, "%d\n", link->clkpm_enabled);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1269) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1270)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1271) static ssize_t clkpm_store(struct device *dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1272) struct device_attribute *attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1273) const char *buf, size_t len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1274) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1275) struct pci_dev *pdev = to_pci_dev(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1276) struct pcie_link_state *link = pcie_aspm_get_link(pdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1277) bool state_enable;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1278)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1279) if (strtobool(buf, &state_enable) < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1280) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1281)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1282) down_read(&pci_bus_sem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1283) mutex_lock(&aspm_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1284)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1285) link->clkpm_disable = !state_enable;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1286) pcie_set_clkpm(link, policy_to_clkpm_state(link));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1287)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1288) mutex_unlock(&aspm_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1289) up_read(&pci_bus_sem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1290)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1291) return len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1292) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1293)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1294) static DEVICE_ATTR_RW(clkpm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1295) static DEVICE_ATTR_RW(l0s_aspm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1296) static DEVICE_ATTR_RW(l1_aspm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1297) static DEVICE_ATTR_RW(l1_1_aspm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1298) static DEVICE_ATTR_RW(l1_2_aspm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1299) static DEVICE_ATTR_RW(l1_1_pcipm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1300) static DEVICE_ATTR_RW(l1_2_pcipm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1301)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1302) static struct attribute *aspm_ctrl_attrs[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1303) &dev_attr_clkpm.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1304) &dev_attr_l0s_aspm.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1305) &dev_attr_l1_aspm.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1306) &dev_attr_l1_1_aspm.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1307) &dev_attr_l1_2_aspm.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1308) &dev_attr_l1_1_pcipm.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1309) &dev_attr_l1_2_pcipm.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1310) NULL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1311) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1312)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1313) static umode_t aspm_ctrl_attrs_are_visible(struct kobject *kobj,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1314) struct attribute *a, int n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1315) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1316) struct device *dev = kobj_to_dev(kobj);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1317) struct pci_dev *pdev = to_pci_dev(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1318) struct pcie_link_state *link = pcie_aspm_get_link(pdev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1319) static const u8 aspm_state_map[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1320) ASPM_STATE_L0S,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1321) ASPM_STATE_L1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1322) ASPM_STATE_L1_1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1323) ASPM_STATE_L1_2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1324) ASPM_STATE_L1_1_PCIPM,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1325) ASPM_STATE_L1_2_PCIPM,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1326) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1327)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1328) if (aspm_disabled || !link)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1329) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1330)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1331) if (n == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1332) return link->clkpm_capable ? a->mode : 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1333)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1334) return link->aspm_capable & aspm_state_map[n - 1] ? a->mode : 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1335) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1336)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1337) const struct attribute_group aspm_ctrl_attr_group = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1338) .name = "link",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1339) .attrs = aspm_ctrl_attrs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1340) .is_visible = aspm_ctrl_attrs_are_visible,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1341) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1342)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1343) static int __init pcie_aspm_disable(char *str)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1344) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1345) if (!strcmp(str, "off")) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1346) aspm_policy = POLICY_DEFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1347) aspm_disabled = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1348) aspm_support_enabled = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1349) printk(KERN_INFO "PCIe ASPM is disabled\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1350) } else if (!strcmp(str, "force")) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1351) aspm_force = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1352) printk(KERN_INFO "PCIe ASPM is forcibly enabled\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1353) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1354) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1355) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1356)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1357) __setup("pcie_aspm=", pcie_aspm_disable);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1358)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1359) void pcie_no_aspm(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1360) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1361) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1362) * Disabling ASPM is intended to prevent the kernel from modifying
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1363) * existing hardware state, not to clear existing state. To that end:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1364) * (a) set policy to POLICY_DEFAULT in order to avoid changing state
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1365) * (b) prevent userspace from changing policy
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1366) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1367) if (!aspm_force) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1368) aspm_policy = POLICY_DEFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1369) aspm_disabled = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1370) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1371) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1372)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1373) bool pcie_aspm_support_enabled(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1374) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1375) return aspm_support_enabled;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1376) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1377) EXPORT_SYMBOL(pcie_aspm_support_enabled);
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags