^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) /* Copyright 2008 - 2016 Freescale Semiconductor, Inc.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Redistribution and use in source and binary forms, with or without
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * modification, are permitted provided that the following conditions are met:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * * Redistributions of source code must retain the above copyright
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * notice, this list of conditions and the following disclaimer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * * Redistributions in binary form must reproduce the above copyright
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) * notice, this list of conditions and the following disclaimer in the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) * documentation and/or other materials provided with the distribution.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) * * Neither the name of Freescale Semiconductor nor the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) * names of its contributors may be used to endorse or promote products
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) * derived from this software without specific prior written permission.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) * ALTERNATIVELY, this software may be distributed under the terms of the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) * GNU General Public License ("GPL") as published by the Free Software
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) * Foundation, either version 2 of that License or (at your option) any
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) * later version.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) #include "qman_priv.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) #define DQRR_MAXFILL 15
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) #define EQCR_ITHRESH 4 /* if EQCR congests, interrupt threshold */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) #define IRQNAME "QMan portal %d"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) #define MAX_IRQNAME 16 /* big enough for "QMan portal %d" */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) #define QMAN_POLL_LIMIT 32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) #define QMAN_PIRQ_DQRR_ITHRESH 12
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) #define QMAN_DQRR_IT_MAX 15
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) #define QMAN_ITP_MAX 0xFFF
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) #define QMAN_PIRQ_MR_ITHRESH 4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) #define QMAN_PIRQ_IPERIOD 100
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) /* Portal register assists */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) #if defined(CONFIG_ARM) || defined(CONFIG_ARM64)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) /* Cache-inhibited register offsets */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) #define QM_REG_EQCR_PI_CINH 0x3000
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) #define QM_REG_EQCR_CI_CINH 0x3040
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) #define QM_REG_EQCR_ITR 0x3080
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) #define QM_REG_DQRR_PI_CINH 0x3100
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) #define QM_REG_DQRR_CI_CINH 0x3140
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) #define QM_REG_DQRR_ITR 0x3180
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) #define QM_REG_DQRR_DCAP 0x31C0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) #define QM_REG_DQRR_SDQCR 0x3200
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) #define QM_REG_DQRR_VDQCR 0x3240
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) #define QM_REG_DQRR_PDQCR 0x3280
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) #define QM_REG_MR_PI_CINH 0x3300
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) #define QM_REG_MR_CI_CINH 0x3340
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) #define QM_REG_MR_ITR 0x3380
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) #define QM_REG_CFG 0x3500
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) #define QM_REG_ISR 0x3600
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) #define QM_REG_IER 0x3640
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) #define QM_REG_ISDR 0x3680
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) #define QM_REG_IIR 0x36C0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) #define QM_REG_ITPR 0x3740
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) /* Cache-enabled register offsets */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) #define QM_CL_EQCR 0x0000
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) #define QM_CL_DQRR 0x1000
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) #define QM_CL_MR 0x2000
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) #define QM_CL_EQCR_PI_CENA 0x3000
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) #define QM_CL_EQCR_CI_CENA 0x3040
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) #define QM_CL_DQRR_PI_CENA 0x3100
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) #define QM_CL_DQRR_CI_CENA 0x3140
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) #define QM_CL_MR_PI_CENA 0x3300
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) #define QM_CL_MR_CI_CENA 0x3340
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) #define QM_CL_CR 0x3800
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) #define QM_CL_RR0 0x3900
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) #define QM_CL_RR1 0x3940
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) /* Cache-inhibited register offsets */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) #define QM_REG_EQCR_PI_CINH 0x0000
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) #define QM_REG_EQCR_CI_CINH 0x0004
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) #define QM_REG_EQCR_ITR 0x0008
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) #define QM_REG_DQRR_PI_CINH 0x0040
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) #define QM_REG_DQRR_CI_CINH 0x0044
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) #define QM_REG_DQRR_ITR 0x0048
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) #define QM_REG_DQRR_DCAP 0x0050
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) #define QM_REG_DQRR_SDQCR 0x0054
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) #define QM_REG_DQRR_VDQCR 0x0058
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) #define QM_REG_DQRR_PDQCR 0x005c
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) #define QM_REG_MR_PI_CINH 0x0080
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) #define QM_REG_MR_CI_CINH 0x0084
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) #define QM_REG_MR_ITR 0x0088
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) #define QM_REG_CFG 0x0100
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) #define QM_REG_ISR 0x0e00
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) #define QM_REG_IER 0x0e04
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) #define QM_REG_ISDR 0x0e08
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) #define QM_REG_IIR 0x0e0c
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) #define QM_REG_ITPR 0x0e14
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) /* Cache-enabled register offsets */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) #define QM_CL_EQCR 0x0000
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) #define QM_CL_DQRR 0x1000
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) #define QM_CL_MR 0x2000
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) #define QM_CL_EQCR_PI_CENA 0x3000
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) #define QM_CL_EQCR_CI_CENA 0x3100
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) #define QM_CL_DQRR_PI_CENA 0x3200
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) #define QM_CL_DQRR_CI_CENA 0x3300
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) #define QM_CL_MR_PI_CENA 0x3400
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) #define QM_CL_MR_CI_CENA 0x3500
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) #define QM_CL_CR 0x3800
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) #define QM_CL_RR0 0x3900
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) #define QM_CL_RR1 0x3940
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) * BTW, the drivers (and h/w programming model) already obtain the required
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) * synchronisation for portal accesses and data-dependencies. Use of barrier()s
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) * or other order-preserving primitives simply degrade performance. Hence the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) * use of the __raw_*() interfaces, which simply ensure that the compiler treats
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) * the portal registers as volatile
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) /* Cache-enabled ring access */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) #define qm_cl(base, idx) ((void *)base + ((idx) << 6))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) * Portal modes.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) * Enum types;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) * pmode == production mode
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) * cmode == consumption mode,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) * dmode == h/w dequeue mode.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) * Enum values use 3 letter codes. First letter matches the portal mode,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) * remaining two letters indicate;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) * ci == cache-inhibited portal register
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) * ce == cache-enabled portal register
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) * vb == in-band valid-bit (cache-enabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) * dc == DCA (Discrete Consumption Acknowledgment), DQRR-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) * As for "enum qm_dqrr_dmode", it should be self-explanatory.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) enum qm_eqcr_pmode { /* matches QCSP_CFG::EPM */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) qm_eqcr_pci = 0, /* PI index, cache-inhibited */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) qm_eqcr_pce = 1, /* PI index, cache-enabled */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) qm_eqcr_pvb = 2 /* valid-bit */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) enum qm_dqrr_dmode { /* matches QCSP_CFG::DP */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) qm_dqrr_dpush = 0, /* SDQCR + VDQCR */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) qm_dqrr_dpull = 1 /* PDQCR */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) enum qm_dqrr_pmode { /* s/w-only */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) qm_dqrr_pci, /* reads DQRR_PI_CINH */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) qm_dqrr_pce, /* reads DQRR_PI_CENA */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) qm_dqrr_pvb /* reads valid-bit */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) enum qm_dqrr_cmode { /* matches QCSP_CFG::DCM */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) qm_dqrr_cci = 0, /* CI index, cache-inhibited */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) qm_dqrr_cce = 1, /* CI index, cache-enabled */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) qm_dqrr_cdc = 2 /* Discrete Consumption Acknowledgment */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) enum qm_mr_pmode { /* s/w-only */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) qm_mr_pci, /* reads MR_PI_CINH */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) qm_mr_pce, /* reads MR_PI_CENA */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) qm_mr_pvb /* reads valid-bit */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) enum qm_mr_cmode { /* matches QCSP_CFG::MM */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) qm_mr_cci = 0, /* CI index, cache-inhibited */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) qm_mr_cce = 1 /* CI index, cache-enabled */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) /* --- Portal structures --- */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) #define QM_EQCR_SIZE 8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) #define QM_DQRR_SIZE 16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) #define QM_MR_SIZE 8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) /* "Enqueue Command" */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) struct qm_eqcr_entry {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) u8 _ncw_verb; /* writes to this are non-coherent */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) u8 dca;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) __be16 seqnum;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) u8 __reserved[4];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) __be32 fqid; /* 24-bit */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) __be32 tag;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) struct qm_fd fd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) u8 __reserved3[32];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) } __packed __aligned(8);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) #define QM_EQCR_VERB_VBIT 0x80
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) #define QM_EQCR_VERB_CMD_MASK 0x61 /* but only one value; */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) #define QM_EQCR_VERB_CMD_ENQUEUE 0x01
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) #define QM_EQCR_SEQNUM_NESN 0x8000 /* Advance NESN */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) #define QM_EQCR_SEQNUM_NLIS 0x4000 /* More fragments to come */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) #define QM_EQCR_SEQNUM_SEQMASK 0x3fff /* sequence number goes here */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) struct qm_eqcr {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) struct qm_eqcr_entry *ring, *cursor;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) u8 ci, available, ithresh, vbit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) #ifdef CONFIG_FSL_DPAA_CHECKING
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) u32 busy;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) enum qm_eqcr_pmode pmode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) struct qm_dqrr {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) const struct qm_dqrr_entry *ring, *cursor;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) u8 pi, ci, fill, ithresh, vbit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) #ifdef CONFIG_FSL_DPAA_CHECKING
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) enum qm_dqrr_dmode dmode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) enum qm_dqrr_pmode pmode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) enum qm_dqrr_cmode cmode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) struct qm_mr {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) union qm_mr_entry *ring, *cursor;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) u8 pi, ci, fill, ithresh, vbit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) #ifdef CONFIG_FSL_DPAA_CHECKING
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) enum qm_mr_pmode pmode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) enum qm_mr_cmode cmode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) /* MC (Management Command) command */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) /* "FQ" command layout */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) struct qm_mcc_fq {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) u8 _ncw_verb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) u8 __reserved1[3];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) __be32 fqid; /* 24-bit */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) u8 __reserved2[56];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) } __packed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) /* "CGR" command layout */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) struct qm_mcc_cgr {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) u8 _ncw_verb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) u8 __reserved1[30];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) u8 cgid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) u8 __reserved2[32];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) #define QM_MCC_VERB_VBIT 0x80
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) #define QM_MCC_VERB_MASK 0x7f /* where the verb contains; */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) #define QM_MCC_VERB_INITFQ_PARKED 0x40
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) #define QM_MCC_VERB_INITFQ_SCHED 0x41
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) #define QM_MCC_VERB_QUERYFQ 0x44
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) #define QM_MCC_VERB_QUERYFQ_NP 0x45 /* "non-programmable" fields */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) #define QM_MCC_VERB_QUERYWQ 0x46
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) #define QM_MCC_VERB_QUERYWQ_DEDICATED 0x47
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) #define QM_MCC_VERB_ALTER_SCHED 0x48 /* Schedule FQ */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) #define QM_MCC_VERB_ALTER_FE 0x49 /* Force Eligible FQ */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) #define QM_MCC_VERB_ALTER_RETIRE 0x4a /* Retire FQ */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) #define QM_MCC_VERB_ALTER_OOS 0x4b /* Take FQ out of service */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) #define QM_MCC_VERB_ALTER_FQXON 0x4d /* FQ XON */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) #define QM_MCC_VERB_ALTER_FQXOFF 0x4e /* FQ XOFF */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) #define QM_MCC_VERB_INITCGR 0x50
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) #define QM_MCC_VERB_MODIFYCGR 0x51
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) #define QM_MCC_VERB_CGRTESTWRITE 0x52
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) #define QM_MCC_VERB_QUERYCGR 0x58
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) #define QM_MCC_VERB_QUERYCONGESTION 0x59
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) union qm_mc_command {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) struct {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) u8 _ncw_verb; /* writes to this are non-coherent */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) u8 __reserved[63];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) struct qm_mcc_initfq initfq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) struct qm_mcc_initcgr initcgr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) struct qm_mcc_fq fq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) struct qm_mcc_cgr cgr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) /* MC (Management Command) result */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) /* "Query FQ" */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) struct qm_mcr_queryfq {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) u8 verb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) u8 result;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) u8 __reserved1[8];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) struct qm_fqd fqd; /* the FQD fields are here */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) u8 __reserved2[30];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) } __packed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) /* "Alter FQ State Commands" */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) struct qm_mcr_alterfq {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) u8 verb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) u8 result;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) u8 fqs; /* Frame Queue Status */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) u8 __reserved1[61];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) #define QM_MCR_VERB_RRID 0x80
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) #define QM_MCR_VERB_MASK QM_MCC_VERB_MASK
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) #define QM_MCR_VERB_INITFQ_PARKED QM_MCC_VERB_INITFQ_PARKED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) #define QM_MCR_VERB_INITFQ_SCHED QM_MCC_VERB_INITFQ_SCHED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) #define QM_MCR_VERB_QUERYFQ QM_MCC_VERB_QUERYFQ
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) #define QM_MCR_VERB_QUERYFQ_NP QM_MCC_VERB_QUERYFQ_NP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) #define QM_MCR_VERB_QUERYWQ QM_MCC_VERB_QUERYWQ
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) #define QM_MCR_VERB_QUERYWQ_DEDICATED QM_MCC_VERB_QUERYWQ_DEDICATED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) #define QM_MCR_VERB_ALTER_SCHED QM_MCC_VERB_ALTER_SCHED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) #define QM_MCR_VERB_ALTER_FE QM_MCC_VERB_ALTER_FE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) #define QM_MCR_VERB_ALTER_RETIRE QM_MCC_VERB_ALTER_RETIRE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) #define QM_MCR_VERB_ALTER_OOS QM_MCC_VERB_ALTER_OOS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) #define QM_MCR_RESULT_NULL 0x00
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) #define QM_MCR_RESULT_OK 0xf0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) #define QM_MCR_RESULT_ERR_FQID 0xf1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) #define QM_MCR_RESULT_ERR_FQSTATE 0xf2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) #define QM_MCR_RESULT_ERR_NOTEMPTY 0xf3 /* OOS fails if FQ is !empty */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) #define QM_MCR_RESULT_ERR_BADCHANNEL 0xf4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) #define QM_MCR_RESULT_PENDING 0xf8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) #define QM_MCR_RESULT_ERR_BADCOMMAND 0xff
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) #define QM_MCR_FQS_ORLPRESENT 0x02 /* ORL fragments to come */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) #define QM_MCR_FQS_NOTEMPTY 0x01 /* FQ has enqueued frames */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) #define QM_MCR_TIMEOUT 10000 /* us */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) union qm_mc_result {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) struct {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) u8 verb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) u8 result;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) u8 __reserved1[62];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) struct qm_mcr_queryfq queryfq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) struct qm_mcr_alterfq alterfq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) struct qm_mcr_querycgr querycgr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) struct qm_mcr_querycongestion querycongestion;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) struct qm_mcr_querywq querywq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) struct qm_mcr_queryfq_np queryfq_np;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) struct qm_mc {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) union qm_mc_command *cr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) union qm_mc_result *rr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) u8 rridx, vbit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) #ifdef CONFIG_FSL_DPAA_CHECKING
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) enum {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) /* Can be _mc_start()ed */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) qman_mc_idle,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) /* Can be _mc_commit()ed or _mc_abort()ed */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) qman_mc_user,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) /* Can only be _mc_retry()ed */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) qman_mc_hw
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) } state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) struct qm_addr {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) void *ce; /* cache-enabled */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) __be32 *ce_be; /* same value as above but for direct access */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) void __iomem *ci; /* cache-inhibited */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) struct qm_portal {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) * In the non-CONFIG_FSL_DPAA_CHECKING case, the following stuff up to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) * and including 'mc' fits within a cacheline (yay!). The 'config' part
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) * is setup-only, so isn't a cause for a concern. In other words, don't
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) * rearrange this structure on a whim, there be dragons ...
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) struct qm_addr addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) struct qm_eqcr eqcr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) struct qm_dqrr dqrr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) struct qm_mr mr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) struct qm_mc mc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) } ____cacheline_aligned;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) /* Cache-inhibited register access. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) static inline u32 qm_in(struct qm_portal *p, u32 offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) return ioread32be(p->addr.ci + offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) static inline void qm_out(struct qm_portal *p, u32 offset, u32 val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) iowrite32be(val, p->addr.ci + offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) /* Cache Enabled Portal Access */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) static inline void qm_cl_invalidate(struct qm_portal *p, u32 offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) dpaa_invalidate(p->addr.ce + offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) static inline void qm_cl_touch_ro(struct qm_portal *p, u32 offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) dpaa_touch_ro(p->addr.ce + offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) static inline u32 qm_ce_in(struct qm_portal *p, u32 offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) return be32_to_cpu(*(p->addr.ce_be + (offset/4)));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) /* --- EQCR API --- */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) #define EQCR_SHIFT ilog2(sizeof(struct qm_eqcr_entry))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) #define EQCR_CARRY (uintptr_t)(QM_EQCR_SIZE << EQCR_SHIFT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) /* Bit-wise logic to wrap a ring pointer by clearing the "carry bit" */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) static struct qm_eqcr_entry *eqcr_carryclear(struct qm_eqcr_entry *p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) uintptr_t addr = (uintptr_t)p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) addr &= ~EQCR_CARRY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) return (struct qm_eqcr_entry *)addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) /* Bit-wise logic to convert a ring pointer to a ring index */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) static int eqcr_ptr2idx(struct qm_eqcr_entry *e)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) return ((uintptr_t)e >> EQCR_SHIFT) & (QM_EQCR_SIZE - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) /* Increment the 'cursor' ring pointer, taking 'vbit' into account */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) static inline void eqcr_inc(struct qm_eqcr *eqcr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) /* increment to the next EQCR pointer and handle overflow and 'vbit' */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) struct qm_eqcr_entry *partial = eqcr->cursor + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) eqcr->cursor = eqcr_carryclear(partial);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) if (partial != eqcr->cursor)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) eqcr->vbit ^= QM_EQCR_VERB_VBIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) static inline int qm_eqcr_init(struct qm_portal *portal,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) enum qm_eqcr_pmode pmode,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) unsigned int eq_stash_thresh,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) int eq_stash_prio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) struct qm_eqcr *eqcr = &portal->eqcr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) u32 cfg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) u8 pi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) eqcr->ring = portal->addr.ce + QM_CL_EQCR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) eqcr->ci = qm_in(portal, QM_REG_EQCR_CI_CINH) & (QM_EQCR_SIZE - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) qm_cl_invalidate(portal, QM_CL_EQCR_CI_CENA);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) pi = qm_in(portal, QM_REG_EQCR_PI_CINH) & (QM_EQCR_SIZE - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) eqcr->cursor = eqcr->ring + pi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) eqcr->vbit = (qm_in(portal, QM_REG_EQCR_PI_CINH) & QM_EQCR_SIZE) ?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) QM_EQCR_VERB_VBIT : 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) eqcr->available = QM_EQCR_SIZE - 1 -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) dpaa_cyc_diff(QM_EQCR_SIZE, eqcr->ci, pi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) eqcr->ithresh = qm_in(portal, QM_REG_EQCR_ITR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) #ifdef CONFIG_FSL_DPAA_CHECKING
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) eqcr->busy = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) eqcr->pmode = pmode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) cfg = (qm_in(portal, QM_REG_CFG) & 0x00ffffff) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) (eq_stash_thresh << 28) | /* QCSP_CFG: EST */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) (eq_stash_prio << 26) | /* QCSP_CFG: EP */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) ((pmode & 0x3) << 24); /* QCSP_CFG::EPM */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) qm_out(portal, QM_REG_CFG, cfg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) static inline void qm_eqcr_finish(struct qm_portal *portal)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) struct qm_eqcr *eqcr = &portal->eqcr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) u8 pi = qm_in(portal, QM_REG_EQCR_PI_CINH) & (QM_EQCR_SIZE - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) u8 ci = qm_in(portal, QM_REG_EQCR_CI_CINH) & (QM_EQCR_SIZE - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) DPAA_ASSERT(!eqcr->busy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) if (pi != eqcr_ptr2idx(eqcr->cursor))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) pr_crit("losing uncommitted EQCR entries\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) if (ci != eqcr->ci)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) pr_crit("missing existing EQCR completions\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) if (eqcr->ci != eqcr_ptr2idx(eqcr->cursor))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) pr_crit("EQCR destroyed unquiesced\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) static inline struct qm_eqcr_entry *qm_eqcr_start_no_stash(struct qm_portal
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) *portal)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) struct qm_eqcr *eqcr = &portal->eqcr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) DPAA_ASSERT(!eqcr->busy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) if (!eqcr->available)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) #ifdef CONFIG_FSL_DPAA_CHECKING
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) eqcr->busy = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) dpaa_zero(eqcr->cursor);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) return eqcr->cursor;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) static inline struct qm_eqcr_entry *qm_eqcr_start_stash(struct qm_portal
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) *portal)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) struct qm_eqcr *eqcr = &portal->eqcr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) u8 diff, old_ci;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) DPAA_ASSERT(!eqcr->busy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) if (!eqcr->available) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) old_ci = eqcr->ci;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) eqcr->ci = qm_ce_in(portal, QM_CL_EQCR_CI_CENA) &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) (QM_EQCR_SIZE - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) diff = dpaa_cyc_diff(QM_EQCR_SIZE, old_ci, eqcr->ci);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) eqcr->available += diff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) if (!diff)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) #ifdef CONFIG_FSL_DPAA_CHECKING
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) eqcr->busy = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) dpaa_zero(eqcr->cursor);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) return eqcr->cursor;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) static inline void eqcr_commit_checks(struct qm_eqcr *eqcr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) DPAA_ASSERT(eqcr->busy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) DPAA_ASSERT(!(be32_to_cpu(eqcr->cursor->fqid) & ~QM_FQID_MASK));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) DPAA_ASSERT(eqcr->available >= 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) static inline void qm_eqcr_pvb_commit(struct qm_portal *portal, u8 myverb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) struct qm_eqcr *eqcr = &portal->eqcr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) struct qm_eqcr_entry *eqcursor;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) eqcr_commit_checks(eqcr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) DPAA_ASSERT(eqcr->pmode == qm_eqcr_pvb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) dma_wmb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) eqcursor = eqcr->cursor;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) eqcursor->_ncw_verb = myverb | eqcr->vbit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) dpaa_flush(eqcursor);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) eqcr_inc(eqcr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) eqcr->available--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) #ifdef CONFIG_FSL_DPAA_CHECKING
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) eqcr->busy = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) static inline void qm_eqcr_cce_prefetch(struct qm_portal *portal)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) qm_cl_touch_ro(portal, QM_CL_EQCR_CI_CENA);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) static inline u8 qm_eqcr_cce_update(struct qm_portal *portal)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) struct qm_eqcr *eqcr = &portal->eqcr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) u8 diff, old_ci = eqcr->ci;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) eqcr->ci = qm_ce_in(portal, QM_CL_EQCR_CI_CENA) & (QM_EQCR_SIZE - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) qm_cl_invalidate(portal, QM_CL_EQCR_CI_CENA);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) diff = dpaa_cyc_diff(QM_EQCR_SIZE, old_ci, eqcr->ci);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) eqcr->available += diff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) return diff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) static inline void qm_eqcr_set_ithresh(struct qm_portal *portal, u8 ithresh)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) struct qm_eqcr *eqcr = &portal->eqcr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) eqcr->ithresh = ithresh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) qm_out(portal, QM_REG_EQCR_ITR, ithresh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) static inline u8 qm_eqcr_get_avail(struct qm_portal *portal)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) struct qm_eqcr *eqcr = &portal->eqcr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) return eqcr->available;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) static inline u8 qm_eqcr_get_fill(struct qm_portal *portal)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) struct qm_eqcr *eqcr = &portal->eqcr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) return QM_EQCR_SIZE - 1 - eqcr->available;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) /* --- DQRR API --- */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) #define DQRR_SHIFT ilog2(sizeof(struct qm_dqrr_entry))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) #define DQRR_CARRY (uintptr_t)(QM_DQRR_SIZE << DQRR_SHIFT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) static const struct qm_dqrr_entry *dqrr_carryclear(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) const struct qm_dqrr_entry *p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) uintptr_t addr = (uintptr_t)p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) addr &= ~DQRR_CARRY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) return (const struct qm_dqrr_entry *)addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) static inline int dqrr_ptr2idx(const struct qm_dqrr_entry *e)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) return ((uintptr_t)e >> DQRR_SHIFT) & (QM_DQRR_SIZE - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) static const struct qm_dqrr_entry *dqrr_inc(const struct qm_dqrr_entry *e)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) return dqrr_carryclear(e + 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) static inline void qm_dqrr_set_maxfill(struct qm_portal *portal, u8 mf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) qm_out(portal, QM_REG_CFG, (qm_in(portal, QM_REG_CFG) & 0xff0fffff) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) ((mf & (QM_DQRR_SIZE - 1)) << 20));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) static inline int qm_dqrr_init(struct qm_portal *portal,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) const struct qm_portal_config *config,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) enum qm_dqrr_dmode dmode,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) enum qm_dqrr_pmode pmode,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) enum qm_dqrr_cmode cmode, u8 max_fill)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) struct qm_dqrr *dqrr = &portal->dqrr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) u32 cfg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) /* Make sure the DQRR will be idle when we enable */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) qm_out(portal, QM_REG_DQRR_SDQCR, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) qm_out(portal, QM_REG_DQRR_VDQCR, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) qm_out(portal, QM_REG_DQRR_PDQCR, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) dqrr->ring = portal->addr.ce + QM_CL_DQRR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) dqrr->pi = qm_in(portal, QM_REG_DQRR_PI_CINH) & (QM_DQRR_SIZE - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) dqrr->ci = qm_in(portal, QM_REG_DQRR_CI_CINH) & (QM_DQRR_SIZE - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) dqrr->cursor = dqrr->ring + dqrr->ci;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) dqrr->fill = dpaa_cyc_diff(QM_DQRR_SIZE, dqrr->ci, dqrr->pi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) dqrr->vbit = (qm_in(portal, QM_REG_DQRR_PI_CINH) & QM_DQRR_SIZE) ?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) QM_DQRR_VERB_VBIT : 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) dqrr->ithresh = qm_in(portal, QM_REG_DQRR_ITR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) #ifdef CONFIG_FSL_DPAA_CHECKING
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) dqrr->dmode = dmode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) dqrr->pmode = pmode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) dqrr->cmode = cmode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) /* Invalidate every ring entry before beginning */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) for (cfg = 0; cfg < QM_DQRR_SIZE; cfg++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) dpaa_invalidate(qm_cl(dqrr->ring, cfg));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) cfg = (qm_in(portal, QM_REG_CFG) & 0xff000f00) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) ((max_fill & (QM_DQRR_SIZE - 1)) << 20) | /* DQRR_MF */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) ((dmode & 1) << 18) | /* DP */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) ((cmode & 3) << 16) | /* DCM */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) 0xa0 | /* RE+SE */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) (0 ? 0x40 : 0) | /* Ignore RP */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) (0 ? 0x10 : 0); /* Ignore SP */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) qm_out(portal, QM_REG_CFG, cfg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) qm_dqrr_set_maxfill(portal, max_fill);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) static inline void qm_dqrr_finish(struct qm_portal *portal)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) #ifdef CONFIG_FSL_DPAA_CHECKING
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) struct qm_dqrr *dqrr = &portal->dqrr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) if (dqrr->cmode != qm_dqrr_cdc &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) dqrr->ci != dqrr_ptr2idx(dqrr->cursor))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) pr_crit("Ignoring completed DQRR entries\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) static inline const struct qm_dqrr_entry *qm_dqrr_current(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) struct qm_portal *portal)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) struct qm_dqrr *dqrr = &portal->dqrr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) if (!dqrr->fill)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) return dqrr->cursor;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) static inline u8 qm_dqrr_next(struct qm_portal *portal)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) struct qm_dqrr *dqrr = &portal->dqrr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) DPAA_ASSERT(dqrr->fill);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) dqrr->cursor = dqrr_inc(dqrr->cursor);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) return --dqrr->fill;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) static inline void qm_dqrr_pvb_update(struct qm_portal *portal)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) struct qm_dqrr *dqrr = &portal->dqrr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) struct qm_dqrr_entry *res = qm_cl(dqrr->ring, dqrr->pi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) DPAA_ASSERT(dqrr->pmode == qm_dqrr_pvb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) #ifndef CONFIG_FSL_PAMU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) * If PAMU is not available we need to invalidate the cache.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) * When PAMU is available the cache is updated by stash
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) dpaa_invalidate_touch_ro(res);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) if ((res->verb & QM_DQRR_VERB_VBIT) == dqrr->vbit) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) dqrr->pi = (dqrr->pi + 1) & (QM_DQRR_SIZE - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) if (!dqrr->pi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) dqrr->vbit ^= QM_DQRR_VERB_VBIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) dqrr->fill++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) static inline void qm_dqrr_cdc_consume_1ptr(struct qm_portal *portal,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) const struct qm_dqrr_entry *dq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) int park)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) __maybe_unused struct qm_dqrr *dqrr = &portal->dqrr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) int idx = dqrr_ptr2idx(dq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) DPAA_ASSERT(dqrr->cmode == qm_dqrr_cdc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) DPAA_ASSERT((dqrr->ring + idx) == dq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) DPAA_ASSERT(idx < QM_DQRR_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) qm_out(portal, QM_REG_DQRR_DCAP, (0 << 8) | /* DQRR_DCAP::S */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) ((park ? 1 : 0) << 6) | /* DQRR_DCAP::PK */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) idx); /* DQRR_DCAP::DCAP_CI */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) static inline void qm_dqrr_cdc_consume_n(struct qm_portal *portal, u32 bitmask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) __maybe_unused struct qm_dqrr *dqrr = &portal->dqrr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) DPAA_ASSERT(dqrr->cmode == qm_dqrr_cdc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) qm_out(portal, QM_REG_DQRR_DCAP, (1 << 8) | /* DQRR_DCAP::S */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) (bitmask << 16)); /* DQRR_DCAP::DCAP_CI */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) static inline void qm_dqrr_sdqcr_set(struct qm_portal *portal, u32 sdqcr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) qm_out(portal, QM_REG_DQRR_SDQCR, sdqcr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) static inline void qm_dqrr_vdqcr_set(struct qm_portal *portal, u32 vdqcr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) qm_out(portal, QM_REG_DQRR_VDQCR, vdqcr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) static inline int qm_dqrr_set_ithresh(struct qm_portal *portal, u8 ithresh)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) if (ithresh > QMAN_DQRR_IT_MAX)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) qm_out(portal, QM_REG_DQRR_ITR, ithresh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) /* --- MR API --- */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) #define MR_SHIFT ilog2(sizeof(union qm_mr_entry))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) #define MR_CARRY (uintptr_t)(QM_MR_SIZE << MR_SHIFT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) static union qm_mr_entry *mr_carryclear(union qm_mr_entry *p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) uintptr_t addr = (uintptr_t)p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) addr &= ~MR_CARRY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) return (union qm_mr_entry *)addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) static inline int mr_ptr2idx(const union qm_mr_entry *e)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) return ((uintptr_t)e >> MR_SHIFT) & (QM_MR_SIZE - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) static inline union qm_mr_entry *mr_inc(union qm_mr_entry *e)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) return mr_carryclear(e + 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) static inline int qm_mr_init(struct qm_portal *portal, enum qm_mr_pmode pmode,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) enum qm_mr_cmode cmode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) struct qm_mr *mr = &portal->mr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) u32 cfg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) mr->ring = portal->addr.ce + QM_CL_MR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) mr->pi = qm_in(portal, QM_REG_MR_PI_CINH) & (QM_MR_SIZE - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) mr->ci = qm_in(portal, QM_REG_MR_CI_CINH) & (QM_MR_SIZE - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) mr->cursor = mr->ring + mr->ci;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) mr->fill = dpaa_cyc_diff(QM_MR_SIZE, mr->ci, mr->pi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) mr->vbit = (qm_in(portal, QM_REG_MR_PI_CINH) & QM_MR_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) ? QM_MR_VERB_VBIT : 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) mr->ithresh = qm_in(portal, QM_REG_MR_ITR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) #ifdef CONFIG_FSL_DPAA_CHECKING
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) mr->pmode = pmode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) mr->cmode = cmode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) cfg = (qm_in(portal, QM_REG_CFG) & 0xfffff0ff) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) ((cmode & 1) << 8); /* QCSP_CFG:MM */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) qm_out(portal, QM_REG_CFG, cfg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) static inline void qm_mr_finish(struct qm_portal *portal)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) struct qm_mr *mr = &portal->mr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) if (mr->ci != mr_ptr2idx(mr->cursor))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) pr_crit("Ignoring completed MR entries\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) static inline const union qm_mr_entry *qm_mr_current(struct qm_portal *portal)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) struct qm_mr *mr = &portal->mr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) if (!mr->fill)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) return mr->cursor;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) static inline int qm_mr_next(struct qm_portal *portal)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) struct qm_mr *mr = &portal->mr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) DPAA_ASSERT(mr->fill);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) mr->cursor = mr_inc(mr->cursor);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) return --mr->fill;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) static inline void qm_mr_pvb_update(struct qm_portal *portal)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) struct qm_mr *mr = &portal->mr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) union qm_mr_entry *res = qm_cl(mr->ring, mr->pi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) DPAA_ASSERT(mr->pmode == qm_mr_pvb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) if ((res->verb & QM_MR_VERB_VBIT) == mr->vbit) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) mr->pi = (mr->pi + 1) & (QM_MR_SIZE - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) if (!mr->pi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) mr->vbit ^= QM_MR_VERB_VBIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) mr->fill++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) res = mr_inc(res);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) dpaa_invalidate_touch_ro(res);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) static inline void qm_mr_cci_consume(struct qm_portal *portal, u8 num)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) struct qm_mr *mr = &portal->mr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) DPAA_ASSERT(mr->cmode == qm_mr_cci);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) mr->ci = (mr->ci + num) & (QM_MR_SIZE - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) qm_out(portal, QM_REG_MR_CI_CINH, mr->ci);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) static inline void qm_mr_cci_consume_to_current(struct qm_portal *portal)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) struct qm_mr *mr = &portal->mr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) DPAA_ASSERT(mr->cmode == qm_mr_cci);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) mr->ci = mr_ptr2idx(mr->cursor);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) qm_out(portal, QM_REG_MR_CI_CINH, mr->ci);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) static inline void qm_mr_set_ithresh(struct qm_portal *portal, u8 ithresh)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) qm_out(portal, QM_REG_MR_ITR, ithresh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) /* --- Management command API --- */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) static inline int qm_mc_init(struct qm_portal *portal)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) u8 rr0, rr1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) struct qm_mc *mc = &portal->mc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) mc->cr = portal->addr.ce + QM_CL_CR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) mc->rr = portal->addr.ce + QM_CL_RR0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862) * The expected valid bit polarity for the next CR command is 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) * if RR1 contains a valid response, and is 1 if RR0 contains a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) * valid response. If both RR contain all 0, this indicates either
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) * that no command has been executed since reset (in which case the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) * expected valid bit polarity is 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) rr0 = mc->rr->verb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) rr1 = (mc->rr+1)->verb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) if ((rr0 == 0 && rr1 == 0) || rr0 != 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) mc->rridx = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) mc->rridx = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) mc->vbit = mc->rridx ? QM_MCC_VERB_VBIT : 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) #ifdef CONFIG_FSL_DPAA_CHECKING
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) mc->state = qman_mc_idle;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) static inline void qm_mc_finish(struct qm_portal *portal)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) #ifdef CONFIG_FSL_DPAA_CHECKING
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) struct qm_mc *mc = &portal->mc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) DPAA_ASSERT(mc->state == qman_mc_idle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) if (mc->state != qman_mc_idle)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888) pr_crit("Losing incomplete MC command\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) static inline union qm_mc_command *qm_mc_start(struct qm_portal *portal)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) struct qm_mc *mc = &portal->mc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) DPAA_ASSERT(mc->state == qman_mc_idle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) #ifdef CONFIG_FSL_DPAA_CHECKING
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) mc->state = qman_mc_user;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) dpaa_zero(mc->cr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) return mc->cr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) static inline void qm_mc_commit(struct qm_portal *portal, u8 myverb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) struct qm_mc *mc = &portal->mc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907) union qm_mc_result *rr = mc->rr + mc->rridx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) DPAA_ASSERT(mc->state == qman_mc_user);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) dma_wmb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) mc->cr->_ncw_verb = myverb | mc->vbit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) dpaa_flush(mc->cr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) dpaa_invalidate_touch_ro(rr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) #ifdef CONFIG_FSL_DPAA_CHECKING
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) mc->state = qman_mc_hw;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) static inline union qm_mc_result *qm_mc_result(struct qm_portal *portal)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) struct qm_mc *mc = &portal->mc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) union qm_mc_result *rr = mc->rr + mc->rridx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924) DPAA_ASSERT(mc->state == qman_mc_hw);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926) * The inactive response register's verb byte always returns zero until
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927) * its command is submitted and completed. This includes the valid-bit,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) * in case you were wondering...
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) if (!rr->verb) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) dpaa_invalidate_touch_ro(rr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) mc->rridx ^= 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) mc->vbit ^= QM_MCC_VERB_VBIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) #ifdef CONFIG_FSL_DPAA_CHECKING
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) mc->state = qman_mc_idle;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) return rr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) static inline int qm_mc_result_timeout(struct qm_portal *portal,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943) union qm_mc_result **mcr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) int timeout = QM_MCR_TIMEOUT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) *mcr = qm_mc_result(portal);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949) if (*mcr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) udelay(1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952) } while (--timeout);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) return timeout;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957) static inline void fq_set(struct qman_fq *fq, u32 mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959) fq->flags |= mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962) static inline void fq_clear(struct qman_fq *fq, u32 mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964) fq->flags &= ~mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967) static inline int fq_isset(struct qman_fq *fq, u32 mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969) return fq->flags & mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972) static inline int fq_isclear(struct qman_fq *fq, u32 mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974) return !(fq->flags & mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977) struct qman_portal {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978) struct qm_portal p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979) /* PORTAL_BITS_*** - dynamic, strictly internal */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980) unsigned long bits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981) /* interrupt sources processed by portal_isr(), configurable */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982) unsigned long irq_sources;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983) u32 use_eqcr_ci_stashing;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984) /* only 1 volatile dequeue at a time */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985) struct qman_fq *vdqcr_owned;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986) u32 sdqcr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987) /* probing time config params for cpu-affine portals */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988) const struct qm_portal_config *config;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989) /* 2-element array. cgrs[0] is mask, cgrs[1] is snapshot. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990) struct qman_cgrs *cgrs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991) /* linked-list of CSCN handlers. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992) struct list_head cgr_cbs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993) /* list lock */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994) spinlock_t cgr_lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995) struct work_struct congestion_work;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996) struct work_struct mr_work;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997) char irqname[MAX_IRQNAME];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000) static cpumask_t affine_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001) static DEFINE_SPINLOCK(affine_mask_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002) static u16 affine_channels[NR_CPUS];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003) static DEFINE_PER_CPU(struct qman_portal, qman_affine_portal);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004) struct qman_portal *affine_portals[NR_CPUS];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006) static inline struct qman_portal *get_affine_portal(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008) return &get_cpu_var(qman_affine_portal);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011) static inline void put_affine_portal(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013) put_cpu_var(qman_affine_portal);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017) static inline struct qman_portal *get_portal_for_channel(u16 channel)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021) for (i = 0; i < num_possible_cpus(); i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022) if (affine_portals[i] &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023) affine_portals[i]->config->channel == channel)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024) return affine_portals[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030) static struct workqueue_struct *qm_portal_wq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032) int qman_dqrr_set_ithresh(struct qman_portal *portal, u8 ithresh)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034) int res;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036) if (!portal)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039) res = qm_dqrr_set_ithresh(&portal->p, ithresh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040) if (res)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041) return res;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043) portal->p.dqrr.ithresh = ithresh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047) EXPORT_SYMBOL(qman_dqrr_set_ithresh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049) void qman_dqrr_get_ithresh(struct qman_portal *portal, u8 *ithresh)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051) if (portal && ithresh)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052) *ithresh = qm_in(&portal->p, QM_REG_DQRR_ITR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054) EXPORT_SYMBOL(qman_dqrr_get_ithresh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056) void qman_portal_get_iperiod(struct qman_portal *portal, u32 *iperiod)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058) if (portal && iperiod)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059) *iperiod = qm_in(&portal->p, QM_REG_ITPR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1061) EXPORT_SYMBOL(qman_portal_get_iperiod);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1062)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1063) int qman_portal_set_iperiod(struct qman_portal *portal, u32 iperiod)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1064) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1065) if (!portal || iperiod > QMAN_ITP_MAX)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1066) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1067)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1068) qm_out(&portal->p, QM_REG_ITPR, iperiod);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1069)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1070) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1071) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1072) EXPORT_SYMBOL(qman_portal_set_iperiod);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1073)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1074) int qman_wq_alloc(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1075) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1076) qm_portal_wq = alloc_workqueue("qman_portal_wq", 0, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1077) if (!qm_portal_wq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1078) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1079) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1080) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1081)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1082)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1083) void qman_enable_irqs(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1084) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1085) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1086)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1087) for (i = 0; i < num_possible_cpus(); i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1088) if (affine_portals[i]) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1089) qm_out(&affine_portals[i]->p, QM_REG_ISR, 0xffffffff);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1090) qm_out(&affine_portals[i]->p, QM_REG_IIR, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1091) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1092)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1093) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1094) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1095)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1096) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1097) * This is what everything can wait on, even if it migrates to a different cpu
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1098) * to the one whose affine portal it is waiting on.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1099) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1100) static DECLARE_WAIT_QUEUE_HEAD(affine_queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1101)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1102) static struct qman_fq **fq_table;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1103) static u32 num_fqids;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1104)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1105) int qman_alloc_fq_table(u32 _num_fqids)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1106) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1107) num_fqids = _num_fqids;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1108)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1109) fq_table = vzalloc(array3_size(sizeof(struct qman_fq *),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1110) num_fqids, 2));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1111) if (!fq_table)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1112) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1113)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1114) pr_debug("Allocated fq lookup table at %p, entry count %u\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1115) fq_table, num_fqids * 2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1116) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1117) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1118)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1119) static struct qman_fq *idx_to_fq(u32 idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1120) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1121) struct qman_fq *fq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1122)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1123) #ifdef CONFIG_FSL_DPAA_CHECKING
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1124) if (WARN_ON(idx >= num_fqids * 2))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1125) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1126) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1127) fq = fq_table[idx];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1128) DPAA_ASSERT(!fq || idx == fq->idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1129)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1130) return fq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1131) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1132)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1133) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1134) * Only returns full-service fq objects, not enqueue-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1135) * references (QMAN_FQ_FLAG_NO_MODIFY).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1136) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1137) static struct qman_fq *fqid_to_fq(u32 fqid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1138) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1139) return idx_to_fq(fqid * 2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1140) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1141)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1142) static struct qman_fq *tag_to_fq(u32 tag)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1143) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1144) #if BITS_PER_LONG == 64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1145) return idx_to_fq(tag);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1146) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1147) return (struct qman_fq *)tag;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1148) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1149) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1150)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1151) static u32 fq_to_tag(struct qman_fq *fq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1152) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1153) #if BITS_PER_LONG == 64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1154) return fq->idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1155) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1156) return (u32)fq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1157) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1158) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1159)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1160) static u32 __poll_portal_slow(struct qman_portal *p, u32 is);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1161) static inline unsigned int __poll_portal_fast(struct qman_portal *p,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1162) unsigned int poll_limit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1163) static void qm_congestion_task(struct work_struct *work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1164) static void qm_mr_process_task(struct work_struct *work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1165)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1166) static irqreturn_t portal_isr(int irq, void *ptr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1167) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1168) struct qman_portal *p = ptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1169) u32 is = qm_in(&p->p, QM_REG_ISR) & p->irq_sources;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1170) u32 clear = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1171)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1172) if (unlikely(!is))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1173) return IRQ_NONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1174)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1175) /* DQRR-handling if it's interrupt-driven */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1176) if (is & QM_PIRQ_DQRI) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1177) __poll_portal_fast(p, QMAN_POLL_LIMIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1178) clear = QM_DQAVAIL_MASK | QM_PIRQ_DQRI;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1179) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1180) /* Handling of anything else that's interrupt-driven */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1181) clear |= __poll_portal_slow(p, is) & QM_PIRQ_SLOW;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1182) qm_out(&p->p, QM_REG_ISR, clear);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1183) return IRQ_HANDLED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1184) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1185)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1186) static int drain_mr_fqrni(struct qm_portal *p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1187) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1188) const union qm_mr_entry *msg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1189) loop:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1190) qm_mr_pvb_update(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1191) msg = qm_mr_current(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1192) if (!msg) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1193) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1194) * if MR was full and h/w had other FQRNI entries to produce, we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1195) * need to allow it time to produce those entries once the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1196) * existing entries are consumed. A worst-case situation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1197) * (fully-loaded system) means h/w sequencers may have to do 3-4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1198) * other things before servicing the portal's MR pump, each of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1199) * which (if slow) may take ~50 qman cycles (which is ~200
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1200) * processor cycles). So rounding up and then multiplying this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1201) * worst-case estimate by a factor of 10, just to be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1202) * ultra-paranoid, goes as high as 10,000 cycles. NB, we consume
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1203) * one entry at a time, so h/w has an opportunity to produce new
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1204) * entries well before the ring has been fully consumed, so
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1205) * we're being *really* paranoid here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1206) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1207) mdelay(1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1208) qm_mr_pvb_update(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1209) msg = qm_mr_current(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1210) if (!msg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1211) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1212) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1213) if ((msg->verb & QM_MR_VERB_TYPE_MASK) != QM_MR_VERB_FQRNI) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1214) /* We aren't draining anything but FQRNIs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1215) pr_err("Found verb 0x%x in MR\n", msg->verb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1216) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1217) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1218) qm_mr_next(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1219) qm_mr_cci_consume(p, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1220) goto loop;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1221) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1222)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1223) static int qman_create_portal(struct qman_portal *portal,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1224) const struct qm_portal_config *c,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1225) const struct qman_cgrs *cgrs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1226) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1227) struct qm_portal *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1228) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1229) u32 isdr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1230)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1231) p = &portal->p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1232)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1233) #ifdef CONFIG_FSL_PAMU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1234) /* PAMU is required for stashing */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1235) portal->use_eqcr_ci_stashing = ((qman_ip_rev >= QMAN_REV30) ? 1 : 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1236) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1237) portal->use_eqcr_ci_stashing = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1238) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1239) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1240) * prep the low-level portal struct with the mapped addresses from the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1241) * config, everything that follows depends on it and "config" is more
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1242) * for (de)reference
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1243) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1244) p->addr.ce = c->addr_virt_ce;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1245) p->addr.ce_be = c->addr_virt_ce;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1246) p->addr.ci = c->addr_virt_ci;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1247) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1248) * If CI-stashing is used, the current defaults use a threshold of 3,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1249) * and stash with high-than-DQRR priority.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1250) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1251) if (qm_eqcr_init(p, qm_eqcr_pvb,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1252) portal->use_eqcr_ci_stashing ? 3 : 0, 1)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1253) dev_err(c->dev, "EQCR initialisation failed\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1254) goto fail_eqcr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1255) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1256) if (qm_dqrr_init(p, c, qm_dqrr_dpush, qm_dqrr_pvb,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1257) qm_dqrr_cdc, DQRR_MAXFILL)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1258) dev_err(c->dev, "DQRR initialisation failed\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1259) goto fail_dqrr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1260) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1261) if (qm_mr_init(p, qm_mr_pvb, qm_mr_cci)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1262) dev_err(c->dev, "MR initialisation failed\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1263) goto fail_mr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1264) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1265) if (qm_mc_init(p)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1266) dev_err(c->dev, "MC initialisation failed\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1267) goto fail_mc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1268) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1269) /* static interrupt-gating controls */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1270) qm_dqrr_set_ithresh(p, QMAN_PIRQ_DQRR_ITHRESH);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1271) qm_mr_set_ithresh(p, QMAN_PIRQ_MR_ITHRESH);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1272) qm_out(p, QM_REG_ITPR, QMAN_PIRQ_IPERIOD);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1273) portal->cgrs = kmalloc_array(2, sizeof(*cgrs), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1274) if (!portal->cgrs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1275) goto fail_cgrs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1276) /* initial snapshot is no-depletion */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1277) qman_cgrs_init(&portal->cgrs[1]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1278) if (cgrs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1279) portal->cgrs[0] = *cgrs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1280) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1281) /* if the given mask is NULL, assume all CGRs can be seen */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1282) qman_cgrs_fill(&portal->cgrs[0]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1283) INIT_LIST_HEAD(&portal->cgr_cbs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1284) spin_lock_init(&portal->cgr_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1285) INIT_WORK(&portal->congestion_work, qm_congestion_task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1286) INIT_WORK(&portal->mr_work, qm_mr_process_task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1287) portal->bits = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1288) portal->sdqcr = QM_SDQCR_SOURCE_CHANNELS | QM_SDQCR_COUNT_UPTO3 |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1289) QM_SDQCR_DEDICATED_PRECEDENCE | QM_SDQCR_TYPE_PRIO_QOS |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1290) QM_SDQCR_TOKEN_SET(0xab) | QM_SDQCR_CHANNELS_DEDICATED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1291) isdr = 0xffffffff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1292) qm_out(p, QM_REG_ISDR, isdr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1293) portal->irq_sources = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1294) qm_out(p, QM_REG_IER, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1295) snprintf(portal->irqname, MAX_IRQNAME, IRQNAME, c->cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1296) qm_out(p, QM_REG_IIR, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1297) if (request_irq(c->irq, portal_isr, 0, portal->irqname, portal)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1298) dev_err(c->dev, "request_irq() failed\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1299) goto fail_irq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1300) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1301)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1302) if (dpaa_set_portal_irq_affinity(c->dev, c->irq, c->cpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1303) goto fail_affinity;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1304)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1305) /* Need EQCR to be empty before continuing */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1306) isdr &= ~QM_PIRQ_EQCI;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1307) qm_out(p, QM_REG_ISDR, isdr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1308) ret = qm_eqcr_get_fill(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1309) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1310) dev_err(c->dev, "EQCR unclean\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1311) goto fail_eqcr_empty;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1312) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1313) isdr &= ~(QM_PIRQ_DQRI | QM_PIRQ_MRI);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1314) qm_out(p, QM_REG_ISDR, isdr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1315) if (qm_dqrr_current(p)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1316) dev_dbg(c->dev, "DQRR unclean\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1317) qm_dqrr_cdc_consume_n(p, 0xffff);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1318) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1319) if (qm_mr_current(p) && drain_mr_fqrni(p)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1320) /* special handling, drain just in case it's a few FQRNIs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1321) const union qm_mr_entry *e = qm_mr_current(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1322)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1323) dev_err(c->dev, "MR dirty, VB 0x%x, rc 0x%x, addr 0x%llx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1324) e->verb, e->ern.rc, qm_fd_addr_get64(&e->ern.fd));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1325) goto fail_dqrr_mr_empty;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1326) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1327) /* Success */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1328) portal->config = c;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1329) qm_out(p, QM_REG_ISR, 0xffffffff);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1330) qm_out(p, QM_REG_ISDR, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1331) if (!qman_requires_cleanup())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1332) qm_out(p, QM_REG_IIR, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1333) /* Write a sane SDQCR */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1334) qm_dqrr_sdqcr_set(p, portal->sdqcr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1335) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1336)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1337) fail_dqrr_mr_empty:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1338) fail_eqcr_empty:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1339) fail_affinity:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1340) free_irq(c->irq, portal);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1341) fail_irq:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1342) kfree(portal->cgrs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1343) fail_cgrs:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1344) qm_mc_finish(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1345) fail_mc:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1346) qm_mr_finish(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1347) fail_mr:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1348) qm_dqrr_finish(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1349) fail_dqrr:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1350) qm_eqcr_finish(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1351) fail_eqcr:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1352) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1353) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1354)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1355) struct qman_portal *qman_create_affine_portal(const struct qm_portal_config *c,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1356) const struct qman_cgrs *cgrs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1357) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1358) struct qman_portal *portal;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1359) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1360)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1361) portal = &per_cpu(qman_affine_portal, c->cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1362) err = qman_create_portal(portal, c, cgrs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1363) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1364) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1365)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1366) spin_lock(&affine_mask_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1367) cpumask_set_cpu(c->cpu, &affine_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1368) affine_channels[c->cpu] = c->channel;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1369) affine_portals[c->cpu] = portal;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1370) spin_unlock(&affine_mask_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1371)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1372) return portal;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1373) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1374)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1375) static void qman_destroy_portal(struct qman_portal *qm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1376) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1377) const struct qm_portal_config *pcfg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1378)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1379) /* Stop dequeues on the portal */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1380) qm_dqrr_sdqcr_set(&qm->p, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1381)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1382) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1383) * NB we do this to "quiesce" EQCR. If we add enqueue-completions or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1384) * something related to QM_PIRQ_EQCI, this may need fixing.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1385) * Also, due to the prefetching model used for CI updates in the enqueue
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1386) * path, this update will only invalidate the CI cacheline *after*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1387) * working on it, so we need to call this twice to ensure a full update
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1388) * irrespective of where the enqueue processing was at when the teardown
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1389) * began.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1390) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1391) qm_eqcr_cce_update(&qm->p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1392) qm_eqcr_cce_update(&qm->p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1393) pcfg = qm->config;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1394)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1395) free_irq(pcfg->irq, qm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1396)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1397) kfree(qm->cgrs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1398) qm_mc_finish(&qm->p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1399) qm_mr_finish(&qm->p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1400) qm_dqrr_finish(&qm->p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1401) qm_eqcr_finish(&qm->p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1402)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1403) qm->config = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1404) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1405)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1406) const struct qm_portal_config *qman_destroy_affine_portal(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1407) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1408) struct qman_portal *qm = get_affine_portal();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1409) const struct qm_portal_config *pcfg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1410) int cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1411)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1412) pcfg = qm->config;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1413) cpu = pcfg->cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1414)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1415) qman_destroy_portal(qm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1416)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1417) spin_lock(&affine_mask_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1418) cpumask_clear_cpu(cpu, &affine_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1419) spin_unlock(&affine_mask_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1420) put_affine_portal();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1421) return pcfg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1422) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1423)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1424) /* Inline helper to reduce nesting in __poll_portal_slow() */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1425) static inline void fq_state_change(struct qman_portal *p, struct qman_fq *fq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1426) const union qm_mr_entry *msg, u8 verb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1427) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1428) switch (verb) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1429) case QM_MR_VERB_FQRL:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1430) DPAA_ASSERT(fq_isset(fq, QMAN_FQ_STATE_ORL));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1431) fq_clear(fq, QMAN_FQ_STATE_ORL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1432) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1433) case QM_MR_VERB_FQRN:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1434) DPAA_ASSERT(fq->state == qman_fq_state_parked ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1435) fq->state == qman_fq_state_sched);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1436) DPAA_ASSERT(fq_isset(fq, QMAN_FQ_STATE_CHANGING));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1437) fq_clear(fq, QMAN_FQ_STATE_CHANGING);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1438) if (msg->fq.fqs & QM_MR_FQS_NOTEMPTY)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1439) fq_set(fq, QMAN_FQ_STATE_NE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1440) if (msg->fq.fqs & QM_MR_FQS_ORLPRESENT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1441) fq_set(fq, QMAN_FQ_STATE_ORL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1442) fq->state = qman_fq_state_retired;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1443) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1444) case QM_MR_VERB_FQPN:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1445) DPAA_ASSERT(fq->state == qman_fq_state_sched);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1446) DPAA_ASSERT(fq_isclear(fq, QMAN_FQ_STATE_CHANGING));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1447) fq->state = qman_fq_state_parked;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1448) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1449) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1450)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1451) static void qm_congestion_task(struct work_struct *work)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1452) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1453) struct qman_portal *p = container_of(work, struct qman_portal,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1454) congestion_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1455) struct qman_cgrs rr, c;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1456) union qm_mc_result *mcr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1457) struct qman_cgr *cgr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1458)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1459) spin_lock(&p->cgr_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1460) qm_mc_start(&p->p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1461) qm_mc_commit(&p->p, QM_MCC_VERB_QUERYCONGESTION);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1462) if (!qm_mc_result_timeout(&p->p, &mcr)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1463) spin_unlock(&p->cgr_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1464) dev_crit(p->config->dev, "QUERYCONGESTION timeout\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1465) qman_p_irqsource_add(p, QM_PIRQ_CSCI);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1466) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1467) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1468) /* mask out the ones I'm not interested in */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1469) qman_cgrs_and(&rr, (struct qman_cgrs *)&mcr->querycongestion.state,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1470) &p->cgrs[0]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1471) /* check previous snapshot for delta, enter/exit congestion */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1472) qman_cgrs_xor(&c, &rr, &p->cgrs[1]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1473) /* update snapshot */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1474) qman_cgrs_cp(&p->cgrs[1], &rr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1475) /* Invoke callback */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1476) list_for_each_entry(cgr, &p->cgr_cbs, node)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1477) if (cgr->cb && qman_cgrs_get(&c, cgr->cgrid))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1478) cgr->cb(p, cgr, qman_cgrs_get(&rr, cgr->cgrid));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1479) spin_unlock(&p->cgr_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1480) qman_p_irqsource_add(p, QM_PIRQ_CSCI);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1481) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1482)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1483) static void qm_mr_process_task(struct work_struct *work)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1484) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1485) struct qman_portal *p = container_of(work, struct qman_portal,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1486) mr_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1487) const union qm_mr_entry *msg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1488) struct qman_fq *fq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1489) u8 verb, num = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1490)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1491) preempt_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1492)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1493) while (1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1494) qm_mr_pvb_update(&p->p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1495) msg = qm_mr_current(&p->p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1496) if (!msg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1497) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1498)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1499) verb = msg->verb & QM_MR_VERB_TYPE_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1500) /* The message is a software ERN iff the 0x20 bit is clear */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1501) if (verb & 0x20) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1502) switch (verb) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1503) case QM_MR_VERB_FQRNI:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1504) /* nada, we drop FQRNIs on the floor */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1505) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1506) case QM_MR_VERB_FQRN:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1507) case QM_MR_VERB_FQRL:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1508) /* Lookup in the retirement table */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1509) fq = fqid_to_fq(qm_fqid_get(&msg->fq));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1510) if (WARN_ON(!fq))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1511) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1512) fq_state_change(p, fq, msg, verb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1513) if (fq->cb.fqs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1514) fq->cb.fqs(p, fq, msg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1515) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1516) case QM_MR_VERB_FQPN:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1517) /* Parked */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1518) fq = tag_to_fq(be32_to_cpu(msg->fq.context_b));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1519) fq_state_change(p, fq, msg, verb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1520) if (fq->cb.fqs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1521) fq->cb.fqs(p, fq, msg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1522) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1523) case QM_MR_VERB_DC_ERN:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1524) /* DCP ERN */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1525) pr_crit_once("Leaking DCP ERNs!\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1526) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1527) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1528) pr_crit("Invalid MR verb 0x%02x\n", verb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1529) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1530) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1531) /* Its a software ERN */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1532) fq = tag_to_fq(be32_to_cpu(msg->ern.tag));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1533) fq->cb.ern(p, fq, msg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1534) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1535) num++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1536) qm_mr_next(&p->p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1537) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1538)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1539) qm_mr_cci_consume(&p->p, num);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1540) qman_p_irqsource_add(p, QM_PIRQ_MRI);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1541) preempt_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1542) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1543)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1544) static u32 __poll_portal_slow(struct qman_portal *p, u32 is)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1545) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1546) if (is & QM_PIRQ_CSCI) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1547) qman_p_irqsource_remove(p, QM_PIRQ_CSCI);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1548) queue_work_on(smp_processor_id(), qm_portal_wq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1549) &p->congestion_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1550) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1551)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1552) if (is & QM_PIRQ_EQRI) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1553) qm_eqcr_cce_update(&p->p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1554) qm_eqcr_set_ithresh(&p->p, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1555) wake_up(&affine_queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1556) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1557)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1558) if (is & QM_PIRQ_MRI) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1559) qman_p_irqsource_remove(p, QM_PIRQ_MRI);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1560) queue_work_on(smp_processor_id(), qm_portal_wq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1561) &p->mr_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1562) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1563)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1564) return is;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1565) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1566)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1567) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1568) * remove some slowish-path stuff from the "fast path" and make sure it isn't
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1569) * inlined.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1570) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1571) static noinline void clear_vdqcr(struct qman_portal *p, struct qman_fq *fq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1572) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1573) p->vdqcr_owned = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1574) fq_clear(fq, QMAN_FQ_STATE_VDQCR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1575) wake_up(&affine_queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1576) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1577)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1578) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1579) * The only states that would conflict with other things if they ran at the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1580) * same time on the same cpu are:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1581) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1582) * (i) setting/clearing vdqcr_owned, and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1583) * (ii) clearing the NE (Not Empty) flag.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1584) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1585) * Both are safe. Because;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1586) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1587) * (i) this clearing can only occur after qman_volatile_dequeue() has set the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1588) * vdqcr_owned field (which it does before setting VDQCR), and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1589) * qman_volatile_dequeue() blocks interrupts and preemption while this is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1590) * done so that we can't interfere.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1591) * (ii) the NE flag is only cleared after qman_retire_fq() has set it, and as
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1592) * with (i) that API prevents us from interfering until it's safe.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1593) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1594) * The good thing is that qman_volatile_dequeue() and qman_retire_fq() run far
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1595) * less frequently (ie. per-FQ) than __poll_portal_fast() does, so the nett
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1596) * advantage comes from this function not having to "lock" anything at all.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1597) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1598) * Note also that the callbacks are invoked at points which are safe against the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1599) * above potential conflicts, but that this function itself is not re-entrant
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1600) * (this is because the function tracks one end of each FIFO in the portal and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1601) * we do *not* want to lock that). So the consequence is that it is safe for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1602) * user callbacks to call into any QMan API.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1603) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1604) static inline unsigned int __poll_portal_fast(struct qman_portal *p,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1605) unsigned int poll_limit)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1606) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1607) const struct qm_dqrr_entry *dq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1608) struct qman_fq *fq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1609) enum qman_cb_dqrr_result res;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1610) unsigned int limit = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1611)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1612) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1613) qm_dqrr_pvb_update(&p->p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1614) dq = qm_dqrr_current(&p->p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1615) if (!dq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1616) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1617)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1618) if (dq->stat & QM_DQRR_STAT_UNSCHEDULED) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1619) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1620) * VDQCR: don't trust context_b as the FQ may have
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1621) * been configured for h/w consumption and we're
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1622) * draining it post-retirement.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1623) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1624) fq = p->vdqcr_owned;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1625) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1626) * We only set QMAN_FQ_STATE_NE when retiring, so we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1627) * only need to check for clearing it when doing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1628) * volatile dequeues. It's one less thing to check
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1629) * in the critical path (SDQCR).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1630) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1631) if (dq->stat & QM_DQRR_STAT_FQ_EMPTY)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1632) fq_clear(fq, QMAN_FQ_STATE_NE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1633) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1634) * This is duplicated from the SDQCR code, but we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1635) * have stuff to do before *and* after this callback,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1636) * and we don't want multiple if()s in the critical
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1637) * path (SDQCR).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1638) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1639) res = fq->cb.dqrr(p, fq, dq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1640) if (res == qman_cb_dqrr_stop)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1641) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1642) /* Check for VDQCR completion */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1643) if (dq->stat & QM_DQRR_STAT_DQCR_EXPIRED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1644) clear_vdqcr(p, fq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1645) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1646) /* SDQCR: context_b points to the FQ */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1647) fq = tag_to_fq(be32_to_cpu(dq->context_b));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1648) /* Now let the callback do its stuff */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1649) res = fq->cb.dqrr(p, fq, dq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1650) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1651) * The callback can request that we exit without
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1652) * consuming this entry nor advancing;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1653) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1654) if (res == qman_cb_dqrr_stop)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1655) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1656) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1657) /* Interpret 'dq' from a driver perspective. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1658) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1659) * Parking isn't possible unless HELDACTIVE was set. NB,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1660) * FORCEELIGIBLE implies HELDACTIVE, so we only need to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1661) * check for HELDACTIVE to cover both.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1662) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1663) DPAA_ASSERT((dq->stat & QM_DQRR_STAT_FQ_HELDACTIVE) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1664) (res != qman_cb_dqrr_park));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1665) /* just means "skip it, I'll consume it myself later on" */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1666) if (res != qman_cb_dqrr_defer)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1667) qm_dqrr_cdc_consume_1ptr(&p->p, dq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1668) res == qman_cb_dqrr_park);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1669) /* Move forward */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1670) qm_dqrr_next(&p->p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1671) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1672) * Entry processed and consumed, increment our counter. The
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1673) * callback can request that we exit after consuming the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1674) * entry, and we also exit if we reach our processing limit,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1675) * so loop back only if neither of these conditions is met.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1676) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1677) } while (++limit < poll_limit && res != qman_cb_dqrr_consume_stop);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1678)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1679) return limit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1680) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1681)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1682) void qman_p_irqsource_add(struct qman_portal *p, u32 bits)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1683) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1684) unsigned long irqflags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1685)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1686) local_irq_save(irqflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1687) p->irq_sources |= bits & QM_PIRQ_VISIBLE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1688) qm_out(&p->p, QM_REG_IER, p->irq_sources);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1689) local_irq_restore(irqflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1690) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1691) EXPORT_SYMBOL(qman_p_irqsource_add);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1692)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1693) void qman_p_irqsource_remove(struct qman_portal *p, u32 bits)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1694) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1695) unsigned long irqflags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1696) u32 ier;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1697)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1698) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1699) * Our interrupt handler only processes+clears status register bits that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1700) * are in p->irq_sources. As we're trimming that mask, if one of them
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1701) * were to assert in the status register just before we remove it from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1702) * the enable register, there would be an interrupt-storm when we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1703) * release the IRQ lock. So we wait for the enable register update to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1704) * take effect in h/w (by reading it back) and then clear all other bits
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1705) * in the status register. Ie. we clear them from ISR once it's certain
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1706) * IER won't allow them to reassert.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1707) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1708) local_irq_save(irqflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1709) bits &= QM_PIRQ_VISIBLE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1710) p->irq_sources &= ~bits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1711) qm_out(&p->p, QM_REG_IER, p->irq_sources);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1712) ier = qm_in(&p->p, QM_REG_IER);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1713) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1714) * Using "~ier" (rather than "bits" or "~p->irq_sources") creates a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1715) * data-dependency, ie. to protect against re-ordering.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1716) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1717) qm_out(&p->p, QM_REG_ISR, ~ier);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1718) local_irq_restore(irqflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1719) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1720) EXPORT_SYMBOL(qman_p_irqsource_remove);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1721)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1722) const cpumask_t *qman_affine_cpus(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1723) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1724) return &affine_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1725) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1726) EXPORT_SYMBOL(qman_affine_cpus);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1727)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1728) u16 qman_affine_channel(int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1729) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1730) if (cpu < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1731) struct qman_portal *portal = get_affine_portal();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1732)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1733) cpu = portal->config->cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1734) put_affine_portal();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1735) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1736) WARN_ON(!cpumask_test_cpu(cpu, &affine_mask));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1737) return affine_channels[cpu];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1738) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1739) EXPORT_SYMBOL(qman_affine_channel);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1740)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1741) struct qman_portal *qman_get_affine_portal(int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1742) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1743) return affine_portals[cpu];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1744) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1745) EXPORT_SYMBOL(qman_get_affine_portal);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1746)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1747) int qman_start_using_portal(struct qman_portal *p, struct device *dev)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1748) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1749) return (!device_link_add(dev, p->config->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1750) DL_FLAG_AUTOREMOVE_CONSUMER)) ? -EINVAL : 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1751) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1752) EXPORT_SYMBOL(qman_start_using_portal);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1753)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1754) int qman_p_poll_dqrr(struct qman_portal *p, unsigned int limit)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1755) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1756) return __poll_portal_fast(p, limit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1757) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1758) EXPORT_SYMBOL(qman_p_poll_dqrr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1759)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1760) void qman_p_static_dequeue_add(struct qman_portal *p, u32 pools)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1761) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1762) unsigned long irqflags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1763)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1764) local_irq_save(irqflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1765) pools &= p->config->pools;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1766) p->sdqcr |= pools;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1767) qm_dqrr_sdqcr_set(&p->p, p->sdqcr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1768) local_irq_restore(irqflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1769) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1770) EXPORT_SYMBOL(qman_p_static_dequeue_add);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1771)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1772) /* Frame queue API */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1773)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1774) static const char *mcr_result_str(u8 result)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1775) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1776) switch (result) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1777) case QM_MCR_RESULT_NULL:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1778) return "QM_MCR_RESULT_NULL";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1779) case QM_MCR_RESULT_OK:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1780) return "QM_MCR_RESULT_OK";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1781) case QM_MCR_RESULT_ERR_FQID:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1782) return "QM_MCR_RESULT_ERR_FQID";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1783) case QM_MCR_RESULT_ERR_FQSTATE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1784) return "QM_MCR_RESULT_ERR_FQSTATE";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1785) case QM_MCR_RESULT_ERR_NOTEMPTY:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1786) return "QM_MCR_RESULT_ERR_NOTEMPTY";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1787) case QM_MCR_RESULT_PENDING:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1788) return "QM_MCR_RESULT_PENDING";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1789) case QM_MCR_RESULT_ERR_BADCOMMAND:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1790) return "QM_MCR_RESULT_ERR_BADCOMMAND";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1791) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1792) return "<unknown MCR result>";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1793) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1794)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1795) int qman_create_fq(u32 fqid, u32 flags, struct qman_fq *fq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1796) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1797) if (flags & QMAN_FQ_FLAG_DYNAMIC_FQID) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1798) int ret = qman_alloc_fqid(&fqid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1799)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1800) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1801) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1802) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1803) fq->fqid = fqid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1804) fq->flags = flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1805) fq->state = qman_fq_state_oos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1806) fq->cgr_groupid = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1807)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1808) /* A context_b of 0 is allegedly special, so don't use that fqid */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1809) if (fqid == 0 || fqid >= num_fqids) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1810) WARN(1, "bad fqid %d\n", fqid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1811) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1812) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1813)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1814) fq->idx = fqid * 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1815) if (flags & QMAN_FQ_FLAG_NO_MODIFY)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1816) fq->idx++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1817)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1818) WARN_ON(fq_table[fq->idx]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1819) fq_table[fq->idx] = fq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1820)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1821) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1822) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1823) EXPORT_SYMBOL(qman_create_fq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1824)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1825) void qman_destroy_fq(struct qman_fq *fq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1826) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1827) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1828) * We don't need to lock the FQ as it is a pre-condition that the FQ be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1829) * quiesced. Instead, run some checks.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1830) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1831) switch (fq->state) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1832) case qman_fq_state_parked:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1833) case qman_fq_state_oos:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1834) if (fq_isset(fq, QMAN_FQ_FLAG_DYNAMIC_FQID))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1835) qman_release_fqid(fq->fqid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1836)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1837) DPAA_ASSERT(fq_table[fq->idx]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1838) fq_table[fq->idx] = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1839) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1840) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1841) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1842) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1843) DPAA_ASSERT(NULL == "qman_free_fq() on unquiesced FQ!");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1844) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1845) EXPORT_SYMBOL(qman_destroy_fq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1846)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1847) u32 qman_fq_fqid(struct qman_fq *fq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1848) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1849) return fq->fqid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1850) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1851) EXPORT_SYMBOL(qman_fq_fqid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1852)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1853) int qman_init_fq(struct qman_fq *fq, u32 flags, struct qm_mcc_initfq *opts)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1854) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1855) union qm_mc_command *mcc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1856) union qm_mc_result *mcr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1857) struct qman_portal *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1858) u8 res, myverb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1859) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1860)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1861) myverb = (flags & QMAN_INITFQ_FLAG_SCHED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1862) ? QM_MCC_VERB_INITFQ_SCHED : QM_MCC_VERB_INITFQ_PARKED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1863)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1864) if (fq->state != qman_fq_state_oos &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1865) fq->state != qman_fq_state_parked)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1866) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1867) #ifdef CONFIG_FSL_DPAA_CHECKING
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1868) if (fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1869) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1870) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1871) if (opts && (be16_to_cpu(opts->we_mask) & QM_INITFQ_WE_OAC)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1872) /* And can't be set at the same time as TDTHRESH */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1873) if (be16_to_cpu(opts->we_mask) & QM_INITFQ_WE_TDTHRESH)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1874) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1875) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1876) /* Issue an INITFQ_[PARKED|SCHED] management command */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1877) p = get_affine_portal();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1878) if (fq_isset(fq, QMAN_FQ_STATE_CHANGING) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1879) (fq->state != qman_fq_state_oos &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1880) fq->state != qman_fq_state_parked)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1881) ret = -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1882) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1883) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1884) mcc = qm_mc_start(&p->p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1885) if (opts)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1886) mcc->initfq = *opts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1887) qm_fqid_set(&mcc->fq, fq->fqid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1888) mcc->initfq.count = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1889) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1890) * If the FQ does *not* have the TO_DCPORTAL flag, context_b is set as a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1891) * demux pointer. Otherwise, the caller-provided value is allowed to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1892) * stand, don't overwrite it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1893) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1894) if (fq_isclear(fq, QMAN_FQ_FLAG_TO_DCPORTAL)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1895) dma_addr_t phys_fq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1896)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1897) mcc->initfq.we_mask |= cpu_to_be16(QM_INITFQ_WE_CONTEXTB);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1898) mcc->initfq.fqd.context_b = cpu_to_be32(fq_to_tag(fq));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1899) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1900) * and the physical address - NB, if the user wasn't trying to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1901) * set CONTEXTA, clear the stashing settings.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1902) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1903) if (!(be16_to_cpu(mcc->initfq.we_mask) &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1904) QM_INITFQ_WE_CONTEXTA)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1905) mcc->initfq.we_mask |=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1906) cpu_to_be16(QM_INITFQ_WE_CONTEXTA);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1907) memset(&mcc->initfq.fqd.context_a, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1908) sizeof(mcc->initfq.fqd.context_a));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1909) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1910) struct qman_portal *p = qman_dma_portal;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1911)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1912) phys_fq = dma_map_single(p->config->dev, fq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1913) sizeof(*fq), DMA_TO_DEVICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1914) if (dma_mapping_error(p->config->dev, phys_fq)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1915) dev_err(p->config->dev, "dma_mapping failed\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1916) ret = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1917) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1918) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1919)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1920) qm_fqd_stashing_set64(&mcc->initfq.fqd, phys_fq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1921) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1922) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1923) if (flags & QMAN_INITFQ_FLAG_LOCAL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1924) int wq = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1925)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1926) if (!(be16_to_cpu(mcc->initfq.we_mask) &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1927) QM_INITFQ_WE_DESTWQ)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1928) mcc->initfq.we_mask |=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1929) cpu_to_be16(QM_INITFQ_WE_DESTWQ);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1930) wq = 4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1931) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1932) qm_fqd_set_destwq(&mcc->initfq.fqd, p->config->channel, wq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1933) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1934) qm_mc_commit(&p->p, myverb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1935) if (!qm_mc_result_timeout(&p->p, &mcr)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1936) dev_err(p->config->dev, "MCR timeout\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1937) ret = -ETIMEDOUT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1938) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1939) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1940)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1941) DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == myverb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1942) res = mcr->result;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1943) if (res != QM_MCR_RESULT_OK) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1944) ret = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1945) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1946) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1947) if (opts) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1948) if (be16_to_cpu(opts->we_mask) & QM_INITFQ_WE_FQCTRL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1949) if (be16_to_cpu(opts->fqd.fq_ctrl) & QM_FQCTRL_CGE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1950) fq_set(fq, QMAN_FQ_STATE_CGR_EN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1951) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1952) fq_clear(fq, QMAN_FQ_STATE_CGR_EN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1953) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1954) if (be16_to_cpu(opts->we_mask) & QM_INITFQ_WE_CGID)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1955) fq->cgr_groupid = opts->fqd.cgid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1956) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1957) fq->state = (flags & QMAN_INITFQ_FLAG_SCHED) ?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1958) qman_fq_state_sched : qman_fq_state_parked;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1959)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1960) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1961) put_affine_portal();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1962) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1963) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1964) EXPORT_SYMBOL(qman_init_fq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1965)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1966) int qman_schedule_fq(struct qman_fq *fq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1967) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1968) union qm_mc_command *mcc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1969) union qm_mc_result *mcr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1970) struct qman_portal *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1971) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1972)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1973) if (fq->state != qman_fq_state_parked)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1974) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1975) #ifdef CONFIG_FSL_DPAA_CHECKING
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1976) if (fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1977) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1978) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1979) /* Issue a ALTERFQ_SCHED management command */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1980) p = get_affine_portal();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1981) if (fq_isset(fq, QMAN_FQ_STATE_CHANGING) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1982) fq->state != qman_fq_state_parked) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1983) ret = -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1984) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1985) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1986) mcc = qm_mc_start(&p->p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1987) qm_fqid_set(&mcc->fq, fq->fqid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1988) qm_mc_commit(&p->p, QM_MCC_VERB_ALTER_SCHED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1989) if (!qm_mc_result_timeout(&p->p, &mcr)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1990) dev_err(p->config->dev, "ALTER_SCHED timeout\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1991) ret = -ETIMEDOUT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1992) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1993) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1994)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1995) DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_ALTER_SCHED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1996) if (mcr->result != QM_MCR_RESULT_OK) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1997) ret = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1998) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1999) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2000) fq->state = qman_fq_state_sched;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2001) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2002) put_affine_portal();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2003) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2004) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2005) EXPORT_SYMBOL(qman_schedule_fq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2006)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2007) int qman_retire_fq(struct qman_fq *fq, u32 *flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2008) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2009) union qm_mc_command *mcc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2010) union qm_mc_result *mcr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2011) struct qman_portal *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2012) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2013) u8 res;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2014)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2015) if (fq->state != qman_fq_state_parked &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2016) fq->state != qman_fq_state_sched)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2017) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2018) #ifdef CONFIG_FSL_DPAA_CHECKING
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2019) if (fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2020) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2021) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2022) p = get_affine_portal();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2023) if (fq_isset(fq, QMAN_FQ_STATE_CHANGING) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2024) fq->state == qman_fq_state_retired ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2025) fq->state == qman_fq_state_oos) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2026) ret = -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2027) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2028) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2029) mcc = qm_mc_start(&p->p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2030) qm_fqid_set(&mcc->fq, fq->fqid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2031) qm_mc_commit(&p->p, QM_MCC_VERB_ALTER_RETIRE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2032) if (!qm_mc_result_timeout(&p->p, &mcr)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2033) dev_crit(p->config->dev, "ALTER_RETIRE timeout\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2034) ret = -ETIMEDOUT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2035) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2036) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2037)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2038) DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_ALTER_RETIRE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2039) res = mcr->result;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2040) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2041) * "Elegant" would be to treat OK/PENDING the same way; set CHANGING,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2042) * and defer the flags until FQRNI or FQRN (respectively) show up. But
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2043) * "Friendly" is to process OK immediately, and not set CHANGING. We do
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2044) * friendly, otherwise the caller doesn't necessarily have a fully
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2045) * "retired" FQ on return even if the retirement was immediate. However
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2046) * this does mean some code duplication between here and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2047) * fq_state_change().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2048) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2049) if (res == QM_MCR_RESULT_OK) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2050) ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2051) /* Process 'fq' right away, we'll ignore FQRNI */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2052) if (mcr->alterfq.fqs & QM_MCR_FQS_NOTEMPTY)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2053) fq_set(fq, QMAN_FQ_STATE_NE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2054) if (mcr->alterfq.fqs & QM_MCR_FQS_ORLPRESENT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2055) fq_set(fq, QMAN_FQ_STATE_ORL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2056) if (flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2057) *flags = fq->flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2058) fq->state = qman_fq_state_retired;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2059) if (fq->cb.fqs) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2060) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2061) * Another issue with supporting "immediate" retirement
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2062) * is that we're forced to drop FQRNIs, because by the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2063) * time they're seen it may already be "too late" (the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2064) * fq may have been OOS'd and free()'d already). But if
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2065) * the upper layer wants a callback whether it's
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2066) * immediate or not, we have to fake a "MR" entry to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2067) * look like an FQRNI...
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2068) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2069) union qm_mr_entry msg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2070)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2071) msg.verb = QM_MR_VERB_FQRNI;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2072) msg.fq.fqs = mcr->alterfq.fqs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2073) qm_fqid_set(&msg.fq, fq->fqid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2074) msg.fq.context_b = cpu_to_be32(fq_to_tag(fq));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2075) fq->cb.fqs(p, fq, &msg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2076) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2077) } else if (res == QM_MCR_RESULT_PENDING) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2078) ret = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2079) fq_set(fq, QMAN_FQ_STATE_CHANGING);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2080) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2081) ret = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2082) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2083) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2084) put_affine_portal();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2085) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2086) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2087) EXPORT_SYMBOL(qman_retire_fq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2088)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2089) int qman_oos_fq(struct qman_fq *fq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2090) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2091) union qm_mc_command *mcc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2092) union qm_mc_result *mcr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2093) struct qman_portal *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2094) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2095)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2096) if (fq->state != qman_fq_state_retired)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2097) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2098) #ifdef CONFIG_FSL_DPAA_CHECKING
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2099) if (fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2100) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2101) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2102) p = get_affine_portal();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2103) if (fq_isset(fq, QMAN_FQ_STATE_BLOCKOOS) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2104) fq->state != qman_fq_state_retired) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2105) ret = -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2106) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2107) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2108) mcc = qm_mc_start(&p->p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2109) qm_fqid_set(&mcc->fq, fq->fqid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2110) qm_mc_commit(&p->p, QM_MCC_VERB_ALTER_OOS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2111) if (!qm_mc_result_timeout(&p->p, &mcr)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2112) ret = -ETIMEDOUT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2113) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2114) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2115) DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_ALTER_OOS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2116) if (mcr->result != QM_MCR_RESULT_OK) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2117) ret = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2118) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2119) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2120) fq->state = qman_fq_state_oos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2121) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2122) put_affine_portal();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2123) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2124) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2125) EXPORT_SYMBOL(qman_oos_fq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2126)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2127) int qman_query_fq(struct qman_fq *fq, struct qm_fqd *fqd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2128) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2129) union qm_mc_command *mcc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2130) union qm_mc_result *mcr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2131) struct qman_portal *p = get_affine_portal();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2132) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2133)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2134) mcc = qm_mc_start(&p->p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2135) qm_fqid_set(&mcc->fq, fq->fqid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2136) qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2137) if (!qm_mc_result_timeout(&p->p, &mcr)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2138) ret = -ETIMEDOUT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2139) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2140) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2141)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2142) DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2143) if (mcr->result == QM_MCR_RESULT_OK)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2144) *fqd = mcr->queryfq.fqd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2145) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2146) ret = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2147) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2148) put_affine_portal();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2149) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2150) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2151)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2152) int qman_query_fq_np(struct qman_fq *fq, struct qm_mcr_queryfq_np *np)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2153) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2154) union qm_mc_command *mcc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2155) union qm_mc_result *mcr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2156) struct qman_portal *p = get_affine_portal();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2157) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2158)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2159) mcc = qm_mc_start(&p->p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2160) qm_fqid_set(&mcc->fq, fq->fqid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2161) qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ_NP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2162) if (!qm_mc_result_timeout(&p->p, &mcr)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2163) ret = -ETIMEDOUT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2164) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2165) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2166)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2167) DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ_NP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2168) if (mcr->result == QM_MCR_RESULT_OK)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2169) *np = mcr->queryfq_np;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2170) else if (mcr->result == QM_MCR_RESULT_ERR_FQID)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2171) ret = -ERANGE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2172) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2173) ret = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2174) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2175) put_affine_portal();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2176) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2177) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2178) EXPORT_SYMBOL(qman_query_fq_np);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2179)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2180) static int qman_query_cgr(struct qman_cgr *cgr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2181) struct qm_mcr_querycgr *cgrd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2182) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2183) union qm_mc_command *mcc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2184) union qm_mc_result *mcr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2185) struct qman_portal *p = get_affine_portal();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2186) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2187)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2188) mcc = qm_mc_start(&p->p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2189) mcc->cgr.cgid = cgr->cgrid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2190) qm_mc_commit(&p->p, QM_MCC_VERB_QUERYCGR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2191) if (!qm_mc_result_timeout(&p->p, &mcr)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2192) ret = -ETIMEDOUT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2193) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2194) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2195) DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCC_VERB_QUERYCGR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2196) if (mcr->result == QM_MCR_RESULT_OK)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2197) *cgrd = mcr->querycgr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2198) else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2199) dev_err(p->config->dev, "QUERY_CGR failed: %s\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2200) mcr_result_str(mcr->result));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2201) ret = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2202) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2203) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2204) put_affine_portal();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2205) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2206) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2207)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2208) int qman_query_cgr_congested(struct qman_cgr *cgr, bool *result)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2209) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2210) struct qm_mcr_querycgr query_cgr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2211) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2212)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2213) err = qman_query_cgr(cgr, &query_cgr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2214) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2215) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2216)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2217) *result = !!query_cgr.cgr.cs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2218) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2219) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2220) EXPORT_SYMBOL(qman_query_cgr_congested);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2221)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2222) /* internal function used as a wait_event() expression */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2223) static int set_p_vdqcr(struct qman_portal *p, struct qman_fq *fq, u32 vdqcr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2224) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2225) unsigned long irqflags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2226) int ret = -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2227)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2228) local_irq_save(irqflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2229) if (p->vdqcr_owned)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2230) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2231) if (fq_isset(fq, QMAN_FQ_STATE_VDQCR))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2232) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2233)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2234) fq_set(fq, QMAN_FQ_STATE_VDQCR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2235) p->vdqcr_owned = fq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2236) qm_dqrr_vdqcr_set(&p->p, vdqcr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2237) ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2238) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2239) local_irq_restore(irqflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2240) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2241) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2242)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2243) static int set_vdqcr(struct qman_portal **p, struct qman_fq *fq, u32 vdqcr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2244) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2245) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2246)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2247) *p = get_affine_portal();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2248) ret = set_p_vdqcr(*p, fq, vdqcr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2249) put_affine_portal();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2250) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2251) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2252)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2253) static int wait_vdqcr_start(struct qman_portal **p, struct qman_fq *fq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2254) u32 vdqcr, u32 flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2255) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2256) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2257)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2258) if (flags & QMAN_VOLATILE_FLAG_WAIT_INT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2259) ret = wait_event_interruptible(affine_queue,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2260) !set_vdqcr(p, fq, vdqcr));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2261) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2262) wait_event(affine_queue, !set_vdqcr(p, fq, vdqcr));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2263) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2264) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2265)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2266) int qman_volatile_dequeue(struct qman_fq *fq, u32 flags, u32 vdqcr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2267) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2268) struct qman_portal *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2269) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2270)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2271) if (fq->state != qman_fq_state_parked &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2272) fq->state != qman_fq_state_retired)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2273) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2274) if (vdqcr & QM_VDQCR_FQID_MASK)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2275) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2276) if (fq_isset(fq, QMAN_FQ_STATE_VDQCR))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2277) return -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2278) vdqcr = (vdqcr & ~QM_VDQCR_FQID_MASK) | fq->fqid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2279) if (flags & QMAN_VOLATILE_FLAG_WAIT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2280) ret = wait_vdqcr_start(&p, fq, vdqcr, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2281) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2282) ret = set_vdqcr(&p, fq, vdqcr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2283) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2284) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2285) /* VDQCR is set */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2286) if (flags & QMAN_VOLATILE_FLAG_FINISH) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2287) if (flags & QMAN_VOLATILE_FLAG_WAIT_INT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2288) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2289) * NB: don't propagate any error - the caller wouldn't
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2290) * know whether the VDQCR was issued or not. A signal
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2291) * could arrive after returning anyway, so the caller
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2292) * can check signal_pending() if that's an issue.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2293) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2294) wait_event_interruptible(affine_queue,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2295) !fq_isset(fq, QMAN_FQ_STATE_VDQCR));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2296) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2297) wait_event(affine_queue,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2298) !fq_isset(fq, QMAN_FQ_STATE_VDQCR));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2299) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2300) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2301) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2302) EXPORT_SYMBOL(qman_volatile_dequeue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2303)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2304) static void update_eqcr_ci(struct qman_portal *p, u8 avail)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2305) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2306) if (avail)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2307) qm_eqcr_cce_prefetch(&p->p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2308) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2309) qm_eqcr_cce_update(&p->p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2310) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2311)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2312) int qman_enqueue(struct qman_fq *fq, const struct qm_fd *fd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2313) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2314) struct qman_portal *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2315) struct qm_eqcr_entry *eq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2316) unsigned long irqflags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2317) u8 avail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2318)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2319) p = get_affine_portal();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2320) local_irq_save(irqflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2321)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2322) if (p->use_eqcr_ci_stashing) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2323) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2324) * The stashing case is easy, only update if we need to in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2325) * order to try and liberate ring entries.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2326) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2327) eq = qm_eqcr_start_stash(&p->p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2328) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2329) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2330) * The non-stashing case is harder, need to prefetch ahead of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2331) * time.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2332) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2333) avail = qm_eqcr_get_avail(&p->p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2334) if (avail < 2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2335) update_eqcr_ci(p, avail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2336) eq = qm_eqcr_start_no_stash(&p->p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2337) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2338)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2339) if (unlikely(!eq))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2340) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2341)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2342) qm_fqid_set(eq, fq->fqid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2343) eq->tag = cpu_to_be32(fq_to_tag(fq));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2344) eq->fd = *fd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2345)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2346) qm_eqcr_pvb_commit(&p->p, QM_EQCR_VERB_CMD_ENQUEUE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2347) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2348) local_irq_restore(irqflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2349) put_affine_portal();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2350) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2351) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2352) EXPORT_SYMBOL(qman_enqueue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2353)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2354) static int qm_modify_cgr(struct qman_cgr *cgr, u32 flags,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2355) struct qm_mcc_initcgr *opts)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2356) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2357) union qm_mc_command *mcc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2358) union qm_mc_result *mcr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2359) struct qman_portal *p = get_affine_portal();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2360) u8 verb = QM_MCC_VERB_MODIFYCGR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2361) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2362)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2363) mcc = qm_mc_start(&p->p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2364) if (opts)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2365) mcc->initcgr = *opts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2366) mcc->initcgr.cgid = cgr->cgrid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2367) if (flags & QMAN_CGR_FLAG_USE_INIT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2368) verb = QM_MCC_VERB_INITCGR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2369) qm_mc_commit(&p->p, verb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2370) if (!qm_mc_result_timeout(&p->p, &mcr)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2371) ret = -ETIMEDOUT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2372) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2373) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2374)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2375) DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == verb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2376) if (mcr->result != QM_MCR_RESULT_OK)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2377) ret = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2378)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2379) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2380) put_affine_portal();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2381) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2382) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2383)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2384) #define PORTAL_IDX(n) (n->config->channel - QM_CHANNEL_SWPORTAL0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2385)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2386) /* congestion state change notification target update control */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2387) static void qm_cgr_cscn_targ_set(struct __qm_mc_cgr *cgr, int pi, u32 val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2388) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2389) if (qman_ip_rev >= QMAN_REV30)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2390) cgr->cscn_targ_upd_ctrl = cpu_to_be16(pi |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2391) QM_CGR_TARG_UDP_CTRL_WRITE_BIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2392) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2393) cgr->cscn_targ = cpu_to_be32(val | QM_CGR_TARG_PORTAL(pi));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2394) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2395)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2396) static void qm_cgr_cscn_targ_clear(struct __qm_mc_cgr *cgr, int pi, u32 val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2397) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2398) if (qman_ip_rev >= QMAN_REV30)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2399) cgr->cscn_targ_upd_ctrl = cpu_to_be16(pi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2400) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2401) cgr->cscn_targ = cpu_to_be32(val & ~QM_CGR_TARG_PORTAL(pi));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2402) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2403)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2404) static u8 qman_cgr_cpus[CGR_NUM];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2405)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2406) void qman_init_cgr_all(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2407) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2408) struct qman_cgr cgr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2409) int err_cnt = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2410)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2411) for (cgr.cgrid = 0; cgr.cgrid < CGR_NUM; cgr.cgrid++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2412) if (qm_modify_cgr(&cgr, QMAN_CGR_FLAG_USE_INIT, NULL))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2413) err_cnt++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2414) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2415)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2416) if (err_cnt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2417) pr_err("Warning: %d error%s while initialising CGR h/w\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2418) err_cnt, (err_cnt > 1) ? "s" : "");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2419) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2420)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2421) int qman_create_cgr(struct qman_cgr *cgr, u32 flags,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2422) struct qm_mcc_initcgr *opts)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2423) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2424) struct qm_mcr_querycgr cgr_state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2425) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2426) struct qman_portal *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2427)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2428) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2429) * We have to check that the provided CGRID is within the limits of the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2430) * data-structures, for obvious reasons. However we'll let h/w take
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2431) * care of determining whether it's within the limits of what exists on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2432) * the SoC.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2433) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2434) if (cgr->cgrid >= CGR_NUM)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2435) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2436)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2437) preempt_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2438) p = get_affine_portal();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2439) qman_cgr_cpus[cgr->cgrid] = smp_processor_id();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2440) preempt_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2441)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2442) cgr->chan = p->config->channel;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2443) spin_lock(&p->cgr_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2444)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2445) if (opts) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2446) struct qm_mcc_initcgr local_opts = *opts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2447)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2448) ret = qman_query_cgr(cgr, &cgr_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2449) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2450) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2451)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2452) qm_cgr_cscn_targ_set(&local_opts.cgr, PORTAL_IDX(p),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2453) be32_to_cpu(cgr_state.cgr.cscn_targ));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2454) local_opts.we_mask |= cpu_to_be16(QM_CGR_WE_CSCN_TARG);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2455)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2456) /* send init if flags indicate so */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2457) if (flags & QMAN_CGR_FLAG_USE_INIT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2458) ret = qm_modify_cgr(cgr, QMAN_CGR_FLAG_USE_INIT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2459) &local_opts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2460) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2461) ret = qm_modify_cgr(cgr, 0, &local_opts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2462) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2463) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2464) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2465)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2466) list_add(&cgr->node, &p->cgr_cbs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2467)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2468) /* Determine if newly added object requires its callback to be called */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2469) ret = qman_query_cgr(cgr, &cgr_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2470) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2471) /* we can't go back, so proceed and return success */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2472) dev_err(p->config->dev, "CGR HW state partially modified\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2473) ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2474) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2475) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2476) if (cgr->cb && cgr_state.cgr.cscn_en &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2477) qman_cgrs_get(&p->cgrs[1], cgr->cgrid))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2478) cgr->cb(p, cgr, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2479) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2480) spin_unlock(&p->cgr_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2481) put_affine_portal();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2482) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2483) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2484) EXPORT_SYMBOL(qman_create_cgr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2485)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2486) int qman_delete_cgr(struct qman_cgr *cgr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2487) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2488) unsigned long irqflags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2489) struct qm_mcr_querycgr cgr_state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2490) struct qm_mcc_initcgr local_opts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2491) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2492) struct qman_cgr *i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2493) struct qman_portal *p = get_affine_portal();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2494)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2495) if (cgr->chan != p->config->channel) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2496) /* attempt to delete from other portal than creator */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2497) dev_err(p->config->dev, "CGR not owned by current portal");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2498) dev_dbg(p->config->dev, " create 0x%x, delete 0x%x\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2499) cgr->chan, p->config->channel);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2500)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2501) ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2502) goto put_portal;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2503) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2504) memset(&local_opts, 0, sizeof(struct qm_mcc_initcgr));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2505) spin_lock_irqsave(&p->cgr_lock, irqflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2506) list_del(&cgr->node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2507) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2508) * If there are no other CGR objects for this CGRID in the list,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2509) * update CSCN_TARG accordingly
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2510) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2511) list_for_each_entry(i, &p->cgr_cbs, node)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2512) if (i->cgrid == cgr->cgrid && i->cb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2513) goto release_lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2514) ret = qman_query_cgr(cgr, &cgr_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2515) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2516) /* add back to the list */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2517) list_add(&cgr->node, &p->cgr_cbs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2518) goto release_lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2519) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2520)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2521) local_opts.we_mask = cpu_to_be16(QM_CGR_WE_CSCN_TARG);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2522) qm_cgr_cscn_targ_clear(&local_opts.cgr, PORTAL_IDX(p),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2523) be32_to_cpu(cgr_state.cgr.cscn_targ));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2524)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2525) ret = qm_modify_cgr(cgr, 0, &local_opts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2526) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2527) /* add back to the list */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2528) list_add(&cgr->node, &p->cgr_cbs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2529) release_lock:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2530) spin_unlock_irqrestore(&p->cgr_lock, irqflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2531) put_portal:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2532) put_affine_portal();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2533) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2534) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2535) EXPORT_SYMBOL(qman_delete_cgr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2536)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2537) struct cgr_comp {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2538) struct qman_cgr *cgr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2539) struct completion completion;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2540) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2541)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2542) static void qman_delete_cgr_smp_call(void *p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2543) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2544) qman_delete_cgr((struct qman_cgr *)p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2545) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2546)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2547) void qman_delete_cgr_safe(struct qman_cgr *cgr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2548) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2549) preempt_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2550) if (qman_cgr_cpus[cgr->cgrid] != smp_processor_id()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2551) smp_call_function_single(qman_cgr_cpus[cgr->cgrid],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2552) qman_delete_cgr_smp_call, cgr, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2553) preempt_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2554) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2555) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2556)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2557) qman_delete_cgr(cgr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2558) preempt_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2559) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2560) EXPORT_SYMBOL(qman_delete_cgr_safe);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2561)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2562) /* Cleanup FQs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2563)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2564) static int _qm_mr_consume_and_match_verb(struct qm_portal *p, int v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2565) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2566) const union qm_mr_entry *msg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2567) int found = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2568)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2569) qm_mr_pvb_update(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2570) msg = qm_mr_current(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2571) while (msg) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2572) if ((msg->verb & QM_MR_VERB_TYPE_MASK) == v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2573) found = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2574) qm_mr_next(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2575) qm_mr_cci_consume_to_current(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2576) qm_mr_pvb_update(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2577) msg = qm_mr_current(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2578) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2579) return found;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2580) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2581)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2582) static int _qm_dqrr_consume_and_match(struct qm_portal *p, u32 fqid, int s,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2583) bool wait)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2584) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2585) const struct qm_dqrr_entry *dqrr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2586) int found = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2587)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2588) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2589) qm_dqrr_pvb_update(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2590) dqrr = qm_dqrr_current(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2591) if (!dqrr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2592) cpu_relax();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2593) } while (wait && !dqrr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2594)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2595) while (dqrr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2596) if (qm_fqid_get(dqrr) == fqid && (dqrr->stat & s))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2597) found = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2598) qm_dqrr_cdc_consume_1ptr(p, dqrr, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2599) qm_dqrr_pvb_update(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2600) qm_dqrr_next(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2601) dqrr = qm_dqrr_current(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2602) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2603) return found;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2604) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2605)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2606) #define qm_mr_drain(p, V) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2607) _qm_mr_consume_and_match_verb(p, QM_MR_VERB_##V)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2608)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2609) #define qm_dqrr_drain(p, f, S) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2610) _qm_dqrr_consume_and_match(p, f, QM_DQRR_STAT_##S, false)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2611)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2612) #define qm_dqrr_drain_wait(p, f, S) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2613) _qm_dqrr_consume_and_match(p, f, QM_DQRR_STAT_##S, true)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2614)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2615) #define qm_dqrr_drain_nomatch(p) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2616) _qm_dqrr_consume_and_match(p, 0, 0, false)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2617)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2618) int qman_shutdown_fq(u32 fqid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2619) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2620) struct qman_portal *p, *channel_portal;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2621) struct device *dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2622) union qm_mc_command *mcc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2623) union qm_mc_result *mcr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2624) int orl_empty, drain = 0, ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2625) u32 channel, wq, res;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2626) u8 state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2627)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2628) p = get_affine_portal();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2629) dev = p->config->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2630) /* Determine the state of the FQID */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2631) mcc = qm_mc_start(&p->p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2632) qm_fqid_set(&mcc->fq, fqid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2633) qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ_NP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2634) if (!qm_mc_result_timeout(&p->p, &mcr)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2635) dev_err(dev, "QUERYFQ_NP timeout\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2636) ret = -ETIMEDOUT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2637) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2638) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2639)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2640) DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ_NP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2641) state = mcr->queryfq_np.state & QM_MCR_NP_STATE_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2642) if (state == QM_MCR_NP_STATE_OOS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2643) goto out; /* Already OOS, no need to do anymore checks */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2644)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2645) /* Query which channel the FQ is using */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2646) mcc = qm_mc_start(&p->p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2647) qm_fqid_set(&mcc->fq, fqid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2648) qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2649) if (!qm_mc_result_timeout(&p->p, &mcr)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2650) dev_err(dev, "QUERYFQ timeout\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2651) ret = -ETIMEDOUT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2652) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2653) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2654)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2655) DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2656) /* Need to store these since the MCR gets reused */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2657) channel = qm_fqd_get_chan(&mcr->queryfq.fqd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2658) wq = qm_fqd_get_wq(&mcr->queryfq.fqd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2659)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2660) if (channel < qm_channel_pool1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2661) channel_portal = get_portal_for_channel(channel);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2662) if (channel_portal == NULL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2663) dev_err(dev, "Can't find portal for dedicated channel 0x%x\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2664) channel);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2665) ret = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2666) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2667) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2668) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2669) channel_portal = p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2670)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2671) switch (state) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2672) case QM_MCR_NP_STATE_TEN_SCHED:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2673) case QM_MCR_NP_STATE_TRU_SCHED:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2674) case QM_MCR_NP_STATE_ACTIVE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2675) case QM_MCR_NP_STATE_PARKED:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2676) orl_empty = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2677) mcc = qm_mc_start(&channel_portal->p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2678) qm_fqid_set(&mcc->fq, fqid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2679) qm_mc_commit(&channel_portal->p, QM_MCC_VERB_ALTER_RETIRE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2680) if (!qm_mc_result_timeout(&channel_portal->p, &mcr)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2681) dev_err(dev, "ALTER_RETIRE timeout\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2682) ret = -ETIMEDOUT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2683) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2684) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2685) DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) ==
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2686) QM_MCR_VERB_ALTER_RETIRE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2687) res = mcr->result; /* Make a copy as we reuse MCR below */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2688)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2689) if (res == QM_MCR_RESULT_OK)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2690) drain_mr_fqrni(&channel_portal->p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2691)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2692) if (res == QM_MCR_RESULT_PENDING) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2693) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2694) * Need to wait for the FQRN in the message ring, which
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2695) * will only occur once the FQ has been drained. In
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2696) * order for the FQ to drain the portal needs to be set
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2697) * to dequeue from the channel the FQ is scheduled on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2698) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2699) int found_fqrn = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2700) u16 dequeue_wq = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2701)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2702) /* Flag that we need to drain FQ */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2703) drain = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2704)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2705) if (channel >= qm_channel_pool1 &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2706) channel < qm_channel_pool1 + 15) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2707) /* Pool channel, enable the bit in the portal */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2708) dequeue_wq = (channel -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2709) qm_channel_pool1 + 1)<<4 | wq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2710) } else if (channel < qm_channel_pool1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2711) /* Dedicated channel */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2712) dequeue_wq = wq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2713) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2714) dev_err(dev, "Can't recover FQ 0x%x, ch: 0x%x",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2715) fqid, channel);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2716) ret = -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2717) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2718) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2719) /* Set the sdqcr to drain this channel */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2720) if (channel < qm_channel_pool1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2721) qm_dqrr_sdqcr_set(&channel_portal->p,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2722) QM_SDQCR_TYPE_ACTIVE |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2723) QM_SDQCR_CHANNELS_DEDICATED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2724) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2725) qm_dqrr_sdqcr_set(&channel_portal->p,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2726) QM_SDQCR_TYPE_ACTIVE |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2727) QM_SDQCR_CHANNELS_POOL_CONV
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2728) (channel));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2729) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2730) /* Keep draining DQRR while checking the MR*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2731) qm_dqrr_drain_nomatch(&channel_portal->p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2732) /* Process message ring too */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2733) found_fqrn = qm_mr_drain(&channel_portal->p,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2734) FQRN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2735) cpu_relax();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2736) } while (!found_fqrn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2737) /* Restore SDQCR */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2738) qm_dqrr_sdqcr_set(&channel_portal->p,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2739) channel_portal->sdqcr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2740)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2741) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2742) if (res != QM_MCR_RESULT_OK &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2743) res != QM_MCR_RESULT_PENDING) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2744) dev_err(dev, "retire_fq failed: FQ 0x%x, res=0x%x\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2745) fqid, res);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2746) ret = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2747) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2748) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2749) if (!(mcr->alterfq.fqs & QM_MCR_FQS_ORLPRESENT)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2750) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2751) * ORL had no entries, no need to wait until the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2752) * ERNs come in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2753) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2754) orl_empty = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2755) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2756) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2757) * Retirement succeeded, check to see if FQ needs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2758) * to be drained
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2759) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2760) if (drain || mcr->alterfq.fqs & QM_MCR_FQS_NOTEMPTY) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2761) /* FQ is Not Empty, drain using volatile DQ commands */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2762) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2763) u32 vdqcr = fqid | QM_VDQCR_NUMFRAMES_SET(3);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2764)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2765) qm_dqrr_vdqcr_set(&p->p, vdqcr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2766) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2767) * Wait for a dequeue and process the dequeues,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2768) * making sure to empty the ring completely
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2769) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2770) } while (!qm_dqrr_drain_wait(&p->p, fqid, FQ_EMPTY));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2771) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2772)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2773) while (!orl_empty) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2774) /* Wait for the ORL to have been completely drained */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2775) orl_empty = qm_mr_drain(&p->p, FQRL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2776) cpu_relax();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2777) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2778) mcc = qm_mc_start(&p->p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2779) qm_fqid_set(&mcc->fq, fqid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2780) qm_mc_commit(&p->p, QM_MCC_VERB_ALTER_OOS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2781) if (!qm_mc_result_timeout(&p->p, &mcr)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2782) ret = -ETIMEDOUT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2783) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2784) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2785)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2786) DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) ==
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2787) QM_MCR_VERB_ALTER_OOS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2788) if (mcr->result != QM_MCR_RESULT_OK) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2789) dev_err(dev, "OOS after drain fail: FQ 0x%x (0x%x)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2790) fqid, mcr->result);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2791) ret = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2792) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2793) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2794) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2795)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2796) case QM_MCR_NP_STATE_RETIRED:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2797) /* Send OOS Command */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2798) mcc = qm_mc_start(&p->p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2799) qm_fqid_set(&mcc->fq, fqid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2800) qm_mc_commit(&p->p, QM_MCC_VERB_ALTER_OOS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2801) if (!qm_mc_result_timeout(&p->p, &mcr)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2802) ret = -ETIMEDOUT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2803) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2804) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2805)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2806) DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) ==
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2807) QM_MCR_VERB_ALTER_OOS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2808) if (mcr->result != QM_MCR_RESULT_OK) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2809) dev_err(dev, "OOS fail: FQ 0x%x (0x%x)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2810) fqid, mcr->result);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2811) ret = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2812) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2813) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2814) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2815)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2816) case QM_MCR_NP_STATE_OOS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2817) /* Done */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2818) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2819)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2820) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2821) ret = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2822) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2823)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2824) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2825) put_affine_portal();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2826) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2827) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2828)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2829) const struct qm_portal_config *qman_get_qm_portal_config(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2830) struct qman_portal *portal)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2831) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2832) return portal->config;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2833) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2834) EXPORT_SYMBOL(qman_get_qm_portal_config);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2835)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2836) struct gen_pool *qm_fqalloc; /* FQID allocator */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2837) struct gen_pool *qm_qpalloc; /* pool-channel allocator */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2838) struct gen_pool *qm_cgralloc; /* CGR ID allocator */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2839)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2840) static int qman_alloc_range(struct gen_pool *p, u32 *result, u32 cnt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2841) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2842) unsigned long addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2843)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2844) if (!p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2845) return -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2846)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2847) addr = gen_pool_alloc(p, cnt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2848) if (!addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2849) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2850)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2851) *result = addr & ~DPAA_GENALLOC_OFF;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2852)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2853) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2854) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2855)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2856) int qman_alloc_fqid_range(u32 *result, u32 count)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2857) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2858) return qman_alloc_range(qm_fqalloc, result, count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2859) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2860) EXPORT_SYMBOL(qman_alloc_fqid_range);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2861)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2862) int qman_alloc_pool_range(u32 *result, u32 count)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2863) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2864) return qman_alloc_range(qm_qpalloc, result, count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2865) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2866) EXPORT_SYMBOL(qman_alloc_pool_range);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2867)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2868) int qman_alloc_cgrid_range(u32 *result, u32 count)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2869) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2870) return qman_alloc_range(qm_cgralloc, result, count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2871) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2872) EXPORT_SYMBOL(qman_alloc_cgrid_range);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2873)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2874) int qman_release_fqid(u32 fqid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2875) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2876) int ret = qman_shutdown_fq(fqid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2877)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2878) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2879) pr_debug("FQID %d leaked\n", fqid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2880) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2881) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2882)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2883) gen_pool_free(qm_fqalloc, fqid | DPAA_GENALLOC_OFF, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2884) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2885) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2886) EXPORT_SYMBOL(qman_release_fqid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2887)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2888) static int qpool_cleanup(u32 qp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2889) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2890) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2891) * We query all FQDs starting from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2892) * FQID 1 until we get an "invalid FQID" error, looking for non-OOS FQDs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2893) * whose destination channel is the pool-channel being released.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2894) * When a non-OOS FQD is found we attempt to clean it up
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2895) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2896) struct qman_fq fq = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2897) .fqid = QM_FQID_RANGE_START
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2898) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2899) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2900)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2901) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2902) struct qm_mcr_queryfq_np np;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2903)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2904) err = qman_query_fq_np(&fq, &np);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2905) if (err == -ERANGE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2906) /* FQID range exceeded, found no problems */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2907) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2908) else if (WARN_ON(err))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2909) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2910)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2911) if ((np.state & QM_MCR_NP_STATE_MASK) != QM_MCR_NP_STATE_OOS) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2912) struct qm_fqd fqd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2913)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2914) err = qman_query_fq(&fq, &fqd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2915) if (WARN_ON(err))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2916) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2917) if (qm_fqd_get_chan(&fqd) == qp) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2918) /* The channel is the FQ's target, clean it */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2919) err = qman_shutdown_fq(fq.fqid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2920) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2921) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2922) * Couldn't shut down the FQ
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2923) * so the pool must be leaked
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2924) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2925) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2926) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2927) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2928) /* Move to the next FQID */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2929) fq.fqid++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2930) } while (1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2931) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2932)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2933) int qman_release_pool(u32 qp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2934) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2935) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2936)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2937) ret = qpool_cleanup(qp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2938) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2939) pr_debug("CHID %d leaked\n", qp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2940) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2941) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2942)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2943) gen_pool_free(qm_qpalloc, qp | DPAA_GENALLOC_OFF, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2944) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2945) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2946) EXPORT_SYMBOL(qman_release_pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2947)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2948) static int cgr_cleanup(u32 cgrid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2949) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2950) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2951) * query all FQDs starting from FQID 1 until we get an "invalid FQID"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2952) * error, looking for non-OOS FQDs whose CGR is the CGR being released
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2953) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2954) struct qman_fq fq = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2955) .fqid = QM_FQID_RANGE_START
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2956) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2957) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2958)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2959) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2960) struct qm_mcr_queryfq_np np;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2961)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2962) err = qman_query_fq_np(&fq, &np);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2963) if (err == -ERANGE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2964) /* FQID range exceeded, found no problems */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2965) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2966) else if (WARN_ON(err))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2967) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2968)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2969) if ((np.state & QM_MCR_NP_STATE_MASK) != QM_MCR_NP_STATE_OOS) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2970) struct qm_fqd fqd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2971)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2972) err = qman_query_fq(&fq, &fqd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2973) if (WARN_ON(err))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2974) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2975) if (be16_to_cpu(fqd.fq_ctrl) & QM_FQCTRL_CGE &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2976) fqd.cgid == cgrid) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2977) pr_err("CRGID 0x%x is being used by FQID 0x%x, CGR will be leaked\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2978) cgrid, fq.fqid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2979) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2980) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2981) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2982) /* Move to the next FQID */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2983) fq.fqid++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2984) } while (1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2985) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2986)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2987) int qman_release_cgrid(u32 cgrid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2988) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2989) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2990)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2991) ret = cgr_cleanup(cgrid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2992) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2993) pr_debug("CGRID %d leaked\n", cgrid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2994) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2995) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2996)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2997) gen_pool_free(qm_cgralloc, cgrid | DPAA_GENALLOC_OFF, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2998) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2999) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3000) EXPORT_SYMBOL(qman_release_cgrid);
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags