^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) /* Copyright 2008 - 2016 Freescale Semiconductor, Inc.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Redistribution and use in source and binary forms, with or without
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * modification, are permitted provided that the following conditions are met:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * * Redistributions of source code must retain the above copyright
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * notice, this list of conditions and the following disclaimer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * * Redistributions in binary form must reproduce the above copyright
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) * notice, this list of conditions and the following disclaimer in the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) * documentation and/or other materials provided with the distribution.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) * * Neither the name of Freescale Semiconductor nor the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) * names of its contributors may be used to endorse or promote products
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) * derived from this software without specific prior written permission.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) * ALTERNATIVELY, this software may be distributed under the terms of the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) * GNU General Public License ("GPL") as published by the Free Software
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) * Foundation, either version 2 of that License or (at your option) any
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) * later version.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) #include "bman_priv.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) #define IRQNAME "BMan portal %d"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) #define MAX_IRQNAME 16 /* big enough for "BMan portal %d" */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) /* Portal register assists */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) #if defined(CONFIG_ARM) || defined(CONFIG_ARM64)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) /* Cache-inhibited register offsets */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) #define BM_REG_RCR_PI_CINH 0x3000
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) #define BM_REG_RCR_CI_CINH 0x3100
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) #define BM_REG_RCR_ITR 0x3200
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) #define BM_REG_CFG 0x3300
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) #define BM_REG_SCN(n) (0x3400 + ((n) << 6))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) #define BM_REG_ISR 0x3e00
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) #define BM_REG_IER 0x3e40
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) #define BM_REG_ISDR 0x3e80
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) #define BM_REG_IIR 0x3ec0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) /* Cache-enabled register offsets */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) #define BM_CL_CR 0x0000
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) #define BM_CL_RR0 0x0100
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) #define BM_CL_RR1 0x0140
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) #define BM_CL_RCR 0x1000
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) #define BM_CL_RCR_PI_CENA 0x3000
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) #define BM_CL_RCR_CI_CENA 0x3100
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) /* Cache-inhibited register offsets */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) #define BM_REG_RCR_PI_CINH 0x0000
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) #define BM_REG_RCR_CI_CINH 0x0004
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) #define BM_REG_RCR_ITR 0x0008
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) #define BM_REG_CFG 0x0100
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) #define BM_REG_SCN(n) (0x0200 + ((n) << 2))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) #define BM_REG_ISR 0x0e00
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) #define BM_REG_IER 0x0e04
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) #define BM_REG_ISDR 0x0e08
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) #define BM_REG_IIR 0x0e0c
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) /* Cache-enabled register offsets */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) #define BM_CL_CR 0x0000
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) #define BM_CL_RR0 0x0100
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) #define BM_CL_RR1 0x0140
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) #define BM_CL_RCR 0x1000
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) #define BM_CL_RCR_PI_CENA 0x3000
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) #define BM_CL_RCR_CI_CENA 0x3100
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) * Portal modes.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) * Enum types;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) * pmode == production mode
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) * cmode == consumption mode,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) * Enum values use 3 letter codes. First letter matches the portal mode,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) * remaining two letters indicate;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) * ci == cache-inhibited portal register
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) * ce == cache-enabled portal register
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) * vb == in-band valid-bit (cache-enabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) enum bm_rcr_pmode { /* matches BCSP_CFG::RPM */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) bm_rcr_pci = 0, /* PI index, cache-inhibited */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) bm_rcr_pce = 1, /* PI index, cache-enabled */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) bm_rcr_pvb = 2 /* valid-bit */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) enum bm_rcr_cmode { /* s/w-only */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) bm_rcr_cci, /* CI index, cache-inhibited */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) bm_rcr_cce /* CI index, cache-enabled */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) /* --- Portal structures --- */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) #define BM_RCR_SIZE 8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) /* Release Command */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) struct bm_rcr_entry {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) union {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) struct {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) u8 _ncw_verb; /* writes to this are non-coherent */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) u8 bpid; /* used with BM_RCR_VERB_CMD_BPID_SINGLE */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) u8 __reserved1[62];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) struct bm_buffer bufs[8];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) #define BM_RCR_VERB_VBIT 0x80
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) #define BM_RCR_VERB_CMD_MASK 0x70 /* one of two values; */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) #define BM_RCR_VERB_CMD_BPID_SINGLE 0x20
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) #define BM_RCR_VERB_CMD_BPID_MULTI 0x30
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) #define BM_RCR_VERB_BUFCOUNT_MASK 0x0f /* values 1..8 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) struct bm_rcr {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) struct bm_rcr_entry *ring, *cursor;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) u8 ci, available, ithresh, vbit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) #ifdef CONFIG_FSL_DPAA_CHECKING
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) u32 busy;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) enum bm_rcr_pmode pmode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) enum bm_rcr_cmode cmode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) /* MC (Management Command) command */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) struct bm_mc_command {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) u8 _ncw_verb; /* writes to this are non-coherent */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) u8 bpid; /* used by acquire command */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) u8 __reserved[62];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) #define BM_MCC_VERB_VBIT 0x80
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) #define BM_MCC_VERB_CMD_MASK 0x70 /* where the verb contains; */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) #define BM_MCC_VERB_CMD_ACQUIRE 0x10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) #define BM_MCC_VERB_CMD_QUERY 0x40
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) #define BM_MCC_VERB_ACQUIRE_BUFCOUNT 0x0f /* values 1..8 go here */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) /* MC result, Acquire and Query Response */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) union bm_mc_result {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) struct {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) u8 verb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) u8 bpid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) u8 __reserved[62];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) struct bm_buffer bufs[8];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) #define BM_MCR_VERB_VBIT 0x80
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) #define BM_MCR_VERB_CMD_MASK BM_MCC_VERB_CMD_MASK
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) #define BM_MCR_VERB_CMD_ACQUIRE BM_MCC_VERB_CMD_ACQUIRE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) #define BM_MCR_VERB_CMD_QUERY BM_MCC_VERB_CMD_QUERY
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) #define BM_MCR_VERB_CMD_ERR_INVALID 0x60
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) #define BM_MCR_VERB_CMD_ERR_ECC 0x70
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) #define BM_MCR_VERB_ACQUIRE_BUFCOUNT BM_MCC_VERB_ACQUIRE_BUFCOUNT /* 0..8 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) #define BM_MCR_TIMEOUT 10000 /* us */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) struct bm_mc {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) struct bm_mc_command *cr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) union bm_mc_result *rr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) u8 rridx, vbit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) #ifdef CONFIG_FSL_DPAA_CHECKING
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) enum {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) /* Can only be _mc_start()ed */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) mc_idle,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) /* Can only be _mc_commit()ed or _mc_abort()ed */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) mc_user,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) /* Can only be _mc_retry()ed */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) mc_hw
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) } state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) struct bm_addr {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) void *ce; /* cache-enabled */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) __be32 *ce_be; /* Same as above but for direct access */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) void __iomem *ci; /* cache-inhibited */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) struct bm_portal {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) struct bm_addr addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) struct bm_rcr rcr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) struct bm_mc mc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) } ____cacheline_aligned;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) /* Cache-inhibited register access. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) static inline u32 bm_in(struct bm_portal *p, u32 offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) return ioread32be(p->addr.ci + offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) static inline void bm_out(struct bm_portal *p, u32 offset, u32 val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) iowrite32be(val, p->addr.ci + offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) /* Cache Enabled Portal Access */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) static inline void bm_cl_invalidate(struct bm_portal *p, u32 offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) dpaa_invalidate(p->addr.ce + offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) static inline void bm_cl_touch_ro(struct bm_portal *p, u32 offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) dpaa_touch_ro(p->addr.ce + offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) static inline u32 bm_ce_in(struct bm_portal *p, u32 offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) return be32_to_cpu(*(p->addr.ce_be + (offset/4)));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) struct bman_portal {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) struct bm_portal p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) /* interrupt sources processed by portal_isr(), configurable */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) unsigned long irq_sources;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) /* probing time config params for cpu-affine portals */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) const struct bm_portal_config *config;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) char irqname[MAX_IRQNAME];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) static cpumask_t affine_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) static DEFINE_SPINLOCK(affine_mask_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) static DEFINE_PER_CPU(struct bman_portal, bman_affine_portal);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) static inline struct bman_portal *get_affine_portal(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) return &get_cpu_var(bman_affine_portal);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) static inline void put_affine_portal(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) put_cpu_var(bman_affine_portal);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) * This object type refers to a pool, it isn't *the* pool. There may be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) * more than one such object per BMan buffer pool, eg. if different users of the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) * pool are operating via different portals.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) struct bman_pool {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) /* index of the buffer pool to encapsulate (0-63) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) u32 bpid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) /* Used for hash-table admin when using depletion notifications. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) struct bman_portal *portal;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) struct bman_pool *next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) static u32 poll_portal_slow(struct bman_portal *p, u32 is);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) static irqreturn_t portal_isr(int irq, void *ptr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) struct bman_portal *p = ptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) struct bm_portal *portal = &p->p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) u32 clear = p->irq_sources;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) u32 is = bm_in(portal, BM_REG_ISR) & p->irq_sources;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) if (unlikely(!is))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) return IRQ_NONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) clear |= poll_portal_slow(p, is);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) bm_out(portal, BM_REG_ISR, clear);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) return IRQ_HANDLED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) /* --- RCR API --- */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) #define RCR_SHIFT ilog2(sizeof(struct bm_rcr_entry))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) #define RCR_CARRY (uintptr_t)(BM_RCR_SIZE << RCR_SHIFT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) /* Bit-wise logic to wrap a ring pointer by clearing the "carry bit" */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) static struct bm_rcr_entry *rcr_carryclear(struct bm_rcr_entry *p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) uintptr_t addr = (uintptr_t)p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) addr &= ~RCR_CARRY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) return (struct bm_rcr_entry *)addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) #ifdef CONFIG_FSL_DPAA_CHECKING
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) /* Bit-wise logic to convert a ring pointer to a ring index */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) static int rcr_ptr2idx(struct bm_rcr_entry *e)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) return ((uintptr_t)e >> RCR_SHIFT) & (BM_RCR_SIZE - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) /* Increment the 'cursor' ring pointer, taking 'vbit' into account */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) static inline void rcr_inc(struct bm_rcr *rcr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) /* increment to the next RCR pointer and handle overflow and 'vbit' */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) struct bm_rcr_entry *partial = rcr->cursor + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) rcr->cursor = rcr_carryclear(partial);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) if (partial != rcr->cursor)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) rcr->vbit ^= BM_RCR_VERB_VBIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) static int bm_rcr_get_avail(struct bm_portal *portal)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) struct bm_rcr *rcr = &portal->rcr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) return rcr->available;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) static int bm_rcr_get_fill(struct bm_portal *portal)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) struct bm_rcr *rcr = &portal->rcr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) return BM_RCR_SIZE - 1 - rcr->available;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) static void bm_rcr_set_ithresh(struct bm_portal *portal, u8 ithresh)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) struct bm_rcr *rcr = &portal->rcr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) rcr->ithresh = ithresh;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) bm_out(portal, BM_REG_RCR_ITR, ithresh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) static void bm_rcr_cce_prefetch(struct bm_portal *portal)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) __maybe_unused struct bm_rcr *rcr = &portal->rcr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) DPAA_ASSERT(rcr->cmode == bm_rcr_cce);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) bm_cl_touch_ro(portal, BM_CL_RCR_CI_CENA);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) static u8 bm_rcr_cce_update(struct bm_portal *portal)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) struct bm_rcr *rcr = &portal->rcr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) u8 diff, old_ci = rcr->ci;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) DPAA_ASSERT(rcr->cmode == bm_rcr_cce);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) rcr->ci = bm_ce_in(portal, BM_CL_RCR_CI_CENA) & (BM_RCR_SIZE - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) bm_cl_invalidate(portal, BM_CL_RCR_CI_CENA);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) diff = dpaa_cyc_diff(BM_RCR_SIZE, old_ci, rcr->ci);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) rcr->available += diff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) return diff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) static inline struct bm_rcr_entry *bm_rcr_start(struct bm_portal *portal)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) struct bm_rcr *rcr = &portal->rcr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) DPAA_ASSERT(!rcr->busy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) if (!rcr->available)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) #ifdef CONFIG_FSL_DPAA_CHECKING
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) rcr->busy = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) dpaa_zero(rcr->cursor);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) return rcr->cursor;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) static inline void bm_rcr_pvb_commit(struct bm_portal *portal, u8 myverb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) struct bm_rcr *rcr = &portal->rcr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) struct bm_rcr_entry *rcursor;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) DPAA_ASSERT(rcr->busy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) DPAA_ASSERT(rcr->pmode == bm_rcr_pvb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) DPAA_ASSERT(rcr->available >= 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) dma_wmb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) rcursor = rcr->cursor;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) rcursor->_ncw_verb = myverb | rcr->vbit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) dpaa_flush(rcursor);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) rcr_inc(rcr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) rcr->available--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) #ifdef CONFIG_FSL_DPAA_CHECKING
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) rcr->busy = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) static int bm_rcr_init(struct bm_portal *portal, enum bm_rcr_pmode pmode,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) enum bm_rcr_cmode cmode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) struct bm_rcr *rcr = &portal->rcr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) u32 cfg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) u8 pi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) rcr->ring = portal->addr.ce + BM_CL_RCR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) rcr->ci = bm_in(portal, BM_REG_RCR_CI_CINH) & (BM_RCR_SIZE - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) pi = bm_in(portal, BM_REG_RCR_PI_CINH) & (BM_RCR_SIZE - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) rcr->cursor = rcr->ring + pi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) rcr->vbit = (bm_in(portal, BM_REG_RCR_PI_CINH) & BM_RCR_SIZE) ?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) BM_RCR_VERB_VBIT : 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) rcr->available = BM_RCR_SIZE - 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) - dpaa_cyc_diff(BM_RCR_SIZE, rcr->ci, pi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) rcr->ithresh = bm_in(portal, BM_REG_RCR_ITR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) #ifdef CONFIG_FSL_DPAA_CHECKING
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) rcr->busy = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) rcr->pmode = pmode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) rcr->cmode = cmode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) cfg = (bm_in(portal, BM_REG_CFG) & 0xffffffe0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) | (pmode & 0x3); /* BCSP_CFG::RPM */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) bm_out(portal, BM_REG_CFG, cfg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) static void bm_rcr_finish(struct bm_portal *portal)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) #ifdef CONFIG_FSL_DPAA_CHECKING
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) struct bm_rcr *rcr = &portal->rcr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) DPAA_ASSERT(!rcr->busy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) i = bm_in(portal, BM_REG_RCR_PI_CINH) & (BM_RCR_SIZE - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) if (i != rcr_ptr2idx(rcr->cursor))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) pr_crit("losing uncommitted RCR entries\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) i = bm_in(portal, BM_REG_RCR_CI_CINH) & (BM_RCR_SIZE - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) if (i != rcr->ci)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) pr_crit("missing existing RCR completions\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) if (rcr->ci != rcr_ptr2idx(rcr->cursor))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) pr_crit("RCR destroyed unquiesced\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) /* --- Management command API --- */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) static int bm_mc_init(struct bm_portal *portal)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) struct bm_mc *mc = &portal->mc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) mc->cr = portal->addr.ce + BM_CL_CR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) mc->rr = portal->addr.ce + BM_CL_RR0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) mc->rridx = (mc->cr->_ncw_verb & BM_MCC_VERB_VBIT) ?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) 0 : 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) mc->vbit = mc->rridx ? BM_MCC_VERB_VBIT : 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) #ifdef CONFIG_FSL_DPAA_CHECKING
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) mc->state = mc_idle;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) static void bm_mc_finish(struct bm_portal *portal)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) #ifdef CONFIG_FSL_DPAA_CHECKING
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) struct bm_mc *mc = &portal->mc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) DPAA_ASSERT(mc->state == mc_idle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) if (mc->state != mc_idle)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) pr_crit("Losing incomplete MC command\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) static inline struct bm_mc_command *bm_mc_start(struct bm_portal *portal)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) struct bm_mc *mc = &portal->mc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) DPAA_ASSERT(mc->state == mc_idle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) #ifdef CONFIG_FSL_DPAA_CHECKING
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) mc->state = mc_user;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) dpaa_zero(mc->cr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) return mc->cr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) static inline void bm_mc_commit(struct bm_portal *portal, u8 myverb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) struct bm_mc *mc = &portal->mc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) union bm_mc_result *rr = mc->rr + mc->rridx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) DPAA_ASSERT(mc->state == mc_user);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) dma_wmb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) mc->cr->_ncw_verb = myverb | mc->vbit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) dpaa_flush(mc->cr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) dpaa_invalidate_touch_ro(rr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) #ifdef CONFIG_FSL_DPAA_CHECKING
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) mc->state = mc_hw;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) static inline union bm_mc_result *bm_mc_result(struct bm_portal *portal)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) struct bm_mc *mc = &portal->mc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) union bm_mc_result *rr = mc->rr + mc->rridx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) DPAA_ASSERT(mc->state == mc_hw);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) * The inactive response register's verb byte always returns zero until
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) * its command is submitted and completed. This includes the valid-bit,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) * in case you were wondering...
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) if (!rr->verb) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) dpaa_invalidate_touch_ro(rr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) mc->rridx ^= 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) mc->vbit ^= BM_MCC_VERB_VBIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) #ifdef CONFIG_FSL_DPAA_CHECKING
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) mc->state = mc_idle;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) return rr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) static inline int bm_mc_result_timeout(struct bm_portal *portal,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) union bm_mc_result **mcr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) int timeout = BM_MCR_TIMEOUT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) *mcr = bm_mc_result(portal);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) if (*mcr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) udelay(1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) } while (--timeout);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) return timeout;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) /* Disable all BSCN interrupts for the portal */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) static void bm_isr_bscn_disable(struct bm_portal *portal)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) bm_out(portal, BM_REG_SCN(0), 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) bm_out(portal, BM_REG_SCN(1), 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) static int bman_create_portal(struct bman_portal *portal,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) const struct bm_portal_config *c)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) struct bm_portal *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) p = &portal->p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) * prep the low-level portal struct with the mapped addresses from the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) * config, everything that follows depends on it and "config" is more
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) * for (de)reference...
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) p->addr.ce = c->addr_virt_ce;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) p->addr.ce_be = c->addr_virt_ce;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) p->addr.ci = c->addr_virt_ci;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) if (bm_rcr_init(p, bm_rcr_pvb, bm_rcr_cce)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) dev_err(c->dev, "RCR initialisation failed\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) goto fail_rcr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) if (bm_mc_init(p)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) dev_err(c->dev, "MC initialisation failed\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) goto fail_mc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) * Default to all BPIDs disabled, we enable as required at
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) * run-time.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) bm_isr_bscn_disable(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) /* Write-to-clear any stale interrupt status bits */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) bm_out(p, BM_REG_ISDR, 0xffffffff);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) portal->irq_sources = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) bm_out(p, BM_REG_IER, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) bm_out(p, BM_REG_ISR, 0xffffffff);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) snprintf(portal->irqname, MAX_IRQNAME, IRQNAME, c->cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) if (request_irq(c->irq, portal_isr, 0, portal->irqname, portal)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) dev_err(c->dev, "request_irq() failed\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) goto fail_irq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) if (dpaa_set_portal_irq_affinity(c->dev, c->irq, c->cpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) goto fail_affinity;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) /* Need RCR to be empty before continuing */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) ret = bm_rcr_get_fill(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) dev_err(c->dev, "RCR unclean\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) goto fail_rcr_empty;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) /* Success */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) portal->config = c;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) bm_out(p, BM_REG_ISDR, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) bm_out(p, BM_REG_IIR, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) fail_rcr_empty:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) fail_affinity:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) free_irq(c->irq, portal);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) fail_irq:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) bm_mc_finish(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) fail_mc:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) bm_rcr_finish(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) fail_rcr:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) struct bman_portal *bman_create_affine_portal(const struct bm_portal_config *c)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) struct bman_portal *portal;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) portal = &per_cpu(bman_affine_portal, c->cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) err = bman_create_portal(portal, c);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) spin_lock(&affine_mask_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) cpumask_set_cpu(c->cpu, &affine_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) spin_unlock(&affine_mask_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) return portal;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) static u32 poll_portal_slow(struct bman_portal *p, u32 is)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) u32 ret = is;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) if (is & BM_PIRQ_RCRI) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) bm_rcr_cce_update(&p->p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) bm_rcr_set_ithresh(&p->p, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) bm_out(&p->p, BM_REG_ISR, BM_PIRQ_RCRI);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) is &= ~BM_PIRQ_RCRI;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) /* There should be no status register bits left undefined */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) DPAA_ASSERT(!is);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) int bman_p_irqsource_add(struct bman_portal *p, u32 bits)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) unsigned long irqflags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) local_irq_save(irqflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) p->irq_sources |= bits & BM_PIRQ_VISIBLE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) bm_out(&p->p, BM_REG_IER, p->irq_sources);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) local_irq_restore(irqflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) int bm_shutdown_pool(u32 bpid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) int err = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) struct bm_mc_command *bm_cmd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) union bm_mc_result *bm_res;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) struct bman_portal *p = get_affine_portal();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) while (1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) /* Acquire buffers until empty */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) bm_cmd = bm_mc_start(&p->p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) bm_cmd->bpid = bpid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) bm_mc_commit(&p->p, BM_MCC_VERB_CMD_ACQUIRE | 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) if (!bm_mc_result_timeout(&p->p, &bm_res)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) pr_crit("BMan Acquire Command timedout\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) err = -ETIMEDOUT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) goto done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) if (!(bm_res->verb & BM_MCR_VERB_ACQUIRE_BUFCOUNT)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) /* Pool is empty */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) goto done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) done:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) put_affine_portal();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) struct gen_pool *bm_bpalloc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) static int bm_alloc_bpid_range(u32 *result, u32 count)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) unsigned long addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) addr = gen_pool_alloc(bm_bpalloc, count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) if (!addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) *result = addr & ~DPAA_GENALLOC_OFF;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) static int bm_release_bpid(u32 bpid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) ret = bm_shutdown_pool(bpid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) pr_debug("BPID %d leaked\n", bpid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) gen_pool_free(bm_bpalloc, bpid | DPAA_GENALLOC_OFF, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) struct bman_pool *bman_new_pool(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) struct bman_pool *pool = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) u32 bpid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) if (bm_alloc_bpid_range(&bpid, 1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) pool = kmalloc(sizeof(*pool), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) if (!pool)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) goto err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) pool->bpid = bpid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) return pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) err:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) bm_release_bpid(bpid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) kfree(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) EXPORT_SYMBOL(bman_new_pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) void bman_free_pool(struct bman_pool *pool)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) bm_release_bpid(pool->bpid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) kfree(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) EXPORT_SYMBOL(bman_free_pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) int bman_get_bpid(const struct bman_pool *pool)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) return pool->bpid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) EXPORT_SYMBOL(bman_get_bpid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) static void update_rcr_ci(struct bman_portal *p, int avail)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) if (avail)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) bm_rcr_cce_prefetch(&p->p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) bm_rcr_cce_update(&p->p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) int bman_release(struct bman_pool *pool, const struct bm_buffer *bufs, u8 num)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) struct bman_portal *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) struct bm_rcr_entry *r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) unsigned long irqflags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) int avail, timeout = 1000; /* 1ms */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) int i = num - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) DPAA_ASSERT(num > 0 && num <= 8);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) p = get_affine_portal();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) local_irq_save(irqflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) avail = bm_rcr_get_avail(&p->p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) if (avail < 2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) update_rcr_ci(p, avail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) r = bm_rcr_start(&p->p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) local_irq_restore(irqflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) put_affine_portal();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) if (likely(r))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) udelay(1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) } while (--timeout);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) if (unlikely(!timeout))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) return -ETIMEDOUT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) p = get_affine_portal();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) local_irq_save(irqflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) * we can copy all but the first entry, as this can trigger badness
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) * with the valid-bit
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) bm_buffer_set64(r->bufs, bm_buffer_get64(bufs));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) bm_buffer_set_bpid(r->bufs, pool->bpid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) if (i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) memcpy(&r->bufs[1], &bufs[1], i * sizeof(bufs[0]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) bm_rcr_pvb_commit(&p->p, BM_RCR_VERB_CMD_BPID_SINGLE |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) (num & BM_RCR_VERB_BUFCOUNT_MASK));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) local_irq_restore(irqflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) put_affine_portal();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) EXPORT_SYMBOL(bman_release);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) int bman_acquire(struct bman_pool *pool, struct bm_buffer *bufs, u8 num)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) struct bman_portal *p = get_affine_portal();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) struct bm_mc_command *mcc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) union bm_mc_result *mcr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) DPAA_ASSERT(num > 0 && num <= 8);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) mcc = bm_mc_start(&p->p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) mcc->bpid = pool->bpid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) bm_mc_commit(&p->p, BM_MCC_VERB_CMD_ACQUIRE |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) (num & BM_MCC_VERB_ACQUIRE_BUFCOUNT));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) if (!bm_mc_result_timeout(&p->p, &mcr)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) put_affine_portal();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) pr_crit("BMan Acquire Timeout\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) return -ETIMEDOUT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) ret = mcr->verb & BM_MCR_VERB_ACQUIRE_BUFCOUNT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) if (bufs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) memcpy(&bufs[0], &mcr->bufs[0], num * sizeof(bufs[0]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) put_affine_portal();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) if (ret != num)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) ret = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) EXPORT_SYMBOL(bman_acquire);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) const struct bm_portal_config *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) bman_get_bm_portal_config(const struct bman_portal *portal)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) return portal->config;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) }
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags