^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) /* Copyright (c) 2015-2018, The Linux Foundation. All rights reserved.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * Copyright (C) 2018-2020 Linaro Ltd.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) #include <linux/types.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include <linux/bits.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <linux/bitfield.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/mutex.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/completion.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/io.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/bug.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/interrupt.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/platform_device.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/netdevice.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include "gsi.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include "gsi_reg.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include "gsi_private.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include "gsi_trans.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include "ipa_gsi.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #include "ipa_data.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) * DOC: The IPA Generic Software Interface
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) * The generic software interface (GSI) is an integral component of the IPA,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) * providing a well-defined communication layer between the AP subsystem
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) * and the IPA core. The modem uses the GSI layer as well.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) * -------- ---------
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) * | | | |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) * | AP +<---. .----+ Modem |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) * | +--. | | .->+ |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) * | | | | | | | |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) * -------- | | | | ---------
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) * v | v |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) * --+-+---+-+--
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) * | GSI |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) * |-----------|
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) * | |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) * | IPA |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) * | |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) * -------------
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) * In the above diagram, the AP and Modem represent "execution environments"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) * (EEs), which are independent operating environments that use the IPA for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) * data transfer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) * Each EE uses a set of unidirectional GSI "channels," which allow transfer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) * of data to or from the IPA. A channel is implemented as a ring buffer,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) * with a DRAM-resident array of "transfer elements" (TREs) available to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) * describe transfers to or from other EEs through the IPA. A transfer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) * element can also contain an immediate command, requesting the IPA perform
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) * actions other than data transfer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) * Each TRE refers to a block of data--also located DRAM. After writing one
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) * or more TREs to a channel, the writer (either the IPA or an EE) writes a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) * doorbell register to inform the receiving side how many elements have
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) * been written.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) * Each channel has a GSI "event ring" associated with it. An event ring
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) * is implemented very much like a channel ring, but is always directed from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) * the IPA to an EE. The IPA notifies an EE (such as the AP) about channel
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) * events by adding an entry to the event ring associated with the channel.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) * The GSI then writes its doorbell for the event ring, causing the target
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) * EE to be interrupted. Each entry in an event ring contains a pointer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) * to the channel TRE whose completion the event represents.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) * Each TRE in a channel ring has a set of flags. One flag indicates whether
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) * the completion of the transfer operation generates an entry (and possibly
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) * an interrupt) in the channel's event ring. Other flags allow transfer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) * elements to be chained together, forming a single logical transaction.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) * TRE flags are used to control whether and when interrupts are generated
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) * to signal completion of channel transfers.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) * Elements in channel and event rings are completed (or consumed) strictly
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) * in order. Completion of one entry implies the completion of all preceding
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) * entries. A single completion interrupt can therefore communicate the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) * completion of many transfers.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) * Note that all GSI registers are little-endian, which is the assumed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) * endianness of I/O space accesses. The accessor functions perform byte
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) * swapping if needed (i.e., for a big endian CPU).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) /* Delay period for interrupt moderation (in 32KHz IPA internal timer ticks) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) #define GSI_EVT_RING_INT_MODT (32 * 1) /* 1ms under 32KHz clock */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) #define GSI_CMD_TIMEOUT 5 /* seconds */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) #define GSI_CHANNEL_STOP_RX_RETRIES 10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) #define GSI_MHI_EVENT_ID_START 10 /* 1st reserved event id */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) #define GSI_MHI_EVENT_ID_END 16 /* Last reserved event id */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) #define GSI_ISR_MAX_ITER 50 /* Detect interrupt storms */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) /* An entry in an event ring */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) struct gsi_event {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) __le64 xfer_ptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) __le16 len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) u8 reserved1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) u8 code;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) __le16 reserved2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) u8 type;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) u8 chid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) /* Hardware values from the error log register error code field */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) enum gsi_err_code {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) GSI_INVALID_TRE_ERR = 0x1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) GSI_OUT_OF_BUFFERS_ERR = 0x2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) GSI_OUT_OF_RESOURCES_ERR = 0x3,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) GSI_UNSUPPORTED_INTER_EE_OP_ERR = 0x4,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) GSI_EVT_RING_EMPTY_ERR = 0x5,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) GSI_NON_ALLOCATED_EVT_ACCESS_ERR = 0x6,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) GSI_HWO_1_ERR = 0x8,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) /* Hardware values from the error log register error type field */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) enum gsi_err_type {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) GSI_ERR_TYPE_GLOB = 0x1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) GSI_ERR_TYPE_CHAN = 0x2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) GSI_ERR_TYPE_EVT = 0x3,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) /* Hardware values used when programming an event ring */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) enum gsi_evt_chtype {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) GSI_EVT_CHTYPE_MHI_EV = 0x0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) GSI_EVT_CHTYPE_XHCI_EV = 0x1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) GSI_EVT_CHTYPE_GPI_EV = 0x2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) GSI_EVT_CHTYPE_XDCI_EV = 0x3,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) /* Hardware values used when programming a channel */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) enum gsi_channel_protocol {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) GSI_CHANNEL_PROTOCOL_MHI = 0x0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) GSI_CHANNEL_PROTOCOL_XHCI = 0x1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) GSI_CHANNEL_PROTOCOL_GPI = 0x2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) GSI_CHANNEL_PROTOCOL_XDCI = 0x3,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) /* Hardware values representing an event ring immediate command opcode */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) enum gsi_evt_cmd_opcode {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) GSI_EVT_ALLOCATE = 0x0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) GSI_EVT_RESET = 0x9,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) GSI_EVT_DE_ALLOC = 0xa,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) /* Hardware values representing a generic immediate command opcode */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) enum gsi_generic_cmd_opcode {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) GSI_GENERIC_HALT_CHANNEL = 0x1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) GSI_GENERIC_ALLOCATE_CHANNEL = 0x2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) /* Hardware values representing a channel immediate command opcode */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) enum gsi_ch_cmd_opcode {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) GSI_CH_ALLOCATE = 0x0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) GSI_CH_START = 0x1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) GSI_CH_STOP = 0x2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) GSI_CH_RESET = 0x9,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) GSI_CH_DE_ALLOC = 0xa,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) /** gsi_channel_scratch_gpi - GPI protocol scratch register
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) * @max_outstanding_tre:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) * Defines the maximum number of TREs allowed in a single transaction
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) * on a channel (in bytes). This determines the amount of prefetch
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) * performed by the hardware. We configure this to equal the size of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) * the TLV FIFO for the channel.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) * @outstanding_threshold:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) * Defines the threshold (in bytes) determining when the sequencer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) * should update the channel doorbell. We configure this to equal
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) * the size of two TREs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) struct gsi_channel_scratch_gpi {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) u64 reserved1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) u16 reserved2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) u16 max_outstanding_tre;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) u16 reserved3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) u16 outstanding_threshold;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) /** gsi_channel_scratch - channel scratch configuration area
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) * The exact interpretation of this register is protocol-specific.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) * We only use GPI channels; see struct gsi_channel_scratch_gpi, above.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) union gsi_channel_scratch {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) struct gsi_channel_scratch_gpi gpi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) struct {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) u32 word1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) u32 word2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) u32 word3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) u32 word4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) } data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) /* Check things that can be validated at build time. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) static void gsi_validate_build(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) /* This is used as a divisor */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) BUILD_BUG_ON(!GSI_RING_ELEMENT_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) /* Code assumes the size of channel and event ring element are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) * the same (and fixed). Make sure the size of an event ring
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) * element is what's expected.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) BUILD_BUG_ON(sizeof(struct gsi_event) != GSI_RING_ELEMENT_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) /* Hardware requires a 2^n ring size. We ensure the number of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) * elements in an event ring is a power of 2 elsewhere; this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) * ensure the elements themselves meet the requirement.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) BUILD_BUG_ON(!is_power_of_2(GSI_RING_ELEMENT_SIZE));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) /* The channel element size must fit in this field */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) BUILD_BUG_ON(GSI_RING_ELEMENT_SIZE > field_max(ELEMENT_SIZE_FMASK));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) /* The event ring element size must fit in this field */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) BUILD_BUG_ON(GSI_RING_ELEMENT_SIZE > field_max(EV_ELEMENT_SIZE_FMASK));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) /* Return the channel id associated with a given channel */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) static u32 gsi_channel_id(struct gsi_channel *channel)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) return channel - &channel->gsi->channel[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) static void gsi_irq_ieob_enable(struct gsi *gsi, u32 evt_ring_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) u32 val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) gsi->event_enable_bitmap |= BIT(evt_ring_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) val = gsi->event_enable_bitmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) iowrite32(val, gsi->virt + GSI_CNTXT_SRC_IEOB_IRQ_MSK_OFFSET);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) static void gsi_irq_ieob_disable(struct gsi *gsi, u32 evt_ring_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) u32 val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) gsi->event_enable_bitmap &= ~BIT(evt_ring_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) val = gsi->event_enable_bitmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) iowrite32(val, gsi->virt + GSI_CNTXT_SRC_IEOB_IRQ_MSK_OFFSET);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) /* Enable all GSI_interrupt types */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) static void gsi_irq_enable(struct gsi *gsi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) u32 val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) /* We don't use inter-EE channel or event interrupts */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) val = GSI_CNTXT_TYPE_IRQ_MSK_ALL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) val &= ~INTER_EE_CH_CTRL_FMASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) val &= ~INTER_EE_EV_CTRL_FMASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) iowrite32(val, gsi->virt + GSI_CNTXT_TYPE_IRQ_MSK_OFFSET);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) val = GENMASK(gsi->channel_count - 1, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) iowrite32(val, gsi->virt + GSI_CNTXT_SRC_CH_IRQ_MSK_OFFSET);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) val = GENMASK(gsi->evt_ring_count - 1, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) iowrite32(val, gsi->virt + GSI_CNTXT_SRC_EV_CH_IRQ_MSK_OFFSET);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) /* Each IEOB interrupt is enabled (later) as needed by channels */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) iowrite32(0, gsi->virt + GSI_CNTXT_SRC_IEOB_IRQ_MSK_OFFSET);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) val = GSI_CNTXT_GLOB_IRQ_ALL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) iowrite32(val, gsi->virt + GSI_CNTXT_GLOB_IRQ_EN_OFFSET);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) /* Never enable GSI_BREAK_POINT */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) val = GSI_CNTXT_GSI_IRQ_ALL & ~BREAK_POINT_FMASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) iowrite32(val, gsi->virt + GSI_CNTXT_GSI_IRQ_EN_OFFSET);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) /* Disable all GSI_interrupt types */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) static void gsi_irq_disable(struct gsi *gsi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) iowrite32(0, gsi->virt + GSI_CNTXT_GSI_IRQ_EN_OFFSET);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) iowrite32(0, gsi->virt + GSI_CNTXT_GLOB_IRQ_EN_OFFSET);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) iowrite32(0, gsi->virt + GSI_CNTXT_SRC_IEOB_IRQ_MSK_OFFSET);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) iowrite32(0, gsi->virt + GSI_CNTXT_SRC_EV_CH_IRQ_MSK_OFFSET);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) iowrite32(0, gsi->virt + GSI_CNTXT_SRC_CH_IRQ_MSK_OFFSET);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) iowrite32(0, gsi->virt + GSI_CNTXT_TYPE_IRQ_MSK_OFFSET);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) /* Return the virtual address associated with a ring index */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) void *gsi_ring_virt(struct gsi_ring *ring, u32 index)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) /* Note: index *must* be used modulo the ring count here */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) return ring->virt + (index % ring->count) * GSI_RING_ELEMENT_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) /* Return the 32-bit DMA address associated with a ring index */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) static u32 gsi_ring_addr(struct gsi_ring *ring, u32 index)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) return (ring->addr & GENMASK(31, 0)) + index * GSI_RING_ELEMENT_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) /* Return the ring index of a 32-bit ring offset */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) static u32 gsi_ring_index(struct gsi_ring *ring, u32 offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) return (offset - gsi_ring_addr(ring, 0)) / GSI_RING_ELEMENT_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) /* Issue a GSI command by writing a value to a register, then wait for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) * completion to be signaled. Returns true if the command completes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) * or false if it times out.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) static bool
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) gsi_command(struct gsi *gsi, u32 reg, u32 val, struct completion *completion)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) reinit_completion(completion);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) iowrite32(val, gsi->virt + reg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) return !!wait_for_completion_timeout(completion, GSI_CMD_TIMEOUT * HZ);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) /* Return the hardware's notion of the current state of an event ring */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) static enum gsi_evt_ring_state
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) gsi_evt_ring_state(struct gsi *gsi, u32 evt_ring_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) u32 val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) val = ioread32(gsi->virt + GSI_EV_CH_E_CNTXT_0_OFFSET(evt_ring_id));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) return u32_get_bits(val, EV_CHSTATE_FMASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) /* Issue an event ring command and wait for it to complete */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) static int evt_ring_command(struct gsi *gsi, u32 evt_ring_id,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) enum gsi_evt_cmd_opcode opcode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) struct gsi_evt_ring *evt_ring = &gsi->evt_ring[evt_ring_id];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) struct completion *completion = &evt_ring->completion;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) struct device *dev = gsi->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) u32 val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) val = u32_encode_bits(evt_ring_id, EV_CHID_FMASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) val |= u32_encode_bits(opcode, EV_OPCODE_FMASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) if (gsi_command(gsi, GSI_EV_CH_CMD_OFFSET, val, completion))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) return 0; /* Success! */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) dev_err(dev, "GSI command %u for event ring %u timed out, state %u\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) opcode, evt_ring_id, evt_ring->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) return -ETIMEDOUT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) /* Allocate an event ring in NOT_ALLOCATED state */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) static int gsi_evt_ring_alloc_command(struct gsi *gsi, u32 evt_ring_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) struct gsi_evt_ring *evt_ring = &gsi->evt_ring[evt_ring_id];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) /* Get initial event ring state */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) evt_ring->state = gsi_evt_ring_state(gsi, evt_ring_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) if (evt_ring->state != GSI_EVT_RING_STATE_NOT_ALLOCATED) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) dev_err(gsi->dev, "bad event ring state %u before alloc\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) evt_ring->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) ret = evt_ring_command(gsi, evt_ring_id, GSI_EVT_ALLOCATE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) if (!ret && evt_ring->state != GSI_EVT_RING_STATE_ALLOCATED) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) dev_err(gsi->dev, "bad event ring state %u after alloc\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) evt_ring->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) ret = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) /* Reset a GSI event ring in ALLOCATED or ERROR state. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) static void gsi_evt_ring_reset_command(struct gsi *gsi, u32 evt_ring_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) struct gsi_evt_ring *evt_ring = &gsi->evt_ring[evt_ring_id];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) enum gsi_evt_ring_state state = evt_ring->state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) if (state != GSI_EVT_RING_STATE_ALLOCATED &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) state != GSI_EVT_RING_STATE_ERROR) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) dev_err(gsi->dev, "bad event ring state %u before reset\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) evt_ring->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) ret = evt_ring_command(gsi, evt_ring_id, GSI_EVT_RESET);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) if (!ret && evt_ring->state != GSI_EVT_RING_STATE_ALLOCATED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) dev_err(gsi->dev, "bad event ring state %u after reset\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) evt_ring->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) /* Issue a hardware de-allocation request for an allocated event ring */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) static void gsi_evt_ring_de_alloc_command(struct gsi *gsi, u32 evt_ring_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) struct gsi_evt_ring *evt_ring = &gsi->evt_ring[evt_ring_id];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) if (evt_ring->state != GSI_EVT_RING_STATE_ALLOCATED) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) dev_err(gsi->dev, "bad event ring state %u before dealloc\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) evt_ring->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) ret = evt_ring_command(gsi, evt_ring_id, GSI_EVT_DE_ALLOC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) if (!ret && evt_ring->state != GSI_EVT_RING_STATE_NOT_ALLOCATED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) dev_err(gsi->dev, "bad event ring state %u after dealloc\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) evt_ring->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) /* Fetch the current state of a channel from hardware */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) static enum gsi_channel_state gsi_channel_state(struct gsi_channel *channel)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) u32 channel_id = gsi_channel_id(channel);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) void *virt = channel->gsi->virt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) u32 val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) val = ioread32(virt + GSI_CH_C_CNTXT_0_OFFSET(channel_id));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) return u32_get_bits(val, CHSTATE_FMASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) /* Issue a channel command and wait for it to complete */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) static int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) gsi_channel_command(struct gsi_channel *channel, enum gsi_ch_cmd_opcode opcode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) struct completion *completion = &channel->completion;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) u32 channel_id = gsi_channel_id(channel);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) struct gsi *gsi = channel->gsi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) struct device *dev = gsi->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) u32 val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) val = u32_encode_bits(channel_id, CH_CHID_FMASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) val |= u32_encode_bits(opcode, CH_OPCODE_FMASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) if (gsi_command(gsi, GSI_CH_CMD_OFFSET, val, completion))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) return 0; /* Success! */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) dev_err(dev, "GSI command %u for channel %u timed out, state %u\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) opcode, channel_id, gsi_channel_state(channel));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) return -ETIMEDOUT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) /* Allocate GSI channel in NOT_ALLOCATED state */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) static int gsi_channel_alloc_command(struct gsi *gsi, u32 channel_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) struct gsi_channel *channel = &gsi->channel[channel_id];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) struct device *dev = gsi->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) enum gsi_channel_state state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) /* Get initial channel state */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) state = gsi_channel_state(channel);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) if (state != GSI_CHANNEL_STATE_NOT_ALLOCATED) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) dev_err(dev, "bad channel state %u before alloc\n", state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) ret = gsi_channel_command(channel, GSI_CH_ALLOCATE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) /* Channel state will normally have been updated */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) state = gsi_channel_state(channel);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) if (!ret && state != GSI_CHANNEL_STATE_ALLOCATED) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) dev_err(dev, "bad channel state %u after alloc\n", state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) ret = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) /* Start an ALLOCATED channel */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) static int gsi_channel_start_command(struct gsi_channel *channel)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) struct device *dev = channel->gsi->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) enum gsi_channel_state state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) state = gsi_channel_state(channel);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) if (state != GSI_CHANNEL_STATE_ALLOCATED &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) state != GSI_CHANNEL_STATE_STOPPED) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) dev_err(dev, "bad channel state %u before start\n", state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) ret = gsi_channel_command(channel, GSI_CH_START);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) /* Channel state will normally have been updated */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) state = gsi_channel_state(channel);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) if (!ret && state != GSI_CHANNEL_STATE_STARTED) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) dev_err(dev, "bad channel state %u after start\n", state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) ret = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) /* Stop a GSI channel in STARTED state */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) static int gsi_channel_stop_command(struct gsi_channel *channel)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) struct device *dev = channel->gsi->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) enum gsi_channel_state state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) state = gsi_channel_state(channel);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) /* Channel could have entered STOPPED state since last call
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) * if it timed out. If so, we're done.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) if (state == GSI_CHANNEL_STATE_STOPPED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) if (state != GSI_CHANNEL_STATE_STARTED &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) state != GSI_CHANNEL_STATE_STOP_IN_PROC) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) dev_err(dev, "bad channel state %u before stop\n", state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) ret = gsi_channel_command(channel, GSI_CH_STOP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) /* Channel state will normally have been updated */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) state = gsi_channel_state(channel);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) if (ret || state == GSI_CHANNEL_STATE_STOPPED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) /* We may have to try again if stop is in progress */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) if (state == GSI_CHANNEL_STATE_STOP_IN_PROC)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) return -EAGAIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) dev_err(dev, "bad channel state %u after stop\n", state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) /* Reset a GSI channel in ALLOCATED or ERROR state. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) static void gsi_channel_reset_command(struct gsi_channel *channel)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) struct device *dev = channel->gsi->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) enum gsi_channel_state state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) msleep(1); /* A short delay is required before a RESET command */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) state = gsi_channel_state(channel);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) if (state != GSI_CHANNEL_STATE_STOPPED &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) state != GSI_CHANNEL_STATE_ERROR) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) dev_err(dev, "bad channel state %u before reset\n", state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) ret = gsi_channel_command(channel, GSI_CH_RESET);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) /* Channel state will normally have been updated */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) state = gsi_channel_state(channel);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) if (!ret && state != GSI_CHANNEL_STATE_ALLOCATED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) dev_err(dev, "bad channel state %u after reset\n", state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) /* Deallocate an ALLOCATED GSI channel */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) static void gsi_channel_de_alloc_command(struct gsi *gsi, u32 channel_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) struct gsi_channel *channel = &gsi->channel[channel_id];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) struct device *dev = gsi->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) enum gsi_channel_state state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) state = gsi_channel_state(channel);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) if (state != GSI_CHANNEL_STATE_ALLOCATED) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) dev_err(dev, "bad channel state %u before dealloc\n", state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) ret = gsi_channel_command(channel, GSI_CH_DE_ALLOC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) /* Channel state will normally have been updated */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) state = gsi_channel_state(channel);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) if (!ret && state != GSI_CHANNEL_STATE_NOT_ALLOCATED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) dev_err(dev, "bad channel state %u after dealloc\n", state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) /* Ring an event ring doorbell, reporting the last entry processed by the AP.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) * The index argument (modulo the ring count) is the first unfilled entry, so
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) * we supply one less than that with the doorbell. Update the event ring
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) * index field with the value provided.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) static void gsi_evt_ring_doorbell(struct gsi *gsi, u32 evt_ring_id, u32 index)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) struct gsi_ring *ring = &gsi->evt_ring[evt_ring_id].ring;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) u32 val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) ring->index = index; /* Next unused entry */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) /* Note: index *must* be used modulo the ring count here */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) val = gsi_ring_addr(ring, (index - 1) % ring->count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) iowrite32(val, gsi->virt + GSI_EV_CH_E_DOORBELL_0_OFFSET(evt_ring_id));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) /* Program an event ring for use */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) static void gsi_evt_ring_program(struct gsi *gsi, u32 evt_ring_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) struct gsi_evt_ring *evt_ring = &gsi->evt_ring[evt_ring_id];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) size_t size = evt_ring->ring.count * GSI_RING_ELEMENT_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) u32 val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) val = u32_encode_bits(GSI_EVT_CHTYPE_GPI_EV, EV_CHTYPE_FMASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) val |= EV_INTYPE_FMASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) val |= u32_encode_bits(GSI_RING_ELEMENT_SIZE, EV_ELEMENT_SIZE_FMASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) iowrite32(val, gsi->virt + GSI_EV_CH_E_CNTXT_0_OFFSET(evt_ring_id));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) val = u32_encode_bits(size, EV_R_LENGTH_FMASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) iowrite32(val, gsi->virt + GSI_EV_CH_E_CNTXT_1_OFFSET(evt_ring_id));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) /* The context 2 and 3 registers store the low-order and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) * high-order 32 bits of the address of the event ring,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) * respectively.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) val = evt_ring->ring.addr & GENMASK(31, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) iowrite32(val, gsi->virt + GSI_EV_CH_E_CNTXT_2_OFFSET(evt_ring_id));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) val = evt_ring->ring.addr >> 32;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) iowrite32(val, gsi->virt + GSI_EV_CH_E_CNTXT_3_OFFSET(evt_ring_id));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) /* Enable interrupt moderation by setting the moderation delay */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) val = u32_encode_bits(GSI_EVT_RING_INT_MODT, MODT_FMASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) val |= u32_encode_bits(1, MODC_FMASK); /* comes from channel */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) iowrite32(val, gsi->virt + GSI_EV_CH_E_CNTXT_8_OFFSET(evt_ring_id));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) /* No MSI write data, and MSI address high and low address is 0 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) iowrite32(0, gsi->virt + GSI_EV_CH_E_CNTXT_9_OFFSET(evt_ring_id));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) iowrite32(0, gsi->virt + GSI_EV_CH_E_CNTXT_10_OFFSET(evt_ring_id));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) iowrite32(0, gsi->virt + GSI_EV_CH_E_CNTXT_11_OFFSET(evt_ring_id));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) /* We don't need to get event read pointer updates */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) iowrite32(0, gsi->virt + GSI_EV_CH_E_CNTXT_12_OFFSET(evt_ring_id));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) iowrite32(0, gsi->virt + GSI_EV_CH_E_CNTXT_13_OFFSET(evt_ring_id));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) /* Finally, tell the hardware we've completed event 0 (arbitrary) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) gsi_evt_ring_doorbell(gsi, evt_ring_id, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) /* Return the last (most recent) transaction completed on a channel. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) static struct gsi_trans *gsi_channel_trans_last(struct gsi_channel *channel)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) struct gsi_trans_info *trans_info = &channel->trans_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) struct gsi_trans *trans;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) spin_lock_bh(&trans_info->spinlock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) if (!list_empty(&trans_info->complete))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) trans = list_last_entry(&trans_info->complete,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) struct gsi_trans, links);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) else if (!list_empty(&trans_info->polled))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) trans = list_last_entry(&trans_info->polled,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) struct gsi_trans, links);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) trans = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) /* Caller will wait for this, so take a reference */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) if (trans)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) refcount_inc(&trans->refcount);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) spin_unlock_bh(&trans_info->spinlock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) return trans;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) /* Wait for transaction activity on a channel to complete */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) static void gsi_channel_trans_quiesce(struct gsi_channel *channel)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) struct gsi_trans *trans;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) /* Get the last transaction, and wait for it to complete */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) trans = gsi_channel_trans_last(channel);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) if (trans) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) wait_for_completion(&trans->completion);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) gsi_trans_free(trans);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) /* Stop channel activity. Transactions may not be allocated until thawed. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) static void gsi_channel_freeze(struct gsi_channel *channel)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) gsi_channel_trans_quiesce(channel);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) napi_disable(&channel->napi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) gsi_irq_ieob_disable(channel->gsi, channel->evt_ring_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) /* Allow transactions to be used on the channel again. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) static void gsi_channel_thaw(struct gsi_channel *channel)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) gsi_irq_ieob_enable(channel->gsi, channel->evt_ring_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) napi_enable(&channel->napi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) /* Program a channel for use */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) static void gsi_channel_program(struct gsi_channel *channel, bool doorbell)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) size_t size = channel->tre_ring.count * GSI_RING_ELEMENT_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) u32 channel_id = gsi_channel_id(channel);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) union gsi_channel_scratch scr = { };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) struct gsi_channel_scratch_gpi *gpi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) struct gsi *gsi = channel->gsi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) u32 wrr_weight = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) u32 val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) /* Arbitrarily pick TRE 0 as the first channel element to use */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) channel->tre_ring.index = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) /* We program all channels to use GPI protocol */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) val = u32_encode_bits(GSI_CHANNEL_PROTOCOL_GPI, CHTYPE_PROTOCOL_FMASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) if (channel->toward_ipa)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) val |= CHTYPE_DIR_FMASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) val |= u32_encode_bits(channel->evt_ring_id, ERINDEX_FMASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) val |= u32_encode_bits(GSI_RING_ELEMENT_SIZE, ELEMENT_SIZE_FMASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) iowrite32(val, gsi->virt + GSI_CH_C_CNTXT_0_OFFSET(channel_id));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) val = u32_encode_bits(size, R_LENGTH_FMASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) iowrite32(val, gsi->virt + GSI_CH_C_CNTXT_1_OFFSET(channel_id));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) /* The context 2 and 3 registers store the low-order and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) * high-order 32 bits of the address of the channel ring,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) * respectively.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) val = channel->tre_ring.addr & GENMASK(31, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) iowrite32(val, gsi->virt + GSI_CH_C_CNTXT_2_OFFSET(channel_id));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) val = channel->tre_ring.addr >> 32;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) iowrite32(val, gsi->virt + GSI_CH_C_CNTXT_3_OFFSET(channel_id));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) /* Command channel gets low weighted round-robin priority */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) if (channel->command)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) wrr_weight = field_max(WRR_WEIGHT_FMASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) val = u32_encode_bits(wrr_weight, WRR_WEIGHT_FMASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) /* Max prefetch is 1 segment (do not set MAX_PREFETCH_FMASK) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) /* Enable the doorbell engine if requested */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) if (doorbell)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) val |= USE_DB_ENG_FMASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) if (!channel->use_prefetch)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) val |= USE_ESCAPE_BUF_ONLY_FMASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) iowrite32(val, gsi->virt + GSI_CH_C_QOS_OFFSET(channel_id));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) /* Now update the scratch registers for GPI protocol */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) gpi = &scr.gpi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) gpi->max_outstanding_tre = gsi_channel_trans_tre_max(gsi, channel_id) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) GSI_RING_ELEMENT_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) gpi->outstanding_threshold = 2 * GSI_RING_ELEMENT_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) val = scr.data.word1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) iowrite32(val, gsi->virt + GSI_CH_C_SCRATCH_0_OFFSET(channel_id));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) val = scr.data.word2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) iowrite32(val, gsi->virt + GSI_CH_C_SCRATCH_1_OFFSET(channel_id));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) val = scr.data.word3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) iowrite32(val, gsi->virt + GSI_CH_C_SCRATCH_2_OFFSET(channel_id));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) /* We must preserve the upper 16 bits of the last scratch register.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) * The next sequence assumes those bits remain unchanged between the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) * read and the write.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) val = ioread32(gsi->virt + GSI_CH_C_SCRATCH_3_OFFSET(channel_id));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) val = (scr.data.word4 & GENMASK(31, 16)) | (val & GENMASK(15, 0));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) iowrite32(val, gsi->virt + GSI_CH_C_SCRATCH_3_OFFSET(channel_id));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) /* All done! */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) static void gsi_channel_deprogram(struct gsi_channel *channel)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) /* Nothing to do */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) /* Start an allocated GSI channel */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) int gsi_channel_start(struct gsi *gsi, u32 channel_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) struct gsi_channel *channel = &gsi->channel[channel_id];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) mutex_lock(&gsi->mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) ret = gsi_channel_start_command(channel);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) mutex_unlock(&gsi->mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) gsi_channel_thaw(channel);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) /* Stop a started channel */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) int gsi_channel_stop(struct gsi *gsi, u32 channel_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) struct gsi_channel *channel = &gsi->channel[channel_id];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) u32 retries;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) gsi_channel_freeze(channel);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) /* RX channels might require a little time to enter STOPPED state */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) retries = channel->toward_ipa ? 0 : GSI_CHANNEL_STOP_RX_RETRIES;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) mutex_lock(&gsi->mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) ret = gsi_channel_stop_command(channel);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) if (ret != -EAGAIN)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) msleep(1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) } while (retries--);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) mutex_unlock(&gsi->mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) /* Thaw the channel if we need to retry (or on error) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) gsi_channel_thaw(channel);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) /* Reset and reconfigure a channel (possibly leaving doorbell disabled) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) void gsi_channel_reset(struct gsi *gsi, u32 channel_id, bool legacy)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) struct gsi_channel *channel = &gsi->channel[channel_id];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) mutex_lock(&gsi->mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) gsi_channel_reset_command(channel);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) /* Due to a hardware quirk we may need to reset RX channels twice. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) if (legacy && !channel->toward_ipa)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) gsi_channel_reset_command(channel);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) gsi_channel_program(channel, legacy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) gsi_channel_trans_cancel_pending(channel);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) mutex_unlock(&gsi->mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) /* Stop a STARTED channel for suspend (using stop if requested) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) int gsi_channel_suspend(struct gsi *gsi, u32 channel_id, bool stop)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) struct gsi_channel *channel = &gsi->channel[channel_id];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) if (stop)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) return gsi_channel_stop(gsi, channel_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) gsi_channel_freeze(channel);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) /* Resume a suspended channel (starting will be requested if STOPPED) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) int gsi_channel_resume(struct gsi *gsi, u32 channel_id, bool start)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) struct gsi_channel *channel = &gsi->channel[channel_id];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) if (start)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) return gsi_channel_start(gsi, channel_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) gsi_channel_thaw(channel);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) * gsi_channel_tx_queued() - Report queued TX transfers for a channel
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) * @channel: Channel for which to report
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) * Report to the network stack the number of bytes and transactions that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) * have been queued to hardware since last call. This and the next function
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) * supply information used by the network stack for throttling.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) * For each channel we track the number of transactions used and bytes of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) * data those transactions represent. We also track what those values are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) * each time this function is called. Subtracting the two tells us
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) * the number of bytes and transactions that have been added between
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888) * successive calls.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) * Calling this each time we ring the channel doorbell allows us to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) * provide accurate information to the network stack about how much
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) * work we've given the hardware at any point in time.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) void gsi_channel_tx_queued(struct gsi_channel *channel)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) u32 trans_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) u32 byte_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) byte_count = channel->byte_count - channel->queued_byte_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) trans_count = channel->trans_count - channel->queued_trans_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) channel->queued_byte_count = channel->byte_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) channel->queued_trans_count = channel->trans_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) ipa_gsi_channel_tx_queued(channel->gsi, gsi_channel_id(channel),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) trans_count, byte_count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) * gsi_channel_tx_update() - Report completed TX transfers
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) * @channel: Channel that has completed transmitting packets
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) * @trans: Last transation known to be complete
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) * Compute the number of transactions and bytes that have been transferred
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) * over a TX channel since the given transaction was committed. Report this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) * information to the network stack.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) * At the time a transaction is committed, we record its channel's
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) * committed transaction and byte counts *in the transaction*.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) * Completions are signaled by the hardware with an interrupt, and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) * we can determine the latest completed transaction at that time.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) * The difference between the byte/transaction count recorded in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) * the transaction and the count last time we recorded a completion
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924) * tells us exactly how much data has been transferred between
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) * completions.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927) * Calling this each time we learn of a newly-completed transaction
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) * allows us to provide accurate information to the network stack
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) * about how much work has been completed by the hardware at a given
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) * point in time.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) gsi_channel_tx_update(struct gsi_channel *channel, struct gsi_trans *trans)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) u64 byte_count = trans->byte_count + trans->len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) u64 trans_count = trans->trans_count + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938) byte_count -= channel->compl_byte_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) channel->compl_byte_count += byte_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) trans_count -= channel->compl_trans_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) channel->compl_trans_count += trans_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943) ipa_gsi_channel_tx_completed(channel->gsi, gsi_channel_id(channel),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) trans_count, byte_count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947) /* Channel control interrupt handler */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) static void gsi_isr_chan_ctrl(struct gsi *gsi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950) u32 channel_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952) channel_mask = ioread32(gsi->virt + GSI_CNTXT_SRC_CH_IRQ_OFFSET);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) iowrite32(channel_mask, gsi->virt + GSI_CNTXT_SRC_CH_IRQ_CLR_OFFSET);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) while (channel_mask) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956) u32 channel_id = __ffs(channel_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957) struct gsi_channel *channel;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959) channel_mask ^= BIT(channel_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961) channel = &gsi->channel[channel_id];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963) complete(&channel->completion);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967) /* Event ring control interrupt handler */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968) static void gsi_isr_evt_ctrl(struct gsi *gsi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970) u32 event_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972) event_mask = ioread32(gsi->virt + GSI_CNTXT_SRC_EV_CH_IRQ_OFFSET);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973) iowrite32(event_mask, gsi->virt + GSI_CNTXT_SRC_EV_CH_IRQ_CLR_OFFSET);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975) while (event_mask) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976) u32 evt_ring_id = __ffs(event_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977) struct gsi_evt_ring *evt_ring;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979) event_mask ^= BIT(evt_ring_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981) evt_ring = &gsi->evt_ring[evt_ring_id];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982) evt_ring->state = gsi_evt_ring_state(gsi, evt_ring_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984) complete(&evt_ring->completion);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988) /* Global channel error interrupt handler */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990) gsi_isr_glob_chan_err(struct gsi *gsi, u32 err_ee, u32 channel_id, u32 code)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992) if (code == GSI_OUT_OF_RESOURCES_ERR) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993) dev_err(gsi->dev, "channel %u out of resources\n", channel_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994) complete(&gsi->channel[channel_id].completion);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998) /* Report, but otherwise ignore all other error codes */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999) dev_err(gsi->dev, "channel %u global error ee 0x%08x code 0x%08x\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000) channel_id, err_ee, code);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003) /* Global event error interrupt handler */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005) gsi_isr_glob_evt_err(struct gsi *gsi, u32 err_ee, u32 evt_ring_id, u32 code)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007) if (code == GSI_OUT_OF_RESOURCES_ERR) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008) struct gsi_evt_ring *evt_ring = &gsi->evt_ring[evt_ring_id];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009) u32 channel_id = gsi_channel_id(evt_ring->channel);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011) complete(&evt_ring->completion);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012) dev_err(gsi->dev, "evt_ring for channel %u out of resources\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013) channel_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017) /* Report, but otherwise ignore all other error codes */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018) dev_err(gsi->dev, "event ring %u global error ee %u code 0x%08x\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019) evt_ring_id, err_ee, code);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022) /* Global error interrupt handler */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023) static void gsi_isr_glob_err(struct gsi *gsi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025) enum gsi_err_type type;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026) enum gsi_err_code code;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027) u32 which;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028) u32 val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029) u32 ee;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031) /* Get the logged error, then reinitialize the log */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032) val = ioread32(gsi->virt + GSI_ERROR_LOG_OFFSET);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033) iowrite32(0, gsi->virt + GSI_ERROR_LOG_OFFSET);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034) iowrite32(~0, gsi->virt + GSI_ERROR_LOG_CLR_OFFSET);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036) ee = u32_get_bits(val, ERR_EE_FMASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037) which = u32_get_bits(val, ERR_VIRT_IDX_FMASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038) type = u32_get_bits(val, ERR_TYPE_FMASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039) code = u32_get_bits(val, ERR_CODE_FMASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041) if (type == GSI_ERR_TYPE_CHAN)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042) gsi_isr_glob_chan_err(gsi, ee, which, code);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043) else if (type == GSI_ERR_TYPE_EVT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044) gsi_isr_glob_evt_err(gsi, ee, which, code);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045) else /* type GSI_ERR_TYPE_GLOB should be fatal */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046) dev_err(gsi->dev, "unexpected global error 0x%08x\n", type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049) /* Generic EE interrupt handler */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050) static void gsi_isr_gp_int1(struct gsi *gsi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052) u32 result;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053) u32 val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055) val = ioread32(gsi->virt + GSI_CNTXT_SCRATCH_0_OFFSET);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056) result = u32_get_bits(val, GENERIC_EE_RESULT_FMASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057) if (result != GENERIC_EE_SUCCESS_FVAL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058) dev_err(gsi->dev, "global INT1 generic result %u\n", result);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060) complete(&gsi->completion);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1061) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1062)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1063) /* Inter-EE interrupt handler */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1064) static void gsi_isr_glob_ee(struct gsi *gsi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1065) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1066) u32 val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1067)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1068) val = ioread32(gsi->virt + GSI_CNTXT_GLOB_IRQ_STTS_OFFSET);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1069)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1070) if (val & ERROR_INT_FMASK)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1071) gsi_isr_glob_err(gsi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1072)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1073) iowrite32(val, gsi->virt + GSI_CNTXT_GLOB_IRQ_CLR_OFFSET);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1074)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1075) val &= ~ERROR_INT_FMASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1076)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1077) if (val & GP_INT1_FMASK) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1078) val ^= GP_INT1_FMASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1079) gsi_isr_gp_int1(gsi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1080) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1081)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1082) if (val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1083) dev_err(gsi->dev, "unexpected global interrupt 0x%08x\n", val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1084) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1085)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1086) /* I/O completion interrupt event */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1087) static void gsi_isr_ieob(struct gsi *gsi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1088) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1089) u32 event_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1090)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1091) event_mask = ioread32(gsi->virt + GSI_CNTXT_SRC_IEOB_IRQ_OFFSET);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1092) iowrite32(event_mask, gsi->virt + GSI_CNTXT_SRC_IEOB_IRQ_CLR_OFFSET);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1093)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1094) while (event_mask) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1095) u32 evt_ring_id = __ffs(event_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1096)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1097) event_mask ^= BIT(evt_ring_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1098)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1099) gsi_irq_ieob_disable(gsi, evt_ring_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1100) napi_schedule(&gsi->evt_ring[evt_ring_id].channel->napi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1101) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1102) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1103)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1104) /* General event interrupts represent serious problems, so report them */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1105) static void gsi_isr_general(struct gsi *gsi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1106) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1107) struct device *dev = gsi->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1108) u32 val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1109)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1110) val = ioread32(gsi->virt + GSI_CNTXT_GSI_IRQ_STTS_OFFSET);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1111) iowrite32(val, gsi->virt + GSI_CNTXT_GSI_IRQ_CLR_OFFSET);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1112)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1113) if (val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1114) dev_err(dev, "unexpected general interrupt 0x%08x\n", val);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1115) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1116)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1117) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1118) * gsi_isr() - Top level GSI interrupt service routine
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1119) * @irq: Interrupt number (ignored)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1120) * @dev_id: GSI pointer supplied to request_irq()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1121) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1122) * This is the main handler function registered for the GSI IRQ. Each type
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1123) * of interrupt has a separate handler function that is called from here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1124) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1125) static irqreturn_t gsi_isr(int irq, void *dev_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1126) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1127) struct gsi *gsi = dev_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1128) u32 intr_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1129) u32 cnt = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1130)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1131) while ((intr_mask = ioread32(gsi->virt + GSI_CNTXT_TYPE_IRQ_OFFSET))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1132) /* intr_mask contains bitmask of pending GSI interrupts */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1133) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1134) u32 gsi_intr = BIT(__ffs(intr_mask));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1135)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1136) intr_mask ^= gsi_intr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1137)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1138) switch (gsi_intr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1139) case CH_CTRL_FMASK:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1140) gsi_isr_chan_ctrl(gsi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1141) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1142) case EV_CTRL_FMASK:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1143) gsi_isr_evt_ctrl(gsi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1144) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1145) case GLOB_EE_FMASK:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1146) gsi_isr_glob_ee(gsi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1147) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1148) case IEOB_FMASK:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1149) gsi_isr_ieob(gsi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1150) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1151) case GENERAL_FMASK:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1152) gsi_isr_general(gsi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1153) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1154) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1155) dev_err(gsi->dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1156) "unrecognized interrupt type 0x%08x\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1157) gsi_intr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1158) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1159) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1160) } while (intr_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1161)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1162) if (++cnt > GSI_ISR_MAX_ITER) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1163) dev_err(gsi->dev, "interrupt flood\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1164) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1165) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1166) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1167)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1168) return IRQ_HANDLED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1169) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1170)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1171) /* Return the transaction associated with a transfer completion event */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1172) static struct gsi_trans *gsi_event_trans(struct gsi_channel *channel,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1173) struct gsi_event *event)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1174) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1175) u32 tre_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1176) u32 tre_index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1177)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1178) /* Event xfer_ptr records the TRE it's associated with */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1179) tre_offset = le64_to_cpu(event->xfer_ptr) & GENMASK(31, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1180) tre_index = gsi_ring_index(&channel->tre_ring, tre_offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1181)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1182) return gsi_channel_trans_mapped(channel, tre_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1183) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1184)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1185) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1186) * gsi_evt_ring_rx_update() - Record lengths of received data
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1187) * @evt_ring: Event ring associated with channel that received packets
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1188) * @index: Event index in ring reported by hardware
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1189) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1190) * Events for RX channels contain the actual number of bytes received into
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1191) * the buffer. Every event has a transaction associated with it, and here
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1192) * we update transactions to record their actual received lengths.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1193) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1194) * This function is called whenever we learn that the GSI hardware has filled
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1195) * new events since the last time we checked. The ring's index field tells
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1196) * the first entry in need of processing. The index provided is the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1197) * first *unfilled* event in the ring (following the last filled one).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1198) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1199) * Events are sequential within the event ring, and transactions are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1200) * sequential within the transaction pool.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1201) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1202) * Note that @index always refers to an element *within* the event ring.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1203) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1204) static void gsi_evt_ring_rx_update(struct gsi_evt_ring *evt_ring, u32 index)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1205) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1206) struct gsi_channel *channel = evt_ring->channel;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1207) struct gsi_ring *ring = &evt_ring->ring;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1208) struct gsi_trans_info *trans_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1209) struct gsi_event *event_done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1210) struct gsi_event *event;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1211) struct gsi_trans *trans;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1212) u32 byte_count = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1213) u32 old_index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1214) u32 event_avail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1215)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1216) trans_info = &channel->trans_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1217)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1218) /* We'll start with the oldest un-processed event. RX channels
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1219) * replenish receive buffers in single-TRE transactions, so we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1220) * can just map that event to its transaction. Transactions
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1221) * associated with completion events are consecutive.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1222) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1223) old_index = ring->index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1224) event = gsi_ring_virt(ring, old_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1225) trans = gsi_event_trans(channel, event);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1226)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1227) /* Compute the number of events to process before we wrap,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1228) * and determine when we'll be done processing events.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1229) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1230) event_avail = ring->count - old_index % ring->count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1231) event_done = gsi_ring_virt(ring, index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1232) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1233) trans->len = __le16_to_cpu(event->len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1234) byte_count += trans->len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1235)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1236) /* Move on to the next event and transaction */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1237) if (--event_avail)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1238) event++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1239) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1240) event = gsi_ring_virt(ring, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1241) trans = gsi_trans_pool_next(&trans_info->pool, trans);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1242) } while (event != event_done);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1243)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1244) /* We record RX bytes when they are received */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1245) channel->byte_count += byte_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1246) channel->trans_count++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1247) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1248)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1249) /* Initialize a ring, including allocating DMA memory for its entries */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1250) static int gsi_ring_alloc(struct gsi *gsi, struct gsi_ring *ring, u32 count)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1251) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1252) size_t size = count * GSI_RING_ELEMENT_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1253) struct device *dev = gsi->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1254) dma_addr_t addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1255)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1256) /* Hardware requires a 2^n ring size, with alignment equal to size */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1257) ring->virt = dma_alloc_coherent(dev, size, &addr, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1258) if (ring->virt && addr % size) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1259) dma_free_coherent(dev, size, ring->virt, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1260) dev_err(dev, "unable to alloc 0x%zx-aligned ring buffer\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1261) size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1262) return -EINVAL; /* Not a good error value, but distinct */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1263) } else if (!ring->virt) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1264) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1265) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1266) ring->addr = addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1267) ring->count = count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1268)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1269) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1270) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1271)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1272) /* Free a previously-allocated ring */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1273) static void gsi_ring_free(struct gsi *gsi, struct gsi_ring *ring)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1274) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1275) size_t size = ring->count * GSI_RING_ELEMENT_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1276)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1277) dma_free_coherent(gsi->dev, size, ring->virt, ring->addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1278) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1279)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1280) /* Allocate an available event ring id */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1281) static int gsi_evt_ring_id_alloc(struct gsi *gsi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1282) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1283) u32 evt_ring_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1284)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1285) if (gsi->event_bitmap == ~0U) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1286) dev_err(gsi->dev, "event rings exhausted\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1287) return -ENOSPC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1288) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1289)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1290) evt_ring_id = ffz(gsi->event_bitmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1291) gsi->event_bitmap |= BIT(evt_ring_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1292)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1293) return (int)evt_ring_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1294) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1295)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1296) /* Free a previously-allocated event ring id */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1297) static void gsi_evt_ring_id_free(struct gsi *gsi, u32 evt_ring_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1298) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1299) gsi->event_bitmap &= ~BIT(evt_ring_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1300) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1301)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1302) /* Ring a channel doorbell, reporting the first un-filled entry */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1303) void gsi_channel_doorbell(struct gsi_channel *channel)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1304) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1305) struct gsi_ring *tre_ring = &channel->tre_ring;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1306) u32 channel_id = gsi_channel_id(channel);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1307) struct gsi *gsi = channel->gsi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1308) u32 val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1309)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1310) /* Note: index *must* be used modulo the ring count here */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1311) val = gsi_ring_addr(tre_ring, tre_ring->index % tre_ring->count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1312) iowrite32(val, gsi->virt + GSI_CH_C_DOORBELL_0_OFFSET(channel_id));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1313) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1314)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1315) /* Consult hardware, move any newly completed transactions to completed list */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1316) static void gsi_channel_update(struct gsi_channel *channel)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1317) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1318) u32 evt_ring_id = channel->evt_ring_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1319) struct gsi *gsi = channel->gsi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1320) struct gsi_evt_ring *evt_ring;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1321) struct gsi_trans *trans;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1322) struct gsi_ring *ring;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1323) u32 offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1324) u32 index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1325)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1326) evt_ring = &gsi->evt_ring[evt_ring_id];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1327) ring = &evt_ring->ring;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1328)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1329) /* See if there's anything new to process; if not, we're done. Note
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1330) * that index always refers to an entry *within* the event ring.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1331) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1332) offset = GSI_EV_CH_E_CNTXT_4_OFFSET(evt_ring_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1333) index = gsi_ring_index(ring, ioread32(gsi->virt + offset));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1334) if (index == ring->index % ring->count)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1335) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1336)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1337) /* Get the transaction for the latest completed event. Take a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1338) * reference to keep it from completing before we give the events
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1339) * for this and previous transactions back to the hardware.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1340) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1341) trans = gsi_event_trans(channel, gsi_ring_virt(ring, index - 1));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1342) refcount_inc(&trans->refcount);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1343)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1344) /* For RX channels, update each completed transaction with the number
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1345) * of bytes that were actually received. For TX channels, report
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1346) * the number of transactions and bytes this completion represents
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1347) * up the network stack.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1348) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1349) if (channel->toward_ipa)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1350) gsi_channel_tx_update(channel, trans);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1351) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1352) gsi_evt_ring_rx_update(evt_ring, index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1353)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1354) gsi_trans_move_complete(trans);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1355)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1356) /* Tell the hardware we've handled these events */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1357) gsi_evt_ring_doorbell(channel->gsi, channel->evt_ring_id, index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1358)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1359) gsi_trans_free(trans);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1360) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1361)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1362) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1363) * gsi_channel_poll_one() - Return a single completed transaction on a channel
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1364) * @channel: Channel to be polled
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1365) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1366) * Return: Transaction pointer, or null if none are available
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1367) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1368) * This function returns the first entry on a channel's completed transaction
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1369) * list. If that list is empty, the hardware is consulted to determine
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1370) * whether any new transactions have completed. If so, they're moved to the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1371) * completed list and the new first entry is returned. If there are no more
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1372) * completed transactions, a null pointer is returned.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1373) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1374) static struct gsi_trans *gsi_channel_poll_one(struct gsi_channel *channel)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1375) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1376) struct gsi_trans *trans;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1377)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1378) /* Get the first transaction from the completed list */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1379) trans = gsi_channel_trans_complete(channel);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1380) if (!trans) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1381) /* List is empty; see if there's more to do */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1382) gsi_channel_update(channel);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1383) trans = gsi_channel_trans_complete(channel);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1384) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1385)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1386) if (trans)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1387) gsi_trans_move_polled(trans);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1388)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1389) return trans;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1390) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1391)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1392) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1393) * gsi_channel_poll() - NAPI poll function for a channel
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1394) * @napi: NAPI structure for the channel
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1395) * @budget: Budget supplied by NAPI core
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1396) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1397) * Return: Number of items polled (<= budget)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1398) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1399) * Single transactions completed by hardware are polled until either
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1400) * the budget is exhausted, or there are no more. Each transaction
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1401) * polled is passed to gsi_trans_complete(), to perform remaining
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1402) * completion processing and retire/free the transaction.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1403) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1404) static int gsi_channel_poll(struct napi_struct *napi, int budget)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1405) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1406) struct gsi_channel *channel;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1407) int count = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1408)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1409) channel = container_of(napi, struct gsi_channel, napi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1410) while (count < budget) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1411) struct gsi_trans *trans;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1412)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1413) count++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1414) trans = gsi_channel_poll_one(channel);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1415) if (!trans)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1416) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1417) gsi_trans_complete(trans);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1418) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1419)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1420) if (count < budget) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1421) napi_complete(&channel->napi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1422) gsi_irq_ieob_enable(channel->gsi, channel->evt_ring_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1423) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1424)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1425) return count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1426) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1427)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1428) /* The event bitmap represents which event ids are available for allocation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1429) * Set bits are not available, clear bits can be used. This function
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1430) * initializes the map so all events supported by the hardware are available,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1431) * then precludes any reserved events from being allocated.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1432) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1433) static u32 gsi_event_bitmap_init(u32 evt_ring_max)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1434) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1435) u32 event_bitmap = GENMASK(BITS_PER_LONG - 1, evt_ring_max);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1436)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1437) event_bitmap |= GENMASK(GSI_MHI_EVENT_ID_END, GSI_MHI_EVENT_ID_START);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1438)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1439) return event_bitmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1440) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1441)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1442) /* Setup function for event rings */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1443) static void gsi_evt_ring_setup(struct gsi *gsi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1444) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1445) /* Nothing to do */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1446) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1447)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1448) /* Inverse of gsi_evt_ring_setup() */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1449) static void gsi_evt_ring_teardown(struct gsi *gsi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1450) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1451) /* Nothing to do */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1452) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1453)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1454) /* Setup function for a single channel */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1455) static int gsi_channel_setup_one(struct gsi *gsi, u32 channel_id,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1456) bool legacy)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1457) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1458) struct gsi_channel *channel = &gsi->channel[channel_id];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1459) u32 evt_ring_id = channel->evt_ring_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1460) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1461)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1462) if (!channel->gsi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1463) return 0; /* Ignore uninitialized channels */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1464)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1465) ret = gsi_evt_ring_alloc_command(gsi, evt_ring_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1466) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1467) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1468)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1469) gsi_evt_ring_program(gsi, evt_ring_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1470)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1471) ret = gsi_channel_alloc_command(gsi, channel_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1472) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1473) goto err_evt_ring_de_alloc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1474)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1475) gsi_channel_program(channel, legacy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1476)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1477) if (channel->toward_ipa)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1478) netif_tx_napi_add(&gsi->dummy_dev, &channel->napi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1479) gsi_channel_poll, NAPI_POLL_WEIGHT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1480) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1481) netif_napi_add(&gsi->dummy_dev, &channel->napi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1482) gsi_channel_poll, NAPI_POLL_WEIGHT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1483)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1484) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1485)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1486) err_evt_ring_de_alloc:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1487) /* We've done nothing with the event ring yet so don't reset */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1488) gsi_evt_ring_de_alloc_command(gsi, evt_ring_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1489)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1490) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1491) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1492)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1493) /* Inverse of gsi_channel_setup_one() */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1494) static void gsi_channel_teardown_one(struct gsi *gsi, u32 channel_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1495) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1496) struct gsi_channel *channel = &gsi->channel[channel_id];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1497) u32 evt_ring_id = channel->evt_ring_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1498)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1499) if (!channel->gsi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1500) return; /* Ignore uninitialized channels */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1501)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1502) netif_napi_del(&channel->napi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1503)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1504) gsi_channel_deprogram(channel);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1505) gsi_channel_de_alloc_command(gsi, channel_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1506) gsi_evt_ring_reset_command(gsi, evt_ring_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1507) gsi_evt_ring_de_alloc_command(gsi, evt_ring_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1508) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1509)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1510) static int gsi_generic_command(struct gsi *gsi, u32 channel_id,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1511) enum gsi_generic_cmd_opcode opcode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1512) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1513) struct completion *completion = &gsi->completion;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1514) u32 val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1515)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1516) /* First zero the result code field */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1517) val = ioread32(gsi->virt + GSI_CNTXT_SCRATCH_0_OFFSET);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1518) val &= ~GENERIC_EE_RESULT_FMASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1519) iowrite32(val, gsi->virt + GSI_CNTXT_SCRATCH_0_OFFSET);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1520)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1521) /* Now issue the command */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1522) val = u32_encode_bits(opcode, GENERIC_OPCODE_FMASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1523) val |= u32_encode_bits(channel_id, GENERIC_CHID_FMASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1524) val |= u32_encode_bits(GSI_EE_MODEM, GENERIC_EE_FMASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1525)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1526) if (gsi_command(gsi, GSI_GENERIC_CMD_OFFSET, val, completion))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1527) return 0; /* Success! */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1528)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1529) dev_err(gsi->dev, "GSI generic command %u to channel %u timed out\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1530) opcode, channel_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1531)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1532) return -ETIMEDOUT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1533) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1534)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1535) static int gsi_modem_channel_alloc(struct gsi *gsi, u32 channel_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1536) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1537) return gsi_generic_command(gsi, channel_id,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1538) GSI_GENERIC_ALLOCATE_CHANNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1539) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1540)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1541) static void gsi_modem_channel_halt(struct gsi *gsi, u32 channel_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1542) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1543) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1544)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1545) ret = gsi_generic_command(gsi, channel_id, GSI_GENERIC_HALT_CHANNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1546) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1547) dev_err(gsi->dev, "error %d halting modem channel %u\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1548) ret, channel_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1549) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1550)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1551) /* Setup function for channels */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1552) static int gsi_channel_setup(struct gsi *gsi, bool legacy)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1553) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1554) u32 channel_id = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1555) u32 mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1556) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1557)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1558) gsi_evt_ring_setup(gsi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1559) gsi_irq_enable(gsi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1560)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1561) mutex_lock(&gsi->mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1562)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1563) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1564) ret = gsi_channel_setup_one(gsi, channel_id, legacy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1565) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1566) goto err_unwind;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1567) } while (++channel_id < gsi->channel_count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1568)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1569) /* Make sure no channels were defined that hardware does not support */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1570) while (channel_id < GSI_CHANNEL_COUNT_MAX) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1571) struct gsi_channel *channel = &gsi->channel[channel_id++];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1572)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1573) if (!channel->gsi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1574) continue; /* Ignore uninitialized channels */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1575)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1576) ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1577) dev_err(gsi->dev, "channel %u not supported by hardware\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1578) channel_id - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1579) channel_id = gsi->channel_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1580) goto err_unwind;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1581) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1582)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1583) /* Allocate modem channels if necessary */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1584) mask = gsi->modem_channel_bitmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1585) while (mask) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1586) u32 modem_channel_id = __ffs(mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1587)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1588) ret = gsi_modem_channel_alloc(gsi, modem_channel_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1589) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1590) goto err_unwind_modem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1591)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1592) /* Clear bit from mask only after success (for unwind) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1593) mask ^= BIT(modem_channel_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1594) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1595)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1596) mutex_unlock(&gsi->mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1597)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1598) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1599)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1600) err_unwind_modem:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1601) /* Compute which modem channels need to be deallocated */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1602) mask ^= gsi->modem_channel_bitmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1603) while (mask) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1604) channel_id = __fls(mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1605)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1606) mask ^= BIT(channel_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1607)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1608) gsi_modem_channel_halt(gsi, channel_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1609) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1610)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1611) err_unwind:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1612) while (channel_id--)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1613) gsi_channel_teardown_one(gsi, channel_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1614)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1615) mutex_unlock(&gsi->mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1616)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1617) gsi_irq_disable(gsi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1618) gsi_evt_ring_teardown(gsi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1619)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1620) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1621) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1622)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1623) /* Inverse of gsi_channel_setup() */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1624) static void gsi_channel_teardown(struct gsi *gsi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1625) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1626) u32 mask = gsi->modem_channel_bitmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1627) u32 channel_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1628)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1629) mutex_lock(&gsi->mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1630)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1631) while (mask) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1632) channel_id = __fls(mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1633)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1634) mask ^= BIT(channel_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1635)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1636) gsi_modem_channel_halt(gsi, channel_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1637) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1638)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1639) channel_id = gsi->channel_count - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1640) do
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1641) gsi_channel_teardown_one(gsi, channel_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1642) while (channel_id--);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1643)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1644) mutex_unlock(&gsi->mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1645)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1646) gsi_irq_disable(gsi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1647) gsi_evt_ring_teardown(gsi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1648) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1649)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1650) /* Setup function for GSI. GSI firmware must be loaded and initialized */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1651) int gsi_setup(struct gsi *gsi, bool legacy)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1652) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1653) struct device *dev = gsi->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1654) u32 val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1655)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1656) /* Here is where we first touch the GSI hardware */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1657) val = ioread32(gsi->virt + GSI_GSI_STATUS_OFFSET);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1658) if (!(val & ENABLED_FMASK)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1659) dev_err(dev, "GSI has not been enabled\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1660) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1661) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1662)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1663) val = ioread32(gsi->virt + GSI_GSI_HW_PARAM_2_OFFSET);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1664)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1665) gsi->channel_count = u32_get_bits(val, NUM_CH_PER_EE_FMASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1666) if (!gsi->channel_count) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1667) dev_err(dev, "GSI reports zero channels supported\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1668) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1669) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1670) if (gsi->channel_count > GSI_CHANNEL_COUNT_MAX) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1671) dev_warn(dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1672) "limiting to %u channels; hardware supports %u\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1673) GSI_CHANNEL_COUNT_MAX, gsi->channel_count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1674) gsi->channel_count = GSI_CHANNEL_COUNT_MAX;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1675) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1676)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1677) gsi->evt_ring_count = u32_get_bits(val, NUM_EV_PER_EE_FMASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1678) if (!gsi->evt_ring_count) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1679) dev_err(dev, "GSI reports zero event rings supported\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1680) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1681) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1682) if (gsi->evt_ring_count > GSI_EVT_RING_COUNT_MAX) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1683) dev_warn(dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1684) "limiting to %u event rings; hardware supports %u\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1685) GSI_EVT_RING_COUNT_MAX, gsi->evt_ring_count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1686) gsi->evt_ring_count = GSI_EVT_RING_COUNT_MAX;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1687) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1688)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1689) /* Initialize the error log */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1690) iowrite32(0, gsi->virt + GSI_ERROR_LOG_OFFSET);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1691)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1692) /* Writing 1 indicates IRQ interrupts; 0 would be MSI */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1693) iowrite32(1, gsi->virt + GSI_CNTXT_INTSET_OFFSET);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1694)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1695) return gsi_channel_setup(gsi, legacy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1696) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1697)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1698) /* Inverse of gsi_setup() */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1699) void gsi_teardown(struct gsi *gsi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1700) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1701) gsi_channel_teardown(gsi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1702) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1703)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1704) /* Initialize a channel's event ring */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1705) static int gsi_channel_evt_ring_init(struct gsi_channel *channel)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1706) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1707) struct gsi *gsi = channel->gsi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1708) struct gsi_evt_ring *evt_ring;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1709) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1710)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1711) ret = gsi_evt_ring_id_alloc(gsi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1712) if (ret < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1713) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1714) channel->evt_ring_id = ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1715)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1716) evt_ring = &gsi->evt_ring[channel->evt_ring_id];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1717) evt_ring->channel = channel;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1718)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1719) ret = gsi_ring_alloc(gsi, &evt_ring->ring, channel->event_count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1720) if (!ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1721) return 0; /* Success! */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1722)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1723) dev_err(gsi->dev, "error %d allocating channel %u event ring\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1724) ret, gsi_channel_id(channel));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1725)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1726) gsi_evt_ring_id_free(gsi, channel->evt_ring_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1727)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1728) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1729) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1730)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1731) /* Inverse of gsi_channel_evt_ring_init() */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1732) static void gsi_channel_evt_ring_exit(struct gsi_channel *channel)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1733) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1734) u32 evt_ring_id = channel->evt_ring_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1735) struct gsi *gsi = channel->gsi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1736) struct gsi_evt_ring *evt_ring;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1737)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1738) evt_ring = &gsi->evt_ring[evt_ring_id];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1739) gsi_ring_free(gsi, &evt_ring->ring);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1740) gsi_evt_ring_id_free(gsi, evt_ring_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1741) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1742)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1743) /* Init function for event rings */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1744) static void gsi_evt_ring_init(struct gsi *gsi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1745) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1746) u32 evt_ring_id = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1747)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1748) gsi->event_bitmap = gsi_event_bitmap_init(GSI_EVT_RING_COUNT_MAX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1749) gsi->event_enable_bitmap = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1750) do
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1751) init_completion(&gsi->evt_ring[evt_ring_id].completion);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1752) while (++evt_ring_id < GSI_EVT_RING_COUNT_MAX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1753) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1754)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1755) /* Inverse of gsi_evt_ring_init() */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1756) static void gsi_evt_ring_exit(struct gsi *gsi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1757) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1758) /* Nothing to do */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1759) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1760)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1761) static bool gsi_channel_data_valid(struct gsi *gsi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1762) const struct ipa_gsi_endpoint_data *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1763) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1764) #ifdef IPA_VALIDATION
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1765) u32 channel_id = data->channel_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1766) struct device *dev = gsi->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1767)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1768) /* Make sure channel ids are in the range driver supports */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1769) if (channel_id >= GSI_CHANNEL_COUNT_MAX) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1770) dev_err(dev, "bad channel id %u; must be less than %u\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1771) channel_id, GSI_CHANNEL_COUNT_MAX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1772) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1773) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1774)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1775) if (data->ee_id != GSI_EE_AP && data->ee_id != GSI_EE_MODEM) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1776) dev_err(dev, "bad EE id %u; not AP or modem\n", data->ee_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1777) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1778) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1779)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1780) if (!data->channel.tlv_count ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1781) data->channel.tlv_count > GSI_TLV_MAX) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1782) dev_err(dev, "channel %u bad tlv_count %u; must be 1..%u\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1783) channel_id, data->channel.tlv_count, GSI_TLV_MAX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1784) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1785) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1786)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1787) /* We have to allow at least one maximally-sized transaction to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1788) * be outstanding (which would use tlv_count TREs). Given how
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1789) * gsi_channel_tre_max() is computed, tre_count has to be almost
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1790) * twice the TLV FIFO size to satisfy this requirement.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1791) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1792) if (data->channel.tre_count < 2 * data->channel.tlv_count - 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1793) dev_err(dev, "channel %u TLV count %u exceeds TRE count %u\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1794) channel_id, data->channel.tlv_count,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1795) data->channel.tre_count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1796) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1797) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1798)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1799) if (!is_power_of_2(data->channel.tre_count)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1800) dev_err(dev, "channel %u bad tre_count %u; not power of 2\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1801) channel_id, data->channel.tre_count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1802) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1803) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1804)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1805) if (!is_power_of_2(data->channel.event_count)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1806) dev_err(dev, "channel %u bad event_count %u; not power of 2\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1807) channel_id, data->channel.event_count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1808) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1809) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1810) #endif /* IPA_VALIDATION */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1811)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1812) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1813) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1814)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1815) /* Init function for a single channel */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1816) static int gsi_channel_init_one(struct gsi *gsi,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1817) const struct ipa_gsi_endpoint_data *data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1818) bool command, bool prefetch)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1819) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1820) struct gsi_channel *channel;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1821) u32 tre_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1822) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1823)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1824) if (!gsi_channel_data_valid(gsi, data))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1825) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1826)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1827) /* Worst case we need an event for every outstanding TRE */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1828) if (data->channel.tre_count > data->channel.event_count) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1829) tre_count = data->channel.event_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1830) dev_warn(gsi->dev, "channel %u limited to %u TREs\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1831) data->channel_id, tre_count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1832) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1833) tre_count = data->channel.tre_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1834) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1835)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1836) channel = &gsi->channel[data->channel_id];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1837) memset(channel, 0, sizeof(*channel));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1838)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1839) channel->gsi = gsi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1840) channel->toward_ipa = data->toward_ipa;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1841) channel->command = command;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1842) channel->use_prefetch = command && prefetch;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1843) channel->tlv_count = data->channel.tlv_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1844) channel->tre_count = tre_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1845) channel->event_count = data->channel.event_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1846) init_completion(&channel->completion);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1847)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1848) ret = gsi_channel_evt_ring_init(channel);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1849) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1850) goto err_clear_gsi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1851)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1852) ret = gsi_ring_alloc(gsi, &channel->tre_ring, data->channel.tre_count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1853) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1854) dev_err(gsi->dev, "error %d allocating channel %u ring\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1855) ret, data->channel_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1856) goto err_channel_evt_ring_exit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1857) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1858)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1859) ret = gsi_channel_trans_init(gsi, data->channel_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1860) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1861) goto err_ring_free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1862)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1863) if (command) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1864) u32 tre_max = gsi_channel_tre_max(gsi, data->channel_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1865)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1866) ret = ipa_cmd_pool_init(channel, tre_max);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1867) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1868) if (!ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1869) return 0; /* Success! */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1870)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1871) gsi_channel_trans_exit(channel);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1872) err_ring_free:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1873) gsi_ring_free(gsi, &channel->tre_ring);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1874) err_channel_evt_ring_exit:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1875) gsi_channel_evt_ring_exit(channel);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1876) err_clear_gsi:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1877) channel->gsi = NULL; /* Mark it not (fully) initialized */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1878)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1879) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1880) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1881)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1882) /* Inverse of gsi_channel_init_one() */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1883) static void gsi_channel_exit_one(struct gsi_channel *channel)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1884) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1885) if (!channel->gsi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1886) return; /* Ignore uninitialized channels */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1887)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1888) if (channel->command)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1889) ipa_cmd_pool_exit(channel);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1890) gsi_channel_trans_exit(channel);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1891) gsi_ring_free(channel->gsi, &channel->tre_ring);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1892) gsi_channel_evt_ring_exit(channel);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1893) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1894)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1895) /* Init function for channels */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1896) static int gsi_channel_init(struct gsi *gsi, bool prefetch, u32 count,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1897) const struct ipa_gsi_endpoint_data *data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1898) bool modem_alloc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1899) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1900) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1901) u32 i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1902)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1903) gsi_evt_ring_init(gsi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1904)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1905) /* The endpoint data array is indexed by endpoint name */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1906) for (i = 0; i < count; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1907) bool command = i == IPA_ENDPOINT_AP_COMMAND_TX;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1908)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1909) if (ipa_gsi_endpoint_data_empty(&data[i]))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1910) continue; /* Skip over empty slots */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1911)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1912) /* Mark modem channels to be allocated (hardware workaround) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1913) if (data[i].ee_id == GSI_EE_MODEM) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1914) if (modem_alloc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1915) gsi->modem_channel_bitmap |=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1916) BIT(data[i].channel_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1917) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1918) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1919)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1920) ret = gsi_channel_init_one(gsi, &data[i], command, prefetch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1921) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1922) goto err_unwind;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1923) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1924)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1925) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1926)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1927) err_unwind:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1928) while (i--) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1929) if (ipa_gsi_endpoint_data_empty(&data[i]))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1930) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1931) if (modem_alloc && data[i].ee_id == GSI_EE_MODEM) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1932) gsi->modem_channel_bitmap &= ~BIT(data[i].channel_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1933) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1934) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1935) gsi_channel_exit_one(&gsi->channel[data->channel_id]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1936) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1937) gsi_evt_ring_exit(gsi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1938)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1939) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1940) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1941)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1942) /* Inverse of gsi_channel_init() */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1943) static void gsi_channel_exit(struct gsi *gsi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1944) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1945) u32 channel_id = GSI_CHANNEL_COUNT_MAX - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1946)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1947) do
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1948) gsi_channel_exit_one(&gsi->channel[channel_id]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1949) while (channel_id--);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1950) gsi->modem_channel_bitmap = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1951)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1952) gsi_evt_ring_exit(gsi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1953) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1954)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1955) /* Init function for GSI. GSI hardware does not need to be "ready" */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1956) int gsi_init(struct gsi *gsi, struct platform_device *pdev, bool prefetch,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1957) u32 count, const struct ipa_gsi_endpoint_data *data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1958) bool modem_alloc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1959) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1960) struct device *dev = &pdev->dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1961) struct resource *res;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1962) resource_size_t size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1963) unsigned int irq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1964) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1965)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1966) gsi_validate_build();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1967)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1968) gsi->dev = dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1969)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1970) /* The GSI layer performs NAPI on all endpoints. NAPI requires a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1971) * network device structure, but the GSI layer does not have one,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1972) * so we must create a dummy network device for this purpose.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1973) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1974) init_dummy_netdev(&gsi->dummy_dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1975)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1976) ret = platform_get_irq_byname(pdev, "gsi");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1977) if (ret <= 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1978) dev_err(dev, "DT error %d getting \"gsi\" IRQ property\n", ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1979) return ret ? : -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1980) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1981) irq = ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1982)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1983) ret = request_irq(irq, gsi_isr, 0, "gsi", gsi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1984) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1985) dev_err(dev, "error %d requesting \"gsi\" IRQ\n", ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1986) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1987) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1988) gsi->irq = irq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1989)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1990) /* Get GSI memory range and map it */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1991) res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "gsi");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1992) if (!res) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1993) dev_err(dev, "DT error getting \"gsi\" memory property\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1994) ret = -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1995) goto err_free_irq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1996) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1997)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1998) size = resource_size(res);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1999) if (res->start > U32_MAX || size > U32_MAX - res->start) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2000) dev_err(dev, "DT memory resource \"gsi\" out of range\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2001) ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2002) goto err_free_irq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2003) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2004)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2005) gsi->virt = ioremap(res->start, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2006) if (!gsi->virt) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2007) dev_err(dev, "unable to remap \"gsi\" memory\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2008) ret = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2009) goto err_free_irq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2010) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2011)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2012) ret = gsi_channel_init(gsi, prefetch, count, data, modem_alloc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2013) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2014) goto err_iounmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2015)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2016) mutex_init(&gsi->mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2017) init_completion(&gsi->completion);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2018)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2019) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2020)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2021) err_iounmap:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2022) iounmap(gsi->virt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2023) err_free_irq:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2024) free_irq(gsi->irq, gsi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2025)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2026) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2027) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2028)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2029) /* Inverse of gsi_init() */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2030) void gsi_exit(struct gsi *gsi)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2031) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2032) mutex_destroy(&gsi->mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2033) gsi_channel_exit(gsi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2034) free_irq(gsi->irq, gsi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2035) iounmap(gsi->virt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2036) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2037)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2038) /* The maximum number of outstanding TREs on a channel. This limits
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2039) * a channel's maximum number of transactions outstanding (worst case
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2040) * is one TRE per transaction).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2041) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2042) * The absolute limit is the number of TREs in the channel's TRE ring,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2043) * and in theory we should be able use all of them. But in practice,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2044) * doing that led to the hardware reporting exhaustion of event ring
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2045) * slots for writing completion information. So the hardware limit
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2046) * would be (tre_count - 1).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2047) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2048) * We reduce it a bit further though. Transaction resource pools are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2049) * sized to be a little larger than this maximum, to allow resource
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2050) * allocations to always be contiguous. The number of entries in a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2051) * TRE ring buffer is a power of 2, and the extra resources in a pool
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2052) * tends to nearly double the memory allocated for it. Reducing the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2053) * maximum number of outstanding TREs allows the number of entries in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2054) * a pool to avoid crossing that power-of-2 boundary, and this can
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2055) * substantially reduce pool memory requirements. The number we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2056) * reduce it by matches the number added in gsi_trans_pool_init().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2057) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2058) u32 gsi_channel_tre_max(struct gsi *gsi, u32 channel_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2059) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2060) struct gsi_channel *channel = &gsi->channel[channel_id];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2061)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2062) /* Hardware limit is channel->tre_count - 1 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2063) return channel->tre_count - (channel->tlv_count - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2064) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2065)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2066) /* Returns the maximum number of TREs in a single transaction for a channel */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2067) u32 gsi_channel_trans_tre_max(struct gsi *gsi, u32 channel_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2068) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2069) struct gsi_channel *channel = &gsi->channel[channel_id];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2070)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2071) return channel->tlv_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2072) }
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags