Orange Pi5 kernel

^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   1) #ifndef _PIO_H
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   2) #define _PIO_H
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   3) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   4)  * Copyright(c) 2015-2017 Intel Corporation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   5)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   6)  * This file is provided under a dual BSD/GPLv2 license.  When using or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   7)  * redistributing this file, you may do so under either license.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   8)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   9)  * GPL LICENSE SUMMARY
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  10)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  11)  * This program is free software; you can redistribute it and/or modify
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  12)  * it under the terms of version 2 of the GNU General Public License as
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  13)  * published by the Free Software Foundation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  14)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  15)  * This program is distributed in the hope that it will be useful, but
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  16)  * WITHOUT ANY WARRANTY; without even the implied warranty of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  17)  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  18)  * General Public License for more details.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  19)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  20)  * BSD LICENSE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  21)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  22)  * Redistribution and use in source and binary forms, with or without
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  23)  * modification, are permitted provided that the following conditions
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  24)  * are met:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  25)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  26)  *  - Redistributions of source code must retain the above copyright
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  27)  *    notice, this list of conditions and the following disclaimer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  28)  *  - Redistributions in binary form must reproduce the above copyright
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  29)  *    notice, this list of conditions and the following disclaimer in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  30)  *    the documentation and/or other materials provided with the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  31)  *    distribution.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  32)  *  - Neither the name of Intel Corporation nor the names of its
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  33)  *    contributors may be used to endorse or promote products derived
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  34)  *    from this software without specific prior written permission.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  35)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  36)  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  37)  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  38)  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  39)  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  40)  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  41)  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  42)  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  43)  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  44)  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  45)  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  46)  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  47)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  48)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  49) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  50) /* send context types */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  51) #define SC_KERNEL 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  52) #define SC_VL15   1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  53) #define SC_ACK    2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  54) #define SC_USER   3	/* must be the last one: it may take all left */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  55) #define SC_MAX    4	/* count of send context types */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  56) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  57) /* invalid send context index */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  58) #define INVALID_SCI 0xff
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  59) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  60) /* PIO buffer release callback function */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  61) typedef void (*pio_release_cb)(void *arg, int code);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  62) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  63) /* PIO release codes - in bits, as there could more than one that apply */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  64) #define PRC_OK		0	/* no known error */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  65) #define PRC_STATUS_ERR	0x01	/* credit return due to status error */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  66) #define PRC_PBC		0x02	/* credit return due to PBC */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  67) #define PRC_THRESHOLD	0x04	/* credit return due to threshold */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  68) #define PRC_FILL_ERR	0x08	/* credit return due fill error */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  69) #define PRC_FORCE	0x10	/* credit return due credit force */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  70) #define PRC_SC_DISABLE	0x20	/* clean-up after a context disable */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  71) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  72) /* byte helper */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  73) union mix {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  74) 	u64 val64;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  75) 	u32 val32[2];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  76) 	u8  val8[8];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  77) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  78) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  79) /* an allocated PIO buffer */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  80) struct pio_buf {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  81) 	struct send_context *sc;/* back pointer to owning send context */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  82) 	pio_release_cb cb;	/* called when the buffer is released */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  83) 	void *arg;		/* argument for cb */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  84) 	void __iomem *start;	/* buffer start address */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  85) 	void __iomem *end;	/* context end address */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  86) 	unsigned long sent_at;	/* buffer is sent when <= free */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  87) 	union mix carry;	/* pending unwritten bytes */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  88) 	u16 qw_written;		/* QW written so far */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  89) 	u8 carry_bytes;	/* number of valid bytes in carry */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  90) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  91) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  92) /* cache line aligned pio buffer array */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  93) union pio_shadow_ring {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  94) 	struct pio_buf pbuf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  95) } ____cacheline_aligned;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  96) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  97) /* per-NUMA send context */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  98) struct send_context {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  99) 	/* read-only after init */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) 	struct hfi1_devdata *dd;		/* device */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) 	union pio_shadow_ring *sr;	/* shadow ring */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) 	void __iomem *base_addr;	/* start of PIO memory */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) 	u32 __percpu *buffers_allocated;/* count of buffers allocated */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) 	u32 size;			/* context size, in bytes */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) 	int node;			/* context home node */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) 	u32 sr_size;			/* size of the shadow ring */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) 	u16 flags;			/* flags */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) 	u8  type;			/* context type */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) 	u8  sw_index;			/* software index number */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) 	u8  hw_context;			/* hardware context number */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) 	u8  group;			/* credit return group */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) 	/* allocator fields */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) 	spinlock_t alloc_lock ____cacheline_aligned_in_smp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) 	u32 sr_head;			/* shadow ring head */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) 	unsigned long fill;		/* official alloc count */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) 	unsigned long alloc_free;	/* copy of free (less cache thrash) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) 	u32 fill_wrap;			/* tracks fill within ring */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) 	u32 credits;			/* number of blocks in context */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) 	/* adding a new field here would make it part of this cacheline */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) 	/* releaser fields */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) 	spinlock_t release_lock ____cacheline_aligned_in_smp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) 	u32 sr_tail;			/* shadow ring tail */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) 	unsigned long free;		/* official free count */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) 	volatile __le64 *hw_free;	/* HW free counter */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) 	/* list for PIO waiters */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) 	struct list_head piowait  ____cacheline_aligned_in_smp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) 	seqlock_t waitlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) 	spinlock_t credit_ctrl_lock ____cacheline_aligned_in_smp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) 	u32 credit_intr_count;		/* count of credit intr users */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) 	u64 credit_ctrl;		/* cache for credit control */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) 	wait_queue_head_t halt_wait;    /* wait until kernel sees interrupt */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) 	struct work_struct halt_work;	/* halted context work queue entry */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) /* send context flags */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) #define SCF_ENABLED 0x01
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) #define SCF_IN_FREE 0x02
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) #define SCF_HALTED  0x04
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) #define SCF_FROZEN  0x08
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) #define SCF_LINK_DOWN 0x10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) struct send_context_info {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) 	struct send_context *sc;	/* allocated working context */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) 	u16 allocated;			/* has this been allocated? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) 	u16 type;			/* context type */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) 	u16 base;			/* base in PIO array */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) 	u16 credits;			/* size in PIO array */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) /* DMA credit return, index is always (context & 0x7) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) struct credit_return {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) 	volatile __le64 cr[8];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) /* NUMA indexed credit return array */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) struct credit_return_base {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) 	struct credit_return *va;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) 	dma_addr_t dma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) /* send context configuration sizes (one per type) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) struct sc_config_sizes {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) 	short int size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) 	short int count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172)  * The diagram below details the relationship of the mapping structures
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174)  * Since the mapping now allows for non-uniform send contexts per vl, the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175)  * number of send contexts for a vl is either the vl_scontexts[vl] or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176)  * a computation based on num_kernel_send_contexts/num_vls:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178)  * For example:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179)  * nactual = vl_scontexts ? vl_scontexts[vl] : num_kernel_send_contexts/num_vls
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181)  * n = roundup to next highest power of 2 using nactual
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183)  * In the case where there are num_kernel_send_contexts/num_vls doesn't divide
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184)  * evenly, the extras are added from the last vl downward.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186)  * For the case where n > nactual, the send contexts are assigned
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187)  * in a round robin fashion wrapping back to the first send context
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188)  * for a particular vl.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190)  *               dd->pio_map
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191)  *                    |                                   pio_map_elem[0]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192)  *                    |                                +--------------------+
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193)  *                    v                                |       mask         |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194)  *               pio_vl_map                            |--------------------|
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195)  *      +--------------------------+                   | ksc[0] -> sc 1     |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196)  *      |    list (RCU)            |                   |--------------------|
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197)  *      |--------------------------|                 ->| ksc[1] -> sc 2     |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198)  *      |    mask                  |              --/  |--------------------|
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199)  *      |--------------------------|            -/     |        *           |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200)  *      |    actual_vls (max 8)    |          -/       |--------------------|
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201)  *      |--------------------------|       --/         | ksc[n-1] -> sc n   |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202)  *      |    vls (max 8)           |     -/            +--------------------+
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203)  *      |--------------------------|  --/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204)  *      |    map[0]                |-/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205)  *      |--------------------------|                   +--------------------+
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206)  *      |    map[1]                |---                |       mask         |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207)  *      |--------------------------|   \----           |--------------------|
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208)  *      |           *              |        \--        | ksc[0] -> sc 1+n   |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209)  *      |           *              |           \----   |--------------------|
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210)  *      |           *              |                \->| ksc[1] -> sc 2+n   |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211)  *      |--------------------------|                   |--------------------|
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212)  *      |   map[vls - 1]           |-                  |         *          |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213)  *      +--------------------------+ \-                |--------------------|
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214)  *                                     \-              | ksc[m-1] -> sc m+n |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215)  *                                       \             +--------------------+
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216)  *                                        \-
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217)  *                                          \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218)  *                                           \-        +----------------------+
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219)  *                                             \-      |       mask           |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220)  *                                               \     |----------------------|
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221)  *                                                \-   | ksc[0] -> sc 1+m+n   |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222)  *                                                  \- |----------------------|
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223)  *                                                    >| ksc[1] -> sc 2+m+n   |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224)  *                                                     |----------------------|
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225)  *                                                     |         *            |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226)  *                                                     |----------------------|
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227)  *                                                     | ksc[o-1] -> sc o+m+n |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228)  *                                                     +----------------------+
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) /* Initial number of send contexts per VL */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) #define INIT_SC_PER_VL 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236)  * struct pio_map_elem - mapping for a vl
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237)  * @mask - selector mask
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238)  * @ksc - array of kernel send contexts for this vl
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240)  * The mask is used to "mod" the selector to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241)  * produce index into the trailing array of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242)  * kscs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) struct pio_map_elem {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) 	u32 mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) 	struct send_context *ksc[];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250)  * struct pio_vl_map - mapping for a vl
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251)  * @list - rcu head for free callback
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252)  * @mask - vl mask to "mod" the vl to produce an index to map array
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253)  * @actual_vls - number of vls
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254)  * @vls - numbers of vls rounded to next power of 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255)  * @map - array of pio_map_elem entries
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257)  * This is the parent mapping structure. The trailing members of the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258)  * struct point to pio_map_elem entries, which in turn point to an
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259)  * array of kscs for that vl.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) struct pio_vl_map {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) 	struct rcu_head list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) 	u32 mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) 	u8 actual_vls;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) 	u8 vls;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) 	struct pio_map_elem *map[];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) int pio_map_init(struct hfi1_devdata *dd, u8 port, u8 num_vls,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) 		 u8 *vl_scontexts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) void free_pio_map(struct hfi1_devdata *dd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) struct send_context *pio_select_send_context_vl(struct hfi1_devdata *dd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) 						u32 selector, u8 vl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) struct send_context *pio_select_send_context_sc(struct hfi1_devdata *dd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) 						u32 selector, u8 sc5);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) /* send context functions */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) int init_credit_return(struct hfi1_devdata *dd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) void free_credit_return(struct hfi1_devdata *dd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) int init_sc_pools_and_sizes(struct hfi1_devdata *dd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) int init_send_contexts(struct hfi1_devdata *dd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) int init_credit_return(struct hfi1_devdata *dd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) int init_pervl_scs(struct hfi1_devdata *dd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) struct send_context *sc_alloc(struct hfi1_devdata *dd, int type,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) 			      uint hdrqentsize, int numa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) void sc_free(struct send_context *sc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) int sc_enable(struct send_context *sc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) void sc_disable(struct send_context *sc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) int sc_restart(struct send_context *sc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) void sc_return_credits(struct send_context *sc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) void sc_flush(struct send_context *sc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) void sc_drop(struct send_context *sc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) void sc_stop(struct send_context *sc, int bit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) struct pio_buf *sc_buffer_alloc(struct send_context *sc, u32 dw_len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) 				pio_release_cb cb, void *arg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) void sc_release_update(struct send_context *sc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) void sc_return_credits(struct send_context *sc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) void sc_group_release_update(struct hfi1_devdata *dd, u32 hw_context);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) void sc_add_credit_return_intr(struct send_context *sc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) void sc_del_credit_return_intr(struct send_context *sc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) void sc_set_cr_threshold(struct send_context *sc, u32 new_threshold);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) u32 sc_percent_to_threshold(struct send_context *sc, u32 percent);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) u32 sc_mtu_to_threshold(struct send_context *sc, u32 mtu, u32 hdrqentsize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) void hfi1_sc_wantpiobuf_intr(struct send_context *sc, u32 needint);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) void sc_wait(struct hfi1_devdata *dd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) void set_pio_integrity(struct send_context *sc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) /* support functions */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) void pio_reset_all(struct hfi1_devdata *dd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) void pio_freeze(struct hfi1_devdata *dd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) void pio_kernel_unfreeze(struct hfi1_devdata *dd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) void pio_kernel_linkup(struct hfi1_devdata *dd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) /* global PIO send control operations */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) #define PSC_GLOBAL_ENABLE 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) #define PSC_GLOBAL_DISABLE 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) #define PSC_GLOBAL_VLARB_ENABLE 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) #define PSC_GLOBAL_VLARB_DISABLE 3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) #define PSC_CM_RESET 4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) #define PSC_DATA_VL_ENABLE 5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) #define PSC_DATA_VL_DISABLE 6
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) void __cm_reset(struct hfi1_devdata *dd, u64 sendctrl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) void pio_send_control(struct hfi1_devdata *dd, int op);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) /* PIO copy routines */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) void pio_copy(struct hfi1_devdata *dd, struct pio_buf *pbuf, u64 pbc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) 	      const void *from, size_t count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) void seg_pio_copy_start(struct pio_buf *pbuf, u64 pbc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) 			const void *from, size_t nbytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) void seg_pio_copy_mid(struct pio_buf *pbuf, const void *from, size_t nbytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) void seg_pio_copy_end(struct pio_buf *pbuf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) void seqfile_dump_sci(struct seq_file *s, u32 i,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) 		      struct send_context_info *sci);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) #endif /* _PIO_H */