Orange Pi5 kernel

^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   1) #ifndef _HFI1_IOWAIT_H
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   2) #define _HFI1_IOWAIT_H
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   3) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   4)  * Copyright(c) 2015 - 2018 Intel Corporation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   5)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   6)  * This file is provided under a dual BSD/GPLv2 license.  When using or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   7)  * redistributing this file, you may do so under either license.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   8)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   9)  * GPL LICENSE SUMMARY
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  10)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  11)  * This program is free software; you can redistribute it and/or modify
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  12)  * it under the terms of version 2 of the GNU General Public License as
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  13)  * published by the Free Software Foundation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  14)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  15)  * This program is distributed in the hope that it will be useful, but
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  16)  * WITHOUT ANY WARRANTY; without even the implied warranty of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  17)  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  18)  * General Public License for more details.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  19)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  20)  * BSD LICENSE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  21)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  22)  * Redistribution and use in source and binary forms, with or without
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  23)  * modification, are permitted provided that the following conditions
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  24)  * are met:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  25)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  26)  *  - Redistributions of source code must retain the above copyright
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  27)  *    notice, this list of conditions and the following disclaimer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  28)  *  - Redistributions in binary form must reproduce the above copyright
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  29)  *    notice, this list of conditions and the following disclaimer in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  30)  *    the documentation and/or other materials provided with the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  31)  *    distribution.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  32)  *  - Neither the name of Intel Corporation nor the names of its
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  33)  *    contributors may be used to endorse or promote products derived
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  34)  *    from this software without specific prior written permission.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  35)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  36)  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  37)  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  38)  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  39)  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  40)  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  41)  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  42)  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  43)  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  44)  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  45)  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  46)  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  47)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  48)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  49) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  50) #include <linux/list.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  51) #include <linux/workqueue.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  52) #include <linux/wait.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  53) #include <linux/sched.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  54) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  55) #include "sdma_txreq.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  56) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  57) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  58)  * typedef (*restart_t)() - restart callback
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  59)  * @work: pointer to work structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  60)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  61) typedef void (*restart_t)(struct work_struct *work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  62) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  63) #define IOWAIT_PENDING_IB  0x0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  64) #define IOWAIT_PENDING_TID 0x1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  65) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  66) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  67)  * A QP can have multiple Send Engines (SEs).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  68)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  69)  * The current use case is for supporting a TID RDMA
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  70)  * packet build/xmit mechanism independent from verbs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  71)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  72) #define IOWAIT_SES 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  73) #define IOWAIT_IB_SE 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  74) #define IOWAIT_TID_SE 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  75) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  76) struct sdma_txreq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  77) struct sdma_engine;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  78) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  79)  * @iowork: the work struct
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  80)  * @tx_head: list of prebuilt packets
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  81)  * @iow: the parent iowait structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  82)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  83)  * This structure is the work item (process) specific
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  84)  * details associated with the each of the two SEs of the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  85)  * QP.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  86)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  87)  * The workstruct and the queued TXs are unique to each
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  88)  * SE.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  89)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  90) struct iowait;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  91) struct iowait_work {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  92) 	struct work_struct iowork;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  93) 	struct list_head tx_head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  94) 	struct iowait *iow;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  95) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  96) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  97) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  98)  * @list: used to add/insert into QP/PQ wait lists
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  99)  * @tx_head: overflow list of sdma_txreq's
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100)  * @sleep: no space callback
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101)  * @wakeup: space callback wakeup
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102)  * @sdma_drained: sdma count drained
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103)  * @init_priority: callback to manipulate priority
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104)  * @lock: lock protected head of wait queue
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105)  * @iowork: workqueue overhead
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106)  * @wait_dma: wait for sdma_busy == 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107)  * @wait_pio: wait for pio_busy == 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108)  * @sdma_busy: # of packets in flight
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109)  * @count: total number of descriptors in tx_head'ed list
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110)  * @tx_limit: limit for overflow queuing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111)  * @tx_count: number of tx entry's in tx_head'ed list
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112)  * @flags: wait flags (one per QP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113)  * @wait: SE array for multiple legs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115)  * This is to be embedded in user's state structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116)  * (QP or PQ).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118)  * The sleep and wakeup members are a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119)  * bit misnamed.   They do not strictly
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120)  * speaking sleep or wake up, but they
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121)  * are callbacks for the ULP to implement
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122)  * what ever queuing/dequeuing of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123)  * the embedded iowait and its containing struct
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124)  * when a resource shortage like SDMA ring space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125)  * or PIO credit space is seen.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127)  * Both potentially have locks help
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128)  * so sleeping is not allowed and it is not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129)  * supported to submit txreqs from the wakeup
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130)  * call directly because of lock conflicts.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132)  * The wait_dma member along with the iow
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134)  * The lock field is used by waiters to record
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135)  * the seqlock_t that guards the list head.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136)  * Waiters explicity know that, but the destroy
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137)  * code that unwaits QPs does not.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) struct iowait {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) 	struct list_head list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) 	int (*sleep)(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) 		struct sdma_engine *sde,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) 		struct iowait_work *wait,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) 		struct sdma_txreq *tx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) 		uint seq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) 		bool pkts_sent
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) 		);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) 	void (*wakeup)(struct iowait *wait, int reason);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) 	void (*sdma_drained)(struct iowait *wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) 	void (*init_priority)(struct iowait *wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) 	seqlock_t *lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) 	wait_queue_head_t wait_dma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) 	wait_queue_head_t wait_pio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) 	atomic_t sdma_busy;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) 	atomic_t pio_busy;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) 	u32 count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) 	u32 tx_limit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) 	u32 tx_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) 	u8 starved_cnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) 	u8 priority;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) 	unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) 	struct iowait_work wait[IOWAIT_SES];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) #define SDMA_AVAIL_REASON 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) void iowait_set_flag(struct iowait *wait, u32 flag);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) bool iowait_flag_set(struct iowait *wait, u32 flag);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) void iowait_clear_flag(struct iowait *wait, u32 flag);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) void iowait_init(struct iowait *wait, u32 tx_limit,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) 		 void (*func)(struct work_struct *work),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) 		 void (*tidfunc)(struct work_struct *work),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) 		 int (*sleep)(struct sdma_engine *sde,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) 			      struct iowait_work *wait,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) 			      struct sdma_txreq *tx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) 			      uint seq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) 			      bool pkts_sent),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) 		 void (*wakeup)(struct iowait *wait, int reason),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) 		 void (*sdma_drained)(struct iowait *wait),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) 		 void (*init_priority)(struct iowait *wait));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184)  * iowait_schedule() - schedule the default send engine work
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185)  * @wait: wait struct to schedule
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186)  * @wq: workqueue for schedule
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187)  * @cpu: cpu
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) static inline bool iowait_schedule(struct iowait *wait,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) 				   struct workqueue_struct *wq, int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) 	return !!queue_work_on(cpu, wq, &wait->wait[IOWAIT_IB_SE].iowork);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196)  * iowait_tid_schedule - schedule the tid SE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197)  * @wait: the iowait structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198)  * @wq: the work queue
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199)  * @cpu: the cpu
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) static inline bool iowait_tid_schedule(struct iowait *wait,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) 				       struct workqueue_struct *wq, int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) 	return !!queue_work_on(cpu, wq, &wait->wait[IOWAIT_TID_SE].iowork);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208)  * iowait_sdma_drain() - wait for DMAs to drain
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210)  * @wait: iowait structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212)  * This will delay until the iowait sdmas have
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213)  * completed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) static inline void iowait_sdma_drain(struct iowait *wait)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) 	wait_event(wait->wait_dma, !atomic_read(&wait->sdma_busy));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221)  * iowait_sdma_pending() - return sdma pending count
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223)  * @wait: iowait structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) static inline int iowait_sdma_pending(struct iowait *wait)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) 	return atomic_read(&wait->sdma_busy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232)  * iowait_sdma_inc - note sdma io pending
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233)  * @wait: iowait structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) static inline void iowait_sdma_inc(struct iowait *wait)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) 	atomic_inc(&wait->sdma_busy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241)  * iowait_sdma_add - add count to pending
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242)  * @wait: iowait structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) static inline void iowait_sdma_add(struct iowait *wait, int count)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) 	atomic_add(count, &wait->sdma_busy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250)  * iowait_sdma_dec - note sdma complete
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251)  * @wait: iowait structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) static inline int iowait_sdma_dec(struct iowait *wait)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) 	if (!wait)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) 		return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) 	return atomic_dec_and_test(&wait->sdma_busy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261)  * iowait_pio_drain() - wait for pios to drain
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263)  * @wait: iowait structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265)  * This will delay until the iowait pios have
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266)  * completed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) static inline void iowait_pio_drain(struct iowait *wait)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) 	wait_event_timeout(wait->wait_pio,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) 			   !atomic_read(&wait->pio_busy),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) 			   HZ);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276)  * iowait_pio_pending() - return pio pending count
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278)  * @wait: iowait structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) static inline int iowait_pio_pending(struct iowait *wait)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) 	return atomic_read(&wait->pio_busy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287)  * iowait_pio_inc - note pio pending
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288)  * @wait: iowait structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) static inline void iowait_pio_inc(struct iowait *wait)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) 	atomic_inc(&wait->pio_busy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296)  * iowait_pio_dec - note pio complete
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297)  * @wait: iowait structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) static inline int iowait_pio_dec(struct iowait *wait)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) 	if (!wait)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) 		return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) 	return atomic_dec_and_test(&wait->pio_busy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307)  * iowait_drain_wakeup() - trigger iowait_drain() waiter
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309)  * @wait: iowait structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311)  * This will trigger any waiters.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) static inline void iowait_drain_wakeup(struct iowait *wait)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) 	wake_up(&wait->wait_dma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) 	wake_up(&wait->wait_pio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) 	if (wait->sdma_drained)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) 		wait->sdma_drained(wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322)  * iowait_get_txhead() - get packet off of iowait list
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324)  * @wait iowait_work struture
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) static inline struct sdma_txreq *iowait_get_txhead(struct iowait_work *wait)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) 	struct sdma_txreq *tx = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) 	if (!list_empty(&wait->tx_head)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) 		tx = list_first_entry(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) 			&wait->tx_head,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) 			struct sdma_txreq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) 			list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) 		list_del_init(&tx->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) 	return tx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) static inline u16 iowait_get_desc(struct iowait_work *w)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) 	u16 num_desc = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) 	struct sdma_txreq *tx = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) 	if (!list_empty(&w->tx_head)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) 		tx = list_first_entry(&w->tx_head, struct sdma_txreq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) 				      list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) 		num_desc = tx->num_desc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) 		if (tx->flags & SDMA_TXREQ_F_VIP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) 			w->iow->priority++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) 	return num_desc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) static inline u32 iowait_get_all_desc(struct iowait *w)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) 	u32 num_desc = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) 	num_desc = iowait_get_desc(&w->wait[IOWAIT_IB_SE]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) 	num_desc += iowait_get_desc(&w->wait[IOWAIT_TID_SE]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) 	return num_desc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) static inline void iowait_update_priority(struct iowait_work *w)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) 	struct sdma_txreq *tx = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) 	if (!list_empty(&w->tx_head)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) 		tx = list_first_entry(&w->tx_head, struct sdma_txreq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) 				      list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) 		if (tx->flags & SDMA_TXREQ_F_VIP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) 			w->iow->priority++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) static inline void iowait_update_all_priority(struct iowait *w)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) 	iowait_update_priority(&w->wait[IOWAIT_IB_SE]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) 	iowait_update_priority(&w->wait[IOWAIT_TID_SE]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) static inline void iowait_init_priority(struct iowait *w)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) 	w->priority = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) 	if (w->init_priority)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) 		w->init_priority(w);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) static inline void iowait_get_priority(struct iowait *w)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) 	iowait_init_priority(w);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) 	iowait_update_all_priority(w);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396)  * iowait_queue - Put the iowait on a wait queue
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397)  * @pkts_sent: have some packets been sent before queuing?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398)  * @w: the iowait struct
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399)  * @wait_head: the wait queue
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401)  * This function is called to insert an iowait struct into a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402)  * wait queue after a resource (eg, sdma descriptor or pio
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403)  * buffer) is run out.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) static inline void iowait_queue(bool pkts_sent, struct iowait *w,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) 				struct list_head *wait_head)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) 	 * To play fair, insert the iowait at the tail of the wait queue if it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) 	 * has already sent some packets; Otherwise, put it at the head.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) 	 * However, if it has priority packets to send, also put it at the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) 	 * head.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) 	if (pkts_sent)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) 		w->starved_cnt = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) 	else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) 		w->starved_cnt++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) 	if (w->priority > 0 || !pkts_sent)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) 		list_add(&w->list, wait_head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) 	else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) 		list_add_tail(&w->list, wait_head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426)  * iowait_starve_clear - clear the wait queue's starve count
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427)  * @pkts_sent: have some packets been sent?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428)  * @w: the iowait struct
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430)  * This function is called to clear the starve count. If no
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431)  * packets have been sent, the starve count will not be cleared.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) static inline void iowait_starve_clear(bool pkts_sent, struct iowait *w)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) 	if (pkts_sent)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) 		w->starved_cnt = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) /* Update the top priority index */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) uint iowait_priority_update_top(struct iowait *w,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) 				struct iowait *top,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) 				uint idx, uint top_idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445)  * iowait_packet_queued() - determine if a packet is queued
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446)  * @wait: the iowait_work structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) static inline bool iowait_packet_queued(struct iowait_work *wait)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) 	return !list_empty(&wait->tx_head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454)  * inc_wait_count - increment wait counts
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455)  * @w: the log work struct
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456)  * @n: the count
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) static inline void iowait_inc_wait_count(struct iowait_work *w, u16 n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) 	if (!w)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) 		return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) 	w->iow->tx_count++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) 	w->iow->count += n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467)  * iowait_get_tid_work - return iowait_work for tid SE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468)  * @w: the iowait struct
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) static inline struct iowait_work *iowait_get_tid_work(struct iowait *w)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) 	return &w->wait[IOWAIT_TID_SE];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476)  * iowait_get_ib_work - return iowait_work for ib SE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477)  * @w: the iowait struct
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) static inline struct iowait_work *iowait_get_ib_work(struct iowait *w)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) 	return &w->wait[IOWAIT_IB_SE];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485)  * iowait_ioww_to_iow - return iowait given iowait_work
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486)  * @w: the iowait_work struct
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) static inline struct iowait *iowait_ioww_to_iow(struct iowait_work *w)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) 	if (likely(w))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) 		return w->iow;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) 	return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) void iowait_cancel_work(struct iowait *w);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) int iowait_set_work_flag(struct iowait_work *w);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) #endif