^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Copyright(c) 2018 Intel Corporation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) #include "iowait.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) #include "trace_iowait.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) /* 1 priority == 16 starve_cnt */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #define IOWAIT_PRIORITY_STARVE_SHIFT 4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) void iowait_set_flag(struct iowait *wait, u32 flag)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) trace_hfi1_iowait_set(wait, flag);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) set_bit(flag, &wait->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) bool iowait_flag_set(struct iowait *wait, u32 flag)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) return test_bit(flag, &wait->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) inline void iowait_clear_flag(struct iowait *wait, u32 flag)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) trace_hfi1_iowait_clear(wait, flag);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) clear_bit(flag, &wait->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) * iowait_init() - initialize wait structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) * @wait: wait struct to initialize
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) * @tx_limit: limit for overflow queuing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) * @func: restart function for workqueue
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) * @sleep: sleep function for no space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) * @resume: wakeup function for no space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) * This function initializes the iowait
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) * structure embedded in the QP or PQ.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) void iowait_init(struct iowait *wait, u32 tx_limit,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) void (*func)(struct work_struct *work),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) void (*tidfunc)(struct work_struct *work),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) int (*sleep)(struct sdma_engine *sde,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) struct iowait_work *wait,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) struct sdma_txreq *tx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) uint seq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) bool pkts_sent),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) void (*wakeup)(struct iowait *wait, int reason),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) void (*sdma_drained)(struct iowait *wait),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) void (*init_priority)(struct iowait *wait))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) wait->count = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) INIT_LIST_HEAD(&wait->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) init_waitqueue_head(&wait->wait_dma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) init_waitqueue_head(&wait->wait_pio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) atomic_set(&wait->sdma_busy, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) atomic_set(&wait->pio_busy, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) wait->tx_limit = tx_limit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) wait->sleep = sleep;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) wait->wakeup = wakeup;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) wait->sdma_drained = sdma_drained;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) wait->init_priority = init_priority;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) wait->flags = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) for (i = 0; i < IOWAIT_SES; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) wait->wait[i].iow = wait;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) INIT_LIST_HEAD(&wait->wait[i].tx_head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) if (i == IOWAIT_IB_SE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) INIT_WORK(&wait->wait[i].iowork, func);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) INIT_WORK(&wait->wait[i].iowork, tidfunc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) * iowait_cancel_work - cancel all work in iowait
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) * @w: the iowait struct
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) void iowait_cancel_work(struct iowait *w)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) cancel_work_sync(&iowait_get_ib_work(w)->iowork);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) /* Make sure that the iowork for TID RDMA is used */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) if (iowait_get_tid_work(w)->iowork.func)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) cancel_work_sync(&iowait_get_tid_work(w)->iowork);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) * iowait_set_work_flag - set work flag based on leg
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) * @w - the iowait work struct
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) int iowait_set_work_flag(struct iowait_work *w)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) if (w == &w->iow->wait[IOWAIT_IB_SE]) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) iowait_set_flag(w->iow, IOWAIT_PENDING_IB);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) return IOWAIT_IB_SE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) iowait_set_flag(w->iow, IOWAIT_PENDING_TID);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) return IOWAIT_TID_SE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) * iowait_priority_update_top - update the top priority entry
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) * @w: the iowait struct
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) * @top: a pointer to the top priority entry
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) * @idx: the index of the current iowait in an array
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) * @top_idx: the array index for the iowait entry that has the top priority
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) * This function is called to compare the priority of a given
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) * iowait with the given top priority entry. The top index will
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) * be returned.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) uint iowait_priority_update_top(struct iowait *w,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) struct iowait *top,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) uint idx, uint top_idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) u8 cnt, tcnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) /* Convert priority into starve_cnt and compare the total.*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) cnt = (w->priority << IOWAIT_PRIORITY_STARVE_SHIFT) + w->starved_cnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) tcnt = (top->priority << IOWAIT_PRIORITY_STARVE_SHIFT) +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) top->starved_cnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) if (cnt > tcnt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) return idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) return top_idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) }
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags