Orange Pi5 kernel

^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   2) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   3) /* net/sched/sch_etf.c  Earliest TxTime First queueing discipline.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   4)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   5)  * Authors:	Jesus Sanchez-Palencia <jesus.sanchez-palencia@intel.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   6)  *		Vinicius Costa Gomes <vinicius.gomes@intel.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   7)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   8) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   9) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  10) #include <linux/types.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  11) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  12) #include <linux/string.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  13) #include <linux/errno.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  14) #include <linux/errqueue.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  15) #include <linux/rbtree.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  16) #include <linux/skbuff.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  17) #include <linux/posix-timers.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  18) #include <net/netlink.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  19) #include <net/sch_generic.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  20) #include <net/pkt_sched.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  21) #include <net/sock.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  22) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  23) #define DEADLINE_MODE_IS_ON(x) ((x)->flags & TC_ETF_DEADLINE_MODE_ON)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  24) #define OFFLOAD_IS_ON(x) ((x)->flags & TC_ETF_OFFLOAD_ON)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  25) #define SKIP_SOCK_CHECK_IS_SET(x) ((x)->flags & TC_ETF_SKIP_SOCK_CHECK)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  26) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  27) struct etf_sched_data {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  28) 	bool offload;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  29) 	bool deadline_mode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  30) 	bool skip_sock_check;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  31) 	int clockid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  32) 	int queue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  33) 	s32 delta; /* in ns */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  34) 	ktime_t last; /* The txtime of the last skb sent to the netdevice. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  35) 	struct rb_root_cached head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  36) 	struct qdisc_watchdog watchdog;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  37) 	ktime_t (*get_time)(void);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  38) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  39) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  40) static const struct nla_policy etf_policy[TCA_ETF_MAX + 1] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  41) 	[TCA_ETF_PARMS]	= { .len = sizeof(struct tc_etf_qopt) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  42) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  43) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  44) static inline int validate_input_params(struct tc_etf_qopt *qopt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  45) 					struct netlink_ext_ack *extack)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  46) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  47) 	/* Check if params comply to the following rules:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  48) 	 *	* Clockid and delta must be valid.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  49) 	 *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  50) 	 *	* Dynamic clockids are not supported.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  51) 	 *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  52) 	 *	* Delta must be a positive integer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  53) 	 *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  54) 	 * Also note that for the HW offload case, we must
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  55) 	 * expect that system clocks have been synchronized to PHC.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  56) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  57) 	if (qopt->clockid < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  58) 		NL_SET_ERR_MSG(extack, "Dynamic clockids are not supported");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  59) 		return -ENOTSUPP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  60) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  61) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  62) 	if (qopt->clockid != CLOCK_TAI) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  63) 		NL_SET_ERR_MSG(extack, "Invalid clockid. CLOCK_TAI must be used");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  64) 		return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  65) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  66) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  67) 	if (qopt->delta < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  68) 		NL_SET_ERR_MSG(extack, "Delta must be positive");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  69) 		return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  70) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  71) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  72) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  73) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  74) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  75) static bool is_packet_valid(struct Qdisc *sch, struct sk_buff *nskb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  76) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  77) 	struct etf_sched_data *q = qdisc_priv(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  78) 	ktime_t txtime = nskb->tstamp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  79) 	struct sock *sk = nskb->sk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  80) 	ktime_t now;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  81) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  82) 	if (q->skip_sock_check)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  83) 		goto skip;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  84) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  85) 	if (!sk || !sk_fullsock(sk))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  86) 		return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  87) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  88) 	if (!sock_flag(sk, SOCK_TXTIME))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  89) 		return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  90) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  91) 	/* We don't perform crosstimestamping.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  92) 	 * Drop if packet's clockid differs from qdisc's.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  93) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  94) 	if (sk->sk_clockid != q->clockid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  95) 		return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  96) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  97) 	if (sk->sk_txtime_deadline_mode != q->deadline_mode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  98) 		return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  99) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) skip:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) 	now = q->get_time();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) 	if (ktime_before(txtime, now) || ktime_before(txtime, q->last))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) 		return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) 	return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) static struct sk_buff *etf_peek_timesortedlist(struct Qdisc *sch)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) 	struct etf_sched_data *q = qdisc_priv(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) 	struct rb_node *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) 	p = rb_first_cached(&q->head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) 	if (!p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) 		return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) 	return rb_to_skb(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) static void reset_watchdog(struct Qdisc *sch)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) 	struct etf_sched_data *q = qdisc_priv(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) 	struct sk_buff *skb = etf_peek_timesortedlist(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) 	ktime_t next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) 	if (!skb) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) 		qdisc_watchdog_cancel(&q->watchdog);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) 		return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) 	next = ktime_sub_ns(skb->tstamp, q->delta);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) 	qdisc_watchdog_schedule_ns(&q->watchdog, ktime_to_ns(next));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) static void report_sock_error(struct sk_buff *skb, u32 err, u8 code)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) 	struct sock_exterr_skb *serr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) 	struct sk_buff *clone;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) 	ktime_t txtime = skb->tstamp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) 	struct sock *sk = skb->sk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) 	if (!sk || !sk_fullsock(sk) || !(sk->sk_txtime_report_errors))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) 		return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) 	clone = skb_clone(skb, GFP_ATOMIC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) 	if (!clone)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) 		return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) 	serr = SKB_EXT_ERR(clone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) 	serr->ee.ee_errno = err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) 	serr->ee.ee_origin = SO_EE_ORIGIN_TXTIME;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) 	serr->ee.ee_type = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) 	serr->ee.ee_code = code;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) 	serr->ee.ee_pad = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) 	serr->ee.ee_data = (txtime >> 32); /* high part of tstamp */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) 	serr->ee.ee_info = txtime; /* low part of tstamp */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) 	if (sock_queue_err_skb(sk, clone))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) 		kfree_skb(clone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) static int etf_enqueue_timesortedlist(struct sk_buff *nskb, struct Qdisc *sch,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) 				      struct sk_buff **to_free)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) 	struct etf_sched_data *q = qdisc_priv(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) 	struct rb_node **p = &q->head.rb_root.rb_node, *parent = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) 	ktime_t txtime = nskb->tstamp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) 	bool leftmost = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) 	if (!is_packet_valid(sch, nskb)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) 		report_sock_error(nskb, EINVAL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) 				  SO_EE_CODE_TXTIME_INVALID_PARAM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) 		return qdisc_drop(nskb, sch, to_free);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) 	while (*p) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) 		struct sk_buff *skb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) 		parent = *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) 		skb = rb_to_skb(parent);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) 		if (ktime_compare(txtime, skb->tstamp) >= 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) 			p = &parent->rb_right;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) 			leftmost = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) 		} else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) 			p = &parent->rb_left;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) 		}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) 	rb_link_node(&nskb->rbnode, parent, p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) 	rb_insert_color_cached(&nskb->rbnode, &q->head, leftmost);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) 	qdisc_qstats_backlog_inc(sch, nskb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) 	sch->q.qlen++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) 	/* Now we may need to re-arm the qdisc watchdog for the next packet. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) 	reset_watchdog(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) 	return NET_XMIT_SUCCESS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) static void timesortedlist_drop(struct Qdisc *sch, struct sk_buff *skb,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) 				ktime_t now)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) 	struct etf_sched_data *q = qdisc_priv(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) 	struct sk_buff *to_free = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) 	struct sk_buff *tmp = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) 	skb_rbtree_walk_from_safe(skb, tmp) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) 		if (ktime_after(skb->tstamp, now))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) 			break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) 		rb_erase_cached(&skb->rbnode, &q->head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) 		/* The rbnode field in the skb re-uses these fields, now that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) 		 * we are done with the rbnode, reset them.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) 		 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) 		skb->next = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) 		skb->prev = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) 		skb->dev = qdisc_dev(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) 		report_sock_error(skb, ECANCELED, SO_EE_CODE_TXTIME_MISSED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) 		qdisc_qstats_backlog_dec(sch, skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) 		qdisc_drop(skb, sch, &to_free);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) 		qdisc_qstats_overlimit(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) 		sch->q.qlen--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) 	kfree_skb_list(to_free);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) static void timesortedlist_remove(struct Qdisc *sch, struct sk_buff *skb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) 	struct etf_sched_data *q = qdisc_priv(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) 	rb_erase_cached(&skb->rbnode, &q->head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) 	/* The rbnode field in the skb re-uses these fields, now that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) 	 * we are done with the rbnode, reset them.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) 	skb->next = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) 	skb->prev = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) 	skb->dev = qdisc_dev(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) 	qdisc_qstats_backlog_dec(sch, skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) 	qdisc_bstats_update(sch, skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) 	q->last = skb->tstamp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) 	sch->q.qlen--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) static struct sk_buff *etf_dequeue_timesortedlist(struct Qdisc *sch)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) 	struct etf_sched_data *q = qdisc_priv(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) 	struct sk_buff *skb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) 	ktime_t now, next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) 	skb = etf_peek_timesortedlist(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) 	if (!skb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) 		return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) 	now = q->get_time();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) 	/* Drop if packet has expired while in queue. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) 	if (ktime_before(skb->tstamp, now)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) 		timesortedlist_drop(sch, skb, now);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) 		skb = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) 		goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) 	/* When in deadline mode, dequeue as soon as possible and change the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) 	 * txtime from deadline to (now + delta).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) 	if (q->deadline_mode) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) 		timesortedlist_remove(sch, skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) 		skb->tstamp = now;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) 		goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) 	next = ktime_sub_ns(skb->tstamp, q->delta);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) 	/* Dequeue only if now is within the [txtime - delta, txtime] range. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) 	if (ktime_after(now, next))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) 		timesortedlist_remove(sch, skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) 	else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) 		skb = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) 	/* Now we may need to re-arm the qdisc watchdog for the next packet. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) 	reset_watchdog(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) 	return skb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) static void etf_disable_offload(struct net_device *dev,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) 				struct etf_sched_data *q)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) 	struct tc_etf_qopt_offload etf = { };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) 	const struct net_device_ops *ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) 	int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) 	if (!q->offload)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) 		return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) 	ops = dev->netdev_ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) 	if (!ops->ndo_setup_tc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) 		return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) 	etf.queue = q->queue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) 	etf.enable = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) 	err = ops->ndo_setup_tc(dev, TC_SETUP_QDISC_ETF, &etf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) 	if (err < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) 		pr_warn("Couldn't disable ETF offload for queue %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) 			etf.queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) static int etf_enable_offload(struct net_device *dev, struct etf_sched_data *q,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) 			      struct netlink_ext_ack *extack)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) 	const struct net_device_ops *ops = dev->netdev_ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) 	struct tc_etf_qopt_offload etf = { };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) 	int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) 	if (q->offload)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) 		return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) 	if (!ops->ndo_setup_tc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) 		NL_SET_ERR_MSG(extack, "Specified device does not support ETF offload");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) 		return -EOPNOTSUPP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) 	etf.queue = q->queue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) 	etf.enable = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) 	err = ops->ndo_setup_tc(dev, TC_SETUP_QDISC_ETF, &etf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) 	if (err < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) 		NL_SET_ERR_MSG(extack, "Specified device failed to setup ETF hardware offload");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) 		return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) static int etf_init(struct Qdisc *sch, struct nlattr *opt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) 		    struct netlink_ext_ack *extack)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) 	struct etf_sched_data *q = qdisc_priv(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) 	struct net_device *dev = qdisc_dev(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) 	struct nlattr *tb[TCA_ETF_MAX + 1];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) 	struct tc_etf_qopt *qopt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) 	int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) 	if (!opt) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) 		NL_SET_ERR_MSG(extack,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) 			       "Missing ETF qdisc options which are mandatory");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) 		return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) 	err = nla_parse_nested_deprecated(tb, TCA_ETF_MAX, opt, etf_policy,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) 					  extack);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) 	if (err < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) 		return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) 	if (!tb[TCA_ETF_PARMS]) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) 		NL_SET_ERR_MSG(extack, "Missing mandatory ETF parameters");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) 		return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) 	qopt = nla_data(tb[TCA_ETF_PARMS]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) 	pr_debug("delta %d clockid %d offload %s deadline %s\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) 		 qopt->delta, qopt->clockid,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) 		 OFFLOAD_IS_ON(qopt) ? "on" : "off",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) 		 DEADLINE_MODE_IS_ON(qopt) ? "on" : "off");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) 	err = validate_input_params(qopt, extack);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) 	if (err < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) 		return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) 	q->queue = sch->dev_queue - netdev_get_tx_queue(dev, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) 	if (OFFLOAD_IS_ON(qopt)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) 		err = etf_enable_offload(dev, q, extack);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) 		if (err < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) 			return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) 	/* Everything went OK, save the parameters used. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) 	q->delta = qopt->delta;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) 	q->clockid = qopt->clockid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) 	q->offload = OFFLOAD_IS_ON(qopt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) 	q->deadline_mode = DEADLINE_MODE_IS_ON(qopt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) 	q->skip_sock_check = SKIP_SOCK_CHECK_IS_SET(qopt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) 	switch (q->clockid) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) 	case CLOCK_REALTIME:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) 		q->get_time = ktime_get_real;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) 		break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) 	case CLOCK_MONOTONIC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) 		q->get_time = ktime_get;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) 		break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) 	case CLOCK_BOOTTIME:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) 		q->get_time = ktime_get_boottime;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) 		break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) 	case CLOCK_TAI:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) 		q->get_time = ktime_get_clocktai;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) 		break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) 	default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) 		NL_SET_ERR_MSG(extack, "Clockid is not supported");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) 		return -ENOTSUPP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) 	qdisc_watchdog_init_clockid(&q->watchdog, sch, q->clockid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) static void timesortedlist_clear(struct Qdisc *sch)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) 	struct etf_sched_data *q = qdisc_priv(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) 	struct rb_node *p = rb_first_cached(&q->head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) 	while (p) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) 		struct sk_buff *skb = rb_to_skb(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) 		p = rb_next(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) 		rb_erase_cached(&skb->rbnode, &q->head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) 		rtnl_kfree_skbs(skb, skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) 		sch->q.qlen--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) static void etf_reset(struct Qdisc *sch)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) 	struct etf_sched_data *q = qdisc_priv(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) 	/* Only cancel watchdog if it's been initialized. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) 	if (q->watchdog.qdisc == sch)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) 		qdisc_watchdog_cancel(&q->watchdog);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) 	/* No matter which mode we are on, it's safe to clear both lists. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) 	timesortedlist_clear(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) 	__qdisc_reset_queue(&sch->q);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) 	sch->qstats.backlog = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) 	sch->q.qlen = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) 	q->last = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) static void etf_destroy(struct Qdisc *sch)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) 	struct etf_sched_data *q = qdisc_priv(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) 	struct net_device *dev = qdisc_dev(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) 	/* Only cancel watchdog if it's been initialized. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) 	if (q->watchdog.qdisc == sch)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) 		qdisc_watchdog_cancel(&q->watchdog);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) 	etf_disable_offload(dev, q);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) static int etf_dump(struct Qdisc *sch, struct sk_buff *skb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) 	struct etf_sched_data *q = qdisc_priv(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) 	struct tc_etf_qopt opt = { };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) 	struct nlattr *nest;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) 	nest = nla_nest_start_noflag(skb, TCA_OPTIONS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) 	if (!nest)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) 		goto nla_put_failure;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) 	opt.delta = q->delta;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) 	opt.clockid = q->clockid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) 	if (q->offload)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) 		opt.flags |= TC_ETF_OFFLOAD_ON;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) 	if (q->deadline_mode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) 		opt.flags |= TC_ETF_DEADLINE_MODE_ON;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) 	if (q->skip_sock_check)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) 		opt.flags |= TC_ETF_SKIP_SOCK_CHECK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) 	if (nla_put(skb, TCA_ETF_PARMS, sizeof(opt), &opt))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) 		goto nla_put_failure;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) 	return nla_nest_end(skb, nest);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) nla_put_failure:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) 	nla_nest_cancel(skb, nest);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) 	return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) static struct Qdisc_ops etf_qdisc_ops __read_mostly = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) 	.id		=	"etf",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) 	.priv_size	=	sizeof(struct etf_sched_data),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) 	.enqueue	=	etf_enqueue_timesortedlist,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) 	.dequeue	=	etf_dequeue_timesortedlist,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) 	.peek		=	etf_peek_timesortedlist,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) 	.init		=	etf_init,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) 	.reset		=	etf_reset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) 	.destroy	=	etf_destroy,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) 	.dump		=	etf_dump,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) 	.owner		=	THIS_MODULE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) static int __init etf_module_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) 	return register_qdisc(&etf_qdisc_ops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) static void __exit etf_module_exit(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) 	unregister_qdisc(&etf_qdisc_ops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) module_init(etf_module_init)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) module_exit(etf_module_exit)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) MODULE_LICENSE("GPL");