^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-or-later
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * net/sched/sch_sfq.c Stochastic Fairness Queueing discipline.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Authors: Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <linux/types.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/jiffies.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/string.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/in.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/errno.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/init.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/skbuff.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/siphash.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <linux/vmalloc.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <net/netlink.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <net/pkt_sched.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include <net/pkt_cls.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #include <net/red.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) /* Stochastic Fairness Queuing algorithm.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) =======================================
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) Source:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) Paul E. McKenney "Stochastic Fairness Queuing",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) IEEE INFOCOMM'90 Proceedings, San Francisco, 1990.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) Paul E. McKenney "Stochastic Fairness Queuing",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) "Interworking: Research and Experience", v.2, 1991, p.113-131.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) See also:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) M. Shreedhar and George Varghese "Efficient Fair
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) Queuing using Deficit Round Robin", Proc. SIGCOMM 95.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) This is not the thing that is usually called (W)FQ nowadays.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) It does not use any timestamp mechanism, but instead
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) processes queues in round-robin order.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) ADVANTAGE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) - It is very cheap. Both CPU and memory requirements are minimal.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) DRAWBACKS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) - "Stochastic" -> It is not 100% fair.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) When hash collisions occur, several flows are considered as one.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) - "Round-robin" -> It introduces larger delays than virtual clock
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) based schemes, and should not be used for isolating interactive
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) traffic from non-interactive. It means, that this scheduler
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) should be used as leaf of CBQ or P3, which put interactive traffic
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) to higher priority band.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) We still need true WFQ for top level CSZ, but using WFQ
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) for the best effort traffic is absolutely pointless:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) SFQ is superior for this purpose.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) IMPLEMENTATION:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) This implementation limits :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) - maximal queue length per flow to 127 packets.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) - max mtu to 2^18-1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) - max 65408 flows,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) - number of hash buckets to 65536.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) It is easy to increase these values, but not in flight. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) #define SFQ_MAX_DEPTH 127 /* max number of packets per flow */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) #define SFQ_DEFAULT_FLOWS 128
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) #define SFQ_MAX_FLOWS (0x10000 - SFQ_MAX_DEPTH - 1) /* max number of flows */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) #define SFQ_EMPTY_SLOT 0xffff
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) #define SFQ_DEFAULT_HASH_DIVISOR 1024
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) /* We use 16 bits to store allot, and want to handle packets up to 64K
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) * Scale allot by 8 (1<<3) so that no overflow occurs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) #define SFQ_ALLOT_SHIFT 3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) #define SFQ_ALLOT_SIZE(X) DIV_ROUND_UP(X, 1 << SFQ_ALLOT_SHIFT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) /* This type should contain at least SFQ_MAX_DEPTH + 1 + SFQ_MAX_FLOWS values */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) typedef u16 sfq_index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) * We dont use pointers to save space.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) * Small indexes [0 ... SFQ_MAX_FLOWS - 1] are 'pointers' to slots[] array
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) * while following values [SFQ_MAX_FLOWS ... SFQ_MAX_FLOWS + SFQ_MAX_DEPTH]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) * are 'pointers' to dep[] array
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) struct sfq_head {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) sfq_index next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) sfq_index prev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) struct sfq_slot {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) struct sk_buff *skblist_next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) struct sk_buff *skblist_prev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) sfq_index qlen; /* number of skbs in skblist */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) sfq_index next; /* next slot in sfq RR chain */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) struct sfq_head dep; /* anchor in dep[] chains */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) unsigned short hash; /* hash value (index in ht[]) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) short allot; /* credit for this slot */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) unsigned int backlog;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) struct red_vars vars;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) struct sfq_sched_data {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) /* frequently used fields */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) int limit; /* limit of total number of packets in this qdisc */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) unsigned int divisor; /* number of slots in hash table */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) u8 headdrop;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) u8 maxdepth; /* limit of packets per flow */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) siphash_key_t perturbation;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) u8 cur_depth; /* depth of longest slot */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) u8 flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) unsigned short scaled_quantum; /* SFQ_ALLOT_SIZE(quantum) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) struct tcf_proto __rcu *filter_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) struct tcf_block *block;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) sfq_index *ht; /* Hash table ('divisor' slots) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) struct sfq_slot *slots; /* Flows table ('maxflows' entries) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) struct red_parms *red_parms;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) struct tc_sfqred_stats stats;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) struct sfq_slot *tail; /* current slot in round */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) struct sfq_head dep[SFQ_MAX_DEPTH + 1];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) /* Linked lists of slots, indexed by depth
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) * dep[0] : list of unused flows
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) * dep[1] : list of flows with 1 packet
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) * dep[X] : list of flows with X packets
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) unsigned int maxflows; /* number of flows in flows array */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) int perturb_period;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) unsigned int quantum; /* Allotment per round: MUST BE >= MTU */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) struct timer_list perturb_timer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) struct Qdisc *sch;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) * sfq_head are either in a sfq_slot or in dep[] array
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) static inline struct sfq_head *sfq_dep_head(struct sfq_sched_data *q, sfq_index val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) if (val < SFQ_MAX_FLOWS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) return &q->slots[val].dep;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) return &q->dep[val - SFQ_MAX_FLOWS];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) static unsigned int sfq_hash(const struct sfq_sched_data *q,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) const struct sk_buff *skb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) return skb_get_hash_perturb(skb, &q->perturbation) & (q->divisor - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) static unsigned int sfq_classify(struct sk_buff *skb, struct Qdisc *sch,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) int *qerr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) struct sfq_sched_data *q = qdisc_priv(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) struct tcf_result res;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) struct tcf_proto *fl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) int result;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) if (TC_H_MAJ(skb->priority) == sch->handle &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) TC_H_MIN(skb->priority) > 0 &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) TC_H_MIN(skb->priority) <= q->divisor)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) return TC_H_MIN(skb->priority);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) fl = rcu_dereference_bh(q->filter_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) if (!fl)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) return sfq_hash(q, skb) + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) *qerr = NET_XMIT_SUCCESS | __NET_XMIT_BYPASS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) result = tcf_classify(skb, fl, &res, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) if (result >= 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) #ifdef CONFIG_NET_CLS_ACT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) switch (result) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) case TC_ACT_STOLEN:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) case TC_ACT_QUEUED:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) case TC_ACT_TRAP:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) *qerr = NET_XMIT_SUCCESS | __NET_XMIT_STOLEN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) fallthrough;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) case TC_ACT_SHOT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) if (TC_H_MIN(res.classid) <= q->divisor)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) return TC_H_MIN(res.classid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) * x : slot number [0 .. SFQ_MAX_FLOWS - 1]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) static inline void sfq_link(struct sfq_sched_data *q, sfq_index x)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) sfq_index p, n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) struct sfq_slot *slot = &q->slots[x];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) int qlen = slot->qlen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) p = qlen + SFQ_MAX_FLOWS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) n = q->dep[qlen].next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) slot->dep.next = n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) slot->dep.prev = p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) q->dep[qlen].next = x; /* sfq_dep_head(q, p)->next = x */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) sfq_dep_head(q, n)->prev = x;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) #define sfq_unlink(q, x, n, p) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) do { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) n = q->slots[x].dep.next; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) p = q->slots[x].dep.prev; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) sfq_dep_head(q, p)->next = n; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) sfq_dep_head(q, n)->prev = p; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) } while (0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) static inline void sfq_dec(struct sfq_sched_data *q, sfq_index x)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) sfq_index p, n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) int d;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) sfq_unlink(q, x, n, p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) d = q->slots[x].qlen--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) if (n == p && q->cur_depth == d)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) q->cur_depth--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) sfq_link(q, x);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) static inline void sfq_inc(struct sfq_sched_data *q, sfq_index x)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) sfq_index p, n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) int d;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) sfq_unlink(q, x, n, p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) d = ++q->slots[x].qlen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) if (q->cur_depth < d)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) q->cur_depth = d;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) sfq_link(q, x);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) /* helper functions : might be changed when/if skb use a standard list_head */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) /* remove one skb from tail of slot queue */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) static inline struct sk_buff *slot_dequeue_tail(struct sfq_slot *slot)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) struct sk_buff *skb = slot->skblist_prev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) slot->skblist_prev = skb->prev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) skb->prev->next = (struct sk_buff *)slot;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) skb->next = skb->prev = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) return skb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) /* remove one skb from head of slot queue */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) static inline struct sk_buff *slot_dequeue_head(struct sfq_slot *slot)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) struct sk_buff *skb = slot->skblist_next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) slot->skblist_next = skb->next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) skb->next->prev = (struct sk_buff *)slot;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) skb->next = skb->prev = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) return skb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) static inline void slot_queue_init(struct sfq_slot *slot)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) memset(slot, 0, sizeof(*slot));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) slot->skblist_prev = slot->skblist_next = (struct sk_buff *)slot;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) /* add skb to slot queue (tail add) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) static inline void slot_queue_add(struct sfq_slot *slot, struct sk_buff *skb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) skb->prev = slot->skblist_prev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) skb->next = (struct sk_buff *)slot;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) slot->skblist_prev->next = skb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) slot->skblist_prev = skb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) static unsigned int sfq_drop(struct Qdisc *sch, struct sk_buff **to_free)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) struct sfq_sched_data *q = qdisc_priv(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) sfq_index x, d = q->cur_depth;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) struct sk_buff *skb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) unsigned int len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) struct sfq_slot *slot;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) /* Queue is full! Find the longest slot and drop tail packet from it */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) if (d > 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) x = q->dep[d].next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) slot = &q->slots[x];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) drop:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) skb = q->headdrop ? slot_dequeue_head(slot) : slot_dequeue_tail(slot);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) len = qdisc_pkt_len(skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) slot->backlog -= len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) sfq_dec(q, x);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) sch->q.qlen--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) qdisc_qstats_backlog_dec(sch, skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) qdisc_drop(skb, sch, to_free);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) return len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) if (d == 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) /* It is difficult to believe, but ALL THE SLOTS HAVE LENGTH 1. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) x = q->tail->next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) slot = &q->slots[x];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) q->tail->next = slot->next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) q->ht[slot->hash] = SFQ_EMPTY_SLOT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) goto drop;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) /* Is ECN parameter configured */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) static int sfq_prob_mark(const struct sfq_sched_data *q)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) return q->flags & TC_RED_ECN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) /* Should packets over max threshold just be marked */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) static int sfq_hard_mark(const struct sfq_sched_data *q)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) return (q->flags & (TC_RED_ECN | TC_RED_HARDDROP)) == TC_RED_ECN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) static int sfq_headdrop(const struct sfq_sched_data *q)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) return q->headdrop;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) static int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) sfq_enqueue(struct sk_buff *skb, struct Qdisc *sch, struct sk_buff **to_free)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) struct sfq_sched_data *q = qdisc_priv(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) unsigned int hash, dropped;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) sfq_index x, qlen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) struct sfq_slot *slot;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) struct sk_buff *head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) int delta;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) hash = sfq_classify(skb, sch, &ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) if (hash == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) if (ret & __NET_XMIT_BYPASS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) qdisc_qstats_drop(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) __qdisc_drop(skb, to_free);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) hash--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) x = q->ht[hash];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) slot = &q->slots[x];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) if (x == SFQ_EMPTY_SLOT) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) x = q->dep[0].next; /* get a free slot */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) if (x >= SFQ_MAX_FLOWS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) return qdisc_drop(skb, sch, to_free);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) q->ht[hash] = x;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) slot = &q->slots[x];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) slot->hash = hash;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) slot->backlog = 0; /* should already be 0 anyway... */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) red_set_vars(&slot->vars);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) goto enqueue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) if (q->red_parms) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) slot->vars.qavg = red_calc_qavg_no_idle_time(q->red_parms,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) &slot->vars,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) slot->backlog);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) switch (red_action(q->red_parms,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) &slot->vars,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) slot->vars.qavg)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) case RED_DONT_MARK:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) case RED_PROB_MARK:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) qdisc_qstats_overlimit(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) if (sfq_prob_mark(q)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) /* We know we have at least one packet in queue */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) if (sfq_headdrop(q) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) INET_ECN_set_ce(slot->skblist_next)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) q->stats.prob_mark_head++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) if (INET_ECN_set_ce(skb)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) q->stats.prob_mark++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) q->stats.prob_drop++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) goto congestion_drop;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) case RED_HARD_MARK:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) qdisc_qstats_overlimit(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) if (sfq_hard_mark(q)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) /* We know we have at least one packet in queue */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) if (sfq_headdrop(q) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) INET_ECN_set_ce(slot->skblist_next)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) q->stats.forced_mark_head++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) if (INET_ECN_set_ce(skb)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) q->stats.forced_mark++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) q->stats.forced_drop++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) goto congestion_drop;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) if (slot->qlen >= q->maxdepth) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) congestion_drop:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) if (!sfq_headdrop(q))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) return qdisc_drop(skb, sch, to_free);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) /* We know we have at least one packet in queue */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) head = slot_dequeue_head(slot);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) delta = qdisc_pkt_len(head) - qdisc_pkt_len(skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) sch->qstats.backlog -= delta;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) slot->backlog -= delta;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) qdisc_drop(head, sch, to_free);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) slot_queue_add(slot, skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) qdisc_tree_reduce_backlog(sch, 0, delta);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) return NET_XMIT_CN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) enqueue:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) qdisc_qstats_backlog_inc(sch, skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) slot->backlog += qdisc_pkt_len(skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) slot_queue_add(slot, skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) sfq_inc(q, x);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) if (slot->qlen == 1) { /* The flow is new */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) if (q->tail == NULL) { /* It is the first flow */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) slot->next = x;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) slot->next = q->tail->next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) q->tail->next = x;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) /* We put this flow at the end of our flow list.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) * This might sound unfair for a new flow to wait after old ones,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) * but we could endup servicing new flows only, and freeze old ones.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) q->tail = slot;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) /* We could use a bigger initial quantum for new flows */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) slot->allot = q->scaled_quantum;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) if (++sch->q.qlen <= q->limit)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) return NET_XMIT_SUCCESS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) qlen = slot->qlen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) dropped = sfq_drop(sch, to_free);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) /* Return Congestion Notification only if we dropped a packet
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) * from this flow.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) if (qlen != slot->qlen) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) qdisc_tree_reduce_backlog(sch, 0, dropped - qdisc_pkt_len(skb));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) return NET_XMIT_CN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) /* As we dropped a packet, better let upper stack know this */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) qdisc_tree_reduce_backlog(sch, 1, dropped);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) return NET_XMIT_SUCCESS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) static struct sk_buff *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) sfq_dequeue(struct Qdisc *sch)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) struct sfq_sched_data *q = qdisc_priv(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) struct sk_buff *skb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) sfq_index a, next_a;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) struct sfq_slot *slot;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) /* No active slots */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) if (q->tail == NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) next_slot:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) a = q->tail->next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) slot = &q->slots[a];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) if (slot->allot <= 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) q->tail = slot;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) slot->allot += q->scaled_quantum;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) goto next_slot;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) skb = slot_dequeue_head(slot);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) sfq_dec(q, a);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) qdisc_bstats_update(sch, skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) sch->q.qlen--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) qdisc_qstats_backlog_dec(sch, skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) slot->backlog -= qdisc_pkt_len(skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) /* Is the slot empty? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) if (slot->qlen == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) q->ht[slot->hash] = SFQ_EMPTY_SLOT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) next_a = slot->next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) if (a == next_a) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) q->tail = NULL; /* no more active slots */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) return skb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) q->tail->next = next_a;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) slot->allot -= SFQ_ALLOT_SIZE(qdisc_pkt_len(skb));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) return skb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) sfq_reset(struct Qdisc *sch)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) struct sk_buff *skb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) while ((skb = sfq_dequeue(sch)) != NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) rtnl_kfree_skbs(skb, skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) * When q->perturbation is changed, we rehash all queued skbs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) * to avoid OOO (Out Of Order) effects.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) * We dont use sfq_dequeue()/sfq_enqueue() because we dont want to change
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) * counters.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) static void sfq_rehash(struct Qdisc *sch)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) struct sfq_sched_data *q = qdisc_priv(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) struct sk_buff *skb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) struct sfq_slot *slot;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) struct sk_buff_head list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) int dropped = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) unsigned int drop_len = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) __skb_queue_head_init(&list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) for (i = 0; i < q->maxflows; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) slot = &q->slots[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) if (!slot->qlen)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) while (slot->qlen) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) skb = slot_dequeue_head(slot);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) sfq_dec(q, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) __skb_queue_tail(&list, skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) slot->backlog = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) red_set_vars(&slot->vars);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) q->ht[slot->hash] = SFQ_EMPTY_SLOT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) q->tail = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) while ((skb = __skb_dequeue(&list)) != NULL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) unsigned int hash = sfq_hash(q, skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) sfq_index x = q->ht[hash];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) slot = &q->slots[x];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) if (x == SFQ_EMPTY_SLOT) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) x = q->dep[0].next; /* get a free slot */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) if (x >= SFQ_MAX_FLOWS) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) drop:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) qdisc_qstats_backlog_dec(sch, skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) drop_len += qdisc_pkt_len(skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) kfree_skb(skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) dropped++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) q->ht[hash] = x;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) slot = &q->slots[x];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) slot->hash = hash;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) if (slot->qlen >= q->maxdepth)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) goto drop;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) slot_queue_add(slot, skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) if (q->red_parms)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) slot->vars.qavg = red_calc_qavg(q->red_parms,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) &slot->vars,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) slot->backlog);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) slot->backlog += qdisc_pkt_len(skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) sfq_inc(q, x);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) if (slot->qlen == 1) { /* The flow is new */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) if (q->tail == NULL) { /* It is the first flow */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) slot->next = x;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) slot->next = q->tail->next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) q->tail->next = x;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) q->tail = slot;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) slot->allot = q->scaled_quantum;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) sch->q.qlen -= dropped;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) qdisc_tree_reduce_backlog(sch, dropped, drop_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) static void sfq_perturbation(struct timer_list *t)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) struct sfq_sched_data *q = from_timer(q, t, perturb_timer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) struct Qdisc *sch = q->sch;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) spinlock_t *root_lock = qdisc_lock(qdisc_root_sleeping(sch));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) siphash_key_t nkey;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) get_random_bytes(&nkey, sizeof(nkey));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) spin_lock(root_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) q->perturbation = nkey;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) if (!q->filter_list && q->tail)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) sfq_rehash(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) spin_unlock(root_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) if (q->perturb_period)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) mod_timer(&q->perturb_timer, jiffies + q->perturb_period);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) static int sfq_change(struct Qdisc *sch, struct nlattr *opt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) struct sfq_sched_data *q = qdisc_priv(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) struct tc_sfq_qopt *ctl = nla_data(opt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) struct tc_sfq_qopt_v1 *ctl_v1 = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) unsigned int qlen, dropped = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) struct red_parms *p = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) struct sk_buff *to_free = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) struct sk_buff *tail = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) if (opt->nla_len < nla_attr_size(sizeof(*ctl)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) if (opt->nla_len >= nla_attr_size(sizeof(*ctl_v1)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) ctl_v1 = nla_data(opt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) if (ctl->divisor &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) (!is_power_of_2(ctl->divisor) || ctl->divisor > 65536))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) /* slot->allot is a short, make sure quantum is not too big. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) if (ctl->quantum) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) unsigned int scaled = SFQ_ALLOT_SIZE(ctl->quantum);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) if (scaled <= 0 || scaled > SHRT_MAX)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) if (ctl_v1 && !red_check_params(ctl_v1->qth_min, ctl_v1->qth_max,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) ctl_v1->Wlog, ctl_v1->Scell_log, NULL))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) if (ctl_v1 && ctl_v1->qth_min) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) p = kmalloc(sizeof(*p), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) if (!p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) sch_tree_lock(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) if (ctl->quantum) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) q->quantum = ctl->quantum;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) q->scaled_quantum = SFQ_ALLOT_SIZE(q->quantum);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) q->perturb_period = ctl->perturb_period * HZ;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) if (ctl->flows)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) q->maxflows = min_t(u32, ctl->flows, SFQ_MAX_FLOWS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) if (ctl->divisor) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) q->divisor = ctl->divisor;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) q->maxflows = min_t(u32, q->maxflows, q->divisor);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) if (ctl_v1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) if (ctl_v1->depth)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) q->maxdepth = min_t(u32, ctl_v1->depth, SFQ_MAX_DEPTH);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) if (p) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) swap(q->red_parms, p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) red_set_parms(q->red_parms,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) ctl_v1->qth_min, ctl_v1->qth_max,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) ctl_v1->Wlog,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) ctl_v1->Plog, ctl_v1->Scell_log,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) ctl_v1->max_P);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) q->flags = ctl_v1->flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) q->headdrop = ctl_v1->headdrop;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) if (ctl->limit) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) q->limit = min_t(u32, ctl->limit, q->maxdepth * q->maxflows);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) q->maxflows = min_t(u32, q->maxflows, q->limit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) qlen = sch->q.qlen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) while (sch->q.qlen > q->limit) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) dropped += sfq_drop(sch, &to_free);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) if (!tail)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) tail = to_free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) rtnl_kfree_skbs(to_free, tail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) qdisc_tree_reduce_backlog(sch, qlen - sch->q.qlen, dropped);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) del_timer(&q->perturb_timer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) if (q->perturb_period) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) mod_timer(&q->perturb_timer, jiffies + q->perturb_period);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) get_random_bytes(&q->perturbation, sizeof(q->perturbation));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) sch_tree_unlock(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) kfree(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) static void *sfq_alloc(size_t sz)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) return kvmalloc(sz, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) static void sfq_free(void *addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) kvfree(addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) static void sfq_destroy(struct Qdisc *sch)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) struct sfq_sched_data *q = qdisc_priv(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) tcf_block_put(q->block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) q->perturb_period = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) del_timer_sync(&q->perturb_timer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) sfq_free(q->ht);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) sfq_free(q->slots);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) kfree(q->red_parms);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) static int sfq_init(struct Qdisc *sch, struct nlattr *opt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) struct netlink_ext_ack *extack)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) struct sfq_sched_data *q = qdisc_priv(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) q->sch = sch;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) timer_setup(&q->perturb_timer, sfq_perturbation, TIMER_DEFERRABLE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) err = tcf_block_get(&q->block, &q->filter_list, sch, extack);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) for (i = 0; i < SFQ_MAX_DEPTH + 1; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) q->dep[i].next = i + SFQ_MAX_FLOWS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) q->dep[i].prev = i + SFQ_MAX_FLOWS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) q->limit = SFQ_MAX_DEPTH;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) q->maxdepth = SFQ_MAX_DEPTH;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) q->cur_depth = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) q->tail = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) q->divisor = SFQ_DEFAULT_HASH_DIVISOR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) q->maxflows = SFQ_DEFAULT_FLOWS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) q->quantum = psched_mtu(qdisc_dev(sch));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) q->scaled_quantum = SFQ_ALLOT_SIZE(q->quantum);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) q->perturb_period = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) get_random_bytes(&q->perturbation, sizeof(q->perturbation));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) if (opt) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) int err = sfq_change(sch, opt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) q->ht = sfq_alloc(sizeof(q->ht[0]) * q->divisor);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) q->slots = sfq_alloc(sizeof(q->slots[0]) * q->maxflows);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) if (!q->ht || !q->slots) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) /* Note: sfq_destroy() will be called by our caller */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) for (i = 0; i < q->divisor; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) q->ht[i] = SFQ_EMPTY_SLOT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) for (i = 0; i < q->maxflows; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) slot_queue_init(&q->slots[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) sfq_link(q, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) if (q->limit >= 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) sch->flags |= TCQ_F_CAN_BYPASS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) sch->flags &= ~TCQ_F_CAN_BYPASS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) static int sfq_dump(struct Qdisc *sch, struct sk_buff *skb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) struct sfq_sched_data *q = qdisc_priv(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) unsigned char *b = skb_tail_pointer(skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) struct tc_sfq_qopt_v1 opt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) struct red_parms *p = q->red_parms;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) memset(&opt, 0, sizeof(opt));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) opt.v0.quantum = q->quantum;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) opt.v0.perturb_period = q->perturb_period / HZ;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) opt.v0.limit = q->limit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) opt.v0.divisor = q->divisor;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) opt.v0.flows = q->maxflows;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) opt.depth = q->maxdepth;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) opt.headdrop = q->headdrop;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) if (p) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) opt.qth_min = p->qth_min >> p->Wlog;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) opt.qth_max = p->qth_max >> p->Wlog;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) opt.Wlog = p->Wlog;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) opt.Plog = p->Plog;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) opt.Scell_log = p->Scell_log;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) opt.max_P = p->max_P;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) memcpy(&opt.stats, &q->stats, sizeof(opt.stats));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) opt.flags = q->flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) if (nla_put(skb, TCA_OPTIONS, sizeof(opt), &opt))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) goto nla_put_failure;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) return skb->len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) nla_put_failure:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) nlmsg_trim(skb, b);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) static struct Qdisc *sfq_leaf(struct Qdisc *sch, unsigned long arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) static unsigned long sfq_find(struct Qdisc *sch, u32 classid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) static unsigned long sfq_bind(struct Qdisc *sch, unsigned long parent,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) u32 classid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) static void sfq_unbind(struct Qdisc *q, unsigned long cl)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) static struct tcf_block *sfq_tcf_block(struct Qdisc *sch, unsigned long cl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) struct netlink_ext_ack *extack)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) struct sfq_sched_data *q = qdisc_priv(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) if (cl)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) return q->block;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) static int sfq_dump_class(struct Qdisc *sch, unsigned long cl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) struct sk_buff *skb, struct tcmsg *tcm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) tcm->tcm_handle |= TC_H_MIN(cl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862) static int sfq_dump_class_stats(struct Qdisc *sch, unsigned long cl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) struct gnet_dump *d)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) struct sfq_sched_data *q = qdisc_priv(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) sfq_index idx = q->ht[cl - 1];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) struct gnet_stats_queue qs = { 0 };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) struct tc_sfq_xstats xstats = { 0 };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) if (idx != SFQ_EMPTY_SLOT) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) const struct sfq_slot *slot = &q->slots[idx];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) xstats.allot = slot->allot << SFQ_ALLOT_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) qs.qlen = slot->qlen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) qs.backlog = slot->backlog;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) if (gnet_stats_copy_queue(d, NULL, &qs, qs.qlen) < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) return gnet_stats_copy_app(d, &xstats, sizeof(xstats));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) static void sfq_walk(struct Qdisc *sch, struct qdisc_walker *arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) struct sfq_sched_data *q = qdisc_priv(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) unsigned int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) if (arg->stop)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) for (i = 0; i < q->divisor; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) if (q->ht[i] == SFQ_EMPTY_SLOT ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) arg->count < arg->skip) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893) arg->count++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) if (arg->fn(sch, i + 1, arg) < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) arg->stop = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) arg->count++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) static const struct Qdisc_class_ops sfq_class_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) .leaf = sfq_leaf,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) .find = sfq_find,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907) .tcf_block = sfq_tcf_block,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) .bind_tcf = sfq_bind,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) .unbind_tcf = sfq_unbind,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) .dump = sfq_dump_class,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) .dump_stats = sfq_dump_class_stats,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) .walk = sfq_walk,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) static struct Qdisc_ops sfq_qdisc_ops __read_mostly = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) .cl_ops = &sfq_class_ops,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) .id = "sfq",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) .priv_size = sizeof(struct sfq_sched_data),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) .enqueue = sfq_enqueue,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) .dequeue = sfq_dequeue,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) .peek = qdisc_peek_dequeued,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) .init = sfq_init,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) .reset = sfq_reset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924) .destroy = sfq_destroy,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) .change = NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926) .dump = sfq_dump,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927) .owner = THIS_MODULE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) static int __init sfq_module_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932) return register_qdisc(&sfq_qdisc_ops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) static void __exit sfq_module_exit(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) unregister_qdisc(&sfq_qdisc_ops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938) module_init(sfq_module_init)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) module_exit(sfq_module_exit)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) MODULE_LICENSE("GPL");
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags