^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-or-later
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Fair Queue CoDel discipline
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Copyright (C) 2012,2015 Eric Dumazet <edumazet@google.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <linux/types.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/jiffies.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/string.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/in.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/errno.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/init.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/skbuff.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <linux/vmalloc.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <net/netlink.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <net/pkt_sched.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <net/pkt_cls.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include <net/codel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #include <net/codel_impl.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #include <net/codel_qdisc.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) /* Fair Queue CoDel.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) * Principles :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) * Packets are classified (internal classifier or external) on flows.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) * This is a Stochastic model (as we use a hash, several flows
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) * might be hashed on same slot)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) * Each flow has a CoDel managed queue.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) * Flows are linked onto two (Round Robin) lists,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) * so that new flows have priority on old ones.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) * For a given flow, packets are not reordered (CoDel uses a FIFO)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) * head drops only.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) * ECN capability is on by default.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) * Low memory footprint (64 bytes per flow)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) struct fq_codel_flow {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) struct sk_buff *head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) struct sk_buff *tail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) struct list_head flowchain;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) int deficit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) struct codel_vars cvars;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) }; /* please try to keep this structure <= 64 bytes */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) struct fq_codel_sched_data {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) struct tcf_proto __rcu *filter_list; /* optional external classifier */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) struct tcf_block *block;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) struct fq_codel_flow *flows; /* Flows table [flows_cnt] */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) u32 *backlogs; /* backlog table [flows_cnt] */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) u32 flows_cnt; /* number of flows */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) u32 quantum; /* psched_mtu(qdisc_dev(sch)); */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) u32 drop_batch_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) u32 memory_limit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) struct codel_params cparams;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) struct codel_stats cstats;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) u32 memory_usage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) u32 drop_overmemory;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) u32 drop_overlimit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) u32 new_flow_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) struct list_head new_flows; /* list of new flows */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) struct list_head old_flows; /* list of old flows */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) static unsigned int fq_codel_hash(const struct fq_codel_sched_data *q,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) struct sk_buff *skb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) return reciprocal_scale(skb_get_hash(skb), q->flows_cnt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) static unsigned int fq_codel_classify(struct sk_buff *skb, struct Qdisc *sch,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) int *qerr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) struct fq_codel_sched_data *q = qdisc_priv(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) struct tcf_proto *filter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) struct tcf_result res;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) int result;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) if (TC_H_MAJ(skb->priority) == sch->handle &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) TC_H_MIN(skb->priority) > 0 &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) TC_H_MIN(skb->priority) <= q->flows_cnt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) return TC_H_MIN(skb->priority);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) filter = rcu_dereference_bh(q->filter_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) if (!filter)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) return fq_codel_hash(q, skb) + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) *qerr = NET_XMIT_SUCCESS | __NET_XMIT_BYPASS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) result = tcf_classify(skb, filter, &res, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) if (result >= 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) #ifdef CONFIG_NET_CLS_ACT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) switch (result) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) case TC_ACT_STOLEN:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) case TC_ACT_QUEUED:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) case TC_ACT_TRAP:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) *qerr = NET_XMIT_SUCCESS | __NET_XMIT_STOLEN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) fallthrough;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) case TC_ACT_SHOT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) if (TC_H_MIN(res.classid) <= q->flows_cnt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) return TC_H_MIN(res.classid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) /* helper functions : might be changed when/if skb use a standard list_head */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) /* remove one skb from head of slot queue */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) static inline struct sk_buff *dequeue_head(struct fq_codel_flow *flow)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) struct sk_buff *skb = flow->head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) flow->head = skb->next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) skb_mark_not_on_list(skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) return skb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) /* add skb to flow queue (tail add) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) static inline void flow_queue_add(struct fq_codel_flow *flow,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) struct sk_buff *skb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) if (flow->head == NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) flow->head = skb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) flow->tail->next = skb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) flow->tail = skb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) skb->next = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) static unsigned int fq_codel_drop(struct Qdisc *sch, unsigned int max_packets,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) struct sk_buff **to_free)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) struct fq_codel_sched_data *q = qdisc_priv(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) struct sk_buff *skb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) unsigned int maxbacklog = 0, idx = 0, i, len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) struct fq_codel_flow *flow;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) unsigned int threshold;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) unsigned int mem = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) /* Queue is full! Find the fat flow and drop packet(s) from it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) * This might sound expensive, but with 1024 flows, we scan
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) * 4KB of memory, and we dont need to handle a complex tree
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) * in fast path (packet queue/enqueue) with many cache misses.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) * In stress mode, we'll try to drop 64 packets from the flow,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) * amortizing this linear lookup to one cache line per drop.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) for (i = 0; i < q->flows_cnt; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) if (q->backlogs[i] > maxbacklog) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) maxbacklog = q->backlogs[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) idx = i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) /* Our goal is to drop half of this fat flow backlog */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) threshold = maxbacklog >> 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) flow = &q->flows[idx];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) len = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) i = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) skb = dequeue_head(flow);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) len += qdisc_pkt_len(skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) mem += get_codel_cb(skb)->mem_usage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) __qdisc_drop(skb, to_free);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) } while (++i < max_packets && len < threshold);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) /* Tell codel to increase its signal strength also */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) flow->cvars.count += i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) q->backlogs[idx] -= len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) q->memory_usage -= mem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) sch->qstats.drops += i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) sch->qstats.backlog -= len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) sch->q.qlen -= i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) return idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) static int fq_codel_enqueue(struct sk_buff *skb, struct Qdisc *sch,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) struct sk_buff **to_free)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) struct fq_codel_sched_data *q = qdisc_priv(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) unsigned int idx, prev_backlog, prev_qlen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) struct fq_codel_flow *flow;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) unsigned int pkt_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) bool memory_limited;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) idx = fq_codel_classify(skb, sch, &ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) if (idx == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) if (ret & __NET_XMIT_BYPASS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) qdisc_qstats_drop(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) __qdisc_drop(skb, to_free);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) idx--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) codel_set_enqueue_time(skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) flow = &q->flows[idx];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) flow_queue_add(flow, skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) q->backlogs[idx] += qdisc_pkt_len(skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) qdisc_qstats_backlog_inc(sch, skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) if (list_empty(&flow->flowchain)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) list_add_tail(&flow->flowchain, &q->new_flows);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) q->new_flow_count++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) flow->deficit = q->quantum;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) get_codel_cb(skb)->mem_usage = skb->truesize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) q->memory_usage += get_codel_cb(skb)->mem_usage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) memory_limited = q->memory_usage > q->memory_limit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) if (++sch->q.qlen <= sch->limit && !memory_limited)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) return NET_XMIT_SUCCESS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) prev_backlog = sch->qstats.backlog;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) prev_qlen = sch->q.qlen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) /* save this packet length as it might be dropped by fq_codel_drop() */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) pkt_len = qdisc_pkt_len(skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) /* fq_codel_drop() is quite expensive, as it performs a linear search
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) * in q->backlogs[] to find a fat flow.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) * So instead of dropping a single packet, drop half of its backlog
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) * with a 64 packets limit to not add a too big cpu spike here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) ret = fq_codel_drop(sch, q->drop_batch_size, to_free);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) prev_qlen -= sch->q.qlen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) prev_backlog -= sch->qstats.backlog;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) q->drop_overlimit += prev_qlen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) if (memory_limited)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) q->drop_overmemory += prev_qlen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) /* As we dropped packet(s), better let upper stack know this.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) * If we dropped a packet for this flow, return NET_XMIT_CN,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) * but in this case, our parents wont increase their backlogs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) if (ret == idx) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) qdisc_tree_reduce_backlog(sch, prev_qlen - 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) prev_backlog - pkt_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) return NET_XMIT_CN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) qdisc_tree_reduce_backlog(sch, prev_qlen, prev_backlog);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) return NET_XMIT_SUCCESS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) /* This is the specific function called from codel_dequeue()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) * to dequeue a packet from queue. Note: backlog is handled in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) * codel, we dont need to reduce it here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) static struct sk_buff *dequeue_func(struct codel_vars *vars, void *ctx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) struct Qdisc *sch = ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) struct fq_codel_sched_data *q = qdisc_priv(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) struct fq_codel_flow *flow;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) struct sk_buff *skb = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) flow = container_of(vars, struct fq_codel_flow, cvars);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) if (flow->head) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) skb = dequeue_head(flow);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) q->backlogs[flow - q->flows] -= qdisc_pkt_len(skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) q->memory_usage -= get_codel_cb(skb)->mem_usage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) sch->q.qlen--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) sch->qstats.backlog -= qdisc_pkt_len(skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) return skb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) static void drop_func(struct sk_buff *skb, void *ctx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) struct Qdisc *sch = ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) kfree_skb(skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) qdisc_qstats_drop(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) static struct sk_buff *fq_codel_dequeue(struct Qdisc *sch)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) struct fq_codel_sched_data *q = qdisc_priv(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) struct sk_buff *skb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) struct fq_codel_flow *flow;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) struct list_head *head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) begin:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) head = &q->new_flows;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) if (list_empty(head)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) head = &q->old_flows;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) if (list_empty(head))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) flow = list_first_entry(head, struct fq_codel_flow, flowchain);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) if (flow->deficit <= 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) flow->deficit += q->quantum;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) list_move_tail(&flow->flowchain, &q->old_flows);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) goto begin;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) skb = codel_dequeue(sch, &sch->qstats.backlog, &q->cparams,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) &flow->cvars, &q->cstats, qdisc_pkt_len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) codel_get_enqueue_time, drop_func, dequeue_func);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) if (!skb) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) /* force a pass through old_flows to prevent starvation */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) if ((head == &q->new_flows) && !list_empty(&q->old_flows))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) list_move_tail(&flow->flowchain, &q->old_flows);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) list_del_init(&flow->flowchain);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) goto begin;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) qdisc_bstats_update(sch, skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) flow->deficit -= qdisc_pkt_len(skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) /* We cant call qdisc_tree_reduce_backlog() if our qlen is 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) * or HTB crashes. Defer it for next round.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) if (q->cstats.drop_count && sch->q.qlen) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) qdisc_tree_reduce_backlog(sch, q->cstats.drop_count,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) q->cstats.drop_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) q->cstats.drop_count = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) q->cstats.drop_len = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) return skb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) static void fq_codel_flow_purge(struct fq_codel_flow *flow)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) rtnl_kfree_skbs(flow->head, flow->tail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) flow->head = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) static void fq_codel_reset(struct Qdisc *sch)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) struct fq_codel_sched_data *q = qdisc_priv(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) INIT_LIST_HEAD(&q->new_flows);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) INIT_LIST_HEAD(&q->old_flows);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) for (i = 0; i < q->flows_cnt; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) struct fq_codel_flow *flow = q->flows + i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) fq_codel_flow_purge(flow);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) INIT_LIST_HEAD(&flow->flowchain);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) codel_vars_init(&flow->cvars);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) memset(q->backlogs, 0, q->flows_cnt * sizeof(u32));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) sch->q.qlen = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) sch->qstats.backlog = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) q->memory_usage = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) static const struct nla_policy fq_codel_policy[TCA_FQ_CODEL_MAX + 1] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) [TCA_FQ_CODEL_TARGET] = { .type = NLA_U32 },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) [TCA_FQ_CODEL_LIMIT] = { .type = NLA_U32 },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) [TCA_FQ_CODEL_INTERVAL] = { .type = NLA_U32 },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) [TCA_FQ_CODEL_ECN] = { .type = NLA_U32 },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) [TCA_FQ_CODEL_FLOWS] = { .type = NLA_U32 },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) [TCA_FQ_CODEL_QUANTUM] = { .type = NLA_U32 },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) [TCA_FQ_CODEL_CE_THRESHOLD] = { .type = NLA_U32 },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) [TCA_FQ_CODEL_DROP_BATCH_SIZE] = { .type = NLA_U32 },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) [TCA_FQ_CODEL_MEMORY_LIMIT] = { .type = NLA_U32 },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) static int fq_codel_change(struct Qdisc *sch, struct nlattr *opt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) struct netlink_ext_ack *extack)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) struct fq_codel_sched_data *q = qdisc_priv(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) struct nlattr *tb[TCA_FQ_CODEL_MAX + 1];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) u32 quantum = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) if (!opt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) err = nla_parse_nested_deprecated(tb, TCA_FQ_CODEL_MAX, opt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) fq_codel_policy, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) if (err < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) if (tb[TCA_FQ_CODEL_FLOWS]) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) if (q->flows)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) q->flows_cnt = nla_get_u32(tb[TCA_FQ_CODEL_FLOWS]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) if (!q->flows_cnt ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) q->flows_cnt > 65536)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) if (tb[TCA_FQ_CODEL_QUANTUM]) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) quantum = max(256U, nla_get_u32(tb[TCA_FQ_CODEL_QUANTUM]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) if (quantum > FQ_CODEL_QUANTUM_MAX) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) NL_SET_ERR_MSG(extack, "Invalid quantum");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) sch_tree_lock(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) if (tb[TCA_FQ_CODEL_TARGET]) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) u64 target = nla_get_u32(tb[TCA_FQ_CODEL_TARGET]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) q->cparams.target = (target * NSEC_PER_USEC) >> CODEL_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) if (tb[TCA_FQ_CODEL_CE_THRESHOLD]) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) u64 val = nla_get_u32(tb[TCA_FQ_CODEL_CE_THRESHOLD]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) q->cparams.ce_threshold = (val * NSEC_PER_USEC) >> CODEL_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) if (tb[TCA_FQ_CODEL_INTERVAL]) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) u64 interval = nla_get_u32(tb[TCA_FQ_CODEL_INTERVAL]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) q->cparams.interval = (interval * NSEC_PER_USEC) >> CODEL_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) if (tb[TCA_FQ_CODEL_LIMIT])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) sch->limit = nla_get_u32(tb[TCA_FQ_CODEL_LIMIT]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) if (tb[TCA_FQ_CODEL_ECN])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) q->cparams.ecn = !!nla_get_u32(tb[TCA_FQ_CODEL_ECN]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) if (quantum)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) q->quantum = quantum;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) if (tb[TCA_FQ_CODEL_DROP_BATCH_SIZE])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) q->drop_batch_size = max(1U, nla_get_u32(tb[TCA_FQ_CODEL_DROP_BATCH_SIZE]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) if (tb[TCA_FQ_CODEL_MEMORY_LIMIT])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) q->memory_limit = min(1U << 31, nla_get_u32(tb[TCA_FQ_CODEL_MEMORY_LIMIT]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) while (sch->q.qlen > sch->limit ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) q->memory_usage > q->memory_limit) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) struct sk_buff *skb = fq_codel_dequeue(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) q->cstats.drop_len += qdisc_pkt_len(skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) rtnl_kfree_skbs(skb, skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) q->cstats.drop_count++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) qdisc_tree_reduce_backlog(sch, q->cstats.drop_count, q->cstats.drop_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) q->cstats.drop_count = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) q->cstats.drop_len = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) sch_tree_unlock(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) static void fq_codel_destroy(struct Qdisc *sch)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) struct fq_codel_sched_data *q = qdisc_priv(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) tcf_block_put(q->block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) kvfree(q->backlogs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) kvfree(q->flows);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) static int fq_codel_init(struct Qdisc *sch, struct nlattr *opt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) struct netlink_ext_ack *extack)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) struct fq_codel_sched_data *q = qdisc_priv(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) sch->limit = 10*1024;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) q->flows_cnt = 1024;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) q->memory_limit = 32 << 20; /* 32 MBytes */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) q->drop_batch_size = 64;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) q->quantum = psched_mtu(qdisc_dev(sch));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) INIT_LIST_HEAD(&q->new_flows);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) INIT_LIST_HEAD(&q->old_flows);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) codel_params_init(&q->cparams);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) codel_stats_init(&q->cstats);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) q->cparams.ecn = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) q->cparams.mtu = psched_mtu(qdisc_dev(sch));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) if (opt) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) err = fq_codel_change(sch, opt, extack);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) goto init_failure;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) err = tcf_block_get(&q->block, &q->filter_list, sch, extack);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) goto init_failure;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) if (!q->flows) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) q->flows = kvcalloc(q->flows_cnt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) sizeof(struct fq_codel_flow),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) if (!q->flows) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) err = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) goto init_failure;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) q->backlogs = kvcalloc(q->flows_cnt, sizeof(u32), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) if (!q->backlogs) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) err = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) goto alloc_failure;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) for (i = 0; i < q->flows_cnt; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) struct fq_codel_flow *flow = q->flows + i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) INIT_LIST_HEAD(&flow->flowchain);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) codel_vars_init(&flow->cvars);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) if (sch->limit >= 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) sch->flags |= TCQ_F_CAN_BYPASS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) sch->flags &= ~TCQ_F_CAN_BYPASS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) alloc_failure:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) kvfree(q->flows);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) q->flows = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) init_failure:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) q->flows_cnt = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) static int fq_codel_dump(struct Qdisc *sch, struct sk_buff *skb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) struct fq_codel_sched_data *q = qdisc_priv(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) struct nlattr *opts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) opts = nla_nest_start_noflag(skb, TCA_OPTIONS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) if (opts == NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) goto nla_put_failure;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) if (nla_put_u32(skb, TCA_FQ_CODEL_TARGET,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) codel_time_to_us(q->cparams.target)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) nla_put_u32(skb, TCA_FQ_CODEL_LIMIT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) sch->limit) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) nla_put_u32(skb, TCA_FQ_CODEL_INTERVAL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) codel_time_to_us(q->cparams.interval)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) nla_put_u32(skb, TCA_FQ_CODEL_ECN,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) q->cparams.ecn) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) nla_put_u32(skb, TCA_FQ_CODEL_QUANTUM,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) q->quantum) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) nla_put_u32(skb, TCA_FQ_CODEL_DROP_BATCH_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) q->drop_batch_size) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) nla_put_u32(skb, TCA_FQ_CODEL_MEMORY_LIMIT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) q->memory_limit) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) nla_put_u32(skb, TCA_FQ_CODEL_FLOWS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) q->flows_cnt))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) goto nla_put_failure;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) if (q->cparams.ce_threshold != CODEL_DISABLED_THRESHOLD &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) nla_put_u32(skb, TCA_FQ_CODEL_CE_THRESHOLD,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) codel_time_to_us(q->cparams.ce_threshold)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) goto nla_put_failure;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) return nla_nest_end(skb, opts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) nla_put_failure:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) static int fq_codel_dump_stats(struct Qdisc *sch, struct gnet_dump *d)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) struct fq_codel_sched_data *q = qdisc_priv(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) struct tc_fq_codel_xstats st = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) .type = TCA_FQ_CODEL_XSTATS_QDISC,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) struct list_head *pos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) st.qdisc_stats.maxpacket = q->cstats.maxpacket;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) st.qdisc_stats.drop_overlimit = q->drop_overlimit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) st.qdisc_stats.ecn_mark = q->cstats.ecn_mark;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) st.qdisc_stats.new_flow_count = q->new_flow_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) st.qdisc_stats.ce_mark = q->cstats.ce_mark;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) st.qdisc_stats.memory_usage = q->memory_usage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) st.qdisc_stats.drop_overmemory = q->drop_overmemory;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) sch_tree_lock(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) list_for_each(pos, &q->new_flows)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) st.qdisc_stats.new_flows_len++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) list_for_each(pos, &q->old_flows)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) st.qdisc_stats.old_flows_len++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) sch_tree_unlock(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) return gnet_stats_copy_app(d, &st, sizeof(st));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) static struct Qdisc *fq_codel_leaf(struct Qdisc *sch, unsigned long arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) static unsigned long fq_codel_find(struct Qdisc *sch, u32 classid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) static unsigned long fq_codel_bind(struct Qdisc *sch, unsigned long parent,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) u32 classid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) static void fq_codel_unbind(struct Qdisc *q, unsigned long cl)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) static struct tcf_block *fq_codel_tcf_block(struct Qdisc *sch, unsigned long cl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) struct netlink_ext_ack *extack)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) struct fq_codel_sched_data *q = qdisc_priv(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) if (cl)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) return q->block;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) static int fq_codel_dump_class(struct Qdisc *sch, unsigned long cl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) struct sk_buff *skb, struct tcmsg *tcm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) tcm->tcm_handle |= TC_H_MIN(cl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) static int fq_codel_dump_class_stats(struct Qdisc *sch, unsigned long cl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) struct gnet_dump *d)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) struct fq_codel_sched_data *q = qdisc_priv(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) u32 idx = cl - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) struct gnet_stats_queue qs = { 0 };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) struct tc_fq_codel_xstats xstats;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) if (idx < q->flows_cnt) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) const struct fq_codel_flow *flow = &q->flows[idx];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) const struct sk_buff *skb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) memset(&xstats, 0, sizeof(xstats));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) xstats.type = TCA_FQ_CODEL_XSTATS_CLASS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) xstats.class_stats.deficit = flow->deficit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) xstats.class_stats.ldelay =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) codel_time_to_us(flow->cvars.ldelay);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) xstats.class_stats.count = flow->cvars.count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) xstats.class_stats.lastcount = flow->cvars.lastcount;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) xstats.class_stats.dropping = flow->cvars.dropping;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) if (flow->cvars.dropping) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) codel_tdiff_t delta = flow->cvars.drop_next -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) codel_get_time();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) xstats.class_stats.drop_next = (delta >= 0) ?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) codel_time_to_us(delta) :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) -codel_time_to_us(-delta);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) if (flow->head) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) sch_tree_lock(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) skb = flow->head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) while (skb) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) qs.qlen++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) skb = skb->next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) sch_tree_unlock(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) qs.backlog = q->backlogs[idx];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) qs.drops = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) if (gnet_stats_copy_queue(d, NULL, &qs, qs.qlen) < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) if (idx < q->flows_cnt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) return gnet_stats_copy_app(d, &xstats, sizeof(xstats));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) static void fq_codel_walk(struct Qdisc *sch, struct qdisc_walker *arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) struct fq_codel_sched_data *q = qdisc_priv(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) unsigned int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) if (arg->stop)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) for (i = 0; i < q->flows_cnt; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) if (list_empty(&q->flows[i].flowchain) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) arg->count < arg->skip) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) arg->count++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) if (arg->fn(sch, i + 1, arg) < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) arg->stop = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) arg->count++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) static const struct Qdisc_class_ops fq_codel_class_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) .leaf = fq_codel_leaf,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) .find = fq_codel_find,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) .tcf_block = fq_codel_tcf_block,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) .bind_tcf = fq_codel_bind,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) .unbind_tcf = fq_codel_unbind,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) .dump = fq_codel_dump_class,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) .dump_stats = fq_codel_dump_class_stats,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) .walk = fq_codel_walk,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) static struct Qdisc_ops fq_codel_qdisc_ops __read_mostly = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) .cl_ops = &fq_codel_class_ops,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) .id = "fq_codel",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) .priv_size = sizeof(struct fq_codel_sched_data),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) .enqueue = fq_codel_enqueue,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) .dequeue = fq_codel_dequeue,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) .peek = qdisc_peek_dequeued,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) .init = fq_codel_init,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) .reset = fq_codel_reset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) .destroy = fq_codel_destroy,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) .change = fq_codel_change,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) .dump = fq_codel_dump,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) .dump_stats = fq_codel_dump_stats,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) .owner = THIS_MODULE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) static int __init fq_codel_module_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) return register_qdisc(&fq_codel_qdisc_ops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) static void __exit fq_codel_module_exit(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) unregister_qdisc(&fq_codel_qdisc_ops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) module_init(fq_codel_module_init)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) module_exit(fq_codel_module_exit)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) MODULE_AUTHOR("Eric Dumazet");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) MODULE_LICENSE("GPL");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) MODULE_DESCRIPTION("Fair Queue CoDel discipline");
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags