^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /* Flow Queue PIE discipline
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * Copyright (C) 2019 Mohit P. Tahiliani <tahiliani@nitk.edu.in>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Copyright (C) 2019 Sachin D. Patil <sdp.sachin@gmail.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * Copyright (C) 2019 V. Saicharan <vsaicharan1998@gmail.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * Copyright (C) 2019 Mohit Bhasi <mohitbhasi1998@gmail.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) * Copyright (C) 2019 Leslie Monis <lesliemonis@gmail.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) * Copyright (C) 2019 Gautam Ramakrishnan <gautamramk@gmail.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/jhash.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/sizes.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/vmalloc.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <net/pkt_cls.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <net/pie.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) /* Flow Queue PIE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) * Principles:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) * - Packets are classified on flows.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) * - This is a Stochastic model (as we use a hash, several flows might
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) * be hashed to the same slot)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) * - Each flow has a PIE managed queue.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) * - Flows are linked onto two (Round Robin) lists,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) * so that new flows have priority on old ones.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) * - For a given flow, packets are not reordered.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) * - Drops during enqueue only.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) * - ECN capability is off by default.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) * - ECN threshold (if ECN is enabled) is at 10% by default.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) * - Uses timestamps to calculate queue delay by default.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) * struct fq_pie_flow - contains data for each flow
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) * @vars: pie vars associated with the flow
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) * @deficit: number of remaining byte credits
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) * @backlog: size of data in the flow
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) * @qlen: number of packets in the flow
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) * @flowchain: flowchain for the flow
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) * @head: first packet in the flow
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) * @tail: last packet in the flow
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) struct fq_pie_flow {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) struct pie_vars vars;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) s32 deficit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) u32 backlog;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) u32 qlen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) struct list_head flowchain;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) struct sk_buff *head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) struct sk_buff *tail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) struct fq_pie_sched_data {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) struct tcf_proto __rcu *filter_list; /* optional external classifier */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) struct tcf_block *block;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) struct fq_pie_flow *flows;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) struct Qdisc *sch;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) struct list_head old_flows;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) struct list_head new_flows;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) struct pie_params p_params;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) u32 ecn_prob;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) u32 flows_cnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) u32 quantum;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) u32 memory_limit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) u32 new_flow_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) u32 memory_usage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) u32 overmemory;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) struct pie_stats stats;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) struct timer_list adapt_timer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) static unsigned int fq_pie_hash(const struct fq_pie_sched_data *q,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) struct sk_buff *skb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) return reciprocal_scale(skb_get_hash(skb), q->flows_cnt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) static unsigned int fq_pie_classify(struct sk_buff *skb, struct Qdisc *sch,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) int *qerr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) struct fq_pie_sched_data *q = qdisc_priv(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) struct tcf_proto *filter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) struct tcf_result res;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) int result;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) if (TC_H_MAJ(skb->priority) == sch->handle &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) TC_H_MIN(skb->priority) > 0 &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) TC_H_MIN(skb->priority) <= q->flows_cnt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) return TC_H_MIN(skb->priority);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) filter = rcu_dereference_bh(q->filter_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) if (!filter)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) return fq_pie_hash(q, skb) + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) *qerr = NET_XMIT_SUCCESS | __NET_XMIT_BYPASS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) result = tcf_classify(skb, filter, &res, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) if (result >= 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) #ifdef CONFIG_NET_CLS_ACT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) switch (result) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) case TC_ACT_STOLEN:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) case TC_ACT_QUEUED:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) case TC_ACT_TRAP:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) *qerr = NET_XMIT_SUCCESS | __NET_XMIT_STOLEN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) fallthrough;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) case TC_ACT_SHOT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) if (TC_H_MIN(res.classid) <= q->flows_cnt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) return TC_H_MIN(res.classid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) /* add skb to flow queue (tail add) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) static inline void flow_queue_add(struct fq_pie_flow *flow,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) struct sk_buff *skb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) if (!flow->head)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) flow->head = skb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) flow->tail->next = skb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) flow->tail = skb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) skb->next = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) static int fq_pie_qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) struct sk_buff **to_free)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) struct fq_pie_sched_data *q = qdisc_priv(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) struct fq_pie_flow *sel_flow;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) u8 memory_limited = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) u8 enqueue = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) u32 pkt_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) u32 idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) /* Classifies packet into corresponding flow */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) idx = fq_pie_classify(skb, sch, &ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) if (idx == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) if (ret & __NET_XMIT_BYPASS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) qdisc_qstats_drop(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) __qdisc_drop(skb, to_free);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) idx--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) sel_flow = &q->flows[idx];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) /* Checks whether adding a new packet would exceed memory limit */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) get_pie_cb(skb)->mem_usage = skb->truesize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) memory_limited = q->memory_usage > q->memory_limit + skb->truesize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) /* Checks if the qdisc is full */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) if (unlikely(qdisc_qlen(sch) >= sch->limit)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) q->stats.overlimit++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) } else if (unlikely(memory_limited)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) q->overmemory++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) if (!pie_drop_early(sch, &q->p_params, &sel_flow->vars,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) sel_flow->backlog, skb->len)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) enqueue = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) } else if (q->p_params.ecn &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) sel_flow->vars.prob <= (MAX_PROB / 100) * q->ecn_prob &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) INET_ECN_set_ce(skb)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) /* If packet is ecn capable, mark it if drop probability
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) * is lower than the parameter ecn_prob, else drop it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) q->stats.ecn_mark++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) enqueue = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) if (enqueue) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) /* Set enqueue time only when dq_rate_estimator is disabled. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) if (!q->p_params.dq_rate_estimator)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) pie_set_enqueue_time(skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) pkt_len = qdisc_pkt_len(skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) q->stats.packets_in++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) q->memory_usage += skb->truesize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) sch->qstats.backlog += pkt_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) sch->q.qlen++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) flow_queue_add(sel_flow, skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) if (list_empty(&sel_flow->flowchain)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) list_add_tail(&sel_flow->flowchain, &q->new_flows);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) q->new_flow_count++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) sel_flow->deficit = q->quantum;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) sel_flow->qlen = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) sel_flow->backlog = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) sel_flow->qlen++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) sel_flow->backlog += pkt_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) return NET_XMIT_SUCCESS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) q->stats.dropped++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) sel_flow->vars.accu_prob = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) __qdisc_drop(skb, to_free);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) qdisc_qstats_drop(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) return NET_XMIT_CN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) static const struct nla_policy fq_pie_policy[TCA_FQ_PIE_MAX + 1] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) [TCA_FQ_PIE_LIMIT] = {.type = NLA_U32},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) [TCA_FQ_PIE_FLOWS] = {.type = NLA_U32},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) [TCA_FQ_PIE_TARGET] = {.type = NLA_U32},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) [TCA_FQ_PIE_TUPDATE] = {.type = NLA_U32},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) [TCA_FQ_PIE_ALPHA] = {.type = NLA_U32},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) [TCA_FQ_PIE_BETA] = {.type = NLA_U32},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) [TCA_FQ_PIE_QUANTUM] = {.type = NLA_U32},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) [TCA_FQ_PIE_MEMORY_LIMIT] = {.type = NLA_U32},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) [TCA_FQ_PIE_ECN_PROB] = {.type = NLA_U32},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) [TCA_FQ_PIE_ECN] = {.type = NLA_U32},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) [TCA_FQ_PIE_BYTEMODE] = {.type = NLA_U32},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) [TCA_FQ_PIE_DQ_RATE_ESTIMATOR] = {.type = NLA_U32},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) static inline struct sk_buff *dequeue_head(struct fq_pie_flow *flow)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) struct sk_buff *skb = flow->head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) flow->head = skb->next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) skb->next = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) return skb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) static struct sk_buff *fq_pie_qdisc_dequeue(struct Qdisc *sch)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) struct fq_pie_sched_data *q = qdisc_priv(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) struct sk_buff *skb = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) struct fq_pie_flow *flow;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) struct list_head *head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) u32 pkt_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) begin:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) head = &q->new_flows;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) if (list_empty(head)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) head = &q->old_flows;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) if (list_empty(head))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) flow = list_first_entry(head, struct fq_pie_flow, flowchain);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) /* Flow has exhausted all its credits */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) if (flow->deficit <= 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) flow->deficit += q->quantum;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) list_move_tail(&flow->flowchain, &q->old_flows);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) goto begin;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) if (flow->head) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) skb = dequeue_head(flow);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) pkt_len = qdisc_pkt_len(skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) sch->qstats.backlog -= pkt_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) sch->q.qlen--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) qdisc_bstats_update(sch, skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) if (!skb) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) /* force a pass through old_flows to prevent starvation */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) if (head == &q->new_flows && !list_empty(&q->old_flows))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) list_move_tail(&flow->flowchain, &q->old_flows);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) list_del_init(&flow->flowchain);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) goto begin;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) flow->qlen--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) flow->deficit -= pkt_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) flow->backlog -= pkt_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) q->memory_usage -= get_pie_cb(skb)->mem_usage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) pie_process_dequeue(skb, &q->p_params, &flow->vars, flow->backlog);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) return skb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) static int fq_pie_change(struct Qdisc *sch, struct nlattr *opt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) struct netlink_ext_ack *extack)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) struct fq_pie_sched_data *q = qdisc_priv(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) struct nlattr *tb[TCA_FQ_PIE_MAX + 1];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) unsigned int len_dropped = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) unsigned int num_dropped = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) if (!opt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) err = nla_parse_nested(tb, TCA_FQ_PIE_MAX, opt, fq_pie_policy, extack);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) if (err < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) sch_tree_lock(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) if (tb[TCA_FQ_PIE_LIMIT]) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) u32 limit = nla_get_u32(tb[TCA_FQ_PIE_LIMIT]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) q->p_params.limit = limit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) sch->limit = limit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) if (tb[TCA_FQ_PIE_FLOWS]) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) if (q->flows) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) NL_SET_ERR_MSG_MOD(extack,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) "Number of flows cannot be changed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) goto flow_error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) q->flows_cnt = nla_get_u32(tb[TCA_FQ_PIE_FLOWS]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) if (!q->flows_cnt || q->flows_cnt > 65536) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) NL_SET_ERR_MSG_MOD(extack,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) "Number of flows must range in [1..65536]");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) goto flow_error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) /* convert from microseconds to pschedtime */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) if (tb[TCA_FQ_PIE_TARGET]) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) /* target is in us */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) u32 target = nla_get_u32(tb[TCA_FQ_PIE_TARGET]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) /* convert to pschedtime */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) q->p_params.target =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) PSCHED_NS2TICKS((u64)target * NSEC_PER_USEC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) /* tupdate is in jiffies */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) if (tb[TCA_FQ_PIE_TUPDATE])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) q->p_params.tupdate =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) usecs_to_jiffies(nla_get_u32(tb[TCA_FQ_PIE_TUPDATE]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) if (tb[TCA_FQ_PIE_ALPHA])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) q->p_params.alpha = nla_get_u32(tb[TCA_FQ_PIE_ALPHA]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) if (tb[TCA_FQ_PIE_BETA])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) q->p_params.beta = nla_get_u32(tb[TCA_FQ_PIE_BETA]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) if (tb[TCA_FQ_PIE_QUANTUM])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) q->quantum = nla_get_u32(tb[TCA_FQ_PIE_QUANTUM]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) if (tb[TCA_FQ_PIE_MEMORY_LIMIT])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) q->memory_limit = nla_get_u32(tb[TCA_FQ_PIE_MEMORY_LIMIT]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) if (tb[TCA_FQ_PIE_ECN_PROB])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) q->ecn_prob = nla_get_u32(tb[TCA_FQ_PIE_ECN_PROB]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) if (tb[TCA_FQ_PIE_ECN])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) q->p_params.ecn = nla_get_u32(tb[TCA_FQ_PIE_ECN]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) if (tb[TCA_FQ_PIE_BYTEMODE])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) q->p_params.bytemode = nla_get_u32(tb[TCA_FQ_PIE_BYTEMODE]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) if (tb[TCA_FQ_PIE_DQ_RATE_ESTIMATOR])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) q->p_params.dq_rate_estimator =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) nla_get_u32(tb[TCA_FQ_PIE_DQ_RATE_ESTIMATOR]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) /* Drop excess packets if new limit is lower */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) while (sch->q.qlen > sch->limit) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) struct sk_buff *skb = fq_pie_qdisc_dequeue(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) len_dropped += qdisc_pkt_len(skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) num_dropped += 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) rtnl_kfree_skbs(skb, skb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) qdisc_tree_reduce_backlog(sch, num_dropped, len_dropped);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) sch_tree_unlock(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) flow_error:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) sch_tree_unlock(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) static void fq_pie_timer(struct timer_list *t)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) struct fq_pie_sched_data *q = from_timer(q, t, adapt_timer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) struct Qdisc *sch = q->sch;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) spinlock_t *root_lock; /* to lock qdisc for probability calculations */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) u32 idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) root_lock = qdisc_lock(qdisc_root_sleeping(sch));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) spin_lock(root_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) for (idx = 0; idx < q->flows_cnt; idx++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) pie_calculate_probability(&q->p_params, &q->flows[idx].vars,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) q->flows[idx].backlog);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) /* reset the timer to fire after 'tupdate' jiffies. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) if (q->p_params.tupdate)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) mod_timer(&q->adapt_timer, jiffies + q->p_params.tupdate);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) spin_unlock(root_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) static int fq_pie_init(struct Qdisc *sch, struct nlattr *opt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) struct netlink_ext_ack *extack)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) struct fq_pie_sched_data *q = qdisc_priv(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) u32 idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) pie_params_init(&q->p_params);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) sch->limit = 10 * 1024;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) q->p_params.limit = sch->limit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) q->quantum = psched_mtu(qdisc_dev(sch));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) q->sch = sch;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) q->ecn_prob = 10;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) q->flows_cnt = 1024;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) q->memory_limit = SZ_32M;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) INIT_LIST_HEAD(&q->new_flows);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) INIT_LIST_HEAD(&q->old_flows);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) timer_setup(&q->adapt_timer, fq_pie_timer, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) if (opt) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) err = fq_pie_change(sch, opt, extack);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) err = tcf_block_get(&q->block, &q->filter_list, sch, extack);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) goto init_failure;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) q->flows = kvcalloc(q->flows_cnt, sizeof(struct fq_pie_flow),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) if (!q->flows) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) err = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) goto init_failure;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) for (idx = 0; idx < q->flows_cnt; idx++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) struct fq_pie_flow *flow = q->flows + idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) INIT_LIST_HEAD(&flow->flowchain);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) pie_vars_init(&flow->vars);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) mod_timer(&q->adapt_timer, jiffies + HZ / 2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) init_failure:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) q->flows_cnt = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) static int fq_pie_dump(struct Qdisc *sch, struct sk_buff *skb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) struct fq_pie_sched_data *q = qdisc_priv(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) struct nlattr *opts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) opts = nla_nest_start(skb, TCA_OPTIONS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) if (!opts)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) return -EMSGSIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) /* convert target from pschedtime to us */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) if (nla_put_u32(skb, TCA_FQ_PIE_LIMIT, sch->limit) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) nla_put_u32(skb, TCA_FQ_PIE_FLOWS, q->flows_cnt) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) nla_put_u32(skb, TCA_FQ_PIE_TARGET,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) ((u32)PSCHED_TICKS2NS(q->p_params.target)) /
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) NSEC_PER_USEC) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) nla_put_u32(skb, TCA_FQ_PIE_TUPDATE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) jiffies_to_usecs(q->p_params.tupdate)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) nla_put_u32(skb, TCA_FQ_PIE_ALPHA, q->p_params.alpha) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) nla_put_u32(skb, TCA_FQ_PIE_BETA, q->p_params.beta) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) nla_put_u32(skb, TCA_FQ_PIE_QUANTUM, q->quantum) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) nla_put_u32(skb, TCA_FQ_PIE_MEMORY_LIMIT, q->memory_limit) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) nla_put_u32(skb, TCA_FQ_PIE_ECN_PROB, q->ecn_prob) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) nla_put_u32(skb, TCA_FQ_PIE_ECN, q->p_params.ecn) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) nla_put_u32(skb, TCA_FQ_PIE_BYTEMODE, q->p_params.bytemode) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) nla_put_u32(skb, TCA_FQ_PIE_DQ_RATE_ESTIMATOR,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) q->p_params.dq_rate_estimator))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) goto nla_put_failure;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) return nla_nest_end(skb, opts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) nla_put_failure:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) nla_nest_cancel(skb, opts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) return -EMSGSIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) static int fq_pie_dump_stats(struct Qdisc *sch, struct gnet_dump *d)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) struct fq_pie_sched_data *q = qdisc_priv(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) struct tc_fq_pie_xstats st = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) .packets_in = q->stats.packets_in,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) .overlimit = q->stats.overlimit,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) .overmemory = q->overmemory,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) .dropped = q->stats.dropped,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) .ecn_mark = q->stats.ecn_mark,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) .new_flow_count = q->new_flow_count,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) .memory_usage = q->memory_usage,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) struct list_head *pos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) sch_tree_lock(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) list_for_each(pos, &q->new_flows)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) st.new_flows_len++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) list_for_each(pos, &q->old_flows)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) st.old_flows_len++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) sch_tree_unlock(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) return gnet_stats_copy_app(d, &st, sizeof(st));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) static void fq_pie_reset(struct Qdisc *sch)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) struct fq_pie_sched_data *q = qdisc_priv(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) u32 idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) INIT_LIST_HEAD(&q->new_flows);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) INIT_LIST_HEAD(&q->old_flows);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) for (idx = 0; idx < q->flows_cnt; idx++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) struct fq_pie_flow *flow = q->flows + idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) /* Removes all packets from flow */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) rtnl_kfree_skbs(flow->head, flow->tail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) flow->head = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) INIT_LIST_HEAD(&flow->flowchain);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) pie_vars_init(&flow->vars);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) sch->q.qlen = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) sch->qstats.backlog = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) static void fq_pie_destroy(struct Qdisc *sch)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) struct fq_pie_sched_data *q = qdisc_priv(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) tcf_block_put(q->block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) q->p_params.tupdate = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) del_timer_sync(&q->adapt_timer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) kvfree(q->flows);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) static struct Qdisc_ops fq_pie_qdisc_ops __read_mostly = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) .id = "fq_pie",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) .priv_size = sizeof(struct fq_pie_sched_data),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) .enqueue = fq_pie_qdisc_enqueue,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) .dequeue = fq_pie_qdisc_dequeue,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) .peek = qdisc_peek_dequeued,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) .init = fq_pie_init,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) .destroy = fq_pie_destroy,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) .reset = fq_pie_reset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) .change = fq_pie_change,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) .dump = fq_pie_dump,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) .dump_stats = fq_pie_dump_stats,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) .owner = THIS_MODULE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) static int __init fq_pie_module_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) return register_qdisc(&fq_pie_qdisc_ops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) static void __exit fq_pie_module_exit(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) unregister_qdisc(&fq_pie_qdisc_ops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) module_init(fq_pie_module_init);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) module_exit(fq_pie_module_exit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) MODULE_DESCRIPTION("Flow Queue Proportional Integral controller Enhanced (FQ-PIE)");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) MODULE_AUTHOR("Mohit P. Tahiliani");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) MODULE_LICENSE("GPL");
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags