Orange Pi5 kernel

^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   3)  * net/sched/sch_mq.c		Classful multiqueue dummy scheduler
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   4)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   5)  * Copyright (c) 2009 Patrick McHardy <kaber@trash.net>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   6)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   7) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   8) #include <linux/types.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   9) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  10) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  11) #include <linux/export.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  12) #include <linux/string.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  13) #include <linux/errno.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  14) #include <linux/skbuff.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  15) #include <net/netlink.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  16) #include <net/pkt_cls.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  17) #include <net/pkt_sched.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  18) #include <net/sch_generic.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  19) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  20) struct mq_sched {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  21) 	struct Qdisc		**qdiscs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  22) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  23) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  24) static int mq_offload(struct Qdisc *sch, enum tc_mq_command cmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  25) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  26) 	struct net_device *dev = qdisc_dev(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  27) 	struct tc_mq_qopt_offload opt = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  28) 		.command = cmd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  29) 		.handle = sch->handle,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  30) 	};
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  31) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  32) 	if (!tc_can_offload(dev) || !dev->netdev_ops->ndo_setup_tc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  33) 		return -EOPNOTSUPP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  34) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  35) 	return dev->netdev_ops->ndo_setup_tc(dev, TC_SETUP_QDISC_MQ, &opt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  36) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  37) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  38) static int mq_offload_stats(struct Qdisc *sch)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  39) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  40) 	struct tc_mq_qopt_offload opt = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  41) 		.command = TC_MQ_STATS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  42) 		.handle = sch->handle,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  43) 		.stats = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  44) 			.bstats = &sch->bstats,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  45) 			.qstats = &sch->qstats,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  46) 		},
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  47) 	};
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  48) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  49) 	return qdisc_offload_dump_helper(sch, TC_SETUP_QDISC_MQ, &opt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  50) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  51) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  52) static void mq_destroy(struct Qdisc *sch)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  53) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  54) 	struct net_device *dev = qdisc_dev(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  55) 	struct mq_sched *priv = qdisc_priv(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  56) 	unsigned int ntx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  57) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  58) 	mq_offload(sch, TC_MQ_DESTROY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  59) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  60) 	if (!priv->qdiscs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  61) 		return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  62) 	for (ntx = 0; ntx < dev->num_tx_queues && priv->qdiscs[ntx]; ntx++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  63) 		qdisc_put(priv->qdiscs[ntx]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  64) 	kfree(priv->qdiscs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  65) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  66) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  67) static int mq_init(struct Qdisc *sch, struct nlattr *opt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  68) 		   struct netlink_ext_ack *extack)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  69) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  70) 	struct net_device *dev = qdisc_dev(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  71) 	struct mq_sched *priv = qdisc_priv(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  72) 	struct netdev_queue *dev_queue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  73) 	struct Qdisc *qdisc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  74) 	unsigned int ntx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  75) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  76) 	if (sch->parent != TC_H_ROOT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  77) 		return -EOPNOTSUPP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  78) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  79) 	if (!netif_is_multiqueue(dev))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  80) 		return -EOPNOTSUPP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  81) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  82) 	/* pre-allocate qdiscs, attachment can't fail */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  83) 	priv->qdiscs = kcalloc(dev->num_tx_queues, sizeof(priv->qdiscs[0]),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  84) 			       GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  85) 	if (!priv->qdiscs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  86) 		return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  87) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  88) 	for (ntx = 0; ntx < dev->num_tx_queues; ntx++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  89) 		dev_queue = netdev_get_tx_queue(dev, ntx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  90) 		qdisc = qdisc_create_dflt(dev_queue, get_default_qdisc_ops(dev, ntx),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  91) 					  TC_H_MAKE(TC_H_MAJ(sch->handle),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  92) 						    TC_H_MIN(ntx + 1)),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  93) 					  extack);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  94) 		if (!qdisc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  95) 			return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  96) 		priv->qdiscs[ntx] = qdisc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  97) 		qdisc->flags |= TCQ_F_ONETXQUEUE | TCQ_F_NOPARENT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  98) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  99) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) 	sch->flags |= TCQ_F_MQROOT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) 	mq_offload(sch, TC_MQ_CREATE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) static void mq_attach(struct Qdisc *sch)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) 	struct net_device *dev = qdisc_dev(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) 	struct mq_sched *priv = qdisc_priv(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) 	struct Qdisc *qdisc, *old;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) 	unsigned int ntx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) 	for (ntx = 0; ntx < dev->num_tx_queues; ntx++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) 		qdisc = priv->qdiscs[ntx];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) 		old = dev_graft_qdisc(qdisc->dev_queue, qdisc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) 		if (old)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) 			qdisc_put(old);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) #ifdef CONFIG_NET_SCHED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) 		if (ntx < dev->real_num_tx_queues)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) 			qdisc_hash_add(qdisc, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) 	kfree(priv->qdiscs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) 	priv->qdiscs = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) static int mq_dump(struct Qdisc *sch, struct sk_buff *skb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) 	struct net_device *dev = qdisc_dev(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) 	struct Qdisc *qdisc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) 	unsigned int ntx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) 	__u32 qlen = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) 	sch->q.qlen = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) 	memset(&sch->bstats, 0, sizeof(sch->bstats));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) 	memset(&sch->qstats, 0, sizeof(sch->qstats));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) 	/* MQ supports lockless qdiscs. However, statistics accounting needs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) 	 * to account for all, none, or a mix of locked and unlocked child
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) 	 * qdiscs. Percpu stats are added to counters in-band and locking
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) 	 * qdisc totals are added at end.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) 	for (ntx = 0; ntx < dev->num_tx_queues; ntx++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) 		qdisc = netdev_get_tx_queue(dev, ntx)->qdisc_sleeping;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) 		spin_lock_bh(qdisc_lock(qdisc));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) 		if (qdisc_is_percpu_stats(qdisc)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) 			qlen = qdisc_qlen_sum(qdisc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) 			__gnet_stats_copy_basic(NULL, &sch->bstats,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) 						qdisc->cpu_bstats,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) 						&qdisc->bstats);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) 			__gnet_stats_copy_queue(&sch->qstats,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) 						qdisc->cpu_qstats,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) 						&qdisc->qstats, qlen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) 			sch->q.qlen		+= qlen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) 		} else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) 			sch->q.qlen		+= qdisc->q.qlen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) 			sch->bstats.bytes	+= qdisc->bstats.bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) 			sch->bstats.packets	+= qdisc->bstats.packets;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) 			sch->qstats.qlen	+= qdisc->qstats.qlen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) 			sch->qstats.backlog	+= qdisc->qstats.backlog;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) 			sch->qstats.drops	+= qdisc->qstats.drops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) 			sch->qstats.requeues	+= qdisc->qstats.requeues;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) 			sch->qstats.overlimits	+= qdisc->qstats.overlimits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) 		}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) 		spin_unlock_bh(qdisc_lock(qdisc));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) 	return mq_offload_stats(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) static struct netdev_queue *mq_queue_get(struct Qdisc *sch, unsigned long cl)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) 	struct net_device *dev = qdisc_dev(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) 	unsigned long ntx = cl - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) 	if (ntx >= dev->num_tx_queues)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) 		return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) 	return netdev_get_tx_queue(dev, ntx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) static struct netdev_queue *mq_select_queue(struct Qdisc *sch,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) 					    struct tcmsg *tcm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) 	return mq_queue_get(sch, TC_H_MIN(tcm->tcm_parent));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) static int mq_graft(struct Qdisc *sch, unsigned long cl, struct Qdisc *new,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) 		    struct Qdisc **old, struct netlink_ext_ack *extack)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) 	struct netdev_queue *dev_queue = mq_queue_get(sch, cl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) 	struct tc_mq_qopt_offload graft_offload;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) 	struct net_device *dev = qdisc_dev(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) 	if (dev->flags & IFF_UP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) 		dev_deactivate(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) 	*old = dev_graft_qdisc(dev_queue, new);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) 	if (new)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) 		new->flags |= TCQ_F_ONETXQUEUE | TCQ_F_NOPARENT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) 	if (dev->flags & IFF_UP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) 		dev_activate(dev);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) 	graft_offload.handle = sch->handle;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) 	graft_offload.graft_params.queue = cl - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) 	graft_offload.graft_params.child_handle = new ? new->handle : 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) 	graft_offload.command = TC_MQ_GRAFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) 	qdisc_offload_graft_helper(qdisc_dev(sch), sch, new, *old,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) 				   TC_SETUP_QDISC_MQ, &graft_offload, extack);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) static struct Qdisc *mq_leaf(struct Qdisc *sch, unsigned long cl)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) 	struct netdev_queue *dev_queue = mq_queue_get(sch, cl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) 	return dev_queue->qdisc_sleeping;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) static unsigned long mq_find(struct Qdisc *sch, u32 classid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) 	unsigned int ntx = TC_H_MIN(classid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) 	if (!mq_queue_get(sch, ntx))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) 		return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) 	return ntx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) static int mq_dump_class(struct Qdisc *sch, unsigned long cl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) 			 struct sk_buff *skb, struct tcmsg *tcm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) 	struct netdev_queue *dev_queue = mq_queue_get(sch, cl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) 	tcm->tcm_parent = TC_H_ROOT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) 	tcm->tcm_handle |= TC_H_MIN(cl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) 	tcm->tcm_info = dev_queue->qdisc_sleeping->handle;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) static int mq_dump_class_stats(struct Qdisc *sch, unsigned long cl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) 			       struct gnet_dump *d)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) 	struct netdev_queue *dev_queue = mq_queue_get(sch, cl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) 	sch = dev_queue->qdisc_sleeping;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) 	if (gnet_stats_copy_basic(&sch->running, d, sch->cpu_bstats,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) 				  &sch->bstats) < 0 ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) 	    qdisc_qstats_copy(d, sch) < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) 		return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) static void mq_walk(struct Qdisc *sch, struct qdisc_walker *arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) 	struct net_device *dev = qdisc_dev(sch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) 	unsigned int ntx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) 	if (arg->stop)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) 		return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) 	arg->count = arg->skip;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) 	for (ntx = arg->skip; ntx < dev->num_tx_queues; ntx++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) 		if (arg->fn(sch, ntx + 1, arg) < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) 			arg->stop = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) 			break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) 		}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) 		arg->count++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) static const struct Qdisc_class_ops mq_class_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) 	.select_queue	= mq_select_queue,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) 	.graft		= mq_graft,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) 	.leaf		= mq_leaf,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) 	.find		= mq_find,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) 	.walk		= mq_walk,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) 	.dump		= mq_dump_class,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) 	.dump_stats	= mq_dump_class_stats,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) struct Qdisc_ops mq_qdisc_ops __read_mostly = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) 	.cl_ops		= &mq_class_ops,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) 	.id		= "mq",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) 	.priv_size	= sizeof(struct mq_sched),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) 	.init		= mq_init,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) 	.destroy	= mq_destroy,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) 	.attach		= mq_attach,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) 	.dump		= mq_dump,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) 	.owner		= THIS_MODULE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) };