Orange Pi5 kernel

^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   1) /* SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   2)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   3)  * Copyright (c) 2019 Facebook
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   4)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   5)  * This program is free software; you can redistribute it and/or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   6)  * modify it under the terms of version 2 of the GNU General Public
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   7)  * License as published by the Free Software Foundation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   8)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   9)  * Include file for sample Host Bandwidth Manager (HBM) BPF programs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  10)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  11) #define KBUILD_MODNAME "foo"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  12) #include <stddef.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  13) #include <stdbool.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  14) #include <uapi/linux/bpf.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  15) #include <uapi/linux/if_ether.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  16) #include <uapi/linux/if_packet.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  17) #include <uapi/linux/ip.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  18) #include <uapi/linux/ipv6.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  19) #include <uapi/linux/in.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  20) #include <uapi/linux/tcp.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  21) #include <uapi/linux/filter.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  22) #include <uapi/linux/pkt_cls.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  23) #include <net/ipv6.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  24) #include <net/inet_ecn.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  25) #include <bpf/bpf_endian.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  26) #include <bpf/bpf_helpers.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  27) #include "hbm.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  28) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  29) #define DROP_PKT	0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  30) #define ALLOW_PKT	1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  31) #define TCP_ECN_OK	1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  32) #define CWR		2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  33) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  34) #ifndef HBM_DEBUG  // Define HBM_DEBUG to enable debugging
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  35) #undef bpf_printk
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  36) #define bpf_printk(fmt, ...)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  37) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  38) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  39) #define INITIAL_CREDIT_PACKETS	100
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  40) #define MAX_BYTES_PER_PACKET	1500
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  41) #define MARK_THRESH		(40 * MAX_BYTES_PER_PACKET)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  42) #define DROP_THRESH		(80 * 5 * MAX_BYTES_PER_PACKET)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  43) #define LARGE_PKT_DROP_THRESH	(DROP_THRESH - (15 * MAX_BYTES_PER_PACKET))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  44) #define MARK_REGION_SIZE	(LARGE_PKT_DROP_THRESH - MARK_THRESH)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  45) #define LARGE_PKT_THRESH	120
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  46) #define MAX_CREDIT		(100 * MAX_BYTES_PER_PACKET)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  47) #define INIT_CREDIT		(INITIAL_CREDIT_PACKETS * MAX_BYTES_PER_PACKET)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  48) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  49) // Time base accounting for fq's EDT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  50) #define BURST_SIZE_NS		100000 // 100us
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  51) #define MARK_THRESH_NS		50000 // 50us
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  52) #define DROP_THRESH_NS		500000 // 500us
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  53) // Reserve 20us of queuing for small packets (less than 120 bytes)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  54) #define LARGE_PKT_DROP_THRESH_NS (DROP_THRESH_NS - 20000)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  55) #define MARK_REGION_SIZE_NS	(LARGE_PKT_DROP_THRESH_NS - MARK_THRESH_NS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  56) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  57) // rate in bytes per ns << 20
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  58) #define CREDIT_PER_NS(delta, rate) ((((u64)(delta)) * (rate)) >> 20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  59) #define BYTES_PER_NS(delta, rate) ((((u64)(delta)) * (rate)) >> 20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  60) #define BYTES_TO_NS(bytes, rate) div64_u64(((u64)(bytes)) << 20, (u64)(rate))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  61) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  62) struct {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  63) 	__uint(type, BPF_MAP_TYPE_CGROUP_STORAGE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  64) 	__type(key, struct bpf_cgroup_storage_key);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  65) 	__type(value, struct hbm_vqueue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  66) } queue_state SEC(".maps");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  67) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  68) struct {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  69) 	__uint(type, BPF_MAP_TYPE_ARRAY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  70) 	__uint(max_entries, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  71) 	__type(key, u32);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  72) 	__type(value, struct hvm_queue_stats);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  73) } queue_stats SEC(".maps");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  74) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  75) struct hbm_pkt_info {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  76) 	int	cwnd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  77) 	int	rtt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  78) 	int	packets_out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  79) 	bool	is_ip;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  80) 	bool	is_tcp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  81) 	short	ecn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  82) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  83) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  84) static int get_tcp_info(struct __sk_buff *skb, struct hbm_pkt_info *pkti)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  85) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  86) 	struct bpf_sock *sk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  87) 	struct bpf_tcp_sock *tp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  88) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  89) 	sk = skb->sk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  90) 	if (sk) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  91) 		sk = bpf_sk_fullsock(sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  92) 		if (sk) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  93) 			if (sk->protocol == IPPROTO_TCP) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  94) 				tp = bpf_tcp_sock(sk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  95) 				if (tp) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  96) 					pkti->cwnd = tp->snd_cwnd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  97) 					pkti->rtt = tp->srtt_us >> 3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  98) 					pkti->packets_out = tp->packets_out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  99) 					return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) 				}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) 			}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) 		}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) 	pkti->cwnd = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) 	pkti->rtt = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) 	pkti->packets_out = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) 	return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) static void hbm_get_pkt_info(struct __sk_buff *skb,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) 			     struct hbm_pkt_info *pkti)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) 	struct iphdr iph;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) 	struct ipv6hdr *ip6h;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) 	pkti->cwnd = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) 	pkti->rtt = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) 	bpf_skb_load_bytes(skb, 0, &iph, 12);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) 	if (iph.version == 6) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) 		ip6h = (struct ipv6hdr *)&iph;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) 		pkti->is_ip = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) 		pkti->is_tcp = (ip6h->nexthdr == 6);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) 		pkti->ecn = (ip6h->flow_lbl[0] >> 4) & INET_ECN_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) 	} else if (iph.version == 4) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) 		pkti->is_ip = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) 		pkti->is_tcp = (iph.protocol == 6);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) 		pkti->ecn = iph.tos & INET_ECN_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) 	} else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) 		pkti->is_ip = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) 		pkti->is_tcp = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) 		pkti->ecn = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) 	if (pkti->is_tcp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) 		get_tcp_info(skb, pkti);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) static __always_inline void hbm_init_vqueue(struct hbm_vqueue *qdp, int rate)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) 	bpf_printk("Initializing queue_state, rate:%d\n", rate * 128);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) 	qdp->lasttime = bpf_ktime_get_ns();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) 	qdp->credit = INIT_CREDIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) 	qdp->rate = rate * 128;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) static __always_inline void hbm_init_edt_vqueue(struct hbm_vqueue *qdp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) 						int rate)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) 	unsigned long long curtime;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) 	curtime = bpf_ktime_get_ns();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) 	bpf_printk("Initializing queue_state, rate:%d\n", rate * 128);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) 	qdp->lasttime = curtime - BURST_SIZE_NS;	// support initial burst
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) 	qdp->credit = 0;				// not used
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) 	qdp->rate = rate * 128;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) static __always_inline void hbm_update_stats(struct hbm_queue_stats *qsp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) 					     int len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) 					     unsigned long long curtime,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) 					     bool congestion_flag,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) 					     bool drop_flag,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) 					     bool cwr_flag,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) 					     bool ecn_ce_flag,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) 					     struct hbm_pkt_info *pkti,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) 					     int credit)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) 	int rv = ALLOW_PKT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) 	if (qsp != NULL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) 		// Following is needed for work conserving
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) 		__sync_add_and_fetch(&(qsp->bytes_total), len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) 		if (qsp->stats) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) 			// Optionally update statistics
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) 			if (qsp->firstPacketTime == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) 				qsp->firstPacketTime = curtime;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) 			qsp->lastPacketTime = curtime;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) 			__sync_add_and_fetch(&(qsp->pkts_total), 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) 			if (congestion_flag) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) 				__sync_add_and_fetch(&(qsp->pkts_marked), 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) 				__sync_add_and_fetch(&(qsp->bytes_marked), len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) 			}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) 			if (drop_flag) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) 				__sync_add_and_fetch(&(qsp->pkts_dropped), 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) 				__sync_add_and_fetch(&(qsp->bytes_dropped),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) 						     len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) 			}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) 			if (ecn_ce_flag)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) 				__sync_add_and_fetch(&(qsp->pkts_ecn_ce), 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) 			if (pkti->cwnd) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) 				__sync_add_and_fetch(&(qsp->sum_cwnd),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) 						     pkti->cwnd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) 				__sync_add_and_fetch(&(qsp->sum_cwnd_cnt), 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) 			}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) 			if (pkti->rtt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) 				__sync_add_and_fetch(&(qsp->sum_rtt),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) 						     pkti->rtt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) 			__sync_add_and_fetch(&(qsp->sum_credit), credit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) 			if (drop_flag)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) 				rv = DROP_PKT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) 			if (cwr_flag)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) 				rv |= 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) 			if (rv == DROP_PKT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) 				__sync_add_and_fetch(&(qsp->returnValCount[0]),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) 						     1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) 			else if (rv == ALLOW_PKT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) 				__sync_add_and_fetch(&(qsp->returnValCount[1]),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) 						     1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) 			else if (rv == 2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) 				__sync_add_and_fetch(&(qsp->returnValCount[2]),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) 						     1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) 			else if (rv == 3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) 				__sync_add_and_fetch(&(qsp->returnValCount[3]),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) 						     1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) 		}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) }