^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Copyright (c) 2007-2014 Nicira, Inc.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) #include "flow.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) #include "datapath.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include "flow_netlink.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <linux/uaccess.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/netdevice.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/etherdevice.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/if_ether.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/if_vlan.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <net/llc_pdu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/jhash.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/jiffies.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <linux/llc.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <linux/in.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <linux/rcupdate.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include <linux/cpumask.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #include <linux/if_arp.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #include <linux/ip.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #include <linux/ipv6.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) #include <linux/sctp.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) #include <linux/tcp.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) #include <linux/udp.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) #include <linux/icmp.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) #include <linux/icmpv6.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) #include <linux/rculist.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) #include <linux/sort.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) #include <net/ip.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) #include <net/ipv6.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) #include <net/ndisc.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) #define TBL_MIN_BUCKETS 1024
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) #define MASK_ARRAY_SIZE_MIN 16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) #define REHASH_INTERVAL (10 * 60 * HZ)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) #define MC_DEFAULT_HASH_ENTRIES 256
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) #define MC_HASH_SHIFT 8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) #define MC_HASH_SEGS ((sizeof(uint32_t) * 8) / MC_HASH_SHIFT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) static struct kmem_cache *flow_cache;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) struct kmem_cache *flow_stats_cache __read_mostly;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) static u16 range_n_bytes(const struct sw_flow_key_range *range)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) return range->end - range->start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) void ovs_flow_mask_key(struct sw_flow_key *dst, const struct sw_flow_key *src,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) bool full, const struct sw_flow_mask *mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) int start = full ? 0 : mask->range.start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) int len = full ? sizeof *dst : range_n_bytes(&mask->range);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) const long *m = (const long *)((const u8 *)&mask->key + start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) const long *s = (const long *)((const u8 *)src + start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) long *d = (long *)((u8 *)dst + start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) /* If 'full' is true then all of 'dst' is fully initialized. Otherwise,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) * if 'full' is false the memory outside of the 'mask->range' is left
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) * uninitialized. This can be used as an optimization when further
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) * operations on 'dst' only use contents within 'mask->range'.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) for (i = 0; i < len; i += sizeof(long))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) *d++ = *s++ & *m++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) struct sw_flow *ovs_flow_alloc(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) struct sw_flow *flow;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) struct sw_flow_stats *stats;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) flow = kmem_cache_zalloc(flow_cache, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) if (!flow)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) return ERR_PTR(-ENOMEM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) flow->stats_last_writer = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) /* Initialize the default stat node. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) stats = kmem_cache_alloc_node(flow_stats_cache,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) GFP_KERNEL | __GFP_ZERO,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) node_online(0) ? 0 : NUMA_NO_NODE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) if (!stats)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) goto err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) spin_lock_init(&stats->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) RCU_INIT_POINTER(flow->stats[0], stats);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) cpumask_set_cpu(0, &flow->cpu_used_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) return flow;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) err:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) kmem_cache_free(flow_cache, flow);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) return ERR_PTR(-ENOMEM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) int ovs_flow_tbl_count(const struct flow_table *table)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) return table->count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) static void flow_free(struct sw_flow *flow)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) int cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) if (ovs_identifier_is_key(&flow->id))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) kfree(flow->id.unmasked_key);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) if (flow->sf_acts)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) ovs_nla_free_flow_actions((struct sw_flow_actions __force *)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) flow->sf_acts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) /* We open code this to make sure cpu 0 is always considered */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) for (cpu = 0; cpu < nr_cpu_ids;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) cpu = cpumask_next(cpu, &flow->cpu_used_mask)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) if (flow->stats[cpu])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) kmem_cache_free(flow_stats_cache,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) (struct sw_flow_stats __force *)flow->stats[cpu]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) kmem_cache_free(flow_cache, flow);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) static void rcu_free_flow_callback(struct rcu_head *rcu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) struct sw_flow *flow = container_of(rcu, struct sw_flow, rcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) flow_free(flow);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) void ovs_flow_free(struct sw_flow *flow, bool deferred)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) if (!flow)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) if (deferred)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) call_rcu(&flow->rcu, rcu_free_flow_callback);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) flow_free(flow);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) static void __table_instance_destroy(struct table_instance *ti)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) kvfree(ti->buckets);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) kfree(ti);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) static struct table_instance *table_instance_alloc(int new_size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) struct table_instance *ti = kmalloc(sizeof(*ti), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) if (!ti)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) ti->buckets = kvmalloc_array(new_size, sizeof(struct hlist_head),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) if (!ti->buckets) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) kfree(ti);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) for (i = 0; i < new_size; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) INIT_HLIST_HEAD(&ti->buckets[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) ti->n_buckets = new_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) ti->node_ver = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) get_random_bytes(&ti->hash_seed, sizeof(u32));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) return ti;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) static void __mask_array_destroy(struct mask_array *ma)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) free_percpu(ma->masks_usage_stats);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) kfree(ma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) static void mask_array_rcu_cb(struct rcu_head *rcu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) struct mask_array *ma = container_of(rcu, struct mask_array, rcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) __mask_array_destroy(ma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) static void tbl_mask_array_reset_counters(struct mask_array *ma)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) int i, cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) /* As the per CPU counters are not atomic we can not go ahead and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) * reset them from another CPU. To be able to still have an approximate
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) * zero based counter we store the value at reset, and subtract it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) * later when processing.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) for (i = 0; i < ma->max; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) ma->masks_usage_zero_cntr[i] = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) for_each_possible_cpu(cpu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) struct mask_array_stats *stats;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) unsigned int start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) u64 counter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) stats = per_cpu_ptr(ma->masks_usage_stats, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) start = u64_stats_fetch_begin_irq(&stats->syncp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) counter = stats->usage_cntrs[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) } while (u64_stats_fetch_retry_irq(&stats->syncp, start));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) ma->masks_usage_zero_cntr[i] += counter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) static struct mask_array *tbl_mask_array_alloc(int size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) struct mask_array *new;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) size = max(MASK_ARRAY_SIZE_MIN, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) new = kzalloc(sizeof(struct mask_array) +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) sizeof(struct sw_flow_mask *) * size +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) sizeof(u64) * size, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) if (!new)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) new->masks_usage_zero_cntr = (u64 *)((u8 *)new +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) sizeof(struct mask_array) +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) sizeof(struct sw_flow_mask *) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) new->masks_usage_stats = __alloc_percpu(sizeof(struct mask_array_stats) +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) sizeof(u64) * size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) __alignof__(u64));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) if (!new->masks_usage_stats) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) kfree(new);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) new->count = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) new->max = size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) return new;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) static int tbl_mask_array_realloc(struct flow_table *tbl, int size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) struct mask_array *old;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) struct mask_array *new;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) new = tbl_mask_array_alloc(size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) if (!new)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) old = ovsl_dereference(tbl->mask_array);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) if (old) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) for (i = 0; i < old->max; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) if (ovsl_dereference(old->masks[i]))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) new->masks[new->count++] = old->masks[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) call_rcu(&old->rcu, mask_array_rcu_cb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) rcu_assign_pointer(tbl->mask_array, new);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) static int tbl_mask_array_add_mask(struct flow_table *tbl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) struct sw_flow_mask *new)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) struct mask_array *ma = ovsl_dereference(tbl->mask_array);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) int err, ma_count = READ_ONCE(ma->count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) if (ma_count >= ma->max) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) err = tbl_mask_array_realloc(tbl, ma->max +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) MASK_ARRAY_SIZE_MIN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) ma = ovsl_dereference(tbl->mask_array);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) /* On every add or delete we need to reset the counters so
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) * every new mask gets a fair chance of being prioritized.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) tbl_mask_array_reset_counters(ma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) BUG_ON(ovsl_dereference(ma->masks[ma_count]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) rcu_assign_pointer(ma->masks[ma_count], new);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) WRITE_ONCE(ma->count, ma_count + 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) static void tbl_mask_array_del_mask(struct flow_table *tbl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) struct sw_flow_mask *mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) struct mask_array *ma = ovsl_dereference(tbl->mask_array);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) int i, ma_count = READ_ONCE(ma->count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) /* Remove the deleted mask pointers from the array */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) for (i = 0; i < ma_count; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) if (mask == ovsl_dereference(ma->masks[i]))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) goto found;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) BUG();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) found:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) WRITE_ONCE(ma->count, ma_count - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) rcu_assign_pointer(ma->masks[i], ma->masks[ma_count - 1]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) RCU_INIT_POINTER(ma->masks[ma_count - 1], NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) kfree_rcu(mask, rcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) /* Shrink the mask array if necessary. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) if (ma->max >= (MASK_ARRAY_SIZE_MIN * 2) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) ma_count <= (ma->max / 3))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) tbl_mask_array_realloc(tbl, ma->max / 2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) tbl_mask_array_reset_counters(ma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) /* Remove 'mask' from the mask list, if it is not needed any more. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) static void flow_mask_remove(struct flow_table *tbl, struct sw_flow_mask *mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) if (mask) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) /* ovs-lock is required to protect mask-refcount and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) * mask list.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) ASSERT_OVSL();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) BUG_ON(!mask->ref_count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) mask->ref_count--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) if (!mask->ref_count)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) tbl_mask_array_del_mask(tbl, mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) static void __mask_cache_destroy(struct mask_cache *mc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) free_percpu(mc->mask_cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) kfree(mc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) static void mask_cache_rcu_cb(struct rcu_head *rcu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) struct mask_cache *mc = container_of(rcu, struct mask_cache, rcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) __mask_cache_destroy(mc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) static struct mask_cache *tbl_mask_cache_alloc(u32 size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) struct mask_cache_entry __percpu *cache = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) struct mask_cache *new;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) /* Only allow size to be 0, or a power of 2, and does not exceed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) * percpu allocation size.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) if ((!is_power_of_2(size) && size != 0) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) (size * sizeof(struct mask_cache_entry)) > PCPU_MIN_UNIT_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) new = kzalloc(sizeof(*new), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) if (!new)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) new->cache_size = size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) if (new->cache_size > 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) cache = __alloc_percpu(array_size(sizeof(struct mask_cache_entry),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) new->cache_size),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) __alignof__(struct mask_cache_entry));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) if (!cache) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) kfree(new);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) new->mask_cache = cache;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) return new;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) int ovs_flow_tbl_masks_cache_resize(struct flow_table *table, u32 size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) struct mask_cache *mc = rcu_dereference_ovsl(table->mask_cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) struct mask_cache *new;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) if (size == mc->cache_size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) if ((!is_power_of_2(size) && size != 0) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) (size * sizeof(struct mask_cache_entry)) > PCPU_MIN_UNIT_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) new = tbl_mask_cache_alloc(size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) if (!new)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) rcu_assign_pointer(table->mask_cache, new);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) call_rcu(&mc->rcu, mask_cache_rcu_cb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) int ovs_flow_tbl_init(struct flow_table *table)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) struct table_instance *ti, *ufid_ti;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) struct mask_cache *mc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) struct mask_array *ma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) mc = tbl_mask_cache_alloc(MC_DEFAULT_HASH_ENTRIES);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) if (!mc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) ma = tbl_mask_array_alloc(MASK_ARRAY_SIZE_MIN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) if (!ma)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) goto free_mask_cache;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) ti = table_instance_alloc(TBL_MIN_BUCKETS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) if (!ti)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) goto free_mask_array;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) ufid_ti = table_instance_alloc(TBL_MIN_BUCKETS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) if (!ufid_ti)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) goto free_ti;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) rcu_assign_pointer(table->ti, ti);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) rcu_assign_pointer(table->ufid_ti, ufid_ti);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) rcu_assign_pointer(table->mask_array, ma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) rcu_assign_pointer(table->mask_cache, mc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) table->last_rehash = jiffies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) table->count = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) table->ufid_count = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) free_ti:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) __table_instance_destroy(ti);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) free_mask_array:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) __mask_array_destroy(ma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) free_mask_cache:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) __mask_cache_destroy(mc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) static void flow_tbl_destroy_rcu_cb(struct rcu_head *rcu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) struct table_instance *ti;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) ti = container_of(rcu, struct table_instance, rcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) __table_instance_destroy(ti);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) static void table_instance_flow_free(struct flow_table *table,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) struct table_instance *ti,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) struct table_instance *ufid_ti,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) struct sw_flow *flow)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) hlist_del_rcu(&flow->flow_table.node[ti->node_ver]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) table->count--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) if (ovs_identifier_is_ufid(&flow->id)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) hlist_del_rcu(&flow->ufid_table.node[ufid_ti->node_ver]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) table->ufid_count--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) flow_mask_remove(table, flow->mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) /* Must be called with OVS mutex held. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) void table_instance_flow_flush(struct flow_table *table,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) struct table_instance *ti,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) struct table_instance *ufid_ti)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) for (i = 0; i < ti->n_buckets; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) struct hlist_head *head = &ti->buckets[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) struct hlist_node *n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) struct sw_flow *flow;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) hlist_for_each_entry_safe(flow, n, head,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) flow_table.node[ti->node_ver]) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) table_instance_flow_free(table, ti, ufid_ti,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) flow);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) ovs_flow_free(flow, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) if (WARN_ON(table->count != 0 ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) table->ufid_count != 0)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) table->count = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) table->ufid_count = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) static void table_instance_destroy(struct table_instance *ti,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) struct table_instance *ufid_ti)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) call_rcu(&ti->rcu, flow_tbl_destroy_rcu_cb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) call_rcu(&ufid_ti->rcu, flow_tbl_destroy_rcu_cb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) /* No need for locking this function is called from RCU callback or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) * error path.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) void ovs_flow_tbl_destroy(struct flow_table *table)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) struct table_instance *ti = rcu_dereference_raw(table->ti);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) struct table_instance *ufid_ti = rcu_dereference_raw(table->ufid_ti);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) struct mask_cache *mc = rcu_dereference_raw(table->mask_cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) struct mask_array *ma = rcu_dereference_raw(table->mask_array);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) call_rcu(&mc->rcu, mask_cache_rcu_cb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) call_rcu(&ma->rcu, mask_array_rcu_cb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) table_instance_destroy(ti, ufid_ti);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) struct sw_flow *ovs_flow_tbl_dump_next(struct table_instance *ti,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) u32 *bucket, u32 *last)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) struct sw_flow *flow;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) struct hlist_head *head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) int ver;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) ver = ti->node_ver;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) while (*bucket < ti->n_buckets) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) i = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) head = &ti->buckets[*bucket];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) hlist_for_each_entry_rcu(flow, head, flow_table.node[ver]) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) if (i < *last) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) i++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) *last = i + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) return flow;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) (*bucket)++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) *last = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) static struct hlist_head *find_bucket(struct table_instance *ti, u32 hash)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) hash = jhash_1word(hash, ti->hash_seed);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) return &ti->buckets[hash & (ti->n_buckets - 1)];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) static void table_instance_insert(struct table_instance *ti,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) struct sw_flow *flow)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) struct hlist_head *head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) head = find_bucket(ti, flow->flow_table.hash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) hlist_add_head_rcu(&flow->flow_table.node[ti->node_ver], head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) static void ufid_table_instance_insert(struct table_instance *ti,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) struct sw_flow *flow)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) struct hlist_head *head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) head = find_bucket(ti, flow->ufid_table.hash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) hlist_add_head_rcu(&flow->ufid_table.node[ti->node_ver], head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) static void flow_table_copy_flows(struct table_instance *old,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) struct table_instance *new, bool ufid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) int old_ver;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) old_ver = old->node_ver;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) new->node_ver = !old_ver;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) /* Insert in new table. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) for (i = 0; i < old->n_buckets; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) struct sw_flow *flow;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) struct hlist_head *head = &old->buckets[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) if (ufid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) hlist_for_each_entry_rcu(flow, head,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) ufid_table.node[old_ver],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) lockdep_ovsl_is_held())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) ufid_table_instance_insert(new, flow);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) hlist_for_each_entry_rcu(flow, head,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) flow_table.node[old_ver],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) lockdep_ovsl_is_held())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) table_instance_insert(new, flow);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) static struct table_instance *table_instance_rehash(struct table_instance *ti,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) int n_buckets, bool ufid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) struct table_instance *new_ti;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) new_ti = table_instance_alloc(n_buckets);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) if (!new_ti)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) flow_table_copy_flows(ti, new_ti, ufid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) return new_ti;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) int ovs_flow_tbl_flush(struct flow_table *flow_table)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) struct table_instance *old_ti, *new_ti;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) struct table_instance *old_ufid_ti, *new_ufid_ti;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) new_ti = table_instance_alloc(TBL_MIN_BUCKETS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) if (!new_ti)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) new_ufid_ti = table_instance_alloc(TBL_MIN_BUCKETS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) if (!new_ufid_ti)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) goto err_free_ti;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) old_ti = ovsl_dereference(flow_table->ti);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) old_ufid_ti = ovsl_dereference(flow_table->ufid_ti);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) rcu_assign_pointer(flow_table->ti, new_ti);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) rcu_assign_pointer(flow_table->ufid_ti, new_ufid_ti);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) flow_table->last_rehash = jiffies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) table_instance_flow_flush(flow_table, old_ti, old_ufid_ti);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) table_instance_destroy(old_ti, old_ufid_ti);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) err_free_ti:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) __table_instance_destroy(new_ti);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) static u32 flow_hash(const struct sw_flow_key *key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) const struct sw_flow_key_range *range)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) const u32 *hash_key = (const u32 *)((const u8 *)key + range->start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) /* Make sure number of hash bytes are multiple of u32. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) int hash_u32s = range_n_bytes(range) >> 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) return jhash2(hash_key, hash_u32s, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) static int flow_key_start(const struct sw_flow_key *key)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) if (key->tun_proto)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) return rounddown(offsetof(struct sw_flow_key, phy),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) sizeof(long));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) static bool cmp_key(const struct sw_flow_key *key1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) const struct sw_flow_key *key2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) int key_start, int key_end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) const long *cp1 = (const long *)((const u8 *)key1 + key_start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) const long *cp2 = (const long *)((const u8 *)key2 + key_start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) long diffs = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) for (i = key_start; i < key_end; i += sizeof(long))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) diffs |= *cp1++ ^ *cp2++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) return diffs == 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) static bool flow_cmp_masked_key(const struct sw_flow *flow,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) const struct sw_flow_key *key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) const struct sw_flow_key_range *range)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) return cmp_key(&flow->key, key, range->start, range->end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) static bool ovs_flow_cmp_unmasked_key(const struct sw_flow *flow,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) const struct sw_flow_match *match)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) struct sw_flow_key *key = match->key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) int key_start = flow_key_start(key);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) int key_end = match->range.end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) BUG_ON(ovs_identifier_is_ufid(&flow->id));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) return cmp_key(flow->id.unmasked_key, key, key_start, key_end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) static struct sw_flow *masked_flow_lookup(struct table_instance *ti,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) const struct sw_flow_key *unmasked,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) const struct sw_flow_mask *mask,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) u32 *n_mask_hit)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) struct sw_flow *flow;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) struct hlist_head *head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) u32 hash;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) struct sw_flow_key masked_key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) ovs_flow_mask_key(&masked_key, unmasked, false, mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) hash = flow_hash(&masked_key, &mask->range);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) head = find_bucket(ti, hash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) (*n_mask_hit)++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) hlist_for_each_entry_rcu(flow, head, flow_table.node[ti->node_ver],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) lockdep_ovsl_is_held()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) if (flow->mask == mask && flow->flow_table.hash == hash &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) flow_cmp_masked_key(flow, &masked_key, &mask->range))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) return flow;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) /* Flow lookup does full lookup on flow table. It starts with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) * mask from index passed in *index.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) * This function MUST be called with BH disabled due to the use
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) * of CPU specific variables.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) static struct sw_flow *flow_lookup(struct flow_table *tbl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) struct table_instance *ti,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) struct mask_array *ma,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) const struct sw_flow_key *key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) u32 *n_mask_hit,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) u32 *n_cache_hit,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) u32 *index)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) struct mask_array_stats *stats = this_cpu_ptr(ma->masks_usage_stats);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) struct sw_flow *flow;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) struct sw_flow_mask *mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) if (likely(*index < ma->max)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) mask = rcu_dereference_ovsl(ma->masks[*index]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) if (mask) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) flow = masked_flow_lookup(ti, key, mask, n_mask_hit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) if (flow) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) u64_stats_update_begin(&stats->syncp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) stats->usage_cntrs[*index]++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) u64_stats_update_end(&stats->syncp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) (*n_cache_hit)++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) return flow;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) for (i = 0; i < ma->max; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) if (i == *index)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) mask = rcu_dereference_ovsl(ma->masks[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) if (unlikely(!mask))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) flow = masked_flow_lookup(ti, key, mask, n_mask_hit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) if (flow) { /* Found */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) *index = i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) u64_stats_update_begin(&stats->syncp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) stats->usage_cntrs[*index]++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) u64_stats_update_end(&stats->syncp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) return flow;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) * mask_cache maps flow to probable mask. This cache is not tightly
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) * coupled cache, It means updates to mask list can result in inconsistent
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) * cache entry in mask cache.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) * This is per cpu cache and is divided in MC_HASH_SEGS segments.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) * In case of a hash collision the entry is hashed in next segment.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) * */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) struct sw_flow *ovs_flow_tbl_lookup_stats(struct flow_table *tbl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) const struct sw_flow_key *key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) u32 skb_hash,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) u32 *n_mask_hit,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) u32 *n_cache_hit)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) struct mask_cache *mc = rcu_dereference(tbl->mask_cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) struct mask_array *ma = rcu_dereference(tbl->mask_array);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) struct table_instance *ti = rcu_dereference(tbl->ti);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) struct mask_cache_entry *entries, *ce;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) struct sw_flow *flow;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) u32 hash;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) int seg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) *n_mask_hit = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) *n_cache_hit = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) if (unlikely(!skb_hash || mc->cache_size == 0)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) u32 mask_index = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) u32 cache = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) return flow_lookup(tbl, ti, ma, key, n_mask_hit, &cache,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) &mask_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) /* Pre and post recirulation flows usually have the same skb_hash
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) * value. To avoid hash collisions, rehash the 'skb_hash' with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) * 'recirc_id'. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) if (key->recirc_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) skb_hash = jhash_1word(skb_hash, key->recirc_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) ce = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) hash = skb_hash;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) entries = this_cpu_ptr(mc->mask_cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) /* Find the cache entry 'ce' to operate on. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) for (seg = 0; seg < MC_HASH_SEGS; seg++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) int index = hash & (mc->cache_size - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) struct mask_cache_entry *e;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) e = &entries[index];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) if (e->skb_hash == skb_hash) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) flow = flow_lookup(tbl, ti, ma, key, n_mask_hit,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) n_cache_hit, &e->mask_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) if (!flow)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) e->skb_hash = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) return flow;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) if (!ce || e->skb_hash < ce->skb_hash)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) ce = e; /* A better replacement cache candidate. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) hash >>= MC_HASH_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) /* Cache miss, do full lookup. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) flow = flow_lookup(tbl, ti, ma, key, n_mask_hit, n_cache_hit,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) &ce->mask_index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) if (flow)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) ce->skb_hash = skb_hash;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) *n_cache_hit = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) return flow;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) struct sw_flow *ovs_flow_tbl_lookup(struct flow_table *tbl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) const struct sw_flow_key *key)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) struct table_instance *ti = rcu_dereference_ovsl(tbl->ti);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) struct mask_array *ma = rcu_dereference_ovsl(tbl->mask_array);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) u32 __always_unused n_mask_hit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) u32 __always_unused n_cache_hit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) struct sw_flow *flow;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) u32 index = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) /* This function gets called trough the netlink interface and therefore
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) * is preemptible. However, flow_lookup() function needs to be called
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) * with BH disabled due to CPU specific variables.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) local_bh_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) flow = flow_lookup(tbl, ti, ma, key, &n_mask_hit, &n_cache_hit, &index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) local_bh_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) return flow;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) struct sw_flow *ovs_flow_tbl_lookup_exact(struct flow_table *tbl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) const struct sw_flow_match *match)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) struct mask_array *ma = ovsl_dereference(tbl->mask_array);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) /* Always called under ovs-mutex. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) for (i = 0; i < ma->max; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) struct table_instance *ti = rcu_dereference_ovsl(tbl->ti);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) u32 __always_unused n_mask_hit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) struct sw_flow_mask *mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) struct sw_flow *flow;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) mask = ovsl_dereference(ma->masks[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) if (!mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) flow = masked_flow_lookup(ti, match->key, mask, &n_mask_hit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) if (flow && ovs_identifier_is_key(&flow->id) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888) ovs_flow_cmp_unmasked_key(flow, match)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) return flow;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) static u32 ufid_hash(const struct sw_flow_id *sfid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) return jhash(sfid->ufid, sfid->ufid_len, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) static bool ovs_flow_cmp_ufid(const struct sw_flow *flow,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) const struct sw_flow_id *sfid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) if (flow->id.ufid_len != sfid->ufid_len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907) return !memcmp(flow->id.ufid, sfid->ufid, sfid->ufid_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) bool ovs_flow_cmp(const struct sw_flow *flow,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) const struct sw_flow_match *match)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) if (ovs_identifier_is_ufid(&flow->id))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) return flow_cmp_masked_key(flow, match->key, &match->range);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) return ovs_flow_cmp_unmasked_key(flow, match);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) struct sw_flow *ovs_flow_tbl_lookup_ufid(struct flow_table *tbl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) const struct sw_flow_id *ufid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) struct table_instance *ti = rcu_dereference_ovsl(tbl->ufid_ti);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) struct sw_flow *flow;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924) struct hlist_head *head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) u32 hash;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927) hash = ufid_hash(ufid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) head = find_bucket(ti, hash);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) hlist_for_each_entry_rcu(flow, head, ufid_table.node[ti->node_ver],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) lockdep_ovsl_is_held()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) if (flow->ufid_table.hash == hash &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932) ovs_flow_cmp_ufid(flow, ufid))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) return flow;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938) int ovs_flow_tbl_num_masks(const struct flow_table *table)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) struct mask_array *ma = rcu_dereference_ovsl(table->mask_array);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) return READ_ONCE(ma->count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) u32 ovs_flow_tbl_masks_cache_size(const struct flow_table *table)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946) struct mask_cache *mc = rcu_dereference_ovsl(table->mask_cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) return READ_ONCE(mc->cache_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) static struct table_instance *table_instance_expand(struct table_instance *ti,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952) bool ufid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) return table_instance_rehash(ti, ti->n_buckets * 2, ufid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957) /* Must be called with OVS mutex held. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) void ovs_flow_tbl_remove(struct flow_table *table, struct sw_flow *flow)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960) struct table_instance *ti = ovsl_dereference(table->ti);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961) struct table_instance *ufid_ti = ovsl_dereference(table->ufid_ti);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963) BUG_ON(table->count == 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964) table_instance_flow_free(table, ti, ufid_ti, flow);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967) static struct sw_flow_mask *mask_alloc(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969) struct sw_flow_mask *mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971) mask = kmalloc(sizeof(*mask), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972) if (mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973) mask->ref_count = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975) return mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978) static bool mask_equal(const struct sw_flow_mask *a,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979) const struct sw_flow_mask *b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981) const u8 *a_ = (const u8 *)&a->key + a->range.start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982) const u8 *b_ = (const u8 *)&b->key + b->range.start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984) return (a->range.end == b->range.end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985) && (a->range.start == b->range.start)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986) && (memcmp(a_, b_, range_n_bytes(&a->range)) == 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989) static struct sw_flow_mask *flow_mask_find(const struct flow_table *tbl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990) const struct sw_flow_mask *mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992) struct mask_array *ma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995) ma = ovsl_dereference(tbl->mask_array);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996) for (i = 0; i < ma->max; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997) struct sw_flow_mask *t;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998) t = ovsl_dereference(ma->masks[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000) if (t && mask_equal(mask, t))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001) return t;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007) /* Add 'mask' into the mask list, if it is not already there. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008) static int flow_mask_insert(struct flow_table *tbl, struct sw_flow *flow,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009) const struct sw_flow_mask *new)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011) struct sw_flow_mask *mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013) mask = flow_mask_find(tbl, new);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014) if (!mask) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015) /* Allocate a new mask if none exsits. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016) mask = mask_alloc();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017) if (!mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019) mask->key = new->key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020) mask->range = new->range;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022) /* Add mask to mask-list. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023) if (tbl_mask_array_add_mask(tbl, mask)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024) kfree(mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028) BUG_ON(!mask->ref_count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029) mask->ref_count++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032) flow->mask = mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036) /* Must be called with OVS mutex held. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037) static void flow_key_insert(struct flow_table *table, struct sw_flow *flow)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039) struct table_instance *new_ti = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040) struct table_instance *ti;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042) flow->flow_table.hash = flow_hash(&flow->key, &flow->mask->range);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043) ti = ovsl_dereference(table->ti);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044) table_instance_insert(ti, flow);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045) table->count++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047) /* Expand table, if necessary, to make room. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048) if (table->count > ti->n_buckets)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049) new_ti = table_instance_expand(ti, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050) else if (time_after(jiffies, table->last_rehash + REHASH_INTERVAL))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051) new_ti = table_instance_rehash(ti, ti->n_buckets, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053) if (new_ti) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054) rcu_assign_pointer(table->ti, new_ti);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055) call_rcu(&ti->rcu, flow_tbl_destroy_rcu_cb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056) table->last_rehash = jiffies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060) /* Must be called with OVS mutex held. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1061) static void flow_ufid_insert(struct flow_table *table, struct sw_flow *flow)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1062) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1063) struct table_instance *ti;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1064)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1065) flow->ufid_table.hash = ufid_hash(&flow->id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1066) ti = ovsl_dereference(table->ufid_ti);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1067) ufid_table_instance_insert(ti, flow);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1068) table->ufid_count++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1069)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1070) /* Expand table, if necessary, to make room. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1071) if (table->ufid_count > ti->n_buckets) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1072) struct table_instance *new_ti;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1073)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1074) new_ti = table_instance_expand(ti, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1075) if (new_ti) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1076) rcu_assign_pointer(table->ufid_ti, new_ti);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1077) call_rcu(&ti->rcu, flow_tbl_destroy_rcu_cb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1078) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1079) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1080) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1081)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1082) /* Must be called with OVS mutex held. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1083) int ovs_flow_tbl_insert(struct flow_table *table, struct sw_flow *flow,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1084) const struct sw_flow_mask *mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1085) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1086) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1087)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1088) err = flow_mask_insert(table, flow, mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1089) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1090) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1091) flow_key_insert(table, flow);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1092) if (ovs_identifier_is_ufid(&flow->id))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1093) flow_ufid_insert(table, flow);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1094)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1095) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1096) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1097)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1098) static int compare_mask_and_count(const void *a, const void *b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1099) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1100) const struct mask_count *mc_a = a;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1101) const struct mask_count *mc_b = b;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1102)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1103) return (s64)mc_b->counter - (s64)mc_a->counter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1104) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1105)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1106) /* Must be called with OVS mutex held. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1107) void ovs_flow_masks_rebalance(struct flow_table *table)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1108) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1109) struct mask_array *ma = rcu_dereference_ovsl(table->mask_array);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1110) struct mask_count *masks_and_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1111) struct mask_array *new;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1112) int masks_entries = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1113) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1114)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1115) /* Build array of all current entries with use counters. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1116) masks_and_count = kmalloc_array(ma->max, sizeof(*masks_and_count),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1117) GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1118) if (!masks_and_count)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1119) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1120)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1121) for (i = 0; i < ma->max; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1122) struct sw_flow_mask *mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1123) int cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1124)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1125) mask = rcu_dereference_ovsl(ma->masks[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1126) if (unlikely(!mask))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1127) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1128)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1129) masks_and_count[i].index = i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1130) masks_and_count[i].counter = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1131)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1132) for_each_possible_cpu(cpu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1133) struct mask_array_stats *stats;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1134) unsigned int start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1135) u64 counter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1136)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1137) stats = per_cpu_ptr(ma->masks_usage_stats, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1138) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1139) start = u64_stats_fetch_begin_irq(&stats->syncp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1140) counter = stats->usage_cntrs[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1141) } while (u64_stats_fetch_retry_irq(&stats->syncp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1142) start));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1143)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1144) masks_and_count[i].counter += counter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1145) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1146)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1147) /* Subtract the zero count value. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1148) masks_and_count[i].counter -= ma->masks_usage_zero_cntr[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1149)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1150) /* Rather than calling tbl_mask_array_reset_counters()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1151) * below when no change is needed, do it inline here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1152) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1153) ma->masks_usage_zero_cntr[i] += masks_and_count[i].counter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1154) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1155)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1156) if (i == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1157) goto free_mask_entries;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1158)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1159) /* Sort the entries */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1160) masks_entries = i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1161) sort(masks_and_count, masks_entries, sizeof(*masks_and_count),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1162) compare_mask_and_count, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1163)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1164) /* If the order is the same, nothing to do... */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1165) for (i = 0; i < masks_entries; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1166) if (i != masks_and_count[i].index)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1167) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1168) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1169) if (i == masks_entries)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1170) goto free_mask_entries;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1171)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1172) /* Rebuilt the new list in order of usage. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1173) new = tbl_mask_array_alloc(ma->max);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1174) if (!new)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1175) goto free_mask_entries;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1176)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1177) for (i = 0; i < masks_entries; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1178) int index = masks_and_count[i].index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1179)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1180) if (ovsl_dereference(ma->masks[index]))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1181) new->masks[new->count++] = ma->masks[index];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1182) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1183)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1184) rcu_assign_pointer(table->mask_array, new);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1185) call_rcu(&ma->rcu, mask_array_rcu_cb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1186)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1187) free_mask_entries:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1188) kfree(masks_and_count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1189) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1190)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1191) /* Initializes the flow module.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1192) * Returns zero if successful or a negative error code. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1193) int ovs_flow_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1194) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1195) BUILD_BUG_ON(__alignof__(struct sw_flow_key) % __alignof__(long));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1196) BUILD_BUG_ON(sizeof(struct sw_flow_key) % sizeof(long));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1197)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1198) flow_cache = kmem_cache_create("sw_flow", sizeof(struct sw_flow)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1199) + (nr_cpu_ids
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1200) * sizeof(struct sw_flow_stats *)),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1201) 0, 0, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1202) if (flow_cache == NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1203) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1204)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1205) flow_stats_cache
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1206) = kmem_cache_create("sw_flow_stats", sizeof(struct sw_flow_stats),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1207) 0, SLAB_HWCACHE_ALIGN, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1208) if (flow_stats_cache == NULL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1209) kmem_cache_destroy(flow_cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1210) flow_cache = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1211) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1212) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1213)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1214) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1215) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1216)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1217) /* Uninitializes the flow module. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1218) void ovs_flow_exit(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1219) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1220) kmem_cache_destroy(flow_stats_cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1221) kmem_cache_destroy(flow_cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1222) }
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags