Orange Pi5 kernel

^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   3)  * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   4)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   5) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   6) #include "peer.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   7) #include "device.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   8) #include "queueing.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   9) #include "timers.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  10) #include "peerlookup.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  11) #include "noise.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  12) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  13) #include <linux/kref.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  14) #include <linux/lockdep.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  15) #include <linux/rcupdate.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  16) #include <linux/list.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  17) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  18) static struct kmem_cache *peer_cache;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  19) static atomic64_t peer_counter = ATOMIC64_INIT(0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  20) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  21) struct wg_peer *wg_peer_create(struct wg_device *wg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  22) 			       const u8 public_key[NOISE_PUBLIC_KEY_LEN],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  23) 			       const u8 preshared_key[NOISE_SYMMETRIC_KEY_LEN])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  24) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  25) 	struct wg_peer *peer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  26) 	int ret = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  27) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  28) 	lockdep_assert_held(&wg->device_update_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  29) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  30) 	if (wg->num_peers >= MAX_PEERS_PER_DEVICE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  31) 		return ERR_PTR(ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  32) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  33) 	peer = kmem_cache_zalloc(peer_cache, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  34) 	if (unlikely(!peer))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  35) 		return ERR_PTR(ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  36) 	if (unlikely(dst_cache_init(&peer->endpoint_cache, GFP_KERNEL)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  37) 		goto err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  38) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  39) 	peer->device = wg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  40) 	wg_noise_handshake_init(&peer->handshake, &wg->static_identity,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  41) 				public_key, preshared_key, peer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  42) 	peer->internal_id = atomic64_inc_return(&peer_counter);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  43) 	peer->serial_work_cpu = nr_cpumask_bits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  44) 	wg_cookie_init(&peer->latest_cookie);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  45) 	wg_timers_init(peer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  46) 	wg_cookie_checker_precompute_peer_keys(peer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  47) 	spin_lock_init(&peer->keypairs.keypair_update_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  48) 	INIT_WORK(&peer->transmit_handshake_work, wg_packet_handshake_send_worker);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  49) 	INIT_WORK(&peer->transmit_packet_work, wg_packet_tx_worker);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  50) 	wg_prev_queue_init(&peer->tx_queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  51) 	wg_prev_queue_init(&peer->rx_queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  52) 	rwlock_init(&peer->endpoint_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  53) 	kref_init(&peer->refcount);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  54) 	skb_queue_head_init(&peer->staged_packet_queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  55) 	wg_noise_reset_last_sent_handshake(&peer->last_sent_handshake);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  56) 	set_bit(NAPI_STATE_NO_BUSY_POLL, &peer->napi.state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  57) 	netif_napi_add(wg->dev, &peer->napi, wg_packet_rx_poll,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  58) 		       NAPI_POLL_WEIGHT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  59) 	napi_enable(&peer->napi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  60) 	list_add_tail(&peer->peer_list, &wg->peer_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  61) 	INIT_LIST_HEAD(&peer->allowedips_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  62) 	wg_pubkey_hashtable_add(wg->peer_hashtable, peer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  63) 	++wg->num_peers;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  64) 	pr_debug("%s: Peer %llu created\n", wg->dev->name, peer->internal_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  65) 	return peer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  66) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  67) err:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  68) 	kmem_cache_free(peer_cache, peer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  69) 	return ERR_PTR(ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  70) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  71) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  72) struct wg_peer *wg_peer_get_maybe_zero(struct wg_peer *peer)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  73) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  74) 	RCU_LOCKDEP_WARN(!rcu_read_lock_bh_held(),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  75) 			 "Taking peer reference without holding the RCU read lock");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  76) 	if (unlikely(!peer || !kref_get_unless_zero(&peer->refcount)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  77) 		return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  78) 	return peer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  79) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  80) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  81) static void peer_make_dead(struct wg_peer *peer)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  82) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  83) 	/* Remove from configuration-time lookup structures. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  84) 	list_del_init(&peer->peer_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  85) 	wg_allowedips_remove_by_peer(&peer->device->peer_allowedips, peer,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  86) 				     &peer->device->device_update_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  87) 	wg_pubkey_hashtable_remove(peer->device->peer_hashtable, peer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  88) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  89) 	/* Mark as dead, so that we don't allow jumping contexts after. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  90) 	WRITE_ONCE(peer->is_dead, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  91) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  92) 	/* The caller must now synchronize_net() for this to take effect. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  93) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  94) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  95) static void peer_remove_after_dead(struct wg_peer *peer)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  96) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  97) 	WARN_ON(!peer->is_dead);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  98) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  99) 	/* No more keypairs can be created for this peer, since is_dead protects
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) 	 * add_new_keypair, so we can now destroy existing ones.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) 	wg_noise_keypairs_clear(&peer->keypairs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) 	/* Destroy all ongoing timers that were in-flight at the beginning of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) 	 * this function.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) 	wg_timers_stop(peer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) 	/* The transition between packet encryption/decryption queues isn't
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) 	 * guarded by is_dead, but each reference's life is strictly bounded by
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) 	 * two generations: once for parallel crypto and once for serial
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) 	 * ingestion, so we can simply flush twice, and be sure that we no
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) 	 * longer have references inside these queues.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) 	/* a) For encrypt/decrypt. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) 	flush_workqueue(peer->device->packet_crypt_wq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) 	/* b.1) For send (but not receive, since that's napi). */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) 	flush_workqueue(peer->device->packet_crypt_wq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) 	/* b.2.1) For receive (but not send, since that's wq). */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) 	napi_disable(&peer->napi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) 	/* b.2.1) It's now safe to remove the napi struct, which must be done
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) 	 * here from process context.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) 	netif_napi_del(&peer->napi);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) 	/* Ensure any workstructs we own (like transmit_handshake_work or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) 	 * clear_peer_work) no longer are in use.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) 	flush_workqueue(peer->device->handshake_send_wq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) 	/* After the above flushes, a peer might still be active in a few
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) 	 * different contexts: 1) from xmit(), before hitting is_dead and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) 	 * returning, 2) from wg_packet_consume_data(), before hitting is_dead
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) 	 * and returning, 3) from wg_receive_handshake_packet() after a point
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) 	 * where it has processed an incoming handshake packet, but where
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) 	 * all calls to pass it off to timers fails because of is_dead. We won't
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) 	 * have new references in (1) eventually, because we're removed from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) 	 * allowedips; we won't have new references in (2) eventually, because
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) 	 * wg_index_hashtable_lookup will always return NULL, since we removed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) 	 * all existing keypairs and no more can be created; we won't have new
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) 	 * references in (3) eventually, because we're removed from the pubkey
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) 	 * hash table, which allows for a maximum of one handshake response,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) 	 * via the still-uncleared index hashtable entry, but not more than one,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) 	 * and in wg_cookie_message_consume, the lookup eventually gets a peer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) 	 * with a refcount of zero, so no new reference is taken.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) 	--peer->device->num_peers;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) 	wg_peer_put(peer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) /* We have a separate "remove" function make sure that all active places where
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154)  * a peer is currently operating will eventually come to an end and not pass
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155)  * their reference onto another context.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) void wg_peer_remove(struct wg_peer *peer)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) 	if (unlikely(!peer))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) 		return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) 	lockdep_assert_held(&peer->device->device_update_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) 	peer_make_dead(peer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) 	synchronize_net();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) 	peer_remove_after_dead(peer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) void wg_peer_remove_all(struct wg_device *wg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) 	struct wg_peer *peer, *temp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) 	LIST_HEAD(dead_peers);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) 	lockdep_assert_held(&wg->device_update_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) 	/* Avoid having to traverse individually for each one. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) 	wg_allowedips_free(&wg->peer_allowedips, &wg->device_update_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) 	list_for_each_entry_safe(peer, temp, &wg->peer_list, peer_list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) 		peer_make_dead(peer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) 		list_add_tail(&peer->peer_list, &dead_peers);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) 	synchronize_net();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) 	list_for_each_entry_safe(peer, temp, &dead_peers, peer_list)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) 		peer_remove_after_dead(peer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) static void rcu_release(struct rcu_head *rcu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) 	struct wg_peer *peer = container_of(rcu, struct wg_peer, rcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) 	dst_cache_destroy(&peer->endpoint_cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) 	WARN_ON(wg_prev_queue_peek(&peer->tx_queue) || wg_prev_queue_peek(&peer->rx_queue));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) 	/* The final zeroing takes care of clearing any remaining handshake key
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) 	 * material and other potentially sensitive information.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) 	memzero_explicit(peer, sizeof(*peer));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) 	kmem_cache_free(peer_cache, peer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) static void kref_release(struct kref *refcount)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) 	struct wg_peer *peer = container_of(refcount, struct wg_peer, refcount);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) 	pr_debug("%s: Peer %llu (%pISpfsc) destroyed\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) 		 peer->device->dev->name, peer->internal_id,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) 		 &peer->endpoint.addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) 	/* Remove ourself from dynamic runtime lookup structures, now that the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) 	 * last reference is gone.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) 	wg_index_hashtable_remove(peer->device->index_hashtable,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) 				  &peer->handshake.entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) 	/* Remove any lingering packets that didn't have a chance to be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) 	 * transmitted.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) 	wg_packet_purge_staged_packets(peer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) 	/* Free the memory used. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) 	call_rcu(&peer->rcu, rcu_release);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) void wg_peer_put(struct wg_peer *peer)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) 	if (unlikely(!peer))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) 		return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) 	kref_put(&peer->refcount, kref_release);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) int __init wg_peer_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) 	peer_cache = KMEM_CACHE(wg_peer, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) 	return peer_cache ? 0 : -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) void wg_peer_uninit(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) 	kmem_cache_destroy(peer_cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) }