^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * linux/net/sunrpc/xprt.c
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * This is a generic RPC call interface supporting congestion avoidance,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * and asynchronous calls.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) * The interface works like this:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) * - When a process places a call, it allocates a request slot if
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) * one is available. Otherwise, it sleeps on the backlog queue
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) * (xprt_reserve).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) * - Next, the caller puts together the RPC message, stuffs it into
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) * the request struct, and calls xprt_transmit().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) * - xprt_transmit sends the message and installs the caller on the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) * transport's wait list. At the same time, if a reply is expected,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) * it installs a timer that is run after the packet's timeout has
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) * expired.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) * - When a packet arrives, the data_ready handler walks the list of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) * pending requests for that transport. If a matching XID is found, the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) * caller is woken up, and the timer removed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) * - When no reply arrives within the timeout interval, the timer is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) * fired by the kernel and runs xprt_timer(). It either adjusts the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) * timeout values (minor timeout) or wakes up the caller with a status
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) * of -ETIMEDOUT.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) * - When the caller receives a notification from RPC that a reply arrived,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) * it should release the RPC slot, and process the reply.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) * If the call timed out, it may choose to retry the operation by
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) * adjusting the initial timeout value, and simply calling rpc_call
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) * again.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) * Support for async RPC is done through a set of RPC-specific scheduling
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) * primitives that `transparently' work for processes as well as async
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) * tasks that rely on callbacks.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) * Copyright (C) 1995-1997, Olaf Kirch <okir@monad.swb.de>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) * Transport switch API copyright (C) 2005, Chuck Lever <cel@netapp.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) #include <linux/types.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) #include <linux/interrupt.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) #include <linux/workqueue.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) #include <linux/net.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) #include <linux/ktime.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) #include <linux/sunrpc/clnt.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) #include <linux/sunrpc/metrics.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) #include <linux/sunrpc/bc_xprt.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) #include <linux/rcupdate.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) #include <linux/sched/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) #include <trace/events/sunrpc.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) #include "sunrpc.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) * Local variables
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) #if IS_ENABLED(CONFIG_SUNRPC_DEBUG)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) # define RPCDBG_FACILITY RPCDBG_XPRT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) * Local functions
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) static void xprt_init(struct rpc_xprt *xprt, struct net *net);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) static __be32 xprt_alloc_xid(struct rpc_xprt *xprt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) static void xprt_destroy(struct rpc_xprt *xprt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) static void xprt_request_init(struct rpc_task *task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) static DEFINE_SPINLOCK(xprt_list_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) static LIST_HEAD(xprt_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) static unsigned long xprt_request_timeout(const struct rpc_rqst *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) unsigned long timeout = jiffies + req->rq_timeout;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) if (time_before(timeout, req->rq_majortimeo))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) return timeout;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) return req->rq_majortimeo;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) * xprt_register_transport - register a transport implementation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) * @transport: transport to register
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) * If a transport implementation is loaded as a kernel module, it can
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) * call this interface to make itself known to the RPC client.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) * Returns:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) * 0: transport successfully registered
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) * -EEXIST: transport already registered
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) * -EINVAL: transport module being unloaded
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) int xprt_register_transport(struct xprt_class *transport)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) struct xprt_class *t;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) int result;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) result = -EEXIST;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) spin_lock(&xprt_list_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) list_for_each_entry(t, &xprt_list, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) /* don't register the same transport class twice */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) if (t->ident == transport->ident)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) list_add_tail(&transport->list, &xprt_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) printk(KERN_INFO "RPC: Registered %s transport module.\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) transport->name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) result = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) spin_unlock(&xprt_list_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) return result;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) EXPORT_SYMBOL_GPL(xprt_register_transport);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) * xprt_unregister_transport - unregister a transport implementation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) * @transport: transport to unregister
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) * Returns:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) * 0: transport successfully unregistered
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) * -ENOENT: transport never registered
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) int xprt_unregister_transport(struct xprt_class *transport)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) struct xprt_class *t;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) int result;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) result = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) spin_lock(&xprt_list_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) list_for_each_entry(t, &xprt_list, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) if (t == transport) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) printk(KERN_INFO
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) "RPC: Unregistered %s transport module.\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) transport->name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) list_del_init(&transport->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) result = -ENOENT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) spin_unlock(&xprt_list_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) return result;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) EXPORT_SYMBOL_GPL(xprt_unregister_transport);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) xprt_class_release(const struct xprt_class *t)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) module_put(t->owner);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) static const struct xprt_class *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) xprt_class_find_by_netid_locked(const char *netid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) const struct xprt_class *t;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) unsigned int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) list_for_each_entry(t, &xprt_list, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) for (i = 0; t->netid[i][0] != '\0'; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) if (strcmp(t->netid[i], netid) != 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) if (!try_module_get(t->owner))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) return t;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) static const struct xprt_class *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) xprt_class_find_by_netid(const char *netid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) const struct xprt_class *t;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) spin_lock(&xprt_list_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) t = xprt_class_find_by_netid_locked(netid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) if (!t) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) spin_unlock(&xprt_list_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) request_module("rpc%s", netid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) spin_lock(&xprt_list_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) t = xprt_class_find_by_netid_locked(netid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) spin_unlock(&xprt_list_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) return t;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) * xprt_load_transport - load a transport implementation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) * @netid: transport to load
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) * Returns:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) * 0: transport successfully loaded
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) * -ENOENT: transport module not available
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) int xprt_load_transport(const char *netid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) const struct xprt_class *t;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) t = xprt_class_find_by_netid(netid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) if (!t)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) return -ENOENT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) xprt_class_release(t);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) EXPORT_SYMBOL_GPL(xprt_load_transport);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) static void xprt_clear_locked(struct rpc_xprt *xprt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) xprt->snd_task = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) if (!test_bit(XPRT_CLOSE_WAIT, &xprt->state)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) smp_mb__before_atomic();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) clear_bit(XPRT_LOCKED, &xprt->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) smp_mb__after_atomic();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) queue_work(xprtiod_workqueue, &xprt->task_cleanup);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) * xprt_reserve_xprt - serialize write access to transports
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) * @task: task that is requesting access to the transport
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) * @xprt: pointer to the target transport
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) * This prevents mixing the payload of separate requests, and prevents
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) * transport connects from colliding with writes. No congestion control
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) * is provided.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) int xprt_reserve_xprt(struct rpc_xprt *xprt, struct rpc_task *task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) struct rpc_rqst *req = task->tk_rqstp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) if (test_and_set_bit(XPRT_LOCKED, &xprt->state)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) if (task == xprt->snd_task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) goto out_locked;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) goto out_sleep;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) if (test_bit(XPRT_WRITE_SPACE, &xprt->state))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) goto out_unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) xprt->snd_task = task;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) out_locked:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) trace_xprt_reserve_xprt(xprt, task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) out_unlock:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) xprt_clear_locked(xprt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) out_sleep:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) task->tk_status = -EAGAIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) if (RPC_IS_SOFT(task))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) rpc_sleep_on_timeout(&xprt->sending, task, NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) xprt_request_timeout(req));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) rpc_sleep_on(&xprt->sending, task, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) EXPORT_SYMBOL_GPL(xprt_reserve_xprt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) static bool
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) xprt_need_congestion_window_wait(struct rpc_xprt *xprt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) return test_bit(XPRT_CWND_WAIT, &xprt->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) xprt_set_congestion_window_wait(struct rpc_xprt *xprt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) if (!list_empty(&xprt->xmit_queue)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) /* Peek at head of queue to see if it can make progress */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) if (list_first_entry(&xprt->xmit_queue, struct rpc_rqst,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) rq_xmit)->rq_cong)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) set_bit(XPRT_CWND_WAIT, &xprt->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) xprt_test_and_clear_congestion_window_wait(struct rpc_xprt *xprt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) if (!RPCXPRT_CONGESTED(xprt))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) clear_bit(XPRT_CWND_WAIT, &xprt->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) * xprt_reserve_xprt_cong - serialize write access to transports
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) * @task: task that is requesting access to the transport
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) * Same as xprt_reserve_xprt, but Van Jacobson congestion control is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) * integrated into the decision of whether a request is allowed to be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) * woken up and given access to the transport.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) * Note that the lock is only granted if we know there are free slots.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) int xprt_reserve_xprt_cong(struct rpc_xprt *xprt, struct rpc_task *task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) struct rpc_rqst *req = task->tk_rqstp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) if (test_and_set_bit(XPRT_LOCKED, &xprt->state)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) if (task == xprt->snd_task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) goto out_locked;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) goto out_sleep;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) if (req == NULL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) xprt->snd_task = task;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) goto out_locked;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) if (test_bit(XPRT_WRITE_SPACE, &xprt->state))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) goto out_unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) if (!xprt_need_congestion_window_wait(xprt)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) xprt->snd_task = task;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) goto out_locked;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) out_unlock:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) xprt_clear_locked(xprt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) out_sleep:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) task->tk_status = -EAGAIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) if (RPC_IS_SOFT(task))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) rpc_sleep_on_timeout(&xprt->sending, task, NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) xprt_request_timeout(req));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) rpc_sleep_on(&xprt->sending, task, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) out_locked:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) trace_xprt_reserve_cong(xprt, task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) EXPORT_SYMBOL_GPL(xprt_reserve_xprt_cong);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) static inline int xprt_lock_write(struct rpc_xprt *xprt, struct rpc_task *task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) int retval;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) if (test_bit(XPRT_LOCKED, &xprt->state) && xprt->snd_task == task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) spin_lock(&xprt->transport_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) retval = xprt->ops->reserve_xprt(xprt, task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) spin_unlock(&xprt->transport_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) return retval;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) static bool __xprt_lock_write_func(struct rpc_task *task, void *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) struct rpc_xprt *xprt = data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) xprt->snd_task = task;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) static void __xprt_lock_write_next(struct rpc_xprt *xprt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) if (test_and_set_bit(XPRT_LOCKED, &xprt->state))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) if (test_bit(XPRT_WRITE_SPACE, &xprt->state))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) goto out_unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) if (rpc_wake_up_first_on_wq(xprtiod_workqueue, &xprt->sending,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) __xprt_lock_write_func, xprt))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) out_unlock:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) xprt_clear_locked(xprt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) static void __xprt_lock_write_next_cong(struct rpc_xprt *xprt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) if (test_and_set_bit(XPRT_LOCKED, &xprt->state))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) if (test_bit(XPRT_WRITE_SPACE, &xprt->state))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) goto out_unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) if (xprt_need_congestion_window_wait(xprt))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) goto out_unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) if (rpc_wake_up_first_on_wq(xprtiod_workqueue, &xprt->sending,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) __xprt_lock_write_func, xprt))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) out_unlock:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) xprt_clear_locked(xprt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) * xprt_release_xprt - allow other requests to use a transport
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) * @xprt: transport with other tasks potentially waiting
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) * @task: task that is releasing access to the transport
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) * Note that "task" can be NULL. No congestion control is provided.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) void xprt_release_xprt(struct rpc_xprt *xprt, struct rpc_task *task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) if (xprt->snd_task == task) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) xprt_clear_locked(xprt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) __xprt_lock_write_next(xprt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) trace_xprt_release_xprt(xprt, task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) EXPORT_SYMBOL_GPL(xprt_release_xprt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) * xprt_release_xprt_cong - allow other requests to use a transport
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) * @xprt: transport with other tasks potentially waiting
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) * @task: task that is releasing access to the transport
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) * Note that "task" can be NULL. Another task is awoken to use the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) * transport if the transport's congestion window allows it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) void xprt_release_xprt_cong(struct rpc_xprt *xprt, struct rpc_task *task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) if (xprt->snd_task == task) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) xprt_clear_locked(xprt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) __xprt_lock_write_next_cong(xprt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) trace_xprt_release_cong(xprt, task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) EXPORT_SYMBOL_GPL(xprt_release_xprt_cong);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) static inline void xprt_release_write(struct rpc_xprt *xprt, struct rpc_task *task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) if (xprt->snd_task != task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) spin_lock(&xprt->transport_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) xprt->ops->release_xprt(xprt, task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) spin_unlock(&xprt->transport_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) * Van Jacobson congestion avoidance. Check if the congestion window
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) * overflowed. Put the task to sleep if this is the case.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) static int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) __xprt_get_cong(struct rpc_xprt *xprt, struct rpc_rqst *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) if (req->rq_cong)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) trace_xprt_get_cong(xprt, req->rq_task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) if (RPCXPRT_CONGESTED(xprt)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) xprt_set_congestion_window_wait(xprt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) req->rq_cong = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) xprt->cong += RPC_CWNDSCALE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) * Adjust the congestion window, and wake up the next task
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) * that has been sleeping due to congestion
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) __xprt_put_cong(struct rpc_xprt *xprt, struct rpc_rqst *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) if (!req->rq_cong)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) req->rq_cong = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) xprt->cong -= RPC_CWNDSCALE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) xprt_test_and_clear_congestion_window_wait(xprt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) trace_xprt_put_cong(xprt, req->rq_task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) __xprt_lock_write_next_cong(xprt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) * xprt_request_get_cong - Request congestion control credits
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) * @xprt: pointer to transport
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) * @req: pointer to RPC request
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) * Useful for transports that require congestion control.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) bool
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) xprt_request_get_cong(struct rpc_xprt *xprt, struct rpc_rqst *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) bool ret = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) if (req->rq_cong)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) spin_lock(&xprt->transport_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) ret = __xprt_get_cong(xprt, req) != 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) spin_unlock(&xprt->transport_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) EXPORT_SYMBOL_GPL(xprt_request_get_cong);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) * xprt_release_rqst_cong - housekeeping when request is complete
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) * @task: RPC request that recently completed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) * Useful for transports that require congestion control.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) void xprt_release_rqst_cong(struct rpc_task *task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) struct rpc_rqst *req = task->tk_rqstp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) __xprt_put_cong(req->rq_xprt, req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) EXPORT_SYMBOL_GPL(xprt_release_rqst_cong);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) static void xprt_clear_congestion_window_wait_locked(struct rpc_xprt *xprt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) if (test_and_clear_bit(XPRT_CWND_WAIT, &xprt->state))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) __xprt_lock_write_next_cong(xprt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) * Clear the congestion window wait flag and wake up the next
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) * entry on xprt->sending
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) xprt_clear_congestion_window_wait(struct rpc_xprt *xprt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) if (test_and_clear_bit(XPRT_CWND_WAIT, &xprt->state)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) spin_lock(&xprt->transport_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) __xprt_lock_write_next_cong(xprt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) spin_unlock(&xprt->transport_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) * xprt_adjust_cwnd - adjust transport congestion window
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) * @xprt: pointer to xprt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) * @task: recently completed RPC request used to adjust window
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) * @result: result code of completed RPC request
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) * The transport code maintains an estimate on the maximum number of out-
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) * standing RPC requests, using a smoothed version of the congestion
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) * avoidance implemented in 44BSD. This is basically the Van Jacobson
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) * congestion algorithm: If a retransmit occurs, the congestion window is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) * halved; otherwise, it is incremented by 1/cwnd when
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) * - a reply is received and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) * - a full number of requests are outstanding and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) * - the congestion window hasn't been updated recently.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) void xprt_adjust_cwnd(struct rpc_xprt *xprt, struct rpc_task *task, int result)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) struct rpc_rqst *req = task->tk_rqstp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) unsigned long cwnd = xprt->cwnd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) if (result >= 0 && cwnd <= xprt->cong) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) /* The (cwnd >> 1) term makes sure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) * the result gets rounded properly. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) cwnd += (RPC_CWNDSCALE * RPC_CWNDSCALE + (cwnd >> 1)) / cwnd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) if (cwnd > RPC_MAXCWND(xprt))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) cwnd = RPC_MAXCWND(xprt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) __xprt_lock_write_next_cong(xprt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) } else if (result == -ETIMEDOUT) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) cwnd >>= 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) if (cwnd < RPC_CWNDSCALE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) cwnd = RPC_CWNDSCALE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) dprintk("RPC: cong %ld, cwnd was %ld, now %ld\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) xprt->cong, xprt->cwnd, cwnd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) xprt->cwnd = cwnd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) __xprt_put_cong(xprt, req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) EXPORT_SYMBOL_GPL(xprt_adjust_cwnd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) * xprt_wake_pending_tasks - wake all tasks on a transport's pending queue
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) * @xprt: transport with waiting tasks
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) * @status: result code to plant in each task before waking it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) void xprt_wake_pending_tasks(struct rpc_xprt *xprt, int status)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) if (status < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) rpc_wake_up_status(&xprt->pending, status);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) rpc_wake_up(&xprt->pending);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) EXPORT_SYMBOL_GPL(xprt_wake_pending_tasks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) * xprt_wait_for_buffer_space - wait for transport output buffer to clear
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) * @xprt: transport
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) * Note that we only set the timer for the case of RPC_IS_SOFT(), since
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) * we don't in general want to force a socket disconnection due to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) * an incomplete RPC call transmission.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) void xprt_wait_for_buffer_space(struct rpc_xprt *xprt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) set_bit(XPRT_WRITE_SPACE, &xprt->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) EXPORT_SYMBOL_GPL(xprt_wait_for_buffer_space);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) static bool
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) xprt_clear_write_space_locked(struct rpc_xprt *xprt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) if (test_and_clear_bit(XPRT_WRITE_SPACE, &xprt->state)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) __xprt_lock_write_next(xprt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) dprintk("RPC: write space: waking waiting task on "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) "xprt %p\n", xprt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) * xprt_write_space - wake the task waiting for transport output buffer space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) * @xprt: transport with waiting tasks
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) * Can be called in a soft IRQ context, so xprt_write_space never sleeps.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) bool xprt_write_space(struct rpc_xprt *xprt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) bool ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) if (!test_bit(XPRT_WRITE_SPACE, &xprt->state))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) spin_lock(&xprt->transport_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) ret = xprt_clear_write_space_locked(xprt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) spin_unlock(&xprt->transport_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) EXPORT_SYMBOL_GPL(xprt_write_space);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) static unsigned long xprt_abs_ktime_to_jiffies(ktime_t abstime)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) s64 delta = ktime_to_ns(ktime_get() - abstime);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) return likely(delta >= 0) ?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) jiffies - nsecs_to_jiffies(delta) :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) jiffies + nsecs_to_jiffies(-delta);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) static unsigned long xprt_calc_majortimeo(struct rpc_rqst *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) const struct rpc_timeout *to = req->rq_task->tk_client->cl_timeout;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) unsigned long majortimeo = req->rq_timeout;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) if (to->to_exponential)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) majortimeo <<= to->to_retries;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) majortimeo += to->to_increment * to->to_retries;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) if (majortimeo > to->to_maxval || majortimeo == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) majortimeo = to->to_maxval;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) return majortimeo;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) static void xprt_reset_majortimeo(struct rpc_rqst *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) req->rq_majortimeo += xprt_calc_majortimeo(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) static void xprt_reset_minortimeo(struct rpc_rqst *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) req->rq_minortimeo += req->rq_timeout;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) static void xprt_init_majortimeo(struct rpc_task *task, struct rpc_rqst *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) unsigned long time_init;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) struct rpc_xprt *xprt = req->rq_xprt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) if (likely(xprt && xprt_connected(xprt)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) time_init = jiffies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) time_init = xprt_abs_ktime_to_jiffies(task->tk_start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) req->rq_timeout = task->tk_client->cl_timeout->to_initval;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) req->rq_majortimeo = time_init + xprt_calc_majortimeo(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) req->rq_minortimeo = time_init + req->rq_timeout;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) * xprt_adjust_timeout - adjust timeout values for next retransmit
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) * @req: RPC request containing parameters to use for the adjustment
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) int xprt_adjust_timeout(struct rpc_rqst *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) struct rpc_xprt *xprt = req->rq_xprt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) const struct rpc_timeout *to = req->rq_task->tk_client->cl_timeout;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) int status = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) if (time_before(jiffies, req->rq_majortimeo)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) if (time_before(jiffies, req->rq_minortimeo))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) return status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) if (to->to_exponential)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) req->rq_timeout <<= 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) req->rq_timeout += to->to_increment;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) if (to->to_maxval && req->rq_timeout >= to->to_maxval)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) req->rq_timeout = to->to_maxval;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) req->rq_retries++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) req->rq_timeout = to->to_initval;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) req->rq_retries = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) xprt_reset_majortimeo(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) /* Reset the RTT counters == "slow start" */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) spin_lock(&xprt->transport_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) rpc_init_rtt(req->rq_task->tk_client->cl_rtt, to->to_initval);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) spin_unlock(&xprt->transport_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) status = -ETIMEDOUT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) xprt_reset_minortimeo(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) if (req->rq_timeout == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) printk(KERN_WARNING "xprt_adjust_timeout: rq_timeout = 0!\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) req->rq_timeout = 5 * HZ;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) return status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) static void xprt_autoclose(struct work_struct *work)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) struct rpc_xprt *xprt =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) container_of(work, struct rpc_xprt, task_cleanup);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) unsigned int pflags = memalloc_nofs_save();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) trace_xprt_disconnect_auto(xprt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) clear_bit(XPRT_CLOSE_WAIT, &xprt->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) xprt->ops->close(xprt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) xprt_release_write(xprt, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) wake_up_bit(&xprt->state, XPRT_LOCKED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) memalloc_nofs_restore(pflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) * xprt_disconnect_done - mark a transport as disconnected
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) * @xprt: transport to flag for disconnect
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) void xprt_disconnect_done(struct rpc_xprt *xprt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) trace_xprt_disconnect_done(xprt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) spin_lock(&xprt->transport_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) xprt_clear_connected(xprt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) xprt_clear_write_space_locked(xprt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) xprt_clear_congestion_window_wait_locked(xprt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) xprt_wake_pending_tasks(xprt, -ENOTCONN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) spin_unlock(&xprt->transport_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) EXPORT_SYMBOL_GPL(xprt_disconnect_done);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) * xprt_force_disconnect - force a transport to disconnect
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) * @xprt: transport to disconnect
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) void xprt_force_disconnect(struct rpc_xprt *xprt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) trace_xprt_disconnect_force(xprt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) /* Don't race with the test_bit() in xprt_clear_locked() */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) spin_lock(&xprt->transport_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) set_bit(XPRT_CLOSE_WAIT, &xprt->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) /* Try to schedule an autoclose RPC call */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) if (test_and_set_bit(XPRT_LOCKED, &xprt->state) == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) queue_work(xprtiod_workqueue, &xprt->task_cleanup);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) else if (xprt->snd_task && !test_bit(XPRT_SND_IS_COOKIE, &xprt->state))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) rpc_wake_up_queued_task_set_status(&xprt->pending,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) xprt->snd_task, -ENOTCONN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) spin_unlock(&xprt->transport_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) EXPORT_SYMBOL_GPL(xprt_force_disconnect);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) static unsigned int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) xprt_connect_cookie(struct rpc_xprt *xprt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) return READ_ONCE(xprt->connect_cookie);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) static bool
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) xprt_request_retransmit_after_disconnect(struct rpc_task *task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) struct rpc_rqst *req = task->tk_rqstp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) struct rpc_xprt *xprt = req->rq_xprt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) return req->rq_connect_cookie != xprt_connect_cookie(xprt) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) !xprt_connected(xprt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) * xprt_conditional_disconnect - force a transport to disconnect
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) * @xprt: transport to disconnect
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) * @cookie: 'connection cookie'
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) * This attempts to break the connection if and only if 'cookie' matches
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) * the current transport 'connection cookie'. It ensures that we don't
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) * try to break the connection more than once when we need to retransmit
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) * a batch of RPC requests.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) void xprt_conditional_disconnect(struct rpc_xprt *xprt, unsigned int cookie)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) /* Don't race with the test_bit() in xprt_clear_locked() */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) spin_lock(&xprt->transport_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) if (cookie != xprt->connect_cookie)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) if (test_bit(XPRT_CLOSING, &xprt->state))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) set_bit(XPRT_CLOSE_WAIT, &xprt->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) /* Try to schedule an autoclose RPC call */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) if (test_and_set_bit(XPRT_LOCKED, &xprt->state) == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) queue_work(xprtiod_workqueue, &xprt->task_cleanup);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) xprt_wake_pending_tasks(xprt, -EAGAIN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) spin_unlock(&xprt->transport_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) static bool
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) xprt_has_timer(const struct rpc_xprt *xprt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) return xprt->idle_timeout != 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) xprt_schedule_autodisconnect(struct rpc_xprt *xprt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) __must_hold(&xprt->transport_lock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) xprt->last_used = jiffies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) if (RB_EMPTY_ROOT(&xprt->recv_queue) && xprt_has_timer(xprt))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) mod_timer(&xprt->timer, xprt->last_used + xprt->idle_timeout);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) xprt_init_autodisconnect(struct timer_list *t)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) struct rpc_xprt *xprt = from_timer(xprt, t, timer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) if (!RB_EMPTY_ROOT(&xprt->recv_queue))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) /* Reset xprt->last_used to avoid connect/autodisconnect cycling */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) xprt->last_used = jiffies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) if (test_and_set_bit(XPRT_LOCKED, &xprt->state))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) queue_work(xprtiod_workqueue, &xprt->task_cleanup);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) bool xprt_lock_connect(struct rpc_xprt *xprt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) struct rpc_task *task,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) void *cookie)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) bool ret = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) spin_lock(&xprt->transport_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) if (!test_bit(XPRT_LOCKED, &xprt->state))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) if (xprt->snd_task != task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) set_bit(XPRT_SND_IS_COOKIE, &xprt->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) xprt->snd_task = cookie;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) ret = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) spin_unlock(&xprt->transport_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) EXPORT_SYMBOL_GPL(xprt_lock_connect);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) void xprt_unlock_connect(struct rpc_xprt *xprt, void *cookie)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) spin_lock(&xprt->transport_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) if (xprt->snd_task != cookie)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) if (!test_bit(XPRT_LOCKED, &xprt->state))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) xprt->snd_task =NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) clear_bit(XPRT_SND_IS_COOKIE, &xprt->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) xprt->ops->release_xprt(xprt, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) xprt_schedule_autodisconnect(xprt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) spin_unlock(&xprt->transport_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862) wake_up_bit(&xprt->state, XPRT_LOCKED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) EXPORT_SYMBOL_GPL(xprt_unlock_connect);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) * xprt_connect - schedule a transport connect operation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) * @task: RPC task that is requesting the connect
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) void xprt_connect(struct rpc_task *task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) struct rpc_xprt *xprt = task->tk_rqstp->rq_xprt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) trace_xprt_connect(xprt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) if (!xprt_bound(xprt)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) task->tk_status = -EAGAIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) if (!xprt_lock_write(xprt, task))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) if (test_and_clear_bit(XPRT_CLOSE_WAIT, &xprt->state)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) trace_xprt_disconnect_cleanup(xprt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) xprt->ops->close(xprt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) if (!xprt_connected(xprt)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) task->tk_rqstp->rq_connect_cookie = xprt->connect_cookie;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) rpc_sleep_on_timeout(&xprt->pending, task, NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) xprt_request_timeout(task->tk_rqstp));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) if (test_bit(XPRT_CLOSING, &xprt->state))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) if (xprt_test_and_set_connecting(xprt))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) /* Race breaker */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) if (!xprt_connected(xprt)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) xprt->stat.connect_start = jiffies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) xprt->ops->connect(xprt, task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903) xprt_clear_connecting(xprt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) task->tk_status = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) rpc_wake_up_queued_task(&xprt->pending, task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) xprt_release_write(xprt, task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) * xprt_reconnect_delay - compute the wait before scheduling a connect
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) * @xprt: transport instance
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) unsigned long xprt_reconnect_delay(const struct rpc_xprt *xprt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) unsigned long start, now = jiffies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) start = xprt->stat.connect_start + xprt->reestablish_timeout;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) if (time_after(start, now))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) return start - now;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) EXPORT_SYMBOL_GPL(xprt_reconnect_delay);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) * xprt_reconnect_backoff - compute the new re-establish timeout
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) * @xprt: transport instance
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) * @init_to: initial reestablish timeout
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) void xprt_reconnect_backoff(struct rpc_xprt *xprt, unsigned long init_to)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) xprt->reestablish_timeout <<= 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) if (xprt->reestablish_timeout > xprt->max_reconnect_timeout)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) xprt->reestablish_timeout = xprt->max_reconnect_timeout;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938) if (xprt->reestablish_timeout < init_to)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) xprt->reestablish_timeout = init_to;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) EXPORT_SYMBOL_GPL(xprt_reconnect_backoff);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943) enum xprt_xid_rb_cmp {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) XID_RB_EQUAL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) XID_RB_LEFT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946) XID_RB_RIGHT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) static enum xprt_xid_rb_cmp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949) xprt_xid_cmp(__be32 xid1, __be32 xid2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) if (xid1 == xid2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952) return XID_RB_EQUAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) if ((__force u32)xid1 < (__force u32)xid2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) return XID_RB_LEFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) return XID_RB_RIGHT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) static struct rpc_rqst *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959) xprt_request_rb_find(struct rpc_xprt *xprt, __be32 xid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961) struct rb_node *n = xprt->recv_queue.rb_node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962) struct rpc_rqst *req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964) while (n != NULL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965) req = rb_entry(n, struct rpc_rqst, rq_recv);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966) switch (xprt_xid_cmp(xid, req->rq_xid)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967) case XID_RB_LEFT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968) n = n->rb_left;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970) case XID_RB_RIGHT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971) n = n->rb_right;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973) case XID_RB_EQUAL:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974) return req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981) xprt_request_rb_insert(struct rpc_xprt *xprt, struct rpc_rqst *new)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983) struct rb_node **p = &xprt->recv_queue.rb_node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984) struct rb_node *n = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985) struct rpc_rqst *req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987) while (*p != NULL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988) n = *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989) req = rb_entry(n, struct rpc_rqst, rq_recv);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990) switch(xprt_xid_cmp(new->rq_xid, req->rq_xid)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991) case XID_RB_LEFT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992) p = &n->rb_left;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994) case XID_RB_RIGHT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995) p = &n->rb_right;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997) case XID_RB_EQUAL:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998) WARN_ON_ONCE(new != req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002) rb_link_node(&new->rq_recv, n, p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003) rb_insert_color(&new->rq_recv, &xprt->recv_queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007) xprt_request_rb_remove(struct rpc_xprt *xprt, struct rpc_rqst *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009) rb_erase(&req->rq_recv, &xprt->recv_queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013) * xprt_lookup_rqst - find an RPC request corresponding to an XID
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014) * @xprt: transport on which the original request was transmitted
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015) * @xid: RPC XID of incoming reply
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017) * Caller holds xprt->queue_lock.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019) struct rpc_rqst *xprt_lookup_rqst(struct rpc_xprt *xprt, __be32 xid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021) struct rpc_rqst *entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023) entry = xprt_request_rb_find(xprt, xid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024) if (entry != NULL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025) trace_xprt_lookup_rqst(xprt, xid, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026) entry->rq_rtt = ktime_sub(ktime_get(), entry->rq_xtime);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027) return entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030) dprintk("RPC: xprt_lookup_rqst did not find xid %08x\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031) ntohl(xid));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032) trace_xprt_lookup_rqst(xprt, xid, -ENOENT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033) xprt->stat.bad_xids++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036) EXPORT_SYMBOL_GPL(xprt_lookup_rqst);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038) static bool
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039) xprt_is_pinned_rqst(struct rpc_rqst *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041) return atomic_read(&req->rq_pin) != 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045) * xprt_pin_rqst - Pin a request on the transport receive list
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046) * @req: Request to pin
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048) * Caller must ensure this is atomic with the call to xprt_lookup_rqst()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049) * so should be holding xprt->queue_lock.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051) void xprt_pin_rqst(struct rpc_rqst *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053) atomic_inc(&req->rq_pin);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055) EXPORT_SYMBOL_GPL(xprt_pin_rqst);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058) * xprt_unpin_rqst - Unpin a request on the transport receive list
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059) * @req: Request to pin
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1061) * Caller should be holding xprt->queue_lock.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1062) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1063) void xprt_unpin_rqst(struct rpc_rqst *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1064) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1065) if (!test_bit(RPC_TASK_MSG_PIN_WAIT, &req->rq_task->tk_runstate)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1066) atomic_dec(&req->rq_pin);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1067) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1068) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1069) if (atomic_dec_and_test(&req->rq_pin))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1070) wake_up_var(&req->rq_pin);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1071) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1072) EXPORT_SYMBOL_GPL(xprt_unpin_rqst);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1073)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1074) static void xprt_wait_on_pinned_rqst(struct rpc_rqst *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1075) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1076) wait_var_event(&req->rq_pin, !xprt_is_pinned_rqst(req));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1077) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1078)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1079) static bool
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1080) xprt_request_data_received(struct rpc_task *task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1081) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1082) return !test_bit(RPC_TASK_NEED_RECV, &task->tk_runstate) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1083) READ_ONCE(task->tk_rqstp->rq_reply_bytes_recvd) != 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1084) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1085)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1086) static bool
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1087) xprt_request_need_enqueue_receive(struct rpc_task *task, struct rpc_rqst *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1088) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1089) return !test_bit(RPC_TASK_NEED_RECV, &task->tk_runstate) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1090) READ_ONCE(task->tk_rqstp->rq_reply_bytes_recvd) == 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1091) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1092)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1093) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1094) * xprt_request_enqueue_receive - Add an request to the receive queue
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1095) * @task: RPC task
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1096) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1097) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1098) void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1099) xprt_request_enqueue_receive(struct rpc_task *task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1100) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1101) struct rpc_rqst *req = task->tk_rqstp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1102) struct rpc_xprt *xprt = req->rq_xprt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1103)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1104) if (!xprt_request_need_enqueue_receive(task, req))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1105) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1106)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1107) xprt_request_prepare(task->tk_rqstp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1108) spin_lock(&xprt->queue_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1109)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1110) /* Update the softirq receive buffer */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1111) memcpy(&req->rq_private_buf, &req->rq_rcv_buf,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1112) sizeof(req->rq_private_buf));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1113)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1114) /* Add request to the receive list */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1115) xprt_request_rb_insert(xprt, req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1116) set_bit(RPC_TASK_NEED_RECV, &task->tk_runstate);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1117) spin_unlock(&xprt->queue_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1118)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1119) /* Turn off autodisconnect */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1120) del_singleshot_timer_sync(&xprt->timer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1121) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1122)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1123) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1124) * xprt_request_dequeue_receive_locked - Remove a request from the receive queue
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1125) * @task: RPC task
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1126) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1127) * Caller must hold xprt->queue_lock.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1128) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1129) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1130) xprt_request_dequeue_receive_locked(struct rpc_task *task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1131) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1132) struct rpc_rqst *req = task->tk_rqstp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1133)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1134) if (test_and_clear_bit(RPC_TASK_NEED_RECV, &task->tk_runstate))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1135) xprt_request_rb_remove(req->rq_xprt, req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1136) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1137)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1138) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1139) * xprt_update_rtt - Update RPC RTT statistics
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1140) * @task: RPC request that recently completed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1141) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1142) * Caller holds xprt->queue_lock.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1143) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1144) void xprt_update_rtt(struct rpc_task *task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1145) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1146) struct rpc_rqst *req = task->tk_rqstp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1147) struct rpc_rtt *rtt = task->tk_client->cl_rtt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1148) unsigned int timer = task->tk_msg.rpc_proc->p_timer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1149) long m = usecs_to_jiffies(ktime_to_us(req->rq_rtt));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1150)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1151) if (timer) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1152) if (req->rq_ntrans == 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1153) rpc_update_rtt(rtt, timer, m);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1154) rpc_set_timeo(rtt, timer, req->rq_ntrans - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1155) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1156) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1157) EXPORT_SYMBOL_GPL(xprt_update_rtt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1158)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1159) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1160) * xprt_complete_rqst - called when reply processing is complete
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1161) * @task: RPC request that recently completed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1162) * @copied: actual number of bytes received from the transport
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1163) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1164) * Caller holds xprt->queue_lock.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1165) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1166) void xprt_complete_rqst(struct rpc_task *task, int copied)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1167) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1168) struct rpc_rqst *req = task->tk_rqstp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1169) struct rpc_xprt *xprt = req->rq_xprt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1170)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1171) xprt->stat.recvs++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1172)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1173) req->rq_private_buf.len = copied;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1174) /* Ensure all writes are done before we update */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1175) /* req->rq_reply_bytes_recvd */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1176) smp_wmb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1177) req->rq_reply_bytes_recvd = copied;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1178) xprt_request_dequeue_receive_locked(task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1179) rpc_wake_up_queued_task(&xprt->pending, task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1180) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1181) EXPORT_SYMBOL_GPL(xprt_complete_rqst);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1182)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1183) static void xprt_timer(struct rpc_task *task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1184) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1185) struct rpc_rqst *req = task->tk_rqstp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1186) struct rpc_xprt *xprt = req->rq_xprt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1187)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1188) if (task->tk_status != -ETIMEDOUT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1189) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1190)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1191) trace_xprt_timer(xprt, req->rq_xid, task->tk_status);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1192) if (!req->rq_reply_bytes_recvd) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1193) if (xprt->ops->timer)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1194) xprt->ops->timer(xprt, task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1195) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1196) task->tk_status = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1197) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1198)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1199) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1200) * xprt_wait_for_reply_request_def - wait for reply
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1201) * @task: pointer to rpc_task
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1202) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1203) * Set a request's retransmit timeout based on the transport's
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1204) * default timeout parameters. Used by transports that don't adjust
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1205) * the retransmit timeout based on round-trip time estimation,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1206) * and put the task to sleep on the pending queue.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1207) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1208) void xprt_wait_for_reply_request_def(struct rpc_task *task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1209) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1210) struct rpc_rqst *req = task->tk_rqstp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1211)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1212) rpc_sleep_on_timeout(&req->rq_xprt->pending, task, xprt_timer,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1213) xprt_request_timeout(req));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1214) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1215) EXPORT_SYMBOL_GPL(xprt_wait_for_reply_request_def);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1216)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1217) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1218) * xprt_wait_for_reply_request_rtt - wait for reply using RTT estimator
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1219) * @task: pointer to rpc_task
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1220) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1221) * Set a request's retransmit timeout using the RTT estimator,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1222) * and put the task to sleep on the pending queue.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1223) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1224) void xprt_wait_for_reply_request_rtt(struct rpc_task *task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1225) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1226) int timer = task->tk_msg.rpc_proc->p_timer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1227) struct rpc_clnt *clnt = task->tk_client;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1228) struct rpc_rtt *rtt = clnt->cl_rtt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1229) struct rpc_rqst *req = task->tk_rqstp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1230) unsigned long max_timeout = clnt->cl_timeout->to_maxval;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1231) unsigned long timeout;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1232)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1233) timeout = rpc_calc_rto(rtt, timer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1234) timeout <<= rpc_ntimeo(rtt, timer) + req->rq_retries;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1235) if (timeout > max_timeout || timeout == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1236) timeout = max_timeout;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1237) rpc_sleep_on_timeout(&req->rq_xprt->pending, task, xprt_timer,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1238) jiffies + timeout);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1239) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1240) EXPORT_SYMBOL_GPL(xprt_wait_for_reply_request_rtt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1241)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1242) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1243) * xprt_request_wait_receive - wait for the reply to an RPC request
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1244) * @task: RPC task about to send a request
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1245) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1246) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1247) void xprt_request_wait_receive(struct rpc_task *task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1248) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1249) struct rpc_rqst *req = task->tk_rqstp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1250) struct rpc_xprt *xprt = req->rq_xprt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1251)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1252) if (!test_bit(RPC_TASK_NEED_RECV, &task->tk_runstate))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1253) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1254) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1255) * Sleep on the pending queue if we're expecting a reply.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1256) * The spinlock ensures atomicity between the test of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1257) * req->rq_reply_bytes_recvd, and the call to rpc_sleep_on().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1258) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1259) spin_lock(&xprt->queue_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1260) if (test_bit(RPC_TASK_NEED_RECV, &task->tk_runstate)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1261) xprt->ops->wait_for_reply_request(task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1262) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1263) * Send an extra queue wakeup call if the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1264) * connection was dropped in case the call to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1265) * rpc_sleep_on() raced.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1266) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1267) if (xprt_request_retransmit_after_disconnect(task))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1268) rpc_wake_up_queued_task_set_status(&xprt->pending,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1269) task, -ENOTCONN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1270) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1271) spin_unlock(&xprt->queue_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1272) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1273)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1274) static bool
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1275) xprt_request_need_enqueue_transmit(struct rpc_task *task, struct rpc_rqst *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1276) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1277) return !test_bit(RPC_TASK_NEED_XMIT, &task->tk_runstate);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1278) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1279)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1280) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1281) * xprt_request_enqueue_transmit - queue a task for transmission
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1282) * @task: pointer to rpc_task
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1283) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1284) * Add a task to the transmission queue.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1285) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1286) void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1287) xprt_request_enqueue_transmit(struct rpc_task *task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1288) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1289) struct rpc_rqst *pos, *req = task->tk_rqstp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1290) struct rpc_xprt *xprt = req->rq_xprt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1291)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1292) if (xprt_request_need_enqueue_transmit(task, req)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1293) req->rq_bytes_sent = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1294) spin_lock(&xprt->queue_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1295) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1296) * Requests that carry congestion control credits are added
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1297) * to the head of the list to avoid starvation issues.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1298) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1299) if (req->rq_cong) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1300) xprt_clear_congestion_window_wait(xprt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1301) list_for_each_entry(pos, &xprt->xmit_queue, rq_xmit) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1302) if (pos->rq_cong)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1303) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1304) /* Note: req is added _before_ pos */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1305) list_add_tail(&req->rq_xmit, &pos->rq_xmit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1306) INIT_LIST_HEAD(&req->rq_xmit2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1307) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1308) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1309) } else if (RPC_IS_SWAPPER(task)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1310) list_for_each_entry(pos, &xprt->xmit_queue, rq_xmit) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1311) if (pos->rq_cong || pos->rq_bytes_sent)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1312) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1313) if (RPC_IS_SWAPPER(pos->rq_task))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1314) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1315) /* Note: req is added _before_ pos */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1316) list_add_tail(&req->rq_xmit, &pos->rq_xmit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1317) INIT_LIST_HEAD(&req->rq_xmit2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1318) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1319) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1320) } else if (!req->rq_seqno) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1321) list_for_each_entry(pos, &xprt->xmit_queue, rq_xmit) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1322) if (pos->rq_task->tk_owner != task->tk_owner)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1323) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1324) list_add_tail(&req->rq_xmit2, &pos->rq_xmit2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1325) INIT_LIST_HEAD(&req->rq_xmit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1326) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1327) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1328) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1329) list_add_tail(&req->rq_xmit, &xprt->xmit_queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1330) INIT_LIST_HEAD(&req->rq_xmit2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1331) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1332) set_bit(RPC_TASK_NEED_XMIT, &task->tk_runstate);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1333) spin_unlock(&xprt->queue_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1334) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1335) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1336)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1337) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1338) * xprt_request_dequeue_transmit_locked - remove a task from the transmission queue
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1339) * @task: pointer to rpc_task
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1340) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1341) * Remove a task from the transmission queue
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1342) * Caller must hold xprt->queue_lock
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1343) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1344) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1345) xprt_request_dequeue_transmit_locked(struct rpc_task *task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1346) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1347) struct rpc_rqst *req = task->tk_rqstp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1348)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1349) if (!test_and_clear_bit(RPC_TASK_NEED_XMIT, &task->tk_runstate))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1350) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1351) if (!list_empty(&req->rq_xmit)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1352) list_del(&req->rq_xmit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1353) if (!list_empty(&req->rq_xmit2)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1354) struct rpc_rqst *next = list_first_entry(&req->rq_xmit2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1355) struct rpc_rqst, rq_xmit2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1356) list_del(&req->rq_xmit2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1357) list_add_tail(&next->rq_xmit, &next->rq_xprt->xmit_queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1358) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1359) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1360) list_del(&req->rq_xmit2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1361) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1362)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1363) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1364) * xprt_request_dequeue_transmit - remove a task from the transmission queue
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1365) * @task: pointer to rpc_task
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1366) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1367) * Remove a task from the transmission queue
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1368) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1369) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1370) xprt_request_dequeue_transmit(struct rpc_task *task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1371) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1372) struct rpc_rqst *req = task->tk_rqstp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1373) struct rpc_xprt *xprt = req->rq_xprt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1374)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1375) spin_lock(&xprt->queue_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1376) xprt_request_dequeue_transmit_locked(task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1377) spin_unlock(&xprt->queue_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1378) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1379)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1380) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1381) * xprt_request_dequeue_xprt - remove a task from the transmit+receive queue
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1382) * @task: pointer to rpc_task
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1383) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1384) * Remove a task from the transmit and receive queues, and ensure that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1385) * it is not pinned by the receive work item.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1386) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1387) void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1388) xprt_request_dequeue_xprt(struct rpc_task *task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1389) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1390) struct rpc_rqst *req = task->tk_rqstp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1391) struct rpc_xprt *xprt = req->rq_xprt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1392)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1393) if (test_bit(RPC_TASK_NEED_XMIT, &task->tk_runstate) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1394) test_bit(RPC_TASK_NEED_RECV, &task->tk_runstate) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1395) xprt_is_pinned_rqst(req)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1396) spin_lock(&xprt->queue_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1397) xprt_request_dequeue_transmit_locked(task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1398) xprt_request_dequeue_receive_locked(task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1399) while (xprt_is_pinned_rqst(req)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1400) set_bit(RPC_TASK_MSG_PIN_WAIT, &task->tk_runstate);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1401) spin_unlock(&xprt->queue_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1402) xprt_wait_on_pinned_rqst(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1403) spin_lock(&xprt->queue_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1404) clear_bit(RPC_TASK_MSG_PIN_WAIT, &task->tk_runstate);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1405) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1406) spin_unlock(&xprt->queue_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1407) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1408) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1409)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1410) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1411) * xprt_request_prepare - prepare an encoded request for transport
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1412) * @req: pointer to rpc_rqst
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1413) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1414) * Calls into the transport layer to do whatever is needed to prepare
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1415) * the request for transmission or receive.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1416) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1417) void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1418) xprt_request_prepare(struct rpc_rqst *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1419) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1420) struct rpc_xprt *xprt = req->rq_xprt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1421)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1422) if (xprt->ops->prepare_request)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1423) xprt->ops->prepare_request(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1424) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1425)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1426) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1427) * xprt_request_need_retransmit - Test if a task needs retransmission
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1428) * @task: pointer to rpc_task
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1429) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1430) * Test for whether a connection breakage requires the task to retransmit
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1431) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1432) bool
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1433) xprt_request_need_retransmit(struct rpc_task *task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1434) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1435) return xprt_request_retransmit_after_disconnect(task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1436) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1437)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1438) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1439) * xprt_prepare_transmit - reserve the transport before sending a request
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1440) * @task: RPC task about to send a request
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1441) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1442) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1443) bool xprt_prepare_transmit(struct rpc_task *task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1444) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1445) struct rpc_rqst *req = task->tk_rqstp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1446) struct rpc_xprt *xprt = req->rq_xprt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1447)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1448) if (!xprt_lock_write(xprt, task)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1449) /* Race breaker: someone may have transmitted us */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1450) if (!test_bit(RPC_TASK_NEED_XMIT, &task->tk_runstate))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1451) rpc_wake_up_queued_task_set_status(&xprt->sending,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1452) task, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1453) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1454)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1455) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1456) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1457) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1458)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1459) void xprt_end_transmit(struct rpc_task *task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1460) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1461) struct rpc_xprt *xprt = task->tk_rqstp->rq_xprt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1462)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1463) xprt_inject_disconnect(xprt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1464) xprt_release_write(xprt, task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1465) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1466)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1467) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1468) * xprt_request_transmit - send an RPC request on a transport
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1469) * @req: pointer to request to transmit
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1470) * @snd_task: RPC task that owns the transport lock
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1471) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1472) * This performs the transmission of a single request.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1473) * Note that if the request is not the same as snd_task, then it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1474) * does need to be pinned.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1475) * Returns '0' on success.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1476) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1477) static int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1478) xprt_request_transmit(struct rpc_rqst *req, struct rpc_task *snd_task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1479) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1480) struct rpc_xprt *xprt = req->rq_xprt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1481) struct rpc_task *task = req->rq_task;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1482) unsigned int connect_cookie;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1483) int is_retrans = RPC_WAS_SENT(task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1484) int status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1485)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1486) if (!req->rq_bytes_sent) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1487) if (xprt_request_data_received(task)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1488) status = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1489) goto out_dequeue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1490) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1491) /* Verify that our message lies in the RPCSEC_GSS window */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1492) if (rpcauth_xmit_need_reencode(task)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1493) status = -EBADMSG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1494) goto out_dequeue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1495) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1496) if (RPC_SIGNALLED(task)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1497) status = -ERESTARTSYS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1498) goto out_dequeue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1499) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1500) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1501)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1502) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1503) * Update req->rq_ntrans before transmitting to avoid races with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1504) * xprt_update_rtt(), which needs to know that it is recording a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1505) * reply to the first transmission.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1506) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1507) req->rq_ntrans++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1508)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1509) trace_rpc_xdr_sendto(task, &req->rq_snd_buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1510) connect_cookie = xprt->connect_cookie;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1511) status = xprt->ops->send_request(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1512) if (status != 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1513) req->rq_ntrans--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1514) trace_xprt_transmit(req, status);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1515) return status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1516) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1517)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1518) if (is_retrans)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1519) task->tk_client->cl_stats->rpcretrans++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1520)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1521) xprt_inject_disconnect(xprt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1522)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1523) task->tk_flags |= RPC_TASK_SENT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1524) spin_lock(&xprt->transport_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1525)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1526) xprt->stat.sends++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1527) xprt->stat.req_u += xprt->stat.sends - xprt->stat.recvs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1528) xprt->stat.bklog_u += xprt->backlog.qlen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1529) xprt->stat.sending_u += xprt->sending.qlen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1530) xprt->stat.pending_u += xprt->pending.qlen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1531) spin_unlock(&xprt->transport_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1532)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1533) req->rq_connect_cookie = connect_cookie;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1534) out_dequeue:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1535) trace_xprt_transmit(req, status);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1536) xprt_request_dequeue_transmit(task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1537) rpc_wake_up_queued_task_set_status(&xprt->sending, task, status);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1538) return status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1539) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1540)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1541) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1542) * xprt_transmit - send an RPC request on a transport
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1543) * @task: controlling RPC task
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1544) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1545) * Attempts to drain the transmit queue. On exit, either the transport
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1546) * signalled an error that needs to be handled before transmission can
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1547) * resume, or @task finished transmitting, and detected that it already
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1548) * received a reply.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1549) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1550) void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1551) xprt_transmit(struct rpc_task *task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1552) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1553) struct rpc_rqst *next, *req = task->tk_rqstp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1554) struct rpc_xprt *xprt = req->rq_xprt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1555) int status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1556)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1557) spin_lock(&xprt->queue_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1558) for (;;) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1559) next = list_first_entry_or_null(&xprt->xmit_queue,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1560) struct rpc_rqst, rq_xmit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1561) if (!next)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1562) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1563) xprt_pin_rqst(next);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1564) spin_unlock(&xprt->queue_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1565) status = xprt_request_transmit(next, task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1566) if (status == -EBADMSG && next != req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1567) status = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1568) spin_lock(&xprt->queue_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1569) xprt_unpin_rqst(next);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1570) if (status < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1571) if (test_bit(RPC_TASK_NEED_XMIT, &task->tk_runstate))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1572) task->tk_status = status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1573) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1574) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1575) /* Was @task transmitted, and has it received a reply? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1576) if (xprt_request_data_received(task) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1577) !test_bit(RPC_TASK_NEED_XMIT, &task->tk_runstate))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1578) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1579) cond_resched_lock(&xprt->queue_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1580) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1581) spin_unlock(&xprt->queue_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1582) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1583)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1584) static void xprt_complete_request_init(struct rpc_task *task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1585) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1586) if (task->tk_rqstp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1587) xprt_request_init(task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1588) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1589)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1590) void xprt_add_backlog(struct rpc_xprt *xprt, struct rpc_task *task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1591) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1592) set_bit(XPRT_CONGESTED, &xprt->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1593) rpc_sleep_on(&xprt->backlog, task, xprt_complete_request_init);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1594) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1595) EXPORT_SYMBOL_GPL(xprt_add_backlog);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1596)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1597) static bool __xprt_set_rq(struct rpc_task *task, void *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1598) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1599) struct rpc_rqst *req = data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1600)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1601) if (task->tk_rqstp == NULL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1602) memset(req, 0, sizeof(*req)); /* mark unused */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1603) task->tk_rqstp = req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1604) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1605) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1606) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1607) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1608)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1609) bool xprt_wake_up_backlog(struct rpc_xprt *xprt, struct rpc_rqst *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1610) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1611) if (rpc_wake_up_first(&xprt->backlog, __xprt_set_rq, req) == NULL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1612) clear_bit(XPRT_CONGESTED, &xprt->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1613) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1614) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1615) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1616) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1617) EXPORT_SYMBOL_GPL(xprt_wake_up_backlog);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1618)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1619) static bool xprt_throttle_congested(struct rpc_xprt *xprt, struct rpc_task *task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1620) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1621) bool ret = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1622)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1623) if (!test_bit(XPRT_CONGESTED, &xprt->state))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1624) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1625) spin_lock(&xprt->reserve_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1626) if (test_bit(XPRT_CONGESTED, &xprt->state)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1627) xprt_add_backlog(xprt, task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1628) ret = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1629) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1630) spin_unlock(&xprt->reserve_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1631) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1632) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1633) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1634)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1635) static struct rpc_rqst *xprt_dynamic_alloc_slot(struct rpc_xprt *xprt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1636) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1637) struct rpc_rqst *req = ERR_PTR(-EAGAIN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1638)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1639) if (xprt->num_reqs >= xprt->max_reqs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1640) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1641) ++xprt->num_reqs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1642) spin_unlock(&xprt->reserve_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1643) req = kzalloc(sizeof(struct rpc_rqst), GFP_NOFS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1644) spin_lock(&xprt->reserve_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1645) if (req != NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1646) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1647) --xprt->num_reqs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1648) req = ERR_PTR(-ENOMEM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1649) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1650) return req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1651) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1652)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1653) static bool xprt_dynamic_free_slot(struct rpc_xprt *xprt, struct rpc_rqst *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1654) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1655) if (xprt->num_reqs > xprt->min_reqs) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1656) --xprt->num_reqs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1657) kfree(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1658) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1659) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1660) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1661) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1662)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1663) void xprt_alloc_slot(struct rpc_xprt *xprt, struct rpc_task *task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1664) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1665) struct rpc_rqst *req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1666)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1667) spin_lock(&xprt->reserve_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1668) if (!list_empty(&xprt->free)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1669) req = list_entry(xprt->free.next, struct rpc_rqst, rq_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1670) list_del(&req->rq_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1671) goto out_init_req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1672) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1673) req = xprt_dynamic_alloc_slot(xprt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1674) if (!IS_ERR(req))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1675) goto out_init_req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1676) switch (PTR_ERR(req)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1677) case -ENOMEM:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1678) dprintk("RPC: dynamic allocation of request slot "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1679) "failed! Retrying\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1680) task->tk_status = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1681) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1682) case -EAGAIN:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1683) xprt_add_backlog(xprt, task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1684) dprintk("RPC: waiting for request slot\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1685) fallthrough;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1686) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1687) task->tk_status = -EAGAIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1688) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1689) spin_unlock(&xprt->reserve_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1690) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1691) out_init_req:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1692) xprt->stat.max_slots = max_t(unsigned int, xprt->stat.max_slots,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1693) xprt->num_reqs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1694) spin_unlock(&xprt->reserve_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1695)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1696) task->tk_status = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1697) task->tk_rqstp = req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1698) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1699) EXPORT_SYMBOL_GPL(xprt_alloc_slot);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1700)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1701) void xprt_free_slot(struct rpc_xprt *xprt, struct rpc_rqst *req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1702) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1703) spin_lock(&xprt->reserve_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1704) if (!xprt_wake_up_backlog(xprt, req) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1705) !xprt_dynamic_free_slot(xprt, req)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1706) memset(req, 0, sizeof(*req)); /* mark unused */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1707) list_add(&req->rq_list, &xprt->free);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1708) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1709) spin_unlock(&xprt->reserve_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1710) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1711) EXPORT_SYMBOL_GPL(xprt_free_slot);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1712)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1713) static void xprt_free_all_slots(struct rpc_xprt *xprt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1714) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1715) struct rpc_rqst *req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1716) while (!list_empty(&xprt->free)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1717) req = list_first_entry(&xprt->free, struct rpc_rqst, rq_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1718) list_del(&req->rq_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1719) kfree(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1720) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1721) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1722)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1723) struct rpc_xprt *xprt_alloc(struct net *net, size_t size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1724) unsigned int num_prealloc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1725) unsigned int max_alloc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1726) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1727) struct rpc_xprt *xprt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1728) struct rpc_rqst *req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1729) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1730)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1731) xprt = kzalloc(size, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1732) if (xprt == NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1733) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1734)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1735) xprt_init(xprt, net);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1736)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1737) for (i = 0; i < num_prealloc; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1738) req = kzalloc(sizeof(struct rpc_rqst), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1739) if (!req)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1740) goto out_free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1741) list_add(&req->rq_list, &xprt->free);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1742) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1743) if (max_alloc > num_prealloc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1744) xprt->max_reqs = max_alloc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1745) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1746) xprt->max_reqs = num_prealloc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1747) xprt->min_reqs = num_prealloc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1748) xprt->num_reqs = num_prealloc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1749)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1750) return xprt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1751)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1752) out_free:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1753) xprt_free(xprt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1754) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1755) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1756) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1757) EXPORT_SYMBOL_GPL(xprt_alloc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1758)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1759) void xprt_free(struct rpc_xprt *xprt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1760) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1761) put_net(xprt->xprt_net);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1762) xprt_free_all_slots(xprt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1763) kfree_rcu(xprt, rcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1764) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1765) EXPORT_SYMBOL_GPL(xprt_free);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1766)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1767) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1768) xprt_init_connect_cookie(struct rpc_rqst *req, struct rpc_xprt *xprt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1769) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1770) req->rq_connect_cookie = xprt_connect_cookie(xprt) - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1771) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1772)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1773) static __be32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1774) xprt_alloc_xid(struct rpc_xprt *xprt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1775) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1776) __be32 xid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1777)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1778) spin_lock(&xprt->reserve_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1779) xid = (__force __be32)xprt->xid++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1780) spin_unlock(&xprt->reserve_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1781) return xid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1782) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1783)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1784) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1785) xprt_init_xid(struct rpc_xprt *xprt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1786) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1787) xprt->xid = prandom_u32();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1788) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1789)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1790) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1791) xprt_request_init(struct rpc_task *task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1792) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1793) struct rpc_xprt *xprt = task->tk_xprt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1794) struct rpc_rqst *req = task->tk_rqstp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1795)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1796) req->rq_task = task;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1797) req->rq_xprt = xprt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1798) req->rq_buffer = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1799) req->rq_xid = xprt_alloc_xid(xprt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1800) xprt_init_connect_cookie(req, xprt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1801) req->rq_snd_buf.len = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1802) req->rq_snd_buf.buflen = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1803) req->rq_rcv_buf.len = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1804) req->rq_rcv_buf.buflen = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1805) req->rq_snd_buf.bvec = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1806) req->rq_rcv_buf.bvec = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1807) req->rq_release_snd_buf = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1808) xprt_init_majortimeo(task, req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1809)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1810) trace_xprt_reserve(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1811) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1812)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1813) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1814) xprt_do_reserve(struct rpc_xprt *xprt, struct rpc_task *task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1815) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1816) xprt->ops->alloc_slot(xprt, task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1817) if (task->tk_rqstp != NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1818) xprt_request_init(task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1819) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1820)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1821) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1822) * xprt_reserve - allocate an RPC request slot
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1823) * @task: RPC task requesting a slot allocation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1824) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1825) * If the transport is marked as being congested, or if no more
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1826) * slots are available, place the task on the transport's
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1827) * backlog queue.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1828) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1829) void xprt_reserve(struct rpc_task *task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1830) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1831) struct rpc_xprt *xprt = task->tk_xprt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1832)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1833) task->tk_status = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1834) if (task->tk_rqstp != NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1835) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1836)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1837) task->tk_status = -EAGAIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1838) if (!xprt_throttle_congested(xprt, task))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1839) xprt_do_reserve(xprt, task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1840) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1841)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1842) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1843) * xprt_retry_reserve - allocate an RPC request slot
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1844) * @task: RPC task requesting a slot allocation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1845) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1846) * If no more slots are available, place the task on the transport's
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1847) * backlog queue.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1848) * Note that the only difference with xprt_reserve is that we now
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1849) * ignore the value of the XPRT_CONGESTED flag.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1850) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1851) void xprt_retry_reserve(struct rpc_task *task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1852) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1853) struct rpc_xprt *xprt = task->tk_xprt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1854)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1855) task->tk_status = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1856) if (task->tk_rqstp != NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1857) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1858)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1859) task->tk_status = -EAGAIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1860) xprt_do_reserve(xprt, task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1861) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1862)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1863) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1864) * xprt_release - release an RPC request slot
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1865) * @task: task which is finished with the slot
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1866) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1867) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1868) void xprt_release(struct rpc_task *task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1869) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1870) struct rpc_xprt *xprt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1871) struct rpc_rqst *req = task->tk_rqstp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1872)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1873) if (req == NULL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1874) if (task->tk_client) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1875) xprt = task->tk_xprt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1876) xprt_release_write(xprt, task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1877) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1878) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1879) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1880)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1881) xprt = req->rq_xprt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1882) xprt_request_dequeue_xprt(task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1883) spin_lock(&xprt->transport_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1884) xprt->ops->release_xprt(xprt, task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1885) if (xprt->ops->release_request)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1886) xprt->ops->release_request(task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1887) xprt_schedule_autodisconnect(xprt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1888) spin_unlock(&xprt->transport_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1889) if (req->rq_buffer)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1890) xprt->ops->buf_free(task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1891) xdr_free_bvec(&req->rq_rcv_buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1892) xdr_free_bvec(&req->rq_snd_buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1893) if (req->rq_cred != NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1894) put_rpccred(req->rq_cred);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1895) if (req->rq_release_snd_buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1896) req->rq_release_snd_buf(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1897)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1898) task->tk_rqstp = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1899) if (likely(!bc_prealloc(req)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1900) xprt->ops->free_slot(xprt, req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1901) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1902) xprt_free_bc_request(req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1903) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1904)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1905) #ifdef CONFIG_SUNRPC_BACKCHANNEL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1906) void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1907) xprt_init_bc_request(struct rpc_rqst *req, struct rpc_task *task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1908) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1909) struct xdr_buf *xbufp = &req->rq_snd_buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1910)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1911) task->tk_rqstp = req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1912) req->rq_task = task;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1913) xprt_init_connect_cookie(req, req->rq_xprt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1914) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1915) * Set up the xdr_buf length.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1916) * This also indicates that the buffer is XDR encoded already.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1917) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1918) xbufp->len = xbufp->head[0].iov_len + xbufp->page_len +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1919) xbufp->tail[0].iov_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1920) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1921) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1922)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1923) static void xprt_init(struct rpc_xprt *xprt, struct net *net)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1924) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1925) kref_init(&xprt->kref);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1926)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1927) spin_lock_init(&xprt->transport_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1928) spin_lock_init(&xprt->reserve_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1929) spin_lock_init(&xprt->queue_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1930)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1931) INIT_LIST_HEAD(&xprt->free);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1932) xprt->recv_queue = RB_ROOT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1933) INIT_LIST_HEAD(&xprt->xmit_queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1934) #if defined(CONFIG_SUNRPC_BACKCHANNEL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1935) spin_lock_init(&xprt->bc_pa_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1936) INIT_LIST_HEAD(&xprt->bc_pa_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1937) #endif /* CONFIG_SUNRPC_BACKCHANNEL */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1938) INIT_LIST_HEAD(&xprt->xprt_switch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1939)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1940) xprt->last_used = jiffies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1941) xprt->cwnd = RPC_INITCWND;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1942) xprt->bind_index = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1943)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1944) rpc_init_wait_queue(&xprt->binding, "xprt_binding");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1945) rpc_init_wait_queue(&xprt->pending, "xprt_pending");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1946) rpc_init_wait_queue(&xprt->sending, "xprt_sending");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1947) rpc_init_priority_wait_queue(&xprt->backlog, "xprt_backlog");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1948)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1949) xprt_init_xid(xprt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1950)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1951) xprt->xprt_net = get_net(net);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1952) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1953)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1954) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1955) * xprt_create_transport - create an RPC transport
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1956) * @args: rpc transport creation arguments
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1957) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1958) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1959) struct rpc_xprt *xprt_create_transport(struct xprt_create *args)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1960) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1961) struct rpc_xprt *xprt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1962) struct xprt_class *t;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1963)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1964) spin_lock(&xprt_list_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1965) list_for_each_entry(t, &xprt_list, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1966) if (t->ident == args->ident) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1967) spin_unlock(&xprt_list_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1968) goto found;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1969) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1970) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1971) spin_unlock(&xprt_list_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1972) dprintk("RPC: transport (%d) not supported\n", args->ident);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1973) return ERR_PTR(-EIO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1974)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1975) found:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1976) xprt = t->setup(args);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1977) if (IS_ERR(xprt))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1978) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1979) if (args->flags & XPRT_CREATE_NO_IDLE_TIMEOUT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1980) xprt->idle_timeout = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1981) INIT_WORK(&xprt->task_cleanup, xprt_autoclose);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1982) if (xprt_has_timer(xprt))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1983) timer_setup(&xprt->timer, xprt_init_autodisconnect, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1984) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1985) timer_setup(&xprt->timer, NULL, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1986)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1987) if (strlen(args->servername) > RPC_MAXNETNAMELEN) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1988) xprt_destroy(xprt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1989) return ERR_PTR(-EINVAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1990) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1991) xprt->servername = kstrdup(args->servername, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1992) if (xprt->servername == NULL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1993) xprt_destroy(xprt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1994) return ERR_PTR(-ENOMEM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1995) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1996)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1997) rpc_xprt_debugfs_register(xprt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1998)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1999) trace_xprt_create(xprt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2000) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2001) return xprt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2002) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2003)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2004) static void xprt_destroy_cb(struct work_struct *work)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2005) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2006) struct rpc_xprt *xprt =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2007) container_of(work, struct rpc_xprt, task_cleanup);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2008)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2009) trace_xprt_destroy(xprt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2010)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2011) rpc_xprt_debugfs_unregister(xprt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2012) rpc_destroy_wait_queue(&xprt->binding);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2013) rpc_destroy_wait_queue(&xprt->pending);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2014) rpc_destroy_wait_queue(&xprt->sending);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2015) rpc_destroy_wait_queue(&xprt->backlog);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2016) kfree(xprt->servername);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2017) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2018) * Destroy any existing back channel
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2019) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2020) xprt_destroy_backchannel(xprt, UINT_MAX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2021)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2022) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2023) * Tear down transport state and free the rpc_xprt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2024) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2025) xprt->ops->destroy(xprt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2026) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2027)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2028) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2029) * xprt_destroy - destroy an RPC transport, killing off all requests.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2030) * @xprt: transport to destroy
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2031) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2032) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2033) static void xprt_destroy(struct rpc_xprt *xprt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2034) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2035) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2036) * Exclude transport connect/disconnect handlers and autoclose
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2037) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2038) wait_on_bit_lock(&xprt->state, XPRT_LOCKED, TASK_UNINTERRUPTIBLE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2039)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2040) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2041) * xprt_schedule_autodisconnect() can run after XPRT_LOCKED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2042) * is cleared. We use ->transport_lock to ensure the mod_timer()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2043) * can only run *before* del_time_sync(), never after.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2044) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2045) spin_lock(&xprt->transport_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2046) del_timer_sync(&xprt->timer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2047) spin_unlock(&xprt->transport_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2048)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2049) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2050) * Destroy sockets etc from the system workqueue so they can
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2051) * safely flush receive work running on rpciod.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2052) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2053) INIT_WORK(&xprt->task_cleanup, xprt_destroy_cb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2054) schedule_work(&xprt->task_cleanup);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2055) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2056)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2057) static void xprt_destroy_kref(struct kref *kref)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2058) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2059) xprt_destroy(container_of(kref, struct rpc_xprt, kref));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2060) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2061)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2062) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2063) * xprt_get - return a reference to an RPC transport.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2064) * @xprt: pointer to the transport
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2065) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2066) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2067) struct rpc_xprt *xprt_get(struct rpc_xprt *xprt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2068) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2069) if (xprt != NULL && kref_get_unless_zero(&xprt->kref))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2070) return xprt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2071) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2072) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2073) EXPORT_SYMBOL_GPL(xprt_get);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2074)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2075) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2076) * xprt_put - release a reference to an RPC transport.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2077) * @xprt: pointer to the transport
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2078) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2079) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2080) void xprt_put(struct rpc_xprt *xprt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2081) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2082) if (xprt != NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2083) kref_put(&xprt->kref, xprt_destroy_kref);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2084) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2085) EXPORT_SYMBOL_GPL(xprt_put);
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags