^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Request reply cache. This is currently a global cache, but this may
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * change in the future and be a per-client cache.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * This code is heavily inspired by the 44BSD implementation, although
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * it does things a bit differently.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) * Copyright (C) 1995, 1996 Olaf Kirch <okir@monad.swb.de>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/sunrpc/svc_xprt.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/vmalloc.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/sunrpc/addr.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/highmem.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/log2.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <linux/hash.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <net/checksum.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include "nfsd.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include "cache.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #include "trace.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) * We use this value to determine the number of hash buckets from the max
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) * cache size, the idea being that when the cache is at its maximum number
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) * of entries, then this should be the average number of entries per bucket.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) #define TARGET_BUCKET_SIZE 64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) struct nfsd_drc_bucket {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) struct rb_root rb_head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) struct list_head lru_head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) spinlock_t cache_lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) static struct kmem_cache *drc_slab;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) static int nfsd_cache_append(struct svc_rqst *rqstp, struct kvec *vec);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) static unsigned long nfsd_reply_cache_count(struct shrinker *shrink,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) struct shrink_control *sc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) static unsigned long nfsd_reply_cache_scan(struct shrinker *shrink,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) struct shrink_control *sc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) * Put a cap on the size of the DRC based on the amount of available
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) * low memory in the machine.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) * 64MB: 8192
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) * 128MB: 11585
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) * 256MB: 16384
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) * 512MB: 23170
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) * 1GB: 32768
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) * 2GB: 46340
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) * 4GB: 65536
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) * 8GB: 92681
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) * 16GB: 131072
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) * ...with a hard cap of 256k entries. In the worst case, each entry will be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) * ~1k, so the above numbers should give a rough max of the amount of memory
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) * used in k.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) * XXX: these limits are per-container, so memory used will increase
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) * linearly with number of containers. Maybe that's OK.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) static unsigned int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) nfsd_cache_size_limit(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) unsigned int limit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) unsigned long low_pages = totalram_pages() - totalhigh_pages();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) limit = (16 * int_sqrt(low_pages)) << (PAGE_SHIFT-10);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) return min_t(unsigned int, limit, 256*1024);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) * Compute the number of hash buckets we need. Divide the max cachesize by
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) * the "target" max bucket size, and round up to next power of two.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) static unsigned int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) nfsd_hashsize(unsigned int limit)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) return roundup_pow_of_two(limit / TARGET_BUCKET_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) static u32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) nfsd_cache_hash(__be32 xid, struct nfsd_net *nn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) return hash_32(be32_to_cpu(xid), nn->maskbits);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) static struct svc_cacherep *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) nfsd_reply_cache_alloc(struct svc_rqst *rqstp, __wsum csum,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) struct nfsd_net *nn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) struct svc_cacherep *rp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) rp = kmem_cache_alloc(drc_slab, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) if (rp) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) rp->c_state = RC_UNUSED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) rp->c_type = RC_NOCACHE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) RB_CLEAR_NODE(&rp->c_node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) INIT_LIST_HEAD(&rp->c_lru);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) memset(&rp->c_key, 0, sizeof(rp->c_key));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) rp->c_key.k_xid = rqstp->rq_xid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) rp->c_key.k_proc = rqstp->rq_proc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) rpc_copy_addr((struct sockaddr *)&rp->c_key.k_addr, svc_addr(rqstp));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) rpc_set_port((struct sockaddr *)&rp->c_key.k_addr, rpc_get_port(svc_addr(rqstp)));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) rp->c_key.k_prot = rqstp->rq_prot;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) rp->c_key.k_vers = rqstp->rq_vers;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) rp->c_key.k_len = rqstp->rq_arg.len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) rp->c_key.k_csum = csum;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) return rp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) nfsd_reply_cache_free_locked(struct nfsd_drc_bucket *b, struct svc_cacherep *rp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) struct nfsd_net *nn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) if (rp->c_type == RC_REPLBUFF && rp->c_replvec.iov_base) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) nn->drc_mem_usage -= rp->c_replvec.iov_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) kfree(rp->c_replvec.iov_base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) if (rp->c_state != RC_UNUSED) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) rb_erase(&rp->c_node, &b->rb_head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) list_del(&rp->c_lru);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) atomic_dec(&nn->num_drc_entries);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) nn->drc_mem_usage -= sizeof(*rp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) kmem_cache_free(drc_slab, rp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) nfsd_reply_cache_free(struct nfsd_drc_bucket *b, struct svc_cacherep *rp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) struct nfsd_net *nn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) spin_lock(&b->cache_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) nfsd_reply_cache_free_locked(b, rp, nn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) spin_unlock(&b->cache_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) int nfsd_drc_slab_create(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) drc_slab = kmem_cache_create("nfsd_drc",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) sizeof(struct svc_cacherep), 0, 0, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) return drc_slab ? 0: -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) void nfsd_drc_slab_free(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) kmem_cache_destroy(drc_slab);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) int nfsd_reply_cache_init(struct nfsd_net *nn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) unsigned int hashsize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) unsigned int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) int status = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) nn->max_drc_entries = nfsd_cache_size_limit();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) atomic_set(&nn->num_drc_entries, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) hashsize = nfsd_hashsize(nn->max_drc_entries);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) nn->maskbits = ilog2(hashsize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) nn->nfsd_reply_cache_shrinker.scan_objects = nfsd_reply_cache_scan;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) nn->nfsd_reply_cache_shrinker.count_objects = nfsd_reply_cache_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) nn->nfsd_reply_cache_shrinker.seeks = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) status = register_shrinker(&nn->nfsd_reply_cache_shrinker);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) if (status)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) goto out_nomem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) nn->drc_hashtbl = kvzalloc(array_size(hashsize,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) sizeof(*nn->drc_hashtbl)), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) if (!nn->drc_hashtbl)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) goto out_shrinker;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) for (i = 0; i < hashsize; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) INIT_LIST_HEAD(&nn->drc_hashtbl[i].lru_head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) spin_lock_init(&nn->drc_hashtbl[i].cache_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) nn->drc_hashsize = hashsize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) out_shrinker:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) unregister_shrinker(&nn->nfsd_reply_cache_shrinker);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) out_nomem:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) printk(KERN_ERR "nfsd: failed to allocate reply cache\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) void nfsd_reply_cache_shutdown(struct nfsd_net *nn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) struct svc_cacherep *rp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) unsigned int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) unregister_shrinker(&nn->nfsd_reply_cache_shrinker);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) for (i = 0; i < nn->drc_hashsize; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) struct list_head *head = &nn->drc_hashtbl[i].lru_head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) while (!list_empty(head)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) rp = list_first_entry(head, struct svc_cacherep, c_lru);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) nfsd_reply_cache_free_locked(&nn->drc_hashtbl[i],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) rp, nn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) kvfree(nn->drc_hashtbl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) nn->drc_hashtbl = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) nn->drc_hashsize = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) * Move cache entry to end of LRU list, and queue the cleaner to run if it's
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) * not already scheduled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) lru_put_end(struct nfsd_drc_bucket *b, struct svc_cacherep *rp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) rp->c_timestamp = jiffies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) list_move_tail(&rp->c_lru, &b->lru_head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) static long
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) prune_bucket(struct nfsd_drc_bucket *b, struct nfsd_net *nn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) struct svc_cacherep *rp, *tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) long freed = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) list_for_each_entry_safe(rp, tmp, &b->lru_head, c_lru) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) * Don't free entries attached to calls that are still
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) * in-progress, but do keep scanning the list.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) if (rp->c_state == RC_INPROG)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) if (atomic_read(&nn->num_drc_entries) <= nn->max_drc_entries &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) time_before(jiffies, rp->c_timestamp + RC_EXPIRE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) nfsd_reply_cache_free_locked(b, rp, nn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) freed++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) return freed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) * Walk the LRU list and prune off entries that are older than RC_EXPIRE.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) * Also prune the oldest ones when the total exceeds the max number of entries.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) static long
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) prune_cache_entries(struct nfsd_net *nn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) unsigned int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) long freed = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) for (i = 0; i < nn->drc_hashsize; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) struct nfsd_drc_bucket *b = &nn->drc_hashtbl[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) if (list_empty(&b->lru_head))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) spin_lock(&b->cache_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) freed += prune_bucket(b, nn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) spin_unlock(&b->cache_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) return freed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) static unsigned long
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) nfsd_reply_cache_count(struct shrinker *shrink, struct shrink_control *sc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) struct nfsd_net *nn = container_of(shrink,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) struct nfsd_net, nfsd_reply_cache_shrinker);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) return atomic_read(&nn->num_drc_entries);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) static unsigned long
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) nfsd_reply_cache_scan(struct shrinker *shrink, struct shrink_control *sc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) struct nfsd_net *nn = container_of(shrink,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) struct nfsd_net, nfsd_reply_cache_shrinker);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) return prune_cache_entries(nn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) * Walk an xdr_buf and get a CRC for at most the first RC_CSUMLEN bytes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) static __wsum
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) nfsd_cache_csum(struct svc_rqst *rqstp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) int idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) unsigned int base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) __wsum csum;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) struct xdr_buf *buf = &rqstp->rq_arg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) const unsigned char *p = buf->head[0].iov_base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) size_t csum_len = min_t(size_t, buf->head[0].iov_len + buf->page_len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) RC_CSUMLEN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) size_t len = min(buf->head[0].iov_len, csum_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) /* rq_arg.head first */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) csum = csum_partial(p, len, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) csum_len -= len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) /* Continue into page array */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) idx = buf->page_base / PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) base = buf->page_base & ~PAGE_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) while (csum_len) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) p = page_address(buf->pages[idx]) + base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) len = min_t(size_t, PAGE_SIZE - base, csum_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) csum = csum_partial(p, len, csum);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) csum_len -= len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) base = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) ++idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) return csum;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) static int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) nfsd_cache_key_cmp(const struct svc_cacherep *key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) const struct svc_cacherep *rp, struct nfsd_net *nn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) if (key->c_key.k_xid == rp->c_key.k_xid &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) key->c_key.k_csum != rp->c_key.k_csum) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) ++nn->payload_misses;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) trace_nfsd_drc_mismatch(nn, key, rp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) return memcmp(&key->c_key, &rp->c_key, sizeof(key->c_key));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) * Search the request hash for an entry that matches the given rqstp.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) * Must be called with cache_lock held. Returns the found entry or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) * inserts an empty key on failure.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) static struct svc_cacherep *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) nfsd_cache_insert(struct nfsd_drc_bucket *b, struct svc_cacherep *key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) struct nfsd_net *nn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) struct svc_cacherep *rp, *ret = key;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) struct rb_node **p = &b->rb_head.rb_node,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) *parent = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) unsigned int entries = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) int cmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) while (*p != NULL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) ++entries;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) parent = *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) rp = rb_entry(parent, struct svc_cacherep, c_node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) cmp = nfsd_cache_key_cmp(key, rp, nn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) if (cmp < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) p = &parent->rb_left;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) else if (cmp > 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) p = &parent->rb_right;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) ret = rp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) rb_link_node(&key->c_node, parent, p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) rb_insert_color(&key->c_node, &b->rb_head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) /* tally hash chain length stats */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) if (entries > nn->longest_chain) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) nn->longest_chain = entries;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) nn->longest_chain_cachesize = atomic_read(&nn->num_drc_entries);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) } else if (entries == nn->longest_chain) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) /* prefer to keep the smallest cachesize possible here */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) nn->longest_chain_cachesize = min_t(unsigned int,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) nn->longest_chain_cachesize,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) atomic_read(&nn->num_drc_entries));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) lru_put_end(b, ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) * nfsd_cache_lookup - Find an entry in the duplicate reply cache
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) * @rqstp: Incoming Call to find
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) * Try to find an entry matching the current call in the cache. When none
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) * is found, we try to grab the oldest expired entry off the LRU list. If
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) * a suitable one isn't there, then drop the cache_lock and allocate a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) * new one, then search again in case one got inserted while this thread
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) * didn't hold the lock.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) * Return values:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) * %RC_DOIT: Process the request normally
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) * %RC_REPLY: Reply from cache
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) * %RC_DROPIT: Do not process the request further
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) int nfsd_cache_lookup(struct svc_rqst *rqstp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) struct svc_cacherep *rp, *found;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) __be32 xid = rqstp->rq_xid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) __wsum csum;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) u32 hash = nfsd_cache_hash(xid, nn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) struct nfsd_drc_bucket *b = &nn->drc_hashtbl[hash];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) int type = rqstp->rq_cachetype;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) int rtn = RC_DOIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) rqstp->rq_cacherep = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) if (type == RC_NOCACHE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) nfsdstats.rcnocache++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) csum = nfsd_cache_csum(rqstp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) * Since the common case is a cache miss followed by an insert,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) * preallocate an entry.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) rp = nfsd_reply_cache_alloc(rqstp, csum, nn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) if (!rp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) spin_lock(&b->cache_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) found = nfsd_cache_insert(b, rp, nn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) if (found != rp) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) nfsd_reply_cache_free_locked(NULL, rp, nn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) rp = found;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) goto found_entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) nfsdstats.rcmisses++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) rqstp->rq_cacherep = rp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) rp->c_state = RC_INPROG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) atomic_inc(&nn->num_drc_entries);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) nn->drc_mem_usage += sizeof(*rp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) /* go ahead and prune the cache */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) prune_bucket(b, nn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) out_unlock:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) spin_unlock(&b->cache_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) return rtn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) found_entry:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) /* We found a matching entry which is either in progress or done. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) nfsdstats.rchits++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) rtn = RC_DROPIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) /* Request being processed */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) if (rp->c_state == RC_INPROG)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) goto out_trace;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) /* From the hall of fame of impractical attacks:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) * Is this a user who tries to snoop on the cache? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) rtn = RC_DOIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) if (!test_bit(RQ_SECURE, &rqstp->rq_flags) && rp->c_secure)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) goto out_trace;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) /* Compose RPC reply header */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) switch (rp->c_type) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) case RC_NOCACHE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) case RC_REPLSTAT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) svc_putu32(&rqstp->rq_res.head[0], rp->c_replstat);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) rtn = RC_REPLY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) case RC_REPLBUFF:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) if (!nfsd_cache_append(rqstp, &rp->c_replvec))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) goto out_unlock; /* should not happen */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) rtn = RC_REPLY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) WARN_ONCE(1, "nfsd: bad repcache type %d\n", rp->c_type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) out_trace:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) trace_nfsd_drc_found(nn, rqstp, rtn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) goto out_unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) * nfsd_cache_update - Update an entry in the duplicate reply cache.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) * @rqstp: svc_rqst with a finished Reply
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) * @cachetype: which cache to update
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) * @statp: Reply's status code
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) * This is called from nfsd_dispatch when the procedure has been
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) * executed and the complete reply is in rqstp->rq_res.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) * We're copying around data here rather than swapping buffers because
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) * the toplevel loop requires max-sized buffers, which would be a waste
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) * of memory for a cache with a max reply size of 100 bytes (diropokres).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) * If we should start to use different types of cache entries tailored
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) * specifically for attrstat and fh's, we may save even more space.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) * Also note that a cachetype of RC_NOCACHE can legally be passed when
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) * nfsd failed to encode a reply that otherwise would have been cached.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) * In this case, nfsd_cache_update is called with statp == NULL.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) void nfsd_cache_update(struct svc_rqst *rqstp, int cachetype, __be32 *statp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) struct svc_cacherep *rp = rqstp->rq_cacherep;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) struct kvec *resv = &rqstp->rq_res.head[0], *cachv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) u32 hash;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) struct nfsd_drc_bucket *b;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) int len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) size_t bufsize = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) if (!rp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) hash = nfsd_cache_hash(rp->c_key.k_xid, nn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) b = &nn->drc_hashtbl[hash];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) len = resv->iov_len - ((char*)statp - (char*)resv->iov_base);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) len >>= 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) /* Don't cache excessive amounts of data and XDR failures */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) if (!statp || len > (256 >> 2)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) nfsd_reply_cache_free(b, rp, nn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) switch (cachetype) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) case RC_REPLSTAT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) if (len != 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) printk("nfsd: RC_REPLSTAT/reply len %d!\n",len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) rp->c_replstat = *statp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) case RC_REPLBUFF:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) cachv = &rp->c_replvec;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) bufsize = len << 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) cachv->iov_base = kmalloc(bufsize, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) if (!cachv->iov_base) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) nfsd_reply_cache_free(b, rp, nn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) cachv->iov_len = bufsize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) memcpy(cachv->iov_base, statp, bufsize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) case RC_NOCACHE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) nfsd_reply_cache_free(b, rp, nn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) spin_lock(&b->cache_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) nn->drc_mem_usage += bufsize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) lru_put_end(b, rp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) rp->c_secure = test_bit(RQ_SECURE, &rqstp->rq_flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) rp->c_type = cachetype;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) rp->c_state = RC_DONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) spin_unlock(&b->cache_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) * Copy cached reply to current reply buffer. Should always fit.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) * FIXME as reply is in a page, we should just attach the page, and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) * keep a refcount....
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) static int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) nfsd_cache_append(struct svc_rqst *rqstp, struct kvec *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) struct kvec *vec = &rqstp->rq_res.head[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) if (vec->iov_len + data->iov_len > PAGE_SIZE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) printk(KERN_WARNING "nfsd: cached reply too large (%zd).\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) data->iov_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) memcpy((char*)vec->iov_base + vec->iov_len, data->iov_base, data->iov_len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) vec->iov_len += data->iov_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) * Note that fields may be added, removed or reordered in the future. Programs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) * scraping this file for info should test the labels to ensure they're
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) * getting the correct field.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) static int nfsd_reply_cache_stats_show(struct seq_file *m, void *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) struct nfsd_net *nn = m->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) seq_printf(m, "max entries: %u\n", nn->max_drc_entries);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) seq_printf(m, "num entries: %u\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) atomic_read(&nn->num_drc_entries));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) seq_printf(m, "hash buckets: %u\n", 1 << nn->maskbits);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) seq_printf(m, "mem usage: %u\n", nn->drc_mem_usage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) seq_printf(m, "cache hits: %u\n", nfsdstats.rchits);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) seq_printf(m, "cache misses: %u\n", nfsdstats.rcmisses);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) seq_printf(m, "not cached: %u\n", nfsdstats.rcnocache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) seq_printf(m, "payload misses: %u\n", nn->payload_misses);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) seq_printf(m, "longest chain len: %u\n", nn->longest_chain);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) seq_printf(m, "cachesize at longest: %u\n", nn->longest_chain_cachesize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) int nfsd_reply_cache_stats_open(struct inode *inode, struct file *file)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) struct nfsd_net *nn = net_generic(file_inode(file)->i_sb->s_fs_info,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) nfsd_net_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) return single_open(file, nfsd_reply_cache_stats_show, nn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) }
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags