^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0+
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) //
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) // Scalability test comparing RCU vs other mechanisms
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) // for acquiring references on objects.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) //
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) // Copyright (C) Google, 2020.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) //
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) // Author: Joel Fernandes <joel@joelfernandes.org>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #define pr_fmt(fmt) fmt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/atomic.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/bitops.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/completion.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/cpu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/delay.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/err.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <linux/init.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <linux/interrupt.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <linux/kthread.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include <linux/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #include <linux/moduleparam.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #include <linux/notifier.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) #include <linux/percpu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) #include <linux/rcupdate.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) #include <linux/rcupdate_trace.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) #include <linux/reboot.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) #include <linux/sched.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) #include <linux/spinlock.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) #include <linux/smp.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) #include <linux/stat.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) #include <linux/srcu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) #include <linux/torture.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) #include <linux/types.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) #include "rcu.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) #define SCALE_FLAG "-ref-scale: "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) #define SCALEOUT(s, x...) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) pr_alert("%s" SCALE_FLAG s, scale_type, ## x)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) #define VERBOSE_SCALEOUT(s, x...) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) do { if (verbose) pr_alert("%s" SCALE_FLAG s, scale_type, ## x); } while (0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) #define VERBOSE_SCALEOUT_ERRSTRING(s, x...) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) do { if (verbose) pr_alert("%s" SCALE_FLAG "!!! " s, scale_type, ## x); } while (0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) MODULE_LICENSE("GPL");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) MODULE_AUTHOR("Joel Fernandes (Google) <joel@joelfernandes.org>");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) static char *scale_type = "rcu";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) module_param(scale_type, charp, 0444);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) MODULE_PARM_DESC(scale_type, "Type of test (rcu, srcu, refcnt, rwsem, rwlock.");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) torture_param(int, verbose, 0, "Enable verbose debugging printk()s");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) // Wait until there are multiple CPUs before starting test.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) torture_param(int, holdoff, IS_BUILTIN(CONFIG_RCU_REF_SCALE_TEST) ? 10 : 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) "Holdoff time before test start (s)");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) // Number of loops per experiment, all readers execute operations concurrently.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) torture_param(long, loops, 10000, "Number of loops per experiment.");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) // Number of readers, with -1 defaulting to about 75% of the CPUs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) torture_param(int, nreaders, -1, "Number of readers, -1 for 75% of CPUs.");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) // Number of runs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) torture_param(int, nruns, 30, "Number of experiments to run.");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) // Reader delay in nanoseconds, 0 for no delay.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) torture_param(int, readdelay, 0, "Read-side delay in nanoseconds.");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) #ifdef MODULE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) # define REFSCALE_SHUTDOWN 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) # define REFSCALE_SHUTDOWN 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) torture_param(bool, shutdown, REFSCALE_SHUTDOWN,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) "Shutdown at end of scalability tests.");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) struct reader_task {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) struct task_struct *task;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) int start_reader;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) wait_queue_head_t wq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) u64 last_duration_ns;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) static struct task_struct *shutdown_task;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) static wait_queue_head_t shutdown_wq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) static struct task_struct *main_task;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) static wait_queue_head_t main_wq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) static int shutdown_start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) static struct reader_task *reader_tasks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) // Number of readers that are part of the current experiment.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) static atomic_t nreaders_exp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) // Use to wait for all threads to start.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) static atomic_t n_init;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) static atomic_t n_started;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) static atomic_t n_warmedup;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) static atomic_t n_cooleddown;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) // Track which experiment is currently running.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) static int exp_idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) // Operations vector for selecting different types of tests.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) struct ref_scale_ops {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) void (*init)(void);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) void (*cleanup)(void);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) void (*readsection)(const int nloops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) void (*delaysection)(const int nloops, const int udl, const int ndl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) const char *name;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) static struct ref_scale_ops *cur_ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) static void un_delay(const int udl, const int ndl)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) if (udl)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) udelay(udl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) if (ndl)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) ndelay(ndl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) static void ref_rcu_read_section(const int nloops)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) for (i = nloops; i >= 0; i--) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) static void ref_rcu_delay_section(const int nloops, const int udl, const int ndl)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) for (i = nloops; i >= 0; i--) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) un_delay(udl, ndl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) static void rcu_sync_scale_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) static struct ref_scale_ops rcu_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) .init = rcu_sync_scale_init,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) .readsection = ref_rcu_read_section,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) .delaysection = ref_rcu_delay_section,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) .name = "rcu"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) // Definitions for SRCU ref scale testing.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) DEFINE_STATIC_SRCU(srcu_refctl_scale);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) static struct srcu_struct *srcu_ctlp = &srcu_refctl_scale;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) static void srcu_ref_scale_read_section(const int nloops)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) int idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) for (i = nloops; i >= 0; i--) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) idx = srcu_read_lock(srcu_ctlp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) srcu_read_unlock(srcu_ctlp, idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) static void srcu_ref_scale_delay_section(const int nloops, const int udl, const int ndl)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) int idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) for (i = nloops; i >= 0; i--) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) idx = srcu_read_lock(srcu_ctlp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) un_delay(udl, ndl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) srcu_read_unlock(srcu_ctlp, idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) static struct ref_scale_ops srcu_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) .init = rcu_sync_scale_init,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) .readsection = srcu_ref_scale_read_section,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) .delaysection = srcu_ref_scale_delay_section,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) .name = "srcu"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) // Definitions for RCU Tasks ref scale testing: Empty read markers.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) // These definitions also work for RCU Rude readers.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) static void rcu_tasks_ref_scale_read_section(const int nloops)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) for (i = nloops; i >= 0; i--)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) static void rcu_tasks_ref_scale_delay_section(const int nloops, const int udl, const int ndl)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) for (i = nloops; i >= 0; i--)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) un_delay(udl, ndl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) static struct ref_scale_ops rcu_tasks_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) .init = rcu_sync_scale_init,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) .readsection = rcu_tasks_ref_scale_read_section,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) .delaysection = rcu_tasks_ref_scale_delay_section,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) .name = "rcu-tasks"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) // Definitions for RCU Tasks Trace ref scale testing.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) static void rcu_trace_ref_scale_read_section(const int nloops)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) for (i = nloops; i >= 0; i--) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) rcu_read_lock_trace();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) rcu_read_unlock_trace();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) static void rcu_trace_ref_scale_delay_section(const int nloops, const int udl, const int ndl)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) for (i = nloops; i >= 0; i--) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) rcu_read_lock_trace();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) un_delay(udl, ndl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) rcu_read_unlock_trace();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) static struct ref_scale_ops rcu_trace_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) .init = rcu_sync_scale_init,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) .readsection = rcu_trace_ref_scale_read_section,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) .delaysection = rcu_trace_ref_scale_delay_section,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) .name = "rcu-trace"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) // Definitions for reference count
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) static atomic_t refcnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) static void ref_refcnt_section(const int nloops)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) for (i = nloops; i >= 0; i--) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) atomic_inc(&refcnt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) atomic_dec(&refcnt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) static void ref_refcnt_delay_section(const int nloops, const int udl, const int ndl)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) for (i = nloops; i >= 0; i--) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) atomic_inc(&refcnt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) un_delay(udl, ndl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) atomic_dec(&refcnt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) static struct ref_scale_ops refcnt_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) .init = rcu_sync_scale_init,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) .readsection = ref_refcnt_section,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) .delaysection = ref_refcnt_delay_section,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) .name = "refcnt"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) // Definitions for rwlock
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) static rwlock_t test_rwlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) static void ref_rwlock_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) rwlock_init(&test_rwlock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) static void ref_rwlock_section(const int nloops)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) for (i = nloops; i >= 0; i--) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) read_lock(&test_rwlock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) read_unlock(&test_rwlock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) static void ref_rwlock_delay_section(const int nloops, const int udl, const int ndl)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) for (i = nloops; i >= 0; i--) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) read_lock(&test_rwlock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) un_delay(udl, ndl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) read_unlock(&test_rwlock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) static struct ref_scale_ops rwlock_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) .init = ref_rwlock_init,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) .readsection = ref_rwlock_section,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) .delaysection = ref_rwlock_delay_section,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) .name = "rwlock"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) // Definitions for rwsem
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) static struct rw_semaphore test_rwsem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) static void ref_rwsem_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) init_rwsem(&test_rwsem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) static void ref_rwsem_section(const int nloops)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) for (i = nloops; i >= 0; i--) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) down_read(&test_rwsem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) up_read(&test_rwsem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) static void ref_rwsem_delay_section(const int nloops, const int udl, const int ndl)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) for (i = nloops; i >= 0; i--) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) down_read(&test_rwsem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) un_delay(udl, ndl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) up_read(&test_rwsem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) static struct ref_scale_ops rwsem_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) .init = ref_rwsem_init,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) .readsection = ref_rwsem_section,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) .delaysection = ref_rwsem_delay_section,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) .name = "rwsem"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) static void rcu_scale_one_reader(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) if (readdelay <= 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) cur_ops->readsection(loops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) cur_ops->delaysection(loops, readdelay / 1000, readdelay % 1000);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) // Reader kthread. Repeatedly does empty RCU read-side
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) // critical section, minimizing update-side interference.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) static int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) ref_scale_reader(void *arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) long me = (long)arg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) struct reader_task *rt = &(reader_tasks[me]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) u64 start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) s64 duration;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) VERBOSE_SCALEOUT("ref_scale_reader %ld: task started", me);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) set_cpus_allowed_ptr(current, cpumask_of(me % nr_cpu_ids));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) set_user_nice(current, MAX_NICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) atomic_inc(&n_init);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) if (holdoff)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) schedule_timeout_interruptible(holdoff * HZ);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) repeat:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) VERBOSE_SCALEOUT("ref_scale_reader %ld: waiting to start next experiment on cpu %d", me, smp_processor_id());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) // Wait for signal that this reader can start.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) wait_event(rt->wq, (atomic_read(&nreaders_exp) && smp_load_acquire(&rt->start_reader)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) torture_must_stop());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) if (torture_must_stop())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) goto end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) // Make sure that the CPU is affinitized appropriately during testing.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) WARN_ON_ONCE(smp_processor_id() != me);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) WRITE_ONCE(rt->start_reader, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) if (!atomic_dec_return(&n_started))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) while (atomic_read_acquire(&n_started))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) cpu_relax();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) VERBOSE_SCALEOUT("ref_scale_reader %ld: experiment %d started", me, exp_idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) // To reduce noise, do an initial cache-warming invocation, check
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) // in, and then keep warming until everyone has checked in.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) rcu_scale_one_reader();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) if (!atomic_dec_return(&n_warmedup))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) while (atomic_read_acquire(&n_warmedup))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) rcu_scale_one_reader();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) // Also keep interrupts disabled. This also has the effect
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) // of preventing entries into slow path for rcu_read_unlock().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) local_irq_save(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) start = ktime_get_mono_fast_ns();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) rcu_scale_one_reader();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) duration = ktime_get_mono_fast_ns() - start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) local_irq_restore(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) rt->last_duration_ns = WARN_ON_ONCE(duration < 0) ? 0 : duration;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) // To reduce runtime-skew noise, do maintain-load invocations until
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) // everyone is done.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) if (!atomic_dec_return(&n_cooleddown))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) while (atomic_read_acquire(&n_cooleddown))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) rcu_scale_one_reader();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) if (atomic_dec_and_test(&nreaders_exp))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) wake_up(&main_wq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) VERBOSE_SCALEOUT("ref_scale_reader %ld: experiment %d ended, (readers remaining=%d)",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) me, exp_idx, atomic_read(&nreaders_exp));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) if (!torture_must_stop())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) goto repeat;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) end:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) torture_kthread_stopping("ref_scale_reader");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) static void reset_readers(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) struct reader_task *rt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) for (i = 0; i < nreaders; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) rt = &(reader_tasks[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) rt->last_duration_ns = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) // Print the results of each reader and return the sum of all their durations.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) static u64 process_durations(int n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) struct reader_task *rt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) char buf1[64];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) char *buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) u64 sum = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) buf = kmalloc(128 + nreaders * 32, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) if (!buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) buf[0] = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) sprintf(buf, "Experiment #%d (Format: <THREAD-NUM>:<Total loop time in ns>)",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) exp_idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) for (i = 0; i < n && !torture_must_stop(); i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) rt = &(reader_tasks[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) sprintf(buf1, "%d: %llu\t", i, rt->last_duration_ns);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) if (i % 5 == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) strcat(buf, "\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) strcat(buf, buf1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) sum += rt->last_duration_ns;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) strcat(buf, "\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) SCALEOUT("%s\n", buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) kfree(buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) return sum;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) // The main_func is the main orchestrator, it performs a bunch of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) // experiments. For every experiment, it orders all the readers
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) // involved to start and waits for them to finish the experiment. It
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) // then reads their timestamps and starts the next experiment. Each
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) // experiment progresses from 1 concurrent reader to N of them at which
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) // point all the timestamps are printed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) static int main_func(void *arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) bool errexit = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) int exp, r;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) char buf1[64];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) char *buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) u64 *result_avg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) set_cpus_allowed_ptr(current, cpumask_of(nreaders % nr_cpu_ids));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) set_user_nice(current, MAX_NICE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) VERBOSE_SCALEOUT("main_func task started");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) result_avg = kzalloc(nruns * sizeof(*result_avg), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) buf = kzalloc(64 + nruns * 32, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) if (!result_avg || !buf) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) VERBOSE_SCALEOUT_ERRSTRING("out of memory");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) errexit = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) if (holdoff)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) schedule_timeout_interruptible(holdoff * HZ);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) // Wait for all threads to start.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) atomic_inc(&n_init);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) while (atomic_read(&n_init) < nreaders + 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) schedule_timeout_uninterruptible(1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) // Start exp readers up per experiment
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) for (exp = 0; exp < nruns && !torture_must_stop(); exp++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) if (errexit)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) if (torture_must_stop())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) goto end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) reset_readers();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) atomic_set(&nreaders_exp, nreaders);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) atomic_set(&n_started, nreaders);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) atomic_set(&n_warmedup, nreaders);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) atomic_set(&n_cooleddown, nreaders);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) exp_idx = exp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) for (r = 0; r < nreaders; r++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) smp_store_release(&reader_tasks[r].start_reader, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) wake_up(&reader_tasks[r].wq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) VERBOSE_SCALEOUT("main_func: experiment started, waiting for %d readers",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) nreaders);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) wait_event(main_wq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) !atomic_read(&nreaders_exp) || torture_must_stop());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) VERBOSE_SCALEOUT("main_func: experiment ended");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) if (torture_must_stop())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) goto end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) result_avg[exp] = div_u64(1000 * process_durations(nreaders), nreaders * loops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) // Print the average of all experiments
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) SCALEOUT("END OF TEST. Calculating average duration per loop (nanoseconds)...\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) if (!errexit) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) buf[0] = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) strcat(buf, "\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) strcat(buf, "Runs\tTime(ns)\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) for (exp = 0; exp < nruns; exp++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) u64 avg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) u32 rem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) if (errexit)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) avg = div_u64_rem(result_avg[exp], 1000, &rem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) sprintf(buf1, "%d\t%llu.%03u\n", exp + 1, avg, rem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) strcat(buf, buf1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) if (!errexit)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) SCALEOUT("%s", buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) // This will shutdown everything including us.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) if (shutdown) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) shutdown_start = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) wake_up(&shutdown_wq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) // Wait for torture to stop us
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) while (!torture_must_stop())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) schedule_timeout_uninterruptible(1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) end:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) torture_kthread_stopping("main_func");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) kfree(result_avg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) kfree(buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) ref_scale_print_module_parms(struct ref_scale_ops *cur_ops, const char *tag)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) pr_alert("%s" SCALE_FLAG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) "--- %s: verbose=%d shutdown=%d holdoff=%d loops=%ld nreaders=%d nruns=%d readdelay=%d\n", scale_type, tag,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) verbose, shutdown, holdoff, loops, nreaders, nruns, readdelay);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) ref_scale_cleanup(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) if (torture_cleanup_begin())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) if (!cur_ops) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) torture_cleanup_end();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) if (reader_tasks) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) for (i = 0; i < nreaders; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) torture_stop_kthread("ref_scale_reader",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) reader_tasks[i].task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) kfree(reader_tasks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) torture_stop_kthread("main_task", main_task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) kfree(main_task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) // Do scale-type-specific cleanup operations.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) if (cur_ops->cleanup != NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) cur_ops->cleanup();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) torture_cleanup_end();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) // Shutdown kthread. Just waits to be awakened, then shuts down system.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) static int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) ref_scale_shutdown(void *arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) wait_event(shutdown_wq, shutdown_start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) smp_mb(); // Wake before output.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) ref_scale_cleanup();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) kernel_power_off();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) static int __init
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) ref_scale_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) long i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) int firsterr = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) static struct ref_scale_ops *scale_ops[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) &rcu_ops, &srcu_ops, &rcu_trace_ops, &rcu_tasks_ops,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) &refcnt_ops, &rwlock_ops, &rwsem_ops,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) if (!torture_init_begin(scale_type, verbose))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) return -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) for (i = 0; i < ARRAY_SIZE(scale_ops); i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) cur_ops = scale_ops[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) if (strcmp(scale_type, cur_ops->name) == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) if (i == ARRAY_SIZE(scale_ops)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) pr_alert("rcu-scale: invalid scale type: \"%s\"\n", scale_type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) pr_alert("rcu-scale types:");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) for (i = 0; i < ARRAY_SIZE(scale_ops); i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) pr_cont(" %s", scale_ops[i]->name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) pr_cont("\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) WARN_ON(!IS_MODULE(CONFIG_RCU_REF_SCALE_TEST));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) firsterr = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) cur_ops = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) goto unwind;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) if (cur_ops->init)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) cur_ops->init();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) ref_scale_print_module_parms(cur_ops, "Start of test");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) // Shutdown task
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) if (shutdown) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) init_waitqueue_head(&shutdown_wq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) firsterr = torture_create_kthread(ref_scale_shutdown, NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) shutdown_task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) if (firsterr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) goto unwind;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) schedule_timeout_uninterruptible(1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) // Reader tasks (default to ~75% of online CPUs).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) if (nreaders < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) nreaders = (num_online_cpus() >> 1) + (num_online_cpus() >> 2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) reader_tasks = kcalloc(nreaders, sizeof(reader_tasks[0]),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) if (!reader_tasks) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) VERBOSE_SCALEOUT_ERRSTRING("out of memory");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) firsterr = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) goto unwind;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) VERBOSE_SCALEOUT("Starting %d reader threads\n", nreaders);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) for (i = 0; i < nreaders; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) firsterr = torture_create_kthread(ref_scale_reader, (void *)i,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) reader_tasks[i].task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) if (firsterr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) goto unwind;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) init_waitqueue_head(&(reader_tasks[i].wq));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) // Main Task
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) init_waitqueue_head(&main_wq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) firsterr = torture_create_kthread(main_func, NULL, main_task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) if (firsterr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) goto unwind;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) torture_init_end();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) unwind:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) torture_init_end();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) ref_scale_cleanup();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) return firsterr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) module_init(ref_scale_init);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) module_exit(ref_scale_cleanup);
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags