14e88ec4aSPaul E. McKenney // SPDX-License-Identifier: GPL-2.0+
24e88ec4aSPaul E. McKenney /*
34e88ec4aSPaul E. McKenney * Read-Copy Update module-based scalability-test facility
44e88ec4aSPaul E. McKenney *
54e88ec4aSPaul E. McKenney * Copyright (C) IBM Corporation, 2015
64e88ec4aSPaul E. McKenney *
74e88ec4aSPaul E. McKenney * Authors: Paul E. McKenney <paulmck@linux.ibm.com>
84e88ec4aSPaul E. McKenney */
94e88ec4aSPaul E. McKenney
104e88ec4aSPaul E. McKenney #define pr_fmt(fmt) fmt
114e88ec4aSPaul E. McKenney
124e88ec4aSPaul E. McKenney #include <linux/types.h>
134e88ec4aSPaul E. McKenney #include <linux/kernel.h>
144e88ec4aSPaul E. McKenney #include <linux/init.h>
154e88ec4aSPaul E. McKenney #include <linux/mm.h>
164e88ec4aSPaul E. McKenney #include <linux/module.h>
174e88ec4aSPaul E. McKenney #include <linux/kthread.h>
184e88ec4aSPaul E. McKenney #include <linux/err.h>
194e88ec4aSPaul E. McKenney #include <linux/spinlock.h>
204e88ec4aSPaul E. McKenney #include <linux/smp.h>
214e88ec4aSPaul E. McKenney #include <linux/rcupdate.h>
224e88ec4aSPaul E. McKenney #include <linux/interrupt.h>
234e88ec4aSPaul E. McKenney #include <linux/sched.h>
244e88ec4aSPaul E. McKenney #include <uapi/linux/sched/types.h>
254e88ec4aSPaul E. McKenney #include <linux/atomic.h>
264e88ec4aSPaul E. McKenney #include <linux/bitops.h>
274e88ec4aSPaul E. McKenney #include <linux/completion.h>
284e88ec4aSPaul E. McKenney #include <linux/moduleparam.h>
294e88ec4aSPaul E. McKenney #include <linux/percpu.h>
304e88ec4aSPaul E. McKenney #include <linux/notifier.h>
314e88ec4aSPaul E. McKenney #include <linux/reboot.h>
324e88ec4aSPaul E. McKenney #include <linux/freezer.h>
334e88ec4aSPaul E. McKenney #include <linux/cpu.h>
344e88ec4aSPaul E. McKenney #include <linux/delay.h>
354e88ec4aSPaul E. McKenney #include <linux/stat.h>
364e88ec4aSPaul E. McKenney #include <linux/srcu.h>
374e88ec4aSPaul E. McKenney #include <linux/slab.h>
384e88ec4aSPaul E. McKenney #include <asm/byteorder.h>
394e88ec4aSPaul E. McKenney #include <linux/torture.h>
404e88ec4aSPaul E. McKenney #include <linux/vmalloc.h>
41899f317eSPaul E. McKenney #include <linux/rcupdate_trace.h>
424e88ec4aSPaul E. McKenney
434e88ec4aSPaul E. McKenney #include "rcu.h"
444e88ec4aSPaul E. McKenney
454e88ec4aSPaul E. McKenney MODULE_LICENSE("GPL");
464e88ec4aSPaul E. McKenney MODULE_AUTHOR("Paul E. McKenney <paulmck@linux.ibm.com>");
474e88ec4aSPaul E. McKenney
484e88ec4aSPaul E. McKenney #define SCALE_FLAG "-scale:"
494e88ec4aSPaul E. McKenney #define SCALEOUT_STRING(s) \
504e88ec4aSPaul E. McKenney pr_alert("%s" SCALE_FLAG " %s\n", scale_type, s)
514e88ec4aSPaul E. McKenney #define VERBOSE_SCALEOUT_STRING(s) \
524e88ec4aSPaul E. McKenney do { if (verbose) pr_alert("%s" SCALE_FLAG " %s\n", scale_type, s); } while (0)
5386e7ed1bSLi Zhijian #define SCALEOUT_ERRSTRING(s) \
5486e7ed1bSLi Zhijian pr_alert("%s" SCALE_FLAG "!!! %s\n", scale_type, s)
554e88ec4aSPaul E. McKenney
564e88ec4aSPaul E. McKenney /*
574e88ec4aSPaul E. McKenney * The intended use cases for the nreaders and nwriters module parameters
584e88ec4aSPaul E. McKenney * are as follows:
594e88ec4aSPaul E. McKenney *
604e88ec4aSPaul E. McKenney * 1. Specify only the nr_cpus kernel boot parameter. This will
614e88ec4aSPaul E. McKenney * set both nreaders and nwriters to the value specified by
624e88ec4aSPaul E. McKenney * nr_cpus for a mixed reader/writer test.
634e88ec4aSPaul E. McKenney *
644e88ec4aSPaul E. McKenney * 2. Specify the nr_cpus kernel boot parameter, but set
654e88ec4aSPaul E. McKenney * rcuscale.nreaders to zero. This will set nwriters to the
664e88ec4aSPaul E. McKenney * value specified by nr_cpus for an update-only test.
674e88ec4aSPaul E. McKenney *
684e88ec4aSPaul E. McKenney * 3. Specify the nr_cpus kernel boot parameter, but set
694e88ec4aSPaul E. McKenney * rcuscale.nwriters to zero. This will set nreaders to the
704e88ec4aSPaul E. McKenney * value specified by nr_cpus for a read-only test.
714e88ec4aSPaul E. McKenney *
724e88ec4aSPaul E. McKenney * Various other use cases may of course be specified.
734e88ec4aSPaul E. McKenney *
744e88ec4aSPaul E. McKenney * Note that this test's readers are intended only as a test load for
754e88ec4aSPaul E. McKenney * the writers. The reader scalability statistics will be overly
764e88ec4aSPaul E. McKenney * pessimistic due to the per-critical-section interrupt disabling,
774e88ec4aSPaul E. McKenney * test-end checks, and the pair of calls through pointers.
784e88ec4aSPaul E. McKenney */
794e88ec4aSPaul E. McKenney
804e88ec4aSPaul E. McKenney #ifdef MODULE
814e88ec4aSPaul E. McKenney # define RCUSCALE_SHUTDOWN 0
824e88ec4aSPaul E. McKenney #else
834e88ec4aSPaul E. McKenney # define RCUSCALE_SHUTDOWN 1
844e88ec4aSPaul E. McKenney #endif
854e88ec4aSPaul E. McKenney
864e88ec4aSPaul E. McKenney torture_param(bool, gp_async, false, "Use asynchronous GP wait primitives");
87ee7516a1SPaul E. McKenney torture_param(int, gp_async_max, 1000, "Max # outstanding waits per writer");
884e88ec4aSPaul E. McKenney torture_param(bool, gp_exp, false, "Use expedited GP wait primitives");
894e88ec4aSPaul E. McKenney torture_param(int, holdoff, 10, "Holdoff time before test start (s)");
907221f493SPaul E. McKenney torture_param(int, minruntime, 0, "Minimum run time (s)");
914e88ec4aSPaul E. McKenney torture_param(int, nreaders, -1, "Number of RCU reader threads");
924e88ec4aSPaul E. McKenney torture_param(int, nwriters, -1, "Number of RCU updater threads");
934e88ec4aSPaul E. McKenney torture_param(bool, shutdown, RCUSCALE_SHUTDOWN,
944e88ec4aSPaul E. McKenney "Shutdown at end of scalability tests.");
954e88ec4aSPaul E. McKenney torture_param(int, verbose, 1, "Enable verbose debugging printk()s");
964e88ec4aSPaul E. McKenney torture_param(int, writer_holdoff, 0, "Holdoff (us) between GPs, zero to disable");
972226f3dcSPaul E. McKenney torture_param(int, writer_holdoff_jiffies, 0, "Holdoff (jiffies) between GPs, zero to disable");
984e88ec4aSPaul E. McKenney torture_param(int, kfree_rcu_test, 0, "Do we run a kfree_rcu() scale test?");
994e88ec4aSPaul E. McKenney torture_param(int, kfree_mult, 1, "Multiple of kfree_obj size to allocate.");
100084e04ffSJoel Fernandes (Google) torture_param(int, kfree_by_call_rcu, 0, "Use call_rcu() to emulate kfree_rcu()?");
1014e88ec4aSPaul E. McKenney
1024e88ec4aSPaul E. McKenney static char *scale_type = "rcu";
1034e88ec4aSPaul E. McKenney module_param(scale_type, charp, 0444);
1044e88ec4aSPaul E. McKenney MODULE_PARM_DESC(scale_type, "Type of RCU to scalability-test (rcu, srcu, ...)");
1054e88ec4aSPaul E. McKenney
1064e88ec4aSPaul E. McKenney static int nrealreaders;
1074e88ec4aSPaul E. McKenney static int nrealwriters;
1084e88ec4aSPaul E. McKenney static struct task_struct **writer_tasks;
1094e88ec4aSPaul E. McKenney static struct task_struct **reader_tasks;
1104e88ec4aSPaul E. McKenney static struct task_struct *shutdown_task;
1114e88ec4aSPaul E. McKenney
1124e88ec4aSPaul E. McKenney static u64 **writer_durations;
1134e88ec4aSPaul E. McKenney static int *writer_n_durations;
1144e88ec4aSPaul E. McKenney static atomic_t n_rcu_scale_reader_started;
1154e88ec4aSPaul E. McKenney static atomic_t n_rcu_scale_writer_started;
1164e88ec4aSPaul E. McKenney static atomic_t n_rcu_scale_writer_finished;
1174e88ec4aSPaul E. McKenney static wait_queue_head_t shutdown_wq;
1184e88ec4aSPaul E. McKenney static u64 t_rcu_scale_writer_started;
1194e88ec4aSPaul E. McKenney static u64 t_rcu_scale_writer_finished;
1204e88ec4aSPaul E. McKenney static unsigned long b_rcu_gp_test_started;
1214e88ec4aSPaul E. McKenney static unsigned long b_rcu_gp_test_finished;
1224e88ec4aSPaul E. McKenney static DEFINE_PER_CPU(atomic_t, n_async_inflight);
1234e88ec4aSPaul E. McKenney
1244e88ec4aSPaul E. McKenney #define MAX_MEAS 10000
1254e88ec4aSPaul E. McKenney #define MIN_MEAS 100
1264e88ec4aSPaul E. McKenney
1274e88ec4aSPaul E. McKenney /*
1284e88ec4aSPaul E. McKenney * Operations vector for selecting different types of tests.
1294e88ec4aSPaul E. McKenney */
1304e88ec4aSPaul E. McKenney
1314e88ec4aSPaul E. McKenney struct rcu_scale_ops {
1324e88ec4aSPaul E. McKenney int ptype;
1334e88ec4aSPaul E. McKenney void (*init)(void);
1344e88ec4aSPaul E. McKenney void (*cleanup)(void);
1354e88ec4aSPaul E. McKenney int (*readlock)(void);
1364e88ec4aSPaul E. McKenney void (*readunlock)(int idx);
1374e88ec4aSPaul E. McKenney unsigned long (*get_gp_seq)(void);
1384e88ec4aSPaul E. McKenney unsigned long (*gp_diff)(unsigned long new, unsigned long old);
1394e88ec4aSPaul E. McKenney unsigned long (*exp_completed)(void);
1404e88ec4aSPaul E. McKenney void (*async)(struct rcu_head *head, rcu_callback_t func);
1414e88ec4aSPaul E. McKenney void (*gp_barrier)(void);
1424e88ec4aSPaul E. McKenney void (*sync)(void);
1434e88ec4aSPaul E. McKenney void (*exp_sync)(void);
1445f8e3202SPaul E. McKenney struct task_struct *(*rso_gp_kthread)(void);
1454e88ec4aSPaul E. McKenney const char *name;
1464e88ec4aSPaul E. McKenney };
1474e88ec4aSPaul E. McKenney
1484e88ec4aSPaul E. McKenney static struct rcu_scale_ops *cur_ops;
1494e88ec4aSPaul E. McKenney
1504e88ec4aSPaul E. McKenney /*
1514e88ec4aSPaul E. McKenney * Definitions for rcu scalability testing.
1524e88ec4aSPaul E. McKenney */
1534e88ec4aSPaul E. McKenney
rcu_scale_read_lock(void)1544e88ec4aSPaul E. McKenney static int rcu_scale_read_lock(void) __acquires(RCU)
1554e88ec4aSPaul E. McKenney {
1564e88ec4aSPaul E. McKenney rcu_read_lock();
1574e88ec4aSPaul E. McKenney return 0;
1584e88ec4aSPaul E. McKenney }
1594e88ec4aSPaul E. McKenney
rcu_scale_read_unlock(int idx)1604e88ec4aSPaul E. McKenney static void rcu_scale_read_unlock(int idx) __releases(RCU)
1614e88ec4aSPaul E. McKenney {
1624e88ec4aSPaul E. McKenney rcu_read_unlock();
1634e88ec4aSPaul E. McKenney }
1644e88ec4aSPaul E. McKenney
rcu_no_completed(void)1654e88ec4aSPaul E. McKenney static unsigned long __maybe_unused rcu_no_completed(void)
1664e88ec4aSPaul E. McKenney {
1674e88ec4aSPaul E. McKenney return 0;
1684e88ec4aSPaul E. McKenney }
1694e88ec4aSPaul E. McKenney
rcu_sync_scale_init(void)1704e88ec4aSPaul E. McKenney static void rcu_sync_scale_init(void)
1714e88ec4aSPaul E. McKenney {
1724e88ec4aSPaul E. McKenney }
1734e88ec4aSPaul E. McKenney
1744e88ec4aSPaul E. McKenney static struct rcu_scale_ops rcu_ops = {
1754e88ec4aSPaul E. McKenney .ptype = RCU_FLAVOR,
1764e88ec4aSPaul E. McKenney .init = rcu_sync_scale_init,
1774e88ec4aSPaul E. McKenney .readlock = rcu_scale_read_lock,
1784e88ec4aSPaul E. McKenney .readunlock = rcu_scale_read_unlock,
1794e88ec4aSPaul E. McKenney .get_gp_seq = rcu_get_gp_seq,
1804e88ec4aSPaul E. McKenney .gp_diff = rcu_seq_diff,
1814e88ec4aSPaul E. McKenney .exp_completed = rcu_exp_batches_completed,
182723df859SJoel Fernandes (Google) .async = call_rcu_hurry,
1834e88ec4aSPaul E. McKenney .gp_barrier = rcu_barrier,
1844e88ec4aSPaul E. McKenney .sync = synchronize_rcu,
1854e88ec4aSPaul E. McKenney .exp_sync = synchronize_rcu_expedited,
1864e88ec4aSPaul E. McKenney .name = "rcu"
1874e88ec4aSPaul E. McKenney };
1884e88ec4aSPaul E. McKenney
1894e88ec4aSPaul E. McKenney /*
1904e88ec4aSPaul E. McKenney * Definitions for srcu scalability testing.
1914e88ec4aSPaul E. McKenney */
1924e88ec4aSPaul E. McKenney
1934e88ec4aSPaul E. McKenney DEFINE_STATIC_SRCU(srcu_ctl_scale);
1944e88ec4aSPaul E. McKenney static struct srcu_struct *srcu_ctlp = &srcu_ctl_scale;
1954e88ec4aSPaul E. McKenney
srcu_scale_read_lock(void)1964e88ec4aSPaul E. McKenney static int srcu_scale_read_lock(void) __acquires(srcu_ctlp)
1974e88ec4aSPaul E. McKenney {
1984e88ec4aSPaul E. McKenney return srcu_read_lock(srcu_ctlp);
1994e88ec4aSPaul E. McKenney }
2004e88ec4aSPaul E. McKenney
srcu_scale_read_unlock(int idx)2014e88ec4aSPaul E. McKenney static void srcu_scale_read_unlock(int idx) __releases(srcu_ctlp)
2024e88ec4aSPaul E. McKenney {
2034e88ec4aSPaul E. McKenney srcu_read_unlock(srcu_ctlp, idx);
2044e88ec4aSPaul E. McKenney }
2054e88ec4aSPaul E. McKenney
srcu_scale_completed(void)2064e88ec4aSPaul E. McKenney static unsigned long srcu_scale_completed(void)
2074e88ec4aSPaul E. McKenney {
2084e88ec4aSPaul E. McKenney return srcu_batches_completed(srcu_ctlp);
2094e88ec4aSPaul E. McKenney }
2104e88ec4aSPaul E. McKenney
srcu_call_rcu(struct rcu_head * head,rcu_callback_t func)2114e88ec4aSPaul E. McKenney static void srcu_call_rcu(struct rcu_head *head, rcu_callback_t func)
2124e88ec4aSPaul E. McKenney {
2134e88ec4aSPaul E. McKenney call_srcu(srcu_ctlp, head, func);
2144e88ec4aSPaul E. McKenney }
2154e88ec4aSPaul E. McKenney
srcu_rcu_barrier(void)2164e88ec4aSPaul E. McKenney static void srcu_rcu_barrier(void)
2174e88ec4aSPaul E. McKenney {
2184e88ec4aSPaul E. McKenney srcu_barrier(srcu_ctlp);
2194e88ec4aSPaul E. McKenney }
2204e88ec4aSPaul E. McKenney
srcu_scale_synchronize(void)2214e88ec4aSPaul E. McKenney static void srcu_scale_synchronize(void)
2224e88ec4aSPaul E. McKenney {
2234e88ec4aSPaul E. McKenney synchronize_srcu(srcu_ctlp);
2244e88ec4aSPaul E. McKenney }
2254e88ec4aSPaul E. McKenney
srcu_scale_synchronize_expedited(void)2264e88ec4aSPaul E. McKenney static void srcu_scale_synchronize_expedited(void)
2274e88ec4aSPaul E. McKenney {
2284e88ec4aSPaul E. McKenney synchronize_srcu_expedited(srcu_ctlp);
2294e88ec4aSPaul E. McKenney }
2304e88ec4aSPaul E. McKenney
2314e88ec4aSPaul E. McKenney static struct rcu_scale_ops srcu_ops = {
2324e88ec4aSPaul E. McKenney .ptype = SRCU_FLAVOR,
2334e88ec4aSPaul E. McKenney .init = rcu_sync_scale_init,
2344e88ec4aSPaul E. McKenney .readlock = srcu_scale_read_lock,
2354e88ec4aSPaul E. McKenney .readunlock = srcu_scale_read_unlock,
2364e88ec4aSPaul E. McKenney .get_gp_seq = srcu_scale_completed,
2374e88ec4aSPaul E. McKenney .gp_diff = rcu_seq_diff,
2384e88ec4aSPaul E. McKenney .exp_completed = srcu_scale_completed,
2394e88ec4aSPaul E. McKenney .async = srcu_call_rcu,
2404e88ec4aSPaul E. McKenney .gp_barrier = srcu_rcu_barrier,
2414e88ec4aSPaul E. McKenney .sync = srcu_scale_synchronize,
2424e88ec4aSPaul E. McKenney .exp_sync = srcu_scale_synchronize_expedited,
2434e88ec4aSPaul E. McKenney .name = "srcu"
2444e88ec4aSPaul E. McKenney };
2454e88ec4aSPaul E. McKenney
2464e88ec4aSPaul E. McKenney static struct srcu_struct srcud;
2474e88ec4aSPaul E. McKenney
srcu_sync_scale_init(void)2484e88ec4aSPaul E. McKenney static void srcu_sync_scale_init(void)
2494e88ec4aSPaul E. McKenney {
2504e88ec4aSPaul E. McKenney srcu_ctlp = &srcud;
2514e88ec4aSPaul E. McKenney init_srcu_struct(srcu_ctlp);
2524e88ec4aSPaul E. McKenney }
2534e88ec4aSPaul E. McKenney
srcu_sync_scale_cleanup(void)2544e88ec4aSPaul E. McKenney static void srcu_sync_scale_cleanup(void)
2554e88ec4aSPaul E. McKenney {
2564e88ec4aSPaul E. McKenney cleanup_srcu_struct(srcu_ctlp);
2574e88ec4aSPaul E. McKenney }
2584e88ec4aSPaul E. McKenney
2594e88ec4aSPaul E. McKenney static struct rcu_scale_ops srcud_ops = {
2604e88ec4aSPaul E. McKenney .ptype = SRCU_FLAVOR,
2614e88ec4aSPaul E. McKenney .init = srcu_sync_scale_init,
2624e88ec4aSPaul E. McKenney .cleanup = srcu_sync_scale_cleanup,
2634e88ec4aSPaul E. McKenney .readlock = srcu_scale_read_lock,
2644e88ec4aSPaul E. McKenney .readunlock = srcu_scale_read_unlock,
2654e88ec4aSPaul E. McKenney .get_gp_seq = srcu_scale_completed,
2664e88ec4aSPaul E. McKenney .gp_diff = rcu_seq_diff,
2674e88ec4aSPaul E. McKenney .exp_completed = srcu_scale_completed,
2684e88ec4aSPaul E. McKenney .async = srcu_call_rcu,
2694e88ec4aSPaul E. McKenney .gp_barrier = srcu_rcu_barrier,
2704e88ec4aSPaul E. McKenney .sync = srcu_scale_synchronize,
2714e88ec4aSPaul E. McKenney .exp_sync = srcu_scale_synchronize_expedited,
2724e88ec4aSPaul E. McKenney .name = "srcud"
2734e88ec4aSPaul E. McKenney };
2744e88ec4aSPaul E. McKenney
2754df002d9SPaul E. McKenney #ifdef CONFIG_TASKS_RCU
2764df002d9SPaul E. McKenney
2774e88ec4aSPaul E. McKenney /*
2784e88ec4aSPaul E. McKenney * Definitions for RCU-tasks scalability testing.
2794e88ec4aSPaul E. McKenney */
2804e88ec4aSPaul E. McKenney
tasks_scale_read_lock(void)2814e88ec4aSPaul E. McKenney static int tasks_scale_read_lock(void)
2824e88ec4aSPaul E. McKenney {
2834e88ec4aSPaul E. McKenney return 0;
2844e88ec4aSPaul E. McKenney }
2854e88ec4aSPaul E. McKenney
tasks_scale_read_unlock(int idx)2864e88ec4aSPaul E. McKenney static void tasks_scale_read_unlock(int idx)
2874e88ec4aSPaul E. McKenney {
2884e88ec4aSPaul E. McKenney }
2894e88ec4aSPaul E. McKenney
2904e88ec4aSPaul E. McKenney static struct rcu_scale_ops tasks_ops = {
2914e88ec4aSPaul E. McKenney .ptype = RCU_TASKS_FLAVOR,
2924e88ec4aSPaul E. McKenney .init = rcu_sync_scale_init,
2934e88ec4aSPaul E. McKenney .readlock = tasks_scale_read_lock,
2944e88ec4aSPaul E. McKenney .readunlock = tasks_scale_read_unlock,
2954e88ec4aSPaul E. McKenney .get_gp_seq = rcu_no_completed,
2964e88ec4aSPaul E. McKenney .gp_diff = rcu_seq_diff,
2974e88ec4aSPaul E. McKenney .async = call_rcu_tasks,
2984e88ec4aSPaul E. McKenney .gp_barrier = rcu_barrier_tasks,
2994e88ec4aSPaul E. McKenney .sync = synchronize_rcu_tasks,
3004e88ec4aSPaul E. McKenney .exp_sync = synchronize_rcu_tasks,
301271a8467SPaul E. McKenney .rso_gp_kthread = get_rcu_tasks_gp_kthread,
3024e88ec4aSPaul E. McKenney .name = "tasks"
3034e88ec4aSPaul E. McKenney };
3044e88ec4aSPaul E. McKenney
3054df002d9SPaul E. McKenney #define TASKS_OPS &tasks_ops,
3064df002d9SPaul E. McKenney
3074df002d9SPaul E. McKenney #else // #ifdef CONFIG_TASKS_RCU
3084df002d9SPaul E. McKenney
3094df002d9SPaul E. McKenney #define TASKS_OPS
3104df002d9SPaul E. McKenney
3114df002d9SPaul E. McKenney #endif // #else // #ifdef CONFIG_TASKS_RCU
3124df002d9SPaul E. McKenney
313a15ec57cSPaul E. McKenney #ifdef CONFIG_TASKS_RUDE_RCU
314a15ec57cSPaul E. McKenney
315a15ec57cSPaul E. McKenney /*
316a15ec57cSPaul E. McKenney * Definitions for RCU-tasks-rude scalability testing.
317a15ec57cSPaul E. McKenney */
318a15ec57cSPaul E. McKenney
tasks_rude_scale_read_lock(void)319e0a34641SArnd Bergmann static int tasks_rude_scale_read_lock(void)
320e0a34641SArnd Bergmann {
321e0a34641SArnd Bergmann return 0;
322e0a34641SArnd Bergmann }
323e0a34641SArnd Bergmann
tasks_rude_scale_read_unlock(int idx)324e0a34641SArnd Bergmann static void tasks_rude_scale_read_unlock(int idx)
325e0a34641SArnd Bergmann {
326e0a34641SArnd Bergmann }
327e0a34641SArnd Bergmann
328a15ec57cSPaul E. McKenney static struct rcu_scale_ops tasks_rude_ops = {
329a15ec57cSPaul E. McKenney .ptype = RCU_TASKS_RUDE_FLAVOR,
330a15ec57cSPaul E. McKenney .init = rcu_sync_scale_init,
331e0a34641SArnd Bergmann .readlock = tasks_rude_scale_read_lock,
332e0a34641SArnd Bergmann .readunlock = tasks_rude_scale_read_unlock,
333a15ec57cSPaul E. McKenney .get_gp_seq = rcu_no_completed,
334a15ec57cSPaul E. McKenney .gp_diff = rcu_seq_diff,
335a15ec57cSPaul E. McKenney .async = call_rcu_tasks_rude,
336a15ec57cSPaul E. McKenney .gp_barrier = rcu_barrier_tasks_rude,
337a15ec57cSPaul E. McKenney .sync = synchronize_rcu_tasks_rude,
338a15ec57cSPaul E. McKenney .exp_sync = synchronize_rcu_tasks_rude,
339a15ec57cSPaul E. McKenney .rso_gp_kthread = get_rcu_tasks_rude_gp_kthread,
340a15ec57cSPaul E. McKenney .name = "tasks-rude"
341a15ec57cSPaul E. McKenney };
342a15ec57cSPaul E. McKenney
343a15ec57cSPaul E. McKenney #define TASKS_RUDE_OPS &tasks_rude_ops,
344a15ec57cSPaul E. McKenney
345a15ec57cSPaul E. McKenney #else // #ifdef CONFIG_TASKS_RUDE_RCU
346a15ec57cSPaul E. McKenney
347a15ec57cSPaul E. McKenney #define TASKS_RUDE_OPS
348a15ec57cSPaul E. McKenney
349a15ec57cSPaul E. McKenney #endif // #else // #ifdef CONFIG_TASKS_RUDE_RCU
350a15ec57cSPaul E. McKenney
3515ce027f4SPaul E. McKenney #ifdef CONFIG_TASKS_TRACE_RCU
3525ce027f4SPaul E. McKenney
353899f317eSPaul E. McKenney /*
354899f317eSPaul E. McKenney * Definitions for RCU-tasks-trace scalability testing.
355899f317eSPaul E. McKenney */
356899f317eSPaul E. McKenney
tasks_trace_scale_read_lock(void)357899f317eSPaul E. McKenney static int tasks_trace_scale_read_lock(void)
358899f317eSPaul E. McKenney {
359899f317eSPaul E. McKenney rcu_read_lock_trace();
360899f317eSPaul E. McKenney return 0;
361899f317eSPaul E. McKenney }
362899f317eSPaul E. McKenney
tasks_trace_scale_read_unlock(int idx)363899f317eSPaul E. McKenney static void tasks_trace_scale_read_unlock(int idx)
364899f317eSPaul E. McKenney {
365899f317eSPaul E. McKenney rcu_read_unlock_trace();
366899f317eSPaul E. McKenney }
367899f317eSPaul E. McKenney
368899f317eSPaul E. McKenney static struct rcu_scale_ops tasks_tracing_ops = {
369899f317eSPaul E. McKenney .ptype = RCU_TASKS_FLAVOR,
370899f317eSPaul E. McKenney .init = rcu_sync_scale_init,
371899f317eSPaul E. McKenney .readlock = tasks_trace_scale_read_lock,
372899f317eSPaul E. McKenney .readunlock = tasks_trace_scale_read_unlock,
373899f317eSPaul E. McKenney .get_gp_seq = rcu_no_completed,
374899f317eSPaul E. McKenney .gp_diff = rcu_seq_diff,
375899f317eSPaul E. McKenney .async = call_rcu_tasks_trace,
376899f317eSPaul E. McKenney .gp_barrier = rcu_barrier_tasks_trace,
377899f317eSPaul E. McKenney .sync = synchronize_rcu_tasks_trace,
378899f317eSPaul E. McKenney .exp_sync = synchronize_rcu_tasks_trace,
3795f8e3202SPaul E. McKenney .rso_gp_kthread = get_rcu_tasks_trace_gp_kthread,
380899f317eSPaul E. McKenney .name = "tasks-tracing"
381899f317eSPaul E. McKenney };
382899f317eSPaul E. McKenney
3835ce027f4SPaul E. McKenney #define TASKS_TRACING_OPS &tasks_tracing_ops,
3845ce027f4SPaul E. McKenney
3855ce027f4SPaul E. McKenney #else // #ifdef CONFIG_TASKS_TRACE_RCU
3865ce027f4SPaul E. McKenney
3875ce027f4SPaul E. McKenney #define TASKS_TRACING_OPS
3885ce027f4SPaul E. McKenney
3895ce027f4SPaul E. McKenney #endif // #else // #ifdef CONFIG_TASKS_TRACE_RCU
3905ce027f4SPaul E. McKenney
rcuscale_seq_diff(unsigned long new,unsigned long old)3914e88ec4aSPaul E. McKenney static unsigned long rcuscale_seq_diff(unsigned long new, unsigned long old)
3924e88ec4aSPaul E. McKenney {
3934e88ec4aSPaul E. McKenney if (!cur_ops->gp_diff)
3944e88ec4aSPaul E. McKenney return new - old;
3954e88ec4aSPaul E. McKenney return cur_ops->gp_diff(new, old);
3964e88ec4aSPaul E. McKenney }
3974e88ec4aSPaul E. McKenney
3984e88ec4aSPaul E. McKenney /*
3994e88ec4aSPaul E. McKenney * If scalability tests complete, wait for shutdown to commence.
4004e88ec4aSPaul E. McKenney */
rcu_scale_wait_shutdown(void)4014e88ec4aSPaul E. McKenney static void rcu_scale_wait_shutdown(void)
4024e88ec4aSPaul E. McKenney {
4034e88ec4aSPaul E. McKenney cond_resched_tasks_rcu_qs();
4044e88ec4aSPaul E. McKenney if (atomic_read(&n_rcu_scale_writer_finished) < nrealwriters)
4054e88ec4aSPaul E. McKenney return;
4064e88ec4aSPaul E. McKenney while (!torture_must_stop())
4074e88ec4aSPaul E. McKenney schedule_timeout_uninterruptible(1);
4084e88ec4aSPaul E. McKenney }
4094e88ec4aSPaul E. McKenney
4104e88ec4aSPaul E. McKenney /*
4114e88ec4aSPaul E. McKenney * RCU scalability reader kthread. Repeatedly does empty RCU read-side
4124e88ec4aSPaul E. McKenney * critical section, minimizing update-side interference. However, the
4134e88ec4aSPaul E. McKenney * point of this test is not to evaluate reader scalability, but instead
4144e88ec4aSPaul E. McKenney * to serve as a test load for update-side scalability testing.
4154e88ec4aSPaul E. McKenney */
4164e88ec4aSPaul E. McKenney static int
rcu_scale_reader(void * arg)4174e88ec4aSPaul E. McKenney rcu_scale_reader(void *arg)
4184e88ec4aSPaul E. McKenney {
4194e88ec4aSPaul E. McKenney unsigned long flags;
4204e88ec4aSPaul E. McKenney int idx;
4214e88ec4aSPaul E. McKenney long me = (long)arg;
4224e88ec4aSPaul E. McKenney
4234e88ec4aSPaul E. McKenney VERBOSE_SCALEOUT_STRING("rcu_scale_reader task started");
4244e88ec4aSPaul E. McKenney set_cpus_allowed_ptr(current, cpumask_of(me % nr_cpu_ids));
4254e88ec4aSPaul E. McKenney set_user_nice(current, MAX_NICE);
4264e88ec4aSPaul E. McKenney atomic_inc(&n_rcu_scale_reader_started);
4274e88ec4aSPaul E. McKenney
4284e88ec4aSPaul E. McKenney do {
4294e88ec4aSPaul E. McKenney local_irq_save(flags);
4304e88ec4aSPaul E. McKenney idx = cur_ops->readlock();
4314e88ec4aSPaul E. McKenney cur_ops->readunlock(idx);
4324e88ec4aSPaul E. McKenney local_irq_restore(flags);
4334e88ec4aSPaul E. McKenney rcu_scale_wait_shutdown();
4344e88ec4aSPaul E. McKenney } while (!torture_must_stop());
4354e88ec4aSPaul E. McKenney torture_kthread_stopping("rcu_scale_reader");
4364e88ec4aSPaul E. McKenney return 0;
4374e88ec4aSPaul E. McKenney }
4384e88ec4aSPaul E. McKenney
4394e88ec4aSPaul E. McKenney /*
4404e88ec4aSPaul E. McKenney * Callback function for asynchronous grace periods from rcu_scale_writer().
4414e88ec4aSPaul E. McKenney */
rcu_scale_async_cb(struct rcu_head * rhp)4424e88ec4aSPaul E. McKenney static void rcu_scale_async_cb(struct rcu_head *rhp)
4434e88ec4aSPaul E. McKenney {
4444e88ec4aSPaul E. McKenney atomic_dec(this_cpu_ptr(&n_async_inflight));
4454e88ec4aSPaul E. McKenney kfree(rhp);
4464e88ec4aSPaul E. McKenney }
4474e88ec4aSPaul E. McKenney
4484e88ec4aSPaul E. McKenney /*
4494e88ec4aSPaul E. McKenney * RCU scale writer kthread. Repeatedly does a grace period.
4504e88ec4aSPaul E. McKenney */
4514e88ec4aSPaul E. McKenney static int
rcu_scale_writer(void * arg)4524e88ec4aSPaul E. McKenney rcu_scale_writer(void *arg)
4534e88ec4aSPaul E. McKenney {
4544e88ec4aSPaul E. McKenney int i = 0;
4554e88ec4aSPaul E. McKenney int i_max;
4567221f493SPaul E. McKenney unsigned long jdone;
4574e88ec4aSPaul E. McKenney long me = (long)arg;
4584e88ec4aSPaul E. McKenney struct rcu_head *rhp = NULL;
4594e88ec4aSPaul E. McKenney bool started = false, done = false, alldone = false;
4604e88ec4aSPaul E. McKenney u64 t;
4612226f3dcSPaul E. McKenney DEFINE_TORTURE_RANDOM(tr);
4624e88ec4aSPaul E. McKenney u64 *wdp;
4634e88ec4aSPaul E. McKenney u64 *wdpp = writer_durations[me];
4644e88ec4aSPaul E. McKenney
4654e88ec4aSPaul E. McKenney VERBOSE_SCALEOUT_STRING("rcu_scale_writer task started");
4664e88ec4aSPaul E. McKenney WARN_ON(!wdpp);
4674e88ec4aSPaul E. McKenney set_cpus_allowed_ptr(current, cpumask_of(me % nr_cpu_ids));
46892366810SZqiang current->flags |= PF_NO_SETAFFINITY;
4694e88ec4aSPaul E. McKenney sched_set_fifo_low(current);
4704e88ec4aSPaul E. McKenney
4714e88ec4aSPaul E. McKenney if (holdoff)
472e60c122aSZqiang schedule_timeout_idle(holdoff * HZ);
4734e88ec4aSPaul E. McKenney
4744e88ec4aSPaul E. McKenney /*
4754e88ec4aSPaul E. McKenney * Wait until rcu_end_inkernel_boot() is called for normal GP tests
4764e88ec4aSPaul E. McKenney * so that RCU is not always expedited for normal GP tests.
4774e88ec4aSPaul E. McKenney * The system_state test is approximate, but works well in practice.
4784e88ec4aSPaul E. McKenney */
4794e88ec4aSPaul E. McKenney while (!gp_exp && system_state != SYSTEM_RUNNING)
4804e88ec4aSPaul E. McKenney schedule_timeout_uninterruptible(1);
4814e88ec4aSPaul E. McKenney
4824e88ec4aSPaul E. McKenney t = ktime_get_mono_fast_ns();
4834e88ec4aSPaul E. McKenney if (atomic_inc_return(&n_rcu_scale_writer_started) >= nrealwriters) {
4844e88ec4aSPaul E. McKenney t_rcu_scale_writer_started = t;
4854e88ec4aSPaul E. McKenney if (gp_exp) {
4864e88ec4aSPaul E. McKenney b_rcu_gp_test_started =
4874e88ec4aSPaul E. McKenney cur_ops->exp_completed() / 2;
4884e88ec4aSPaul E. McKenney } else {
4894e88ec4aSPaul E. McKenney b_rcu_gp_test_started = cur_ops->get_gp_seq();
4904e88ec4aSPaul E. McKenney }
4914e88ec4aSPaul E. McKenney }
4924e88ec4aSPaul E. McKenney
4937221f493SPaul E. McKenney jdone = jiffies + minruntime * HZ;
4944e88ec4aSPaul E. McKenney do {
4954e88ec4aSPaul E. McKenney if (writer_holdoff)
4964e88ec4aSPaul E. McKenney udelay(writer_holdoff);
4972226f3dcSPaul E. McKenney if (writer_holdoff_jiffies)
4982226f3dcSPaul E. McKenney schedule_timeout_idle(torture_random(&tr) % writer_holdoff_jiffies + 1);
4994e88ec4aSPaul E. McKenney wdp = &wdpp[i];
5004e88ec4aSPaul E. McKenney *wdp = ktime_get_mono_fast_ns();
5014e88ec4aSPaul E. McKenney if (gp_async) {
5024e88ec4aSPaul E. McKenney retry:
5034e88ec4aSPaul E. McKenney if (!rhp)
5044e88ec4aSPaul E. McKenney rhp = kmalloc(sizeof(*rhp), GFP_KERNEL);
5054e88ec4aSPaul E. McKenney if (rhp && atomic_read(this_cpu_ptr(&n_async_inflight)) < gp_async_max) {
5064e88ec4aSPaul E. McKenney atomic_inc(this_cpu_ptr(&n_async_inflight));
5074e88ec4aSPaul E. McKenney cur_ops->async(rhp, rcu_scale_async_cb);
5084e88ec4aSPaul E. McKenney rhp = NULL;
5094e88ec4aSPaul E. McKenney } else if (!kthread_should_stop()) {
5104e88ec4aSPaul E. McKenney cur_ops->gp_barrier();
5114e88ec4aSPaul E. McKenney goto retry;
5124e88ec4aSPaul E. McKenney } else {
5134e88ec4aSPaul E. McKenney kfree(rhp); /* Because we are stopping. */
5144e88ec4aSPaul E. McKenney }
5154e88ec4aSPaul E. McKenney } else if (gp_exp) {
5164e88ec4aSPaul E. McKenney cur_ops->exp_sync();
5174e88ec4aSPaul E. McKenney } else {
5184e88ec4aSPaul E. McKenney cur_ops->sync();
5194e88ec4aSPaul E. McKenney }
5204e88ec4aSPaul E. McKenney t = ktime_get_mono_fast_ns();
5214e88ec4aSPaul E. McKenney *wdp = t - *wdp;
5224e88ec4aSPaul E. McKenney i_max = i;
5234e88ec4aSPaul E. McKenney if (!started &&
5244e88ec4aSPaul E. McKenney atomic_read(&n_rcu_scale_writer_started) >= nrealwriters)
5254e88ec4aSPaul E. McKenney started = true;
5267221f493SPaul E. McKenney if (!done && i >= MIN_MEAS && time_after(jiffies, jdone)) {
5274e88ec4aSPaul E. McKenney done = true;
5284e88ec4aSPaul E. McKenney sched_set_normal(current, 0);
5294e88ec4aSPaul E. McKenney pr_alert("%s%s rcu_scale_writer %ld has %d measurements\n",
5304e88ec4aSPaul E. McKenney scale_type, SCALE_FLAG, me, MIN_MEAS);
5314e88ec4aSPaul E. McKenney if (atomic_inc_return(&n_rcu_scale_writer_finished) >=
5324e88ec4aSPaul E. McKenney nrealwriters) {
5334e88ec4aSPaul E. McKenney schedule_timeout_interruptible(10);
5344e88ec4aSPaul E. McKenney rcu_ftrace_dump(DUMP_ALL);
5354e88ec4aSPaul E. McKenney SCALEOUT_STRING("Test complete");
5364e88ec4aSPaul E. McKenney t_rcu_scale_writer_finished = t;
5374e88ec4aSPaul E. McKenney if (gp_exp) {
5384e88ec4aSPaul E. McKenney b_rcu_gp_test_finished =
5394e88ec4aSPaul E. McKenney cur_ops->exp_completed() / 2;
5404e88ec4aSPaul E. McKenney } else {
5414e88ec4aSPaul E. McKenney b_rcu_gp_test_finished =
5424e88ec4aSPaul E. McKenney cur_ops->get_gp_seq();
5434e88ec4aSPaul E. McKenney }
5444e88ec4aSPaul E. McKenney if (shutdown) {
5454e88ec4aSPaul E. McKenney smp_mb(); /* Assign before wake. */
5464e88ec4aSPaul E. McKenney wake_up(&shutdown_wq);
5474e88ec4aSPaul E. McKenney }
5484e88ec4aSPaul E. McKenney }
5494e88ec4aSPaul E. McKenney }
5504e88ec4aSPaul E. McKenney if (done && !alldone &&
5514e88ec4aSPaul E. McKenney atomic_read(&n_rcu_scale_writer_finished) >= nrealwriters)
5524e88ec4aSPaul E. McKenney alldone = true;
5534e88ec4aSPaul E. McKenney if (started && !alldone && i < MAX_MEAS - 1)
5544e88ec4aSPaul E. McKenney i++;
5554e88ec4aSPaul E. McKenney rcu_scale_wait_shutdown();
5564e88ec4aSPaul E. McKenney } while (!torture_must_stop());
5574e88ec4aSPaul E. McKenney if (gp_async) {
5584e88ec4aSPaul E. McKenney cur_ops->gp_barrier();
5594e88ec4aSPaul E. McKenney }
560811192c5SJiangong.Han writer_n_durations[me] = i_max + 1;
5614e88ec4aSPaul E. McKenney torture_kthread_stopping("rcu_scale_writer");
5624e88ec4aSPaul E. McKenney return 0;
5634e88ec4aSPaul E. McKenney }
5644e88ec4aSPaul E. McKenney
5654e88ec4aSPaul E. McKenney static void
rcu_scale_print_module_parms(struct rcu_scale_ops * cur_ops,const char * tag)5664e88ec4aSPaul E. McKenney rcu_scale_print_module_parms(struct rcu_scale_ops *cur_ops, const char *tag)
5674e88ec4aSPaul E. McKenney {
5684e88ec4aSPaul E. McKenney pr_alert("%s" SCALE_FLAG
569c68465dfSPaul E. McKenney "--- %s: gp_async=%d gp_async_max=%d gp_exp=%d holdoff=%d minruntime=%d nreaders=%d nwriters=%d writer_holdoff=%d writer_holdoff_jiffies=%d verbose=%d shutdown=%d\n",
570c68465dfSPaul E. McKenney scale_type, tag, gp_async, gp_async_max, gp_exp, holdoff, minruntime, nrealreaders, nrealwriters, writer_holdoff, writer_holdoff_jiffies, verbose, shutdown);
5714e88ec4aSPaul E. McKenney }
5724e88ec4aSPaul E. McKenney
5734e88ec4aSPaul E. McKenney /*
5744e88ec4aSPaul E. McKenney * Return the number if non-negative. If -1, the number of CPUs.
5754e88ec4aSPaul E. McKenney * If less than -1, that much less than the number of CPUs, but
5764e88ec4aSPaul E. McKenney * at least one.
5774e88ec4aSPaul E. McKenney */
compute_real(int n)5784e88ec4aSPaul E. McKenney static int compute_real(int n)
5794e88ec4aSPaul E. McKenney {
5804e88ec4aSPaul E. McKenney int nr;
5814e88ec4aSPaul E. McKenney
5824e88ec4aSPaul E. McKenney if (n >= 0) {
5834e88ec4aSPaul E. McKenney nr = n;
5844e88ec4aSPaul E. McKenney } else {
5854e88ec4aSPaul E. McKenney nr = num_online_cpus() + 1 + n;
5864e88ec4aSPaul E. McKenney if (nr <= 0)
5874e88ec4aSPaul E. McKenney nr = 1;
5884e88ec4aSPaul E. McKenney }
5894e88ec4aSPaul E. McKenney return nr;
5904e88ec4aSPaul E. McKenney }
5914e88ec4aSPaul E. McKenney
5924e88ec4aSPaul E. McKenney /*
5934e88ec4aSPaul E. McKenney * kfree_rcu() scalability tests: Start a kfree_rcu() loop on all CPUs for number
5944e88ec4aSPaul E. McKenney * of iterations and measure total time and number of GP for all iterations to complete.
5954e88ec4aSPaul E. McKenney */
5964e88ec4aSPaul E. McKenney
5974e88ec4aSPaul E. McKenney torture_param(int, kfree_nthreads, -1, "Number of threads running loops of kfree_rcu().");
5984e88ec4aSPaul E. McKenney torture_param(int, kfree_alloc_num, 8000, "Number of allocations and frees done in an iteration.");
5994e88ec4aSPaul E. McKenney torture_param(int, kfree_loops, 10, "Number of loops doing kfree_alloc_num allocations and frees.");
600686fe1bfSUladzislau Rezki (Sony) torture_param(bool, kfree_rcu_test_double, false, "Do we run a kfree_rcu() double-argument scale test?");
601686fe1bfSUladzislau Rezki (Sony) torture_param(bool, kfree_rcu_test_single, false, "Do we run a kfree_rcu() single-argument scale test?");
6024e88ec4aSPaul E. McKenney
6034e88ec4aSPaul E. McKenney static struct task_struct **kfree_reader_tasks;
6044e88ec4aSPaul E. McKenney static int kfree_nrealthreads;
6054e88ec4aSPaul E. McKenney static atomic_t n_kfree_scale_thread_started;
6064e88ec4aSPaul E. McKenney static atomic_t n_kfree_scale_thread_ended;
6075f8e3202SPaul E. McKenney static struct task_struct *kthread_tp;
6085f8e3202SPaul E. McKenney static u64 kthread_stime;
6094e88ec4aSPaul E. McKenney
6104e88ec4aSPaul E. McKenney struct kfree_obj {
6114e88ec4aSPaul E. McKenney char kfree_obj[8];
6124e88ec4aSPaul E. McKenney struct rcu_head rh;
6134e88ec4aSPaul E. McKenney };
6144e88ec4aSPaul E. McKenney
615084e04ffSJoel Fernandes (Google) /* Used if doing RCU-kfree'ing via call_rcu(). */
kfree_call_rcu(struct rcu_head * rh)616084e04ffSJoel Fernandes (Google) static void kfree_call_rcu(struct rcu_head *rh)
617084e04ffSJoel Fernandes (Google) {
618084e04ffSJoel Fernandes (Google) struct kfree_obj *obj = container_of(rh, struct kfree_obj, rh);
619084e04ffSJoel Fernandes (Google)
620084e04ffSJoel Fernandes (Google) kfree(obj);
621084e04ffSJoel Fernandes (Google) }
622084e04ffSJoel Fernandes (Google)
6234e88ec4aSPaul E. McKenney static int
kfree_scale_thread(void * arg)6244e88ec4aSPaul E. McKenney kfree_scale_thread(void *arg)
6254e88ec4aSPaul E. McKenney {
6264e88ec4aSPaul E. McKenney int i, loop = 0;
6274e88ec4aSPaul E. McKenney long me = (long)arg;
6284e88ec4aSPaul E. McKenney struct kfree_obj *alloc_ptr;
6294e88ec4aSPaul E. McKenney u64 start_time, end_time;
6304e88ec4aSPaul E. McKenney long long mem_begin, mem_during = 0;
631686fe1bfSUladzislau Rezki (Sony) bool kfree_rcu_test_both;
632686fe1bfSUladzislau Rezki (Sony) DEFINE_TORTURE_RANDOM(tr);
6334e88ec4aSPaul E. McKenney
6344e88ec4aSPaul E. McKenney VERBOSE_SCALEOUT_STRING("kfree_scale_thread task started");
6354e88ec4aSPaul E. McKenney set_cpus_allowed_ptr(current, cpumask_of(me % nr_cpu_ids));
6364e88ec4aSPaul E. McKenney set_user_nice(current, MAX_NICE);
637686fe1bfSUladzislau Rezki (Sony) kfree_rcu_test_both = (kfree_rcu_test_single == kfree_rcu_test_double);
6384e88ec4aSPaul E. McKenney
6394e88ec4aSPaul E. McKenney start_time = ktime_get_mono_fast_ns();
6404e88ec4aSPaul E. McKenney
6414e88ec4aSPaul E. McKenney if (atomic_inc_return(&n_kfree_scale_thread_started) >= kfree_nrealthreads) {
6424e88ec4aSPaul E. McKenney if (gp_exp)
6434e88ec4aSPaul E. McKenney b_rcu_gp_test_started = cur_ops->exp_completed() / 2;
6444e88ec4aSPaul E. McKenney else
6454e88ec4aSPaul E. McKenney b_rcu_gp_test_started = cur_ops->get_gp_seq();
6464e88ec4aSPaul E. McKenney }
6474e88ec4aSPaul E. McKenney
6484e88ec4aSPaul E. McKenney do {
6494e88ec4aSPaul E. McKenney if (!mem_during) {
6504e88ec4aSPaul E. McKenney mem_during = mem_begin = si_mem_available();
6514e88ec4aSPaul E. McKenney } else if (loop % (kfree_loops / 4) == 0) {
6524e88ec4aSPaul E. McKenney mem_during = (mem_during + si_mem_available()) / 2;
6534e88ec4aSPaul E. McKenney }
6544e88ec4aSPaul E. McKenney
6554e88ec4aSPaul E. McKenney for (i = 0; i < kfree_alloc_num; i++) {
6564e88ec4aSPaul E. McKenney alloc_ptr = kmalloc(kfree_mult * sizeof(struct kfree_obj), GFP_KERNEL);
6574e88ec4aSPaul E. McKenney if (!alloc_ptr)
6584e88ec4aSPaul E. McKenney return -ENOMEM;
6594e88ec4aSPaul E. McKenney
660084e04ffSJoel Fernandes (Google) if (kfree_by_call_rcu) {
661084e04ffSJoel Fernandes (Google) call_rcu(&(alloc_ptr->rh), kfree_call_rcu);
662084e04ffSJoel Fernandes (Google) continue;
663084e04ffSJoel Fernandes (Google) }
664084e04ffSJoel Fernandes (Google)
665686fe1bfSUladzislau Rezki (Sony) // By default kfree_rcu_test_single and kfree_rcu_test_double are
666686fe1bfSUladzislau Rezki (Sony) // initialized to false. If both have the same value (false or true)
667686fe1bfSUladzislau Rezki (Sony) // both are randomly tested, otherwise only the one with value true
668686fe1bfSUladzislau Rezki (Sony) // is tested.
669686fe1bfSUladzislau Rezki (Sony) if ((kfree_rcu_test_single && !kfree_rcu_test_double) ||
670686fe1bfSUladzislau Rezki (Sony) (kfree_rcu_test_both && torture_random(&tr) & 0x800))
671936c7e19SUladzislau Rezki (Sony) kfree_rcu_mightsleep(alloc_ptr);
672686fe1bfSUladzislau Rezki (Sony) else
6734e88ec4aSPaul E. McKenney kfree_rcu(alloc_ptr, rh);
6744e88ec4aSPaul E. McKenney }
6754e88ec4aSPaul E. McKenney
6764e88ec4aSPaul E. McKenney cond_resched();
6774e88ec4aSPaul E. McKenney } while (!torture_must_stop() && ++loop < kfree_loops);
6784e88ec4aSPaul E. McKenney
6794e88ec4aSPaul E. McKenney if (atomic_inc_return(&n_kfree_scale_thread_ended) >= kfree_nrealthreads) {
6804e88ec4aSPaul E. McKenney end_time = ktime_get_mono_fast_ns();
6814e88ec4aSPaul E. McKenney
6824e88ec4aSPaul E. McKenney if (gp_exp)
6834e88ec4aSPaul E. McKenney b_rcu_gp_test_finished = cur_ops->exp_completed() / 2;
6844e88ec4aSPaul E. McKenney else
6854e88ec4aSPaul E. McKenney b_rcu_gp_test_finished = cur_ops->get_gp_seq();
6864e88ec4aSPaul E. McKenney
6874e88ec4aSPaul E. McKenney pr_alert("Total time taken by all kfree'ers: %llu ns, loops: %d, batches: %ld, memory footprint: %lldMB\n",
6884e88ec4aSPaul E. McKenney (unsigned long long)(end_time - start_time), kfree_loops,
6894e88ec4aSPaul E. McKenney rcuscale_seq_diff(b_rcu_gp_test_finished, b_rcu_gp_test_started),
6904e88ec4aSPaul E. McKenney (mem_begin - mem_during) >> (20 - PAGE_SHIFT));
6914e88ec4aSPaul E. McKenney
6924e88ec4aSPaul E. McKenney if (shutdown) {
6934e88ec4aSPaul E. McKenney smp_mb(); /* Assign before wake. */
6944e88ec4aSPaul E. McKenney wake_up(&shutdown_wq);
6954e88ec4aSPaul E. McKenney }
6964e88ec4aSPaul E. McKenney }
6974e88ec4aSPaul E. McKenney
6984e88ec4aSPaul E. McKenney torture_kthread_stopping("kfree_scale_thread");
6994e88ec4aSPaul E. McKenney return 0;
7004e88ec4aSPaul E. McKenney }
7014e88ec4aSPaul E. McKenney
7024e88ec4aSPaul E. McKenney static void
kfree_scale_cleanup(void)7034e88ec4aSPaul E. McKenney kfree_scale_cleanup(void)
7044e88ec4aSPaul E. McKenney {
7054e88ec4aSPaul E. McKenney int i;
7064e88ec4aSPaul E. McKenney
7074e88ec4aSPaul E. McKenney if (torture_cleanup_begin())
7084e88ec4aSPaul E. McKenney return;
7094e88ec4aSPaul E. McKenney
7104e88ec4aSPaul E. McKenney if (kfree_reader_tasks) {
7114e88ec4aSPaul E. McKenney for (i = 0; i < kfree_nrealthreads; i++)
7124e88ec4aSPaul E. McKenney torture_stop_kthread(kfree_scale_thread,
7134e88ec4aSPaul E. McKenney kfree_reader_tasks[i]);
7144e88ec4aSPaul E. McKenney kfree(kfree_reader_tasks);
7154e88ec4aSPaul E. McKenney }
7164e88ec4aSPaul E. McKenney
7174e88ec4aSPaul E. McKenney torture_cleanup_end();
7184e88ec4aSPaul E. McKenney }
7194e88ec4aSPaul E. McKenney
7204e88ec4aSPaul E. McKenney /*
7214e88ec4aSPaul E. McKenney * shutdown kthread. Just waits to be awakened, then shuts down system.
7224e88ec4aSPaul E. McKenney */
7234e88ec4aSPaul E. McKenney static int
kfree_scale_shutdown(void * arg)7244e88ec4aSPaul E. McKenney kfree_scale_shutdown(void *arg)
7254e88ec4aSPaul E. McKenney {
726ef1ef3d4SPaul E. McKenney wait_event_idle(shutdown_wq,
7274e88ec4aSPaul E. McKenney atomic_read(&n_kfree_scale_thread_ended) >= kfree_nrealthreads);
7284e88ec4aSPaul E. McKenney
7294e88ec4aSPaul E. McKenney smp_mb(); /* Wake before output. */
7304e88ec4aSPaul E. McKenney
7314e88ec4aSPaul E. McKenney kfree_scale_cleanup();
7324e88ec4aSPaul E. McKenney kernel_power_off();
7334e88ec4aSPaul E. McKenney return -EINVAL;
7344e88ec4aSPaul E. McKenney }
7354e88ec4aSPaul E. McKenney
736084e04ffSJoel Fernandes (Google) // Used if doing RCU-kfree'ing via call_rcu().
737084e04ffSJoel Fernandes (Google) static unsigned long jiffies_at_lazy_cb;
738084e04ffSJoel Fernandes (Google) static struct rcu_head lazy_test1_rh;
739084e04ffSJoel Fernandes (Google) static int rcu_lazy_test1_cb_called;
call_rcu_lazy_test1(struct rcu_head * rh)740084e04ffSJoel Fernandes (Google) static void call_rcu_lazy_test1(struct rcu_head *rh)
741084e04ffSJoel Fernandes (Google) {
742084e04ffSJoel Fernandes (Google) jiffies_at_lazy_cb = jiffies;
743084e04ffSJoel Fernandes (Google) WRITE_ONCE(rcu_lazy_test1_cb_called, 1);
744084e04ffSJoel Fernandes (Google) }
745084e04ffSJoel Fernandes (Google)
7464e88ec4aSPaul E. McKenney static int __init
kfree_scale_init(void)7474e88ec4aSPaul E. McKenney kfree_scale_init(void)
7484e88ec4aSPaul E. McKenney {
7494e88ec4aSPaul E. McKenney int firsterr = 0;
750084e04ffSJoel Fernandes (Google) long i;
751084e04ffSJoel Fernandes (Google) unsigned long jif_start;
752084e04ffSJoel Fernandes (Google) unsigned long orig_jif;
753084e04ffSJoel Fernandes (Google)
754bb7bad3dSPaul E. McKenney pr_alert("%s" SCALE_FLAG
755bb7bad3dSPaul E. McKenney "--- kfree_rcu_test: kfree_mult=%d kfree_by_call_rcu=%d kfree_nthreads=%d kfree_alloc_num=%d kfree_loops=%d kfree_rcu_test_double=%d kfree_rcu_test_single=%d\n",
756bb7bad3dSPaul E. McKenney scale_type, kfree_mult, kfree_by_call_rcu, kfree_nthreads, kfree_alloc_num, kfree_loops, kfree_rcu_test_double, kfree_rcu_test_single);
757bb7bad3dSPaul E. McKenney
758084e04ffSJoel Fernandes (Google) // Also, do a quick self-test to ensure laziness is as much as
759084e04ffSJoel Fernandes (Google) // expected.
760084e04ffSJoel Fernandes (Google) if (kfree_by_call_rcu && !IS_ENABLED(CONFIG_RCU_LAZY)) {
761084e04ffSJoel Fernandes (Google) pr_alert("CONFIG_RCU_LAZY is disabled, falling back to kfree_rcu() for delayed RCU kfree'ing\n");
762084e04ffSJoel Fernandes (Google) kfree_by_call_rcu = 0;
763084e04ffSJoel Fernandes (Google) }
764084e04ffSJoel Fernandes (Google)
765084e04ffSJoel Fernandes (Google) if (kfree_by_call_rcu) {
766084e04ffSJoel Fernandes (Google) /* do a test to check the timeout. */
767*499d7e7eSFrederic Weisbecker orig_jif = rcu_get_jiffies_lazy_flush();
768084e04ffSJoel Fernandes (Google)
769*499d7e7eSFrederic Weisbecker rcu_set_jiffies_lazy_flush(2 * HZ);
770084e04ffSJoel Fernandes (Google) rcu_barrier();
771084e04ffSJoel Fernandes (Google)
772084e04ffSJoel Fernandes (Google) jif_start = jiffies;
773084e04ffSJoel Fernandes (Google) jiffies_at_lazy_cb = 0;
774084e04ffSJoel Fernandes (Google) call_rcu(&lazy_test1_rh, call_rcu_lazy_test1);
775084e04ffSJoel Fernandes (Google)
776084e04ffSJoel Fernandes (Google) smp_cond_load_relaxed(&rcu_lazy_test1_cb_called, VAL == 1);
777084e04ffSJoel Fernandes (Google)
778*499d7e7eSFrederic Weisbecker rcu_set_jiffies_lazy_flush(orig_jif);
779084e04ffSJoel Fernandes (Google)
780084e04ffSJoel Fernandes (Google) if (WARN_ON_ONCE(jiffies_at_lazy_cb - jif_start < 2 * HZ)) {
781084e04ffSJoel Fernandes (Google) pr_alert("ERROR: call_rcu() CBs are not being lazy as expected!\n");
782084e04ffSJoel Fernandes (Google) WARN_ON_ONCE(1);
783084e04ffSJoel Fernandes (Google) return -1;
784084e04ffSJoel Fernandes (Google) }
785084e04ffSJoel Fernandes (Google)
786084e04ffSJoel Fernandes (Google) if (WARN_ON_ONCE(jiffies_at_lazy_cb - jif_start > 3 * HZ)) {
787084e04ffSJoel Fernandes (Google) pr_alert("ERROR: call_rcu() CBs are being too lazy!\n");
788084e04ffSJoel Fernandes (Google) WARN_ON_ONCE(1);
789084e04ffSJoel Fernandes (Google) return -1;
790084e04ffSJoel Fernandes (Google) }
791084e04ffSJoel Fernandes (Google) }
7924e88ec4aSPaul E. McKenney
7934e88ec4aSPaul E. McKenney kfree_nrealthreads = compute_real(kfree_nthreads);
7944e88ec4aSPaul E. McKenney /* Start up the kthreads. */
7954e88ec4aSPaul E. McKenney if (shutdown) {
7964e88ec4aSPaul E. McKenney init_waitqueue_head(&shutdown_wq);
7974e88ec4aSPaul E. McKenney firsterr = torture_create_kthread(kfree_scale_shutdown, NULL,
7984e88ec4aSPaul E. McKenney shutdown_task);
799eb77abfdSPaul E. McKenney if (torture_init_error(firsterr))
8004e88ec4aSPaul E. McKenney goto unwind;
8014e88ec4aSPaul E. McKenney schedule_timeout_uninterruptible(1);
8024e88ec4aSPaul E. McKenney }
8034e88ec4aSPaul E. McKenney
804084e04ffSJoel Fernandes (Google) pr_alert("kfree object size=%zu, kfree_by_call_rcu=%d\n",
805084e04ffSJoel Fernandes (Google) kfree_mult * sizeof(struct kfree_obj),
806084e04ffSJoel Fernandes (Google) kfree_by_call_rcu);
8074e88ec4aSPaul E. McKenney
8084e88ec4aSPaul E. McKenney kfree_reader_tasks = kcalloc(kfree_nrealthreads, sizeof(kfree_reader_tasks[0]),
8094e88ec4aSPaul E. McKenney GFP_KERNEL);
8104e88ec4aSPaul E. McKenney if (kfree_reader_tasks == NULL) {
8114e88ec4aSPaul E. McKenney firsterr = -ENOMEM;
8124e88ec4aSPaul E. McKenney goto unwind;
8134e88ec4aSPaul E. McKenney }
8144e88ec4aSPaul E. McKenney
8154e88ec4aSPaul E. McKenney for (i = 0; i < kfree_nrealthreads; i++) {
8164e88ec4aSPaul E. McKenney firsterr = torture_create_kthread(kfree_scale_thread, (void *)i,
8174e88ec4aSPaul E. McKenney kfree_reader_tasks[i]);
818eb77abfdSPaul E. McKenney if (torture_init_error(firsterr))
8194e88ec4aSPaul E. McKenney goto unwind;
8204e88ec4aSPaul E. McKenney }
8214e88ec4aSPaul E. McKenney
8224e88ec4aSPaul E. McKenney while (atomic_read(&n_kfree_scale_thread_started) < kfree_nrealthreads)
8234e88ec4aSPaul E. McKenney schedule_timeout_uninterruptible(1);
8244e88ec4aSPaul E. McKenney
8254e88ec4aSPaul E. McKenney torture_init_end();
8264e88ec4aSPaul E. McKenney return 0;
8274e88ec4aSPaul E. McKenney
8284e88ec4aSPaul E. McKenney unwind:
8294e88ec4aSPaul E. McKenney torture_init_end();
8304e88ec4aSPaul E. McKenney kfree_scale_cleanup();
8314e88ec4aSPaul E. McKenney return firsterr;
8324e88ec4aSPaul E. McKenney }
8334e88ec4aSPaul E. McKenney
834bf5ddd73SQiuxu Zhuo static void
rcu_scale_cleanup(void)835bf5ddd73SQiuxu Zhuo rcu_scale_cleanup(void)
836bf5ddd73SQiuxu Zhuo {
837bf5ddd73SQiuxu Zhuo int i;
838bf5ddd73SQiuxu Zhuo int j;
839bf5ddd73SQiuxu Zhuo int ngps = 0;
840bf5ddd73SQiuxu Zhuo u64 *wdp;
841bf5ddd73SQiuxu Zhuo u64 *wdpp;
842bf5ddd73SQiuxu Zhuo
843bf5ddd73SQiuxu Zhuo /*
844bf5ddd73SQiuxu Zhuo * Would like warning at start, but everything is expedited
845bf5ddd73SQiuxu Zhuo * during the mid-boot phase, so have to wait till the end.
846bf5ddd73SQiuxu Zhuo */
847bf5ddd73SQiuxu Zhuo if (rcu_gp_is_expedited() && !rcu_gp_is_normal() && !gp_exp)
848bf5ddd73SQiuxu Zhuo SCALEOUT_ERRSTRING("All grace periods expedited, no normal ones to measure!");
849bf5ddd73SQiuxu Zhuo if (rcu_gp_is_normal() && gp_exp)
850bf5ddd73SQiuxu Zhuo SCALEOUT_ERRSTRING("All grace periods normal, no expedited ones to measure!");
851bf5ddd73SQiuxu Zhuo if (gp_exp && gp_async)
852bf5ddd73SQiuxu Zhuo SCALEOUT_ERRSTRING("No expedited async GPs, so went with async!");
853bf5ddd73SQiuxu Zhuo
8545f8e3202SPaul E. McKenney // If built-in, just report all of the GP kthread's CPU time.
8555f8e3202SPaul E. McKenney if (IS_BUILTIN(CONFIG_RCU_SCALE_TEST) && !kthread_tp && cur_ops->rso_gp_kthread)
8565f8e3202SPaul E. McKenney kthread_tp = cur_ops->rso_gp_kthread();
8575f8e3202SPaul E. McKenney if (kthread_tp) {
8585f8e3202SPaul E. McKenney u32 ns;
8595f8e3202SPaul E. McKenney u64 us;
8605f8e3202SPaul E. McKenney
8615f8e3202SPaul E. McKenney kthread_stime = kthread_tp->stime - kthread_stime;
8625f8e3202SPaul E. McKenney us = div_u64_rem(kthread_stime, 1000, &ns);
8635f8e3202SPaul E. McKenney pr_info("rcu_scale: Grace-period kthread CPU time: %llu.%03u us\n", us, ns);
8645f8e3202SPaul E. McKenney show_rcu_gp_kthreads();
8655f8e3202SPaul E. McKenney }
86623fc8df2SQiuxu Zhuo if (kfree_rcu_test) {
86723fc8df2SQiuxu Zhuo kfree_scale_cleanup();
86823fc8df2SQiuxu Zhuo return;
86923fc8df2SQiuxu Zhuo }
87023fc8df2SQiuxu Zhuo
871bf5ddd73SQiuxu Zhuo if (torture_cleanup_begin())
872bf5ddd73SQiuxu Zhuo return;
873bf5ddd73SQiuxu Zhuo if (!cur_ops) {
874bf5ddd73SQiuxu Zhuo torture_cleanup_end();
875bf5ddd73SQiuxu Zhuo return;
876bf5ddd73SQiuxu Zhuo }
877bf5ddd73SQiuxu Zhuo
878bf5ddd73SQiuxu Zhuo if (reader_tasks) {
879bf5ddd73SQiuxu Zhuo for (i = 0; i < nrealreaders; i++)
880bf5ddd73SQiuxu Zhuo torture_stop_kthread(rcu_scale_reader,
881bf5ddd73SQiuxu Zhuo reader_tasks[i]);
882bf5ddd73SQiuxu Zhuo kfree(reader_tasks);
883bf5ddd73SQiuxu Zhuo }
884bf5ddd73SQiuxu Zhuo
885bf5ddd73SQiuxu Zhuo if (writer_tasks) {
886bf5ddd73SQiuxu Zhuo for (i = 0; i < nrealwriters; i++) {
887bf5ddd73SQiuxu Zhuo torture_stop_kthread(rcu_scale_writer,
888bf5ddd73SQiuxu Zhuo writer_tasks[i]);
889bf5ddd73SQiuxu Zhuo if (!writer_n_durations)
890bf5ddd73SQiuxu Zhuo continue;
891bf5ddd73SQiuxu Zhuo j = writer_n_durations[i];
892bf5ddd73SQiuxu Zhuo pr_alert("%s%s writer %d gps: %d\n",
893bf5ddd73SQiuxu Zhuo scale_type, SCALE_FLAG, i, j);
894bf5ddd73SQiuxu Zhuo ngps += j;
895bf5ddd73SQiuxu Zhuo }
896bf5ddd73SQiuxu Zhuo pr_alert("%s%s start: %llu end: %llu duration: %llu gps: %d batches: %ld\n",
897bf5ddd73SQiuxu Zhuo scale_type, SCALE_FLAG,
898bf5ddd73SQiuxu Zhuo t_rcu_scale_writer_started, t_rcu_scale_writer_finished,
899bf5ddd73SQiuxu Zhuo t_rcu_scale_writer_finished -
900bf5ddd73SQiuxu Zhuo t_rcu_scale_writer_started,
901bf5ddd73SQiuxu Zhuo ngps,
902bf5ddd73SQiuxu Zhuo rcuscale_seq_diff(b_rcu_gp_test_finished,
903bf5ddd73SQiuxu Zhuo b_rcu_gp_test_started));
904bf5ddd73SQiuxu Zhuo for (i = 0; i < nrealwriters; i++) {
905bf5ddd73SQiuxu Zhuo if (!writer_durations)
906bf5ddd73SQiuxu Zhuo break;
907bf5ddd73SQiuxu Zhuo if (!writer_n_durations)
908bf5ddd73SQiuxu Zhuo continue;
909bf5ddd73SQiuxu Zhuo wdpp = writer_durations[i];
910bf5ddd73SQiuxu Zhuo if (!wdpp)
911bf5ddd73SQiuxu Zhuo continue;
912bf5ddd73SQiuxu Zhuo for (j = 0; j < writer_n_durations[i]; j++) {
913bf5ddd73SQiuxu Zhuo wdp = &wdpp[j];
914bf5ddd73SQiuxu Zhuo pr_alert("%s%s %4d writer-duration: %5d %llu\n",
915bf5ddd73SQiuxu Zhuo scale_type, SCALE_FLAG,
916bf5ddd73SQiuxu Zhuo i, j, *wdp);
917bf5ddd73SQiuxu Zhuo if (j % 100 == 0)
918bf5ddd73SQiuxu Zhuo schedule_timeout_uninterruptible(1);
919bf5ddd73SQiuxu Zhuo }
920bf5ddd73SQiuxu Zhuo kfree(writer_durations[i]);
921bf5ddd73SQiuxu Zhuo }
922bf5ddd73SQiuxu Zhuo kfree(writer_tasks);
923bf5ddd73SQiuxu Zhuo kfree(writer_durations);
924bf5ddd73SQiuxu Zhuo kfree(writer_n_durations);
925bf5ddd73SQiuxu Zhuo }
926bf5ddd73SQiuxu Zhuo
927bf5ddd73SQiuxu Zhuo /* Do torture-type-specific cleanup operations. */
928bf5ddd73SQiuxu Zhuo if (cur_ops->cleanup != NULL)
929bf5ddd73SQiuxu Zhuo cur_ops->cleanup();
930bf5ddd73SQiuxu Zhuo
931bf5ddd73SQiuxu Zhuo torture_cleanup_end();
932bf5ddd73SQiuxu Zhuo }
933bf5ddd73SQiuxu Zhuo
934bf5ddd73SQiuxu Zhuo /*
935bf5ddd73SQiuxu Zhuo * RCU scalability shutdown kthread. Just waits to be awakened, then shuts
936bf5ddd73SQiuxu Zhuo * down system.
937bf5ddd73SQiuxu Zhuo */
938bf5ddd73SQiuxu Zhuo static int
rcu_scale_shutdown(void * arg)939bf5ddd73SQiuxu Zhuo rcu_scale_shutdown(void *arg)
940bf5ddd73SQiuxu Zhuo {
941bf5ddd73SQiuxu Zhuo wait_event_idle(shutdown_wq, atomic_read(&n_rcu_scale_writer_finished) >= nrealwriters);
942bf5ddd73SQiuxu Zhuo smp_mb(); /* Wake before output. */
943bf5ddd73SQiuxu Zhuo rcu_scale_cleanup();
944bf5ddd73SQiuxu Zhuo kernel_power_off();
945bf5ddd73SQiuxu Zhuo return -EINVAL;
946bf5ddd73SQiuxu Zhuo }
947bf5ddd73SQiuxu Zhuo
9484e88ec4aSPaul E. McKenney static int __init
rcu_scale_init(void)9494e88ec4aSPaul E. McKenney rcu_scale_init(void)
9504e88ec4aSPaul E. McKenney {
9514e88ec4aSPaul E. McKenney long i;
9524e88ec4aSPaul E. McKenney int firsterr = 0;
9534e88ec4aSPaul E. McKenney static struct rcu_scale_ops *scale_ops[] = {
954a15ec57cSPaul E. McKenney &rcu_ops, &srcu_ops, &srcud_ops, TASKS_OPS TASKS_RUDE_OPS TASKS_TRACING_OPS
9554e88ec4aSPaul E. McKenney };
9564e88ec4aSPaul E. McKenney
9574e88ec4aSPaul E. McKenney if (!torture_init_begin(scale_type, verbose))
9584e88ec4aSPaul E. McKenney return -EBUSY;
9594e88ec4aSPaul E. McKenney
9604e88ec4aSPaul E. McKenney /* Process args and announce that the scalability'er is on the job. */
9614e88ec4aSPaul E. McKenney for (i = 0; i < ARRAY_SIZE(scale_ops); i++) {
9624e88ec4aSPaul E. McKenney cur_ops = scale_ops[i];
9634e88ec4aSPaul E. McKenney if (strcmp(scale_type, cur_ops->name) == 0)
9644e88ec4aSPaul E. McKenney break;
9654e88ec4aSPaul E. McKenney }
9664e88ec4aSPaul E. McKenney if (i == ARRAY_SIZE(scale_ops)) {
9674e88ec4aSPaul E. McKenney pr_alert("rcu-scale: invalid scale type: \"%s\"\n", scale_type);
9684e88ec4aSPaul E. McKenney pr_alert("rcu-scale types:");
9694e88ec4aSPaul E. McKenney for (i = 0; i < ARRAY_SIZE(scale_ops); i++)
9704e88ec4aSPaul E. McKenney pr_cont(" %s", scale_ops[i]->name);
9714e88ec4aSPaul E. McKenney pr_cont("\n");
9724e88ec4aSPaul E. McKenney firsterr = -EINVAL;
9734e88ec4aSPaul E. McKenney cur_ops = NULL;
9744e88ec4aSPaul E. McKenney goto unwind;
9754e88ec4aSPaul E. McKenney }
9764e88ec4aSPaul E. McKenney if (cur_ops->init)
9774e88ec4aSPaul E. McKenney cur_ops->init();
9784e88ec4aSPaul E. McKenney
9795f8e3202SPaul E. McKenney if (cur_ops->rso_gp_kthread) {
9805f8e3202SPaul E. McKenney kthread_tp = cur_ops->rso_gp_kthread();
9815f8e3202SPaul E. McKenney if (kthread_tp)
9825f8e3202SPaul E. McKenney kthread_stime = kthread_tp->stime;
9835f8e3202SPaul E. McKenney }
9844e88ec4aSPaul E. McKenney if (kfree_rcu_test)
9854e88ec4aSPaul E. McKenney return kfree_scale_init();
9864e88ec4aSPaul E. McKenney
9874e88ec4aSPaul E. McKenney nrealwriters = compute_real(nwriters);
9884e88ec4aSPaul E. McKenney nrealreaders = compute_real(nreaders);
9894e88ec4aSPaul E. McKenney atomic_set(&n_rcu_scale_reader_started, 0);
9904e88ec4aSPaul E. McKenney atomic_set(&n_rcu_scale_writer_started, 0);
9914e88ec4aSPaul E. McKenney atomic_set(&n_rcu_scale_writer_finished, 0);
9924e88ec4aSPaul E. McKenney rcu_scale_print_module_parms(cur_ops, "Start of test");
9934e88ec4aSPaul E. McKenney
9944e88ec4aSPaul E. McKenney /* Start up the kthreads. */
9954e88ec4aSPaul E. McKenney
9964e88ec4aSPaul E. McKenney if (shutdown) {
9974e88ec4aSPaul E. McKenney init_waitqueue_head(&shutdown_wq);
9984e88ec4aSPaul E. McKenney firsterr = torture_create_kthread(rcu_scale_shutdown, NULL,
9994e88ec4aSPaul E. McKenney shutdown_task);
1000eb77abfdSPaul E. McKenney if (torture_init_error(firsterr))
10014e88ec4aSPaul E. McKenney goto unwind;
10024e88ec4aSPaul E. McKenney schedule_timeout_uninterruptible(1);
10034e88ec4aSPaul E. McKenney }
10044e88ec4aSPaul E. McKenney reader_tasks = kcalloc(nrealreaders, sizeof(reader_tasks[0]),
10054e88ec4aSPaul E. McKenney GFP_KERNEL);
10064e88ec4aSPaul E. McKenney if (reader_tasks == NULL) {
100786e7ed1bSLi Zhijian SCALEOUT_ERRSTRING("out of memory");
10084e88ec4aSPaul E. McKenney firsterr = -ENOMEM;
10094e88ec4aSPaul E. McKenney goto unwind;
10104e88ec4aSPaul E. McKenney }
10114e88ec4aSPaul E. McKenney for (i = 0; i < nrealreaders; i++) {
10124e88ec4aSPaul E. McKenney firsterr = torture_create_kthread(rcu_scale_reader, (void *)i,
10134e88ec4aSPaul E. McKenney reader_tasks[i]);
1014eb77abfdSPaul E. McKenney if (torture_init_error(firsterr))
10154e88ec4aSPaul E. McKenney goto unwind;
10164e88ec4aSPaul E. McKenney }
10174e88ec4aSPaul E. McKenney while (atomic_read(&n_rcu_scale_reader_started) < nrealreaders)
10184e88ec4aSPaul E. McKenney schedule_timeout_uninterruptible(1);
10194e88ec4aSPaul E. McKenney writer_tasks = kcalloc(nrealwriters, sizeof(reader_tasks[0]),
10204e88ec4aSPaul E. McKenney GFP_KERNEL);
10214e88ec4aSPaul E. McKenney writer_durations = kcalloc(nrealwriters, sizeof(*writer_durations),
10224e88ec4aSPaul E. McKenney GFP_KERNEL);
10234e88ec4aSPaul E. McKenney writer_n_durations =
10244e88ec4aSPaul E. McKenney kcalloc(nrealwriters, sizeof(*writer_n_durations),
10254e88ec4aSPaul E. McKenney GFP_KERNEL);
10264e88ec4aSPaul E. McKenney if (!writer_tasks || !writer_durations || !writer_n_durations) {
102786e7ed1bSLi Zhijian SCALEOUT_ERRSTRING("out of memory");
10284e88ec4aSPaul E. McKenney firsterr = -ENOMEM;
10294e88ec4aSPaul E. McKenney goto unwind;
10304e88ec4aSPaul E. McKenney }
10314e88ec4aSPaul E. McKenney for (i = 0; i < nrealwriters; i++) {
10324e88ec4aSPaul E. McKenney writer_durations[i] =
10334e88ec4aSPaul E. McKenney kcalloc(MAX_MEAS, sizeof(*writer_durations[i]),
10344e88ec4aSPaul E. McKenney GFP_KERNEL);
10354e88ec4aSPaul E. McKenney if (!writer_durations[i]) {
10364e88ec4aSPaul E. McKenney firsterr = -ENOMEM;
10374e88ec4aSPaul E. McKenney goto unwind;
10384e88ec4aSPaul E. McKenney }
10394e88ec4aSPaul E. McKenney firsterr = torture_create_kthread(rcu_scale_writer, (void *)i,
10404e88ec4aSPaul E. McKenney writer_tasks[i]);
1041eb77abfdSPaul E. McKenney if (torture_init_error(firsterr))
10424e88ec4aSPaul E. McKenney goto unwind;
10434e88ec4aSPaul E. McKenney }
10444e88ec4aSPaul E. McKenney torture_init_end();
10454e88ec4aSPaul E. McKenney return 0;
10464e88ec4aSPaul E. McKenney
10474e88ec4aSPaul E. McKenney unwind:
10484e88ec4aSPaul E. McKenney torture_init_end();
10494e88ec4aSPaul E. McKenney rcu_scale_cleanup();
10502f2214d4SPaul E. McKenney if (shutdown) {
10512f2214d4SPaul E. McKenney WARN_ON(!IS_MODULE(CONFIG_RCU_SCALE_TEST));
10522f2214d4SPaul E. McKenney kernel_power_off();
10532f2214d4SPaul E. McKenney }
10544e88ec4aSPaul E. McKenney return firsterr;
10554e88ec4aSPaul E. McKenney }
10564e88ec4aSPaul E. McKenney
10574e88ec4aSPaul E. McKenney module_init(rcu_scale_init);
10584e88ec4aSPaul E. McKenney module_exit(rcu_scale_cleanup);
1059