11fbeb3a8SPaul E. McKenney // SPDX-License-Identifier: GPL-2.0+ 21fbeb3a8SPaul E. McKenney // 31fbeb3a8SPaul E. McKenney // Scalability test comparing RCU vs other mechanisms 41fbeb3a8SPaul E. McKenney // for acquiring references on objects. 51fbeb3a8SPaul E. McKenney // 61fbeb3a8SPaul E. McKenney // Copyright (C) Google, 2020. 71fbeb3a8SPaul E. McKenney // 81fbeb3a8SPaul E. McKenney // Author: Joel Fernandes <joel@joelfernandes.org> 91fbeb3a8SPaul E. McKenney 101fbeb3a8SPaul E. McKenney #define pr_fmt(fmt) fmt 111fbeb3a8SPaul E. McKenney 121fbeb3a8SPaul E. McKenney #include <linux/atomic.h> 131fbeb3a8SPaul E. McKenney #include <linux/bitops.h> 141fbeb3a8SPaul E. McKenney #include <linux/completion.h> 151fbeb3a8SPaul E. McKenney #include <linux/cpu.h> 161fbeb3a8SPaul E. McKenney #include <linux/delay.h> 171fbeb3a8SPaul E. McKenney #include <linux/err.h> 181fbeb3a8SPaul E. McKenney #include <linux/init.h> 191fbeb3a8SPaul E. McKenney #include <linux/interrupt.h> 201fbeb3a8SPaul E. McKenney #include <linux/kthread.h> 211fbeb3a8SPaul E. McKenney #include <linux/kernel.h> 221fbeb3a8SPaul E. McKenney #include <linux/mm.h> 231fbeb3a8SPaul E. McKenney #include <linux/module.h> 241fbeb3a8SPaul E. McKenney #include <linux/moduleparam.h> 251fbeb3a8SPaul E. McKenney #include <linux/notifier.h> 261fbeb3a8SPaul E. McKenney #include <linux/percpu.h> 271fbeb3a8SPaul E. McKenney #include <linux/rcupdate.h> 281fbeb3a8SPaul E. McKenney #include <linux/rcupdate_trace.h> 291fbeb3a8SPaul E. McKenney #include <linux/reboot.h> 301fbeb3a8SPaul E. McKenney #include <linux/sched.h> 311fbeb3a8SPaul E. McKenney #include <linux/spinlock.h> 321fbeb3a8SPaul E. McKenney #include <linux/smp.h> 331fbeb3a8SPaul E. McKenney #include <linux/stat.h> 341fbeb3a8SPaul E. McKenney #include <linux/srcu.h> 351fbeb3a8SPaul E. McKenney #include <linux/slab.h> 361fbeb3a8SPaul E. McKenney #include <linux/torture.h> 371fbeb3a8SPaul E. McKenney #include <linux/types.h> 381fbeb3a8SPaul E. McKenney 391fbeb3a8SPaul E. McKenney #include "rcu.h" 401fbeb3a8SPaul E. McKenney 411fbeb3a8SPaul E. McKenney #define SCALE_FLAG "-ref-scale: " 421fbeb3a8SPaul E. McKenney 431fbeb3a8SPaul E. McKenney #define SCALEOUT(s, x...) \ 441fbeb3a8SPaul E. McKenney pr_alert("%s" SCALE_FLAG s, scale_type, ## x) 451fbeb3a8SPaul E. McKenney 461fbeb3a8SPaul E. McKenney #define VERBOSE_SCALEOUT(s, x...) \ 47f71f22b6SLi Zhijian do { \ 48f71f22b6SLi Zhijian if (verbose) \ 49f71f22b6SLi Zhijian pr_alert("%s" SCALE_FLAG s "\n", scale_type, ## x); \ 50f71f22b6SLi Zhijian } while (0) 511fbeb3a8SPaul E. McKenney 52e76506f0SPaul E. McKenney static atomic_t verbose_batch_ctr; 53e76506f0SPaul E. McKenney 54e76506f0SPaul E. McKenney #define VERBOSE_SCALEOUT_BATCH(s, x...) \ 55e76506f0SPaul E. McKenney do { \ 56e76506f0SPaul E. McKenney if (verbose && \ 57e76506f0SPaul E. McKenney (verbose_batched <= 0 || \ 58414c116eSPaul E. McKenney !(atomic_inc_return(&verbose_batch_ctr) % verbose_batched))) { \ 59414c116eSPaul E. McKenney schedule_timeout_uninterruptible(1); \ 60f71f22b6SLi Zhijian pr_alert("%s" SCALE_FLAG s "\n", scale_type, ## x); \ 61414c116eSPaul E. McKenney } \ 62e76506f0SPaul E. McKenney } while (0) 63e76506f0SPaul E. McKenney 64f71f22b6SLi Zhijian #define SCALEOUT_ERRSTRING(s, x...) pr_alert("%s" SCALE_FLAG "!!! " s "\n", scale_type, ## x) 651fbeb3a8SPaul E. McKenney 661fbeb3a8SPaul E. McKenney MODULE_LICENSE("GPL"); 671fbeb3a8SPaul E. McKenney MODULE_AUTHOR("Joel Fernandes (Google) <joel@joelfernandes.org>"); 681fbeb3a8SPaul E. McKenney 691fbeb3a8SPaul E. McKenney static char *scale_type = "rcu"; 701fbeb3a8SPaul E. McKenney module_param(scale_type, charp, 0444); 711fbeb3a8SPaul E. McKenney MODULE_PARM_DESC(scale_type, "Type of test (rcu, srcu, refcnt, rwsem, rwlock."); 721fbeb3a8SPaul E. McKenney 731fbeb3a8SPaul E. McKenney torture_param(int, verbose, 0, "Enable verbose debugging printk()s"); 74e76506f0SPaul E. McKenney torture_param(int, verbose_batched, 0, "Batch verbose debugging printk()s"); 751fbeb3a8SPaul E. McKenney 761fbeb3a8SPaul E. McKenney // Wait until there are multiple CPUs before starting test. 771fbeb3a8SPaul E. McKenney torture_param(int, holdoff, IS_BUILTIN(CONFIG_RCU_REF_SCALE_TEST) ? 10 : 0, 781fbeb3a8SPaul E. McKenney "Holdoff time before test start (s)"); 791fbeb3a8SPaul E. McKenney // Number of loops per experiment, all readers execute operations concurrently. 801fbeb3a8SPaul E. McKenney torture_param(long, loops, 10000, "Number of loops per experiment."); 811fbeb3a8SPaul E. McKenney // Number of readers, with -1 defaulting to about 75% of the CPUs. 821fbeb3a8SPaul E. McKenney torture_param(int, nreaders, -1, "Number of readers, -1 for 75% of CPUs."); 831fbeb3a8SPaul E. McKenney // Number of runs. 841fbeb3a8SPaul E. McKenney torture_param(int, nruns, 30, "Number of experiments to run."); 851fbeb3a8SPaul E. McKenney // Reader delay in nanoseconds, 0 for no delay. 861fbeb3a8SPaul E. McKenney torture_param(int, readdelay, 0, "Read-side delay in nanoseconds."); 871fbeb3a8SPaul E. McKenney 881fbeb3a8SPaul E. McKenney #ifdef MODULE 891fbeb3a8SPaul E. McKenney # define REFSCALE_SHUTDOWN 0 901fbeb3a8SPaul E. McKenney #else 911fbeb3a8SPaul E. McKenney # define REFSCALE_SHUTDOWN 1 921fbeb3a8SPaul E. McKenney #endif 931fbeb3a8SPaul E. McKenney 941fbeb3a8SPaul E. McKenney torture_param(bool, shutdown, REFSCALE_SHUTDOWN, 951fbeb3a8SPaul E. McKenney "Shutdown at end of scalability tests."); 961fbeb3a8SPaul E. McKenney 971fbeb3a8SPaul E. McKenney struct reader_task { 981fbeb3a8SPaul E. McKenney struct task_struct *task; 991fbeb3a8SPaul E. McKenney int start_reader; 1001fbeb3a8SPaul E. McKenney wait_queue_head_t wq; 1011fbeb3a8SPaul E. McKenney u64 last_duration_ns; 1021fbeb3a8SPaul E. McKenney }; 1031fbeb3a8SPaul E. McKenney 1041fbeb3a8SPaul E. McKenney static struct task_struct *shutdown_task; 1051fbeb3a8SPaul E. McKenney static wait_queue_head_t shutdown_wq; 1061fbeb3a8SPaul E. McKenney 1071fbeb3a8SPaul E. McKenney static struct task_struct *main_task; 1081fbeb3a8SPaul E. McKenney static wait_queue_head_t main_wq; 1091fbeb3a8SPaul E. McKenney static int shutdown_start; 1101fbeb3a8SPaul E. McKenney 1111fbeb3a8SPaul E. McKenney static struct reader_task *reader_tasks; 1121fbeb3a8SPaul E. McKenney 1131fbeb3a8SPaul E. McKenney // Number of readers that are part of the current experiment. 1141fbeb3a8SPaul E. McKenney static atomic_t nreaders_exp; 1151fbeb3a8SPaul E. McKenney 1161fbeb3a8SPaul E. McKenney // Use to wait for all threads to start. 1171fbeb3a8SPaul E. McKenney static atomic_t n_init; 1181fbeb3a8SPaul E. McKenney static atomic_t n_started; 1191fbeb3a8SPaul E. McKenney static atomic_t n_warmedup; 1201fbeb3a8SPaul E. McKenney static atomic_t n_cooleddown; 1211fbeb3a8SPaul E. McKenney 1221fbeb3a8SPaul E. McKenney // Track which experiment is currently running. 1231fbeb3a8SPaul E. McKenney static int exp_idx; 1241fbeb3a8SPaul E. McKenney 1251fbeb3a8SPaul E. McKenney // Operations vector for selecting different types of tests. 1261fbeb3a8SPaul E. McKenney struct ref_scale_ops { 127*3c6496c8SPaul E. McKenney bool (*init)(void); 1281fbeb3a8SPaul E. McKenney void (*cleanup)(void); 1291fbeb3a8SPaul E. McKenney void (*readsection)(const int nloops); 1301fbeb3a8SPaul E. McKenney void (*delaysection)(const int nloops, const int udl, const int ndl); 1311fbeb3a8SPaul E. McKenney const char *name; 1321fbeb3a8SPaul E. McKenney }; 1331fbeb3a8SPaul E. McKenney 1341fbeb3a8SPaul E. McKenney static struct ref_scale_ops *cur_ops; 1351fbeb3a8SPaul E. McKenney 1361fbeb3a8SPaul E. McKenney static void un_delay(const int udl, const int ndl) 1371fbeb3a8SPaul E. McKenney { 1381fbeb3a8SPaul E. McKenney if (udl) 1391fbeb3a8SPaul E. McKenney udelay(udl); 1401fbeb3a8SPaul E. McKenney if (ndl) 1411fbeb3a8SPaul E. McKenney ndelay(ndl); 1421fbeb3a8SPaul E. McKenney } 1431fbeb3a8SPaul E. McKenney 1441fbeb3a8SPaul E. McKenney static void ref_rcu_read_section(const int nloops) 1451fbeb3a8SPaul E. McKenney { 1461fbeb3a8SPaul E. McKenney int i; 1471fbeb3a8SPaul E. McKenney 1481fbeb3a8SPaul E. McKenney for (i = nloops; i >= 0; i--) { 1491fbeb3a8SPaul E. McKenney rcu_read_lock(); 1501fbeb3a8SPaul E. McKenney rcu_read_unlock(); 1511fbeb3a8SPaul E. McKenney } 1521fbeb3a8SPaul E. McKenney } 1531fbeb3a8SPaul E. McKenney 1541fbeb3a8SPaul E. McKenney static void ref_rcu_delay_section(const int nloops, const int udl, const int ndl) 1551fbeb3a8SPaul E. McKenney { 1561fbeb3a8SPaul E. McKenney int i; 1571fbeb3a8SPaul E. McKenney 1581fbeb3a8SPaul E. McKenney for (i = nloops; i >= 0; i--) { 1591fbeb3a8SPaul E. McKenney rcu_read_lock(); 1601fbeb3a8SPaul E. McKenney un_delay(udl, ndl); 1611fbeb3a8SPaul E. McKenney rcu_read_unlock(); 1621fbeb3a8SPaul E. McKenney } 1631fbeb3a8SPaul E. McKenney } 1641fbeb3a8SPaul E. McKenney 165*3c6496c8SPaul E. McKenney static bool rcu_sync_scale_init(void) 1661fbeb3a8SPaul E. McKenney { 167*3c6496c8SPaul E. McKenney return true; 1681fbeb3a8SPaul E. McKenney } 1691fbeb3a8SPaul E. McKenney 1701fbeb3a8SPaul E. McKenney static struct ref_scale_ops rcu_ops = { 1711fbeb3a8SPaul E. McKenney .init = rcu_sync_scale_init, 1721fbeb3a8SPaul E. McKenney .readsection = ref_rcu_read_section, 1731fbeb3a8SPaul E. McKenney .delaysection = ref_rcu_delay_section, 1741fbeb3a8SPaul E. McKenney .name = "rcu" 1751fbeb3a8SPaul E. McKenney }; 1761fbeb3a8SPaul E. McKenney 1771fbeb3a8SPaul E. McKenney // Definitions for SRCU ref scale testing. 1781fbeb3a8SPaul E. McKenney DEFINE_STATIC_SRCU(srcu_refctl_scale); 1791fbeb3a8SPaul E. McKenney static struct srcu_struct *srcu_ctlp = &srcu_refctl_scale; 1801fbeb3a8SPaul E. McKenney 1811fbeb3a8SPaul E. McKenney static void srcu_ref_scale_read_section(const int nloops) 1821fbeb3a8SPaul E. McKenney { 1831fbeb3a8SPaul E. McKenney int i; 1841fbeb3a8SPaul E. McKenney int idx; 1851fbeb3a8SPaul E. McKenney 1861fbeb3a8SPaul E. McKenney for (i = nloops; i >= 0; i--) { 1871fbeb3a8SPaul E. McKenney idx = srcu_read_lock(srcu_ctlp); 1881fbeb3a8SPaul E. McKenney srcu_read_unlock(srcu_ctlp, idx); 1891fbeb3a8SPaul E. McKenney } 1901fbeb3a8SPaul E. McKenney } 1911fbeb3a8SPaul E. McKenney 1921fbeb3a8SPaul E. McKenney static void srcu_ref_scale_delay_section(const int nloops, const int udl, const int ndl) 1931fbeb3a8SPaul E. McKenney { 1941fbeb3a8SPaul E. McKenney int i; 1951fbeb3a8SPaul E. McKenney int idx; 1961fbeb3a8SPaul E. McKenney 1971fbeb3a8SPaul E. McKenney for (i = nloops; i >= 0; i--) { 1981fbeb3a8SPaul E. McKenney idx = srcu_read_lock(srcu_ctlp); 1991fbeb3a8SPaul E. McKenney un_delay(udl, ndl); 2001fbeb3a8SPaul E. McKenney srcu_read_unlock(srcu_ctlp, idx); 2011fbeb3a8SPaul E. McKenney } 2021fbeb3a8SPaul E. McKenney } 2031fbeb3a8SPaul E. McKenney 2041fbeb3a8SPaul E. McKenney static struct ref_scale_ops srcu_ops = { 2051fbeb3a8SPaul E. McKenney .init = rcu_sync_scale_init, 2061fbeb3a8SPaul E. McKenney .readsection = srcu_ref_scale_read_section, 2071fbeb3a8SPaul E. McKenney .delaysection = srcu_ref_scale_delay_section, 2081fbeb3a8SPaul E. McKenney .name = "srcu" 2091fbeb3a8SPaul E. McKenney }; 2101fbeb3a8SPaul E. McKenney 2115f654af1SPaul E. McKenney #ifdef CONFIG_TASKS_RCU 2125f654af1SPaul E. McKenney 2131fbeb3a8SPaul E. McKenney // Definitions for RCU Tasks ref scale testing: Empty read markers. 2141fbeb3a8SPaul E. McKenney // These definitions also work for RCU Rude readers. 2151fbeb3a8SPaul E. McKenney static void rcu_tasks_ref_scale_read_section(const int nloops) 2161fbeb3a8SPaul E. McKenney { 2171fbeb3a8SPaul E. McKenney int i; 2181fbeb3a8SPaul E. McKenney 2191fbeb3a8SPaul E. McKenney for (i = nloops; i >= 0; i--) 2201fbeb3a8SPaul E. McKenney continue; 2211fbeb3a8SPaul E. McKenney } 2221fbeb3a8SPaul E. McKenney 2231fbeb3a8SPaul E. McKenney static void rcu_tasks_ref_scale_delay_section(const int nloops, const int udl, const int ndl) 2241fbeb3a8SPaul E. McKenney { 2251fbeb3a8SPaul E. McKenney int i; 2261fbeb3a8SPaul E. McKenney 2271fbeb3a8SPaul E. McKenney for (i = nloops; i >= 0; i--) 2281fbeb3a8SPaul E. McKenney un_delay(udl, ndl); 2291fbeb3a8SPaul E. McKenney } 2301fbeb3a8SPaul E. McKenney 2311fbeb3a8SPaul E. McKenney static struct ref_scale_ops rcu_tasks_ops = { 2321fbeb3a8SPaul E. McKenney .init = rcu_sync_scale_init, 2331fbeb3a8SPaul E. McKenney .readsection = rcu_tasks_ref_scale_read_section, 2341fbeb3a8SPaul E. McKenney .delaysection = rcu_tasks_ref_scale_delay_section, 2351fbeb3a8SPaul E. McKenney .name = "rcu-tasks" 2361fbeb3a8SPaul E. McKenney }; 2371fbeb3a8SPaul E. McKenney 2385f654af1SPaul E. McKenney #define RCU_TASKS_OPS &rcu_tasks_ops, 2395f654af1SPaul E. McKenney 2405f654af1SPaul E. McKenney #else // #ifdef CONFIG_TASKS_RCU 2415f654af1SPaul E. McKenney 2425f654af1SPaul E. McKenney #define RCU_TASKS_OPS 2435f654af1SPaul E. McKenney 2445f654af1SPaul E. McKenney #endif // #else // #ifdef CONFIG_TASKS_RCU 2455f654af1SPaul E. McKenney 246dec86781SPaul E. McKenney #ifdef CONFIG_TASKS_TRACE_RCU 247dec86781SPaul E. McKenney 2481fbeb3a8SPaul E. McKenney // Definitions for RCU Tasks Trace ref scale testing. 2491fbeb3a8SPaul E. McKenney static void rcu_trace_ref_scale_read_section(const int nloops) 2501fbeb3a8SPaul E. McKenney { 2511fbeb3a8SPaul E. McKenney int i; 2521fbeb3a8SPaul E. McKenney 2531fbeb3a8SPaul E. McKenney for (i = nloops; i >= 0; i--) { 2541fbeb3a8SPaul E. McKenney rcu_read_lock_trace(); 2551fbeb3a8SPaul E. McKenney rcu_read_unlock_trace(); 2561fbeb3a8SPaul E. McKenney } 2571fbeb3a8SPaul E. McKenney } 2581fbeb3a8SPaul E. McKenney 2591fbeb3a8SPaul E. McKenney static void rcu_trace_ref_scale_delay_section(const int nloops, const int udl, const int ndl) 2601fbeb3a8SPaul E. McKenney { 2611fbeb3a8SPaul E. McKenney int i; 2621fbeb3a8SPaul E. McKenney 2631fbeb3a8SPaul E. McKenney for (i = nloops; i >= 0; i--) { 2641fbeb3a8SPaul E. McKenney rcu_read_lock_trace(); 2651fbeb3a8SPaul E. McKenney un_delay(udl, ndl); 2661fbeb3a8SPaul E. McKenney rcu_read_unlock_trace(); 2671fbeb3a8SPaul E. McKenney } 2681fbeb3a8SPaul E. McKenney } 2691fbeb3a8SPaul E. McKenney 2701fbeb3a8SPaul E. McKenney static struct ref_scale_ops rcu_trace_ops = { 2711fbeb3a8SPaul E. McKenney .init = rcu_sync_scale_init, 2721fbeb3a8SPaul E. McKenney .readsection = rcu_trace_ref_scale_read_section, 2731fbeb3a8SPaul E. McKenney .delaysection = rcu_trace_ref_scale_delay_section, 2741fbeb3a8SPaul E. McKenney .name = "rcu-trace" 2751fbeb3a8SPaul E. McKenney }; 2761fbeb3a8SPaul E. McKenney 277dec86781SPaul E. McKenney #define RCU_TRACE_OPS &rcu_trace_ops, 278dec86781SPaul E. McKenney 279dec86781SPaul E. McKenney #else // #ifdef CONFIG_TASKS_TRACE_RCU 280dec86781SPaul E. McKenney 281dec86781SPaul E. McKenney #define RCU_TRACE_OPS 282dec86781SPaul E. McKenney 283dec86781SPaul E. McKenney #endif // #else // #ifdef CONFIG_TASKS_TRACE_RCU 284dec86781SPaul E. McKenney 2851fbeb3a8SPaul E. McKenney // Definitions for reference count 2861fbeb3a8SPaul E. McKenney static atomic_t refcnt; 2871fbeb3a8SPaul E. McKenney 2881fbeb3a8SPaul E. McKenney static void ref_refcnt_section(const int nloops) 2891fbeb3a8SPaul E. McKenney { 2901fbeb3a8SPaul E. McKenney int i; 2911fbeb3a8SPaul E. McKenney 2921fbeb3a8SPaul E. McKenney for (i = nloops; i >= 0; i--) { 2931fbeb3a8SPaul E. McKenney atomic_inc(&refcnt); 2941fbeb3a8SPaul E. McKenney atomic_dec(&refcnt); 2951fbeb3a8SPaul E. McKenney } 2961fbeb3a8SPaul E. McKenney } 2971fbeb3a8SPaul E. McKenney 2981fbeb3a8SPaul E. McKenney static void ref_refcnt_delay_section(const int nloops, const int udl, const int ndl) 2991fbeb3a8SPaul E. McKenney { 3001fbeb3a8SPaul E. McKenney int i; 3011fbeb3a8SPaul E. McKenney 3021fbeb3a8SPaul E. McKenney for (i = nloops; i >= 0; i--) { 3031fbeb3a8SPaul E. McKenney atomic_inc(&refcnt); 3041fbeb3a8SPaul E. McKenney un_delay(udl, ndl); 3051fbeb3a8SPaul E. McKenney atomic_dec(&refcnt); 3061fbeb3a8SPaul E. McKenney } 3071fbeb3a8SPaul E. McKenney } 3081fbeb3a8SPaul E. McKenney 3091fbeb3a8SPaul E. McKenney static struct ref_scale_ops refcnt_ops = { 3101fbeb3a8SPaul E. McKenney .init = rcu_sync_scale_init, 3111fbeb3a8SPaul E. McKenney .readsection = ref_refcnt_section, 3121fbeb3a8SPaul E. McKenney .delaysection = ref_refcnt_delay_section, 3131fbeb3a8SPaul E. McKenney .name = "refcnt" 3141fbeb3a8SPaul E. McKenney }; 3151fbeb3a8SPaul E. McKenney 3161fbeb3a8SPaul E. McKenney // Definitions for rwlock 3171fbeb3a8SPaul E. McKenney static rwlock_t test_rwlock; 3181fbeb3a8SPaul E. McKenney 319*3c6496c8SPaul E. McKenney static bool ref_rwlock_init(void) 3201fbeb3a8SPaul E. McKenney { 3211fbeb3a8SPaul E. McKenney rwlock_init(&test_rwlock); 322*3c6496c8SPaul E. McKenney return true; 3231fbeb3a8SPaul E. McKenney } 3241fbeb3a8SPaul E. McKenney 3251fbeb3a8SPaul E. McKenney static void ref_rwlock_section(const int nloops) 3261fbeb3a8SPaul E. McKenney { 3271fbeb3a8SPaul E. McKenney int i; 3281fbeb3a8SPaul E. McKenney 3291fbeb3a8SPaul E. McKenney for (i = nloops; i >= 0; i--) { 3301fbeb3a8SPaul E. McKenney read_lock(&test_rwlock); 3311fbeb3a8SPaul E. McKenney read_unlock(&test_rwlock); 3321fbeb3a8SPaul E. McKenney } 3331fbeb3a8SPaul E. McKenney } 3341fbeb3a8SPaul E. McKenney 3351fbeb3a8SPaul E. McKenney static void ref_rwlock_delay_section(const int nloops, const int udl, const int ndl) 3361fbeb3a8SPaul E. McKenney { 3371fbeb3a8SPaul E. McKenney int i; 3381fbeb3a8SPaul E. McKenney 3391fbeb3a8SPaul E. McKenney for (i = nloops; i >= 0; i--) { 3401fbeb3a8SPaul E. McKenney read_lock(&test_rwlock); 3411fbeb3a8SPaul E. McKenney un_delay(udl, ndl); 3421fbeb3a8SPaul E. McKenney read_unlock(&test_rwlock); 3431fbeb3a8SPaul E. McKenney } 3441fbeb3a8SPaul E. McKenney } 3451fbeb3a8SPaul E. McKenney 3461fbeb3a8SPaul E. McKenney static struct ref_scale_ops rwlock_ops = { 3471fbeb3a8SPaul E. McKenney .init = ref_rwlock_init, 3481fbeb3a8SPaul E. McKenney .readsection = ref_rwlock_section, 3491fbeb3a8SPaul E. McKenney .delaysection = ref_rwlock_delay_section, 3501fbeb3a8SPaul E. McKenney .name = "rwlock" 3511fbeb3a8SPaul E. McKenney }; 3521fbeb3a8SPaul E. McKenney 3531fbeb3a8SPaul E. McKenney // Definitions for rwsem 3541fbeb3a8SPaul E. McKenney static struct rw_semaphore test_rwsem; 3551fbeb3a8SPaul E. McKenney 356*3c6496c8SPaul E. McKenney static bool ref_rwsem_init(void) 3571fbeb3a8SPaul E. McKenney { 3581fbeb3a8SPaul E. McKenney init_rwsem(&test_rwsem); 359*3c6496c8SPaul E. McKenney return true; 3601fbeb3a8SPaul E. McKenney } 3611fbeb3a8SPaul E. McKenney 3621fbeb3a8SPaul E. McKenney static void ref_rwsem_section(const int nloops) 3631fbeb3a8SPaul E. McKenney { 3641fbeb3a8SPaul E. McKenney int i; 3651fbeb3a8SPaul E. McKenney 3661fbeb3a8SPaul E. McKenney for (i = nloops; i >= 0; i--) { 3671fbeb3a8SPaul E. McKenney down_read(&test_rwsem); 3681fbeb3a8SPaul E. McKenney up_read(&test_rwsem); 3691fbeb3a8SPaul E. McKenney } 3701fbeb3a8SPaul E. McKenney } 3711fbeb3a8SPaul E. McKenney 3721fbeb3a8SPaul E. McKenney static void ref_rwsem_delay_section(const int nloops, const int udl, const int ndl) 3731fbeb3a8SPaul E. McKenney { 3741fbeb3a8SPaul E. McKenney int i; 3751fbeb3a8SPaul E. McKenney 3761fbeb3a8SPaul E. McKenney for (i = nloops; i >= 0; i--) { 3771fbeb3a8SPaul E. McKenney down_read(&test_rwsem); 3781fbeb3a8SPaul E. McKenney un_delay(udl, ndl); 3791fbeb3a8SPaul E. McKenney up_read(&test_rwsem); 3801fbeb3a8SPaul E. McKenney } 3811fbeb3a8SPaul E. McKenney } 3821fbeb3a8SPaul E. McKenney 3831fbeb3a8SPaul E. McKenney static struct ref_scale_ops rwsem_ops = { 3841fbeb3a8SPaul E. McKenney .init = ref_rwsem_init, 3851fbeb3a8SPaul E. McKenney .readsection = ref_rwsem_section, 3861fbeb3a8SPaul E. McKenney .delaysection = ref_rwsem_delay_section, 3871fbeb3a8SPaul E. McKenney .name = "rwsem" 3881fbeb3a8SPaul E. McKenney }; 3891fbeb3a8SPaul E. McKenney 390e9b800dbSPaul E. McKenney // Definitions for global spinlock 3917bf336fbSZqiang static DEFINE_RAW_SPINLOCK(test_lock); 392e9b800dbSPaul E. McKenney 393e9b800dbSPaul E. McKenney static void ref_lock_section(const int nloops) 394e9b800dbSPaul E. McKenney { 395e9b800dbSPaul E. McKenney int i; 396e9b800dbSPaul E. McKenney 397e9b800dbSPaul E. McKenney preempt_disable(); 398e9b800dbSPaul E. McKenney for (i = nloops; i >= 0; i--) { 3997bf336fbSZqiang raw_spin_lock(&test_lock); 4007bf336fbSZqiang raw_spin_unlock(&test_lock); 401e9b800dbSPaul E. McKenney } 402e9b800dbSPaul E. McKenney preempt_enable(); 403e9b800dbSPaul E. McKenney } 404e9b800dbSPaul E. McKenney 405e9b800dbSPaul E. McKenney static void ref_lock_delay_section(const int nloops, const int udl, const int ndl) 406e9b800dbSPaul E. McKenney { 407e9b800dbSPaul E. McKenney int i; 408e9b800dbSPaul E. McKenney 409e9b800dbSPaul E. McKenney preempt_disable(); 410e9b800dbSPaul E. McKenney for (i = nloops; i >= 0; i--) { 4117bf336fbSZqiang raw_spin_lock(&test_lock); 412e9b800dbSPaul E. McKenney un_delay(udl, ndl); 4137bf336fbSZqiang raw_spin_unlock(&test_lock); 414e9b800dbSPaul E. McKenney } 415e9b800dbSPaul E. McKenney preempt_enable(); 416e9b800dbSPaul E. McKenney } 417e9b800dbSPaul E. McKenney 418e9b800dbSPaul E. McKenney static struct ref_scale_ops lock_ops = { 419e9b800dbSPaul E. McKenney .readsection = ref_lock_section, 420e9b800dbSPaul E. McKenney .delaysection = ref_lock_delay_section, 421e9b800dbSPaul E. McKenney .name = "lock" 422e9b800dbSPaul E. McKenney }; 423e9b800dbSPaul E. McKenney 424e9b800dbSPaul E. McKenney // Definitions for global irq-save spinlock 425e9b800dbSPaul E. McKenney 426e9b800dbSPaul E. McKenney static void ref_lock_irq_section(const int nloops) 427e9b800dbSPaul E. McKenney { 428e9b800dbSPaul E. McKenney unsigned long flags; 429e9b800dbSPaul E. McKenney int i; 430e9b800dbSPaul E. McKenney 431e9b800dbSPaul E. McKenney preempt_disable(); 432e9b800dbSPaul E. McKenney for (i = nloops; i >= 0; i--) { 4337bf336fbSZqiang raw_spin_lock_irqsave(&test_lock, flags); 4347bf336fbSZqiang raw_spin_unlock_irqrestore(&test_lock, flags); 435e9b800dbSPaul E. McKenney } 436e9b800dbSPaul E. McKenney preempt_enable(); 437e9b800dbSPaul E. McKenney } 438e9b800dbSPaul E. McKenney 439e9b800dbSPaul E. McKenney static void ref_lock_irq_delay_section(const int nloops, const int udl, const int ndl) 440e9b800dbSPaul E. McKenney { 441e9b800dbSPaul E. McKenney unsigned long flags; 442e9b800dbSPaul E. McKenney int i; 443e9b800dbSPaul E. McKenney 444e9b800dbSPaul E. McKenney preempt_disable(); 445e9b800dbSPaul E. McKenney for (i = nloops; i >= 0; i--) { 4467bf336fbSZqiang raw_spin_lock_irqsave(&test_lock, flags); 447e9b800dbSPaul E. McKenney un_delay(udl, ndl); 4487bf336fbSZqiang raw_spin_unlock_irqrestore(&test_lock, flags); 449e9b800dbSPaul E. McKenney } 450e9b800dbSPaul E. McKenney preempt_enable(); 451e9b800dbSPaul E. McKenney } 452e9b800dbSPaul E. McKenney 453e9b800dbSPaul E. McKenney static struct ref_scale_ops lock_irq_ops = { 454e9b800dbSPaul E. McKenney .readsection = ref_lock_irq_section, 455e9b800dbSPaul E. McKenney .delaysection = ref_lock_irq_delay_section, 456e9b800dbSPaul E. McKenney .name = "lock-irq" 457e9b800dbSPaul E. McKenney }; 458e9b800dbSPaul E. McKenney 459e9b800dbSPaul E. McKenney // Definitions acquire-release. 460e9b800dbSPaul E. McKenney static DEFINE_PER_CPU(unsigned long, test_acqrel); 461e9b800dbSPaul E. McKenney 462e9b800dbSPaul E. McKenney static void ref_acqrel_section(const int nloops) 463e9b800dbSPaul E. McKenney { 464e9b800dbSPaul E. McKenney unsigned long x; 465e9b800dbSPaul E. McKenney int i; 466e9b800dbSPaul E. McKenney 467e9b800dbSPaul E. McKenney preempt_disable(); 468e9b800dbSPaul E. McKenney for (i = nloops; i >= 0; i--) { 469e9b800dbSPaul E. McKenney x = smp_load_acquire(this_cpu_ptr(&test_acqrel)); 470e9b800dbSPaul E. McKenney smp_store_release(this_cpu_ptr(&test_acqrel), x + 1); 471e9b800dbSPaul E. McKenney } 472e9b800dbSPaul E. McKenney preempt_enable(); 473e9b800dbSPaul E. McKenney } 474e9b800dbSPaul E. McKenney 475e9b800dbSPaul E. McKenney static void ref_acqrel_delay_section(const int nloops, const int udl, const int ndl) 476e9b800dbSPaul E. McKenney { 477e9b800dbSPaul E. McKenney unsigned long x; 478e9b800dbSPaul E. McKenney int i; 479e9b800dbSPaul E. McKenney 480e9b800dbSPaul E. McKenney preempt_disable(); 481e9b800dbSPaul E. McKenney for (i = nloops; i >= 0; i--) { 482e9b800dbSPaul E. McKenney x = smp_load_acquire(this_cpu_ptr(&test_acqrel)); 483e9b800dbSPaul E. McKenney un_delay(udl, ndl); 484e9b800dbSPaul E. McKenney smp_store_release(this_cpu_ptr(&test_acqrel), x + 1); 485e9b800dbSPaul E. McKenney } 486e9b800dbSPaul E. McKenney preempt_enable(); 487e9b800dbSPaul E. McKenney } 488e9b800dbSPaul E. McKenney 489e9b800dbSPaul E. McKenney static struct ref_scale_ops acqrel_ops = { 490e9b800dbSPaul E. McKenney .readsection = ref_acqrel_section, 491e9b800dbSPaul E. McKenney .delaysection = ref_acqrel_delay_section, 492e9b800dbSPaul E. McKenney .name = "acqrel" 493e9b800dbSPaul E. McKenney }; 494e9b800dbSPaul E. McKenney 49525f6fa53SPaul E. McKenney static volatile u64 stopopts; 49625f6fa53SPaul E. McKenney 49725f6fa53SPaul E. McKenney static void ref_clock_section(const int nloops) 49825f6fa53SPaul E. McKenney { 49925f6fa53SPaul E. McKenney u64 x = 0; 50025f6fa53SPaul E. McKenney int i; 50125f6fa53SPaul E. McKenney 50225f6fa53SPaul E. McKenney preempt_disable(); 50325f6fa53SPaul E. McKenney for (i = nloops; i >= 0; i--) 50425f6fa53SPaul E. McKenney x += ktime_get_real_fast_ns(); 50525f6fa53SPaul E. McKenney preempt_enable(); 50625f6fa53SPaul E. McKenney stopopts = x; 50725f6fa53SPaul E. McKenney } 50825f6fa53SPaul E. McKenney 50925f6fa53SPaul E. McKenney static void ref_clock_delay_section(const int nloops, const int udl, const int ndl) 51025f6fa53SPaul E. McKenney { 51125f6fa53SPaul E. McKenney u64 x = 0; 51225f6fa53SPaul E. McKenney int i; 51325f6fa53SPaul E. McKenney 51425f6fa53SPaul E. McKenney preempt_disable(); 51525f6fa53SPaul E. McKenney for (i = nloops; i >= 0; i--) { 51625f6fa53SPaul E. McKenney x += ktime_get_real_fast_ns(); 51725f6fa53SPaul E. McKenney un_delay(udl, ndl); 51825f6fa53SPaul E. McKenney } 51925f6fa53SPaul E. McKenney preempt_enable(); 52025f6fa53SPaul E. McKenney stopopts = x; 52125f6fa53SPaul E. McKenney } 52225f6fa53SPaul E. McKenney 52325f6fa53SPaul E. McKenney static struct ref_scale_ops clock_ops = { 52425f6fa53SPaul E. McKenney .readsection = ref_clock_section, 52525f6fa53SPaul E. McKenney .delaysection = ref_clock_delay_section, 52625f6fa53SPaul E. McKenney .name = "clock" 52725f6fa53SPaul E. McKenney }; 52825f6fa53SPaul E. McKenney 5291fbeb3a8SPaul E. McKenney static void rcu_scale_one_reader(void) 5301fbeb3a8SPaul E. McKenney { 5311fbeb3a8SPaul E. McKenney if (readdelay <= 0) 5321fbeb3a8SPaul E. McKenney cur_ops->readsection(loops); 5331fbeb3a8SPaul E. McKenney else 5341fbeb3a8SPaul E. McKenney cur_ops->delaysection(loops, readdelay / 1000, readdelay % 1000); 5351fbeb3a8SPaul E. McKenney } 5361fbeb3a8SPaul E. McKenney 5371fbeb3a8SPaul E. McKenney // Reader kthread. Repeatedly does empty RCU read-side 5381fbeb3a8SPaul E. McKenney // critical section, minimizing update-side interference. 5391fbeb3a8SPaul E. McKenney static int 5401fbeb3a8SPaul E. McKenney ref_scale_reader(void *arg) 5411fbeb3a8SPaul E. McKenney { 5421fbeb3a8SPaul E. McKenney unsigned long flags; 5431fbeb3a8SPaul E. McKenney long me = (long)arg; 5441fbeb3a8SPaul E. McKenney struct reader_task *rt = &(reader_tasks[me]); 5451fbeb3a8SPaul E. McKenney u64 start; 5461fbeb3a8SPaul E. McKenney s64 duration; 5471fbeb3a8SPaul E. McKenney 548e76506f0SPaul E. McKenney VERBOSE_SCALEOUT_BATCH("ref_scale_reader %ld: task started", me); 54905bc276cSPaul E. McKenney WARN_ON_ONCE(set_cpus_allowed_ptr(current, cpumask_of(me % nr_cpu_ids))); 5501fbeb3a8SPaul E. McKenney set_user_nice(current, MAX_NICE); 5511fbeb3a8SPaul E. McKenney atomic_inc(&n_init); 5521fbeb3a8SPaul E. McKenney if (holdoff) 5531fbeb3a8SPaul E. McKenney schedule_timeout_interruptible(holdoff * HZ); 5541fbeb3a8SPaul E. McKenney repeat: 55505bc276cSPaul E. McKenney VERBOSE_SCALEOUT_BATCH("ref_scale_reader %ld: waiting to start next experiment on cpu %d", me, raw_smp_processor_id()); 5561fbeb3a8SPaul E. McKenney 5571fbeb3a8SPaul E. McKenney // Wait for signal that this reader can start. 5581fbeb3a8SPaul E. McKenney wait_event(rt->wq, (atomic_read(&nreaders_exp) && smp_load_acquire(&rt->start_reader)) || 5591fbeb3a8SPaul E. McKenney torture_must_stop()); 5601fbeb3a8SPaul E. McKenney 5611fbeb3a8SPaul E. McKenney if (torture_must_stop()) 5621fbeb3a8SPaul E. McKenney goto end; 5631fbeb3a8SPaul E. McKenney 5641fbeb3a8SPaul E. McKenney // Make sure that the CPU is affinitized appropriately during testing. 56505bc276cSPaul E. McKenney WARN_ON_ONCE(raw_smp_processor_id() != me); 5661fbeb3a8SPaul E. McKenney 5671fbeb3a8SPaul E. McKenney WRITE_ONCE(rt->start_reader, 0); 5681fbeb3a8SPaul E. McKenney if (!atomic_dec_return(&n_started)) 5691fbeb3a8SPaul E. McKenney while (atomic_read_acquire(&n_started)) 5701fbeb3a8SPaul E. McKenney cpu_relax(); 5711fbeb3a8SPaul E. McKenney 572e76506f0SPaul E. McKenney VERBOSE_SCALEOUT_BATCH("ref_scale_reader %ld: experiment %d started", me, exp_idx); 5731fbeb3a8SPaul E. McKenney 5741fbeb3a8SPaul E. McKenney 5751fbeb3a8SPaul E. McKenney // To reduce noise, do an initial cache-warming invocation, check 5761fbeb3a8SPaul E. McKenney // in, and then keep warming until everyone has checked in. 5771fbeb3a8SPaul E. McKenney rcu_scale_one_reader(); 5781fbeb3a8SPaul E. McKenney if (!atomic_dec_return(&n_warmedup)) 5791fbeb3a8SPaul E. McKenney while (atomic_read_acquire(&n_warmedup)) 5801fbeb3a8SPaul E. McKenney rcu_scale_one_reader(); 5811fbeb3a8SPaul E. McKenney // Also keep interrupts disabled. This also has the effect 5821fbeb3a8SPaul E. McKenney // of preventing entries into slow path for rcu_read_unlock(). 5831fbeb3a8SPaul E. McKenney local_irq_save(flags); 5841fbeb3a8SPaul E. McKenney start = ktime_get_mono_fast_ns(); 5851fbeb3a8SPaul E. McKenney 5861fbeb3a8SPaul E. McKenney rcu_scale_one_reader(); 5871fbeb3a8SPaul E. McKenney 5881fbeb3a8SPaul E. McKenney duration = ktime_get_mono_fast_ns() - start; 5891fbeb3a8SPaul E. McKenney local_irq_restore(flags); 5901fbeb3a8SPaul E. McKenney 5911fbeb3a8SPaul E. McKenney rt->last_duration_ns = WARN_ON_ONCE(duration < 0) ? 0 : duration; 5921fbeb3a8SPaul E. McKenney // To reduce runtime-skew noise, do maintain-load invocations until 5931fbeb3a8SPaul E. McKenney // everyone is done. 5941fbeb3a8SPaul E. McKenney if (!atomic_dec_return(&n_cooleddown)) 5951fbeb3a8SPaul E. McKenney while (atomic_read_acquire(&n_cooleddown)) 5961fbeb3a8SPaul E. McKenney rcu_scale_one_reader(); 5971fbeb3a8SPaul E. McKenney 5981fbeb3a8SPaul E. McKenney if (atomic_dec_and_test(&nreaders_exp)) 5991fbeb3a8SPaul E. McKenney wake_up(&main_wq); 6001fbeb3a8SPaul E. McKenney 601e76506f0SPaul E. McKenney VERBOSE_SCALEOUT_BATCH("ref_scale_reader %ld: experiment %d ended, (readers remaining=%d)", 6021fbeb3a8SPaul E. McKenney me, exp_idx, atomic_read(&nreaders_exp)); 6031fbeb3a8SPaul E. McKenney 6041fbeb3a8SPaul E. McKenney if (!torture_must_stop()) 6051fbeb3a8SPaul E. McKenney goto repeat; 6061fbeb3a8SPaul E. McKenney end: 6071fbeb3a8SPaul E. McKenney torture_kthread_stopping("ref_scale_reader"); 6081fbeb3a8SPaul E. McKenney return 0; 6091fbeb3a8SPaul E. McKenney } 6101fbeb3a8SPaul E. McKenney 6111fbeb3a8SPaul E. McKenney static void reset_readers(void) 6121fbeb3a8SPaul E. McKenney { 6131fbeb3a8SPaul E. McKenney int i; 6141fbeb3a8SPaul E. McKenney struct reader_task *rt; 6151fbeb3a8SPaul E. McKenney 6161fbeb3a8SPaul E. McKenney for (i = 0; i < nreaders; i++) { 6171fbeb3a8SPaul E. McKenney rt = &(reader_tasks[i]); 6181fbeb3a8SPaul E. McKenney 6191fbeb3a8SPaul E. McKenney rt->last_duration_ns = 0; 6201fbeb3a8SPaul E. McKenney } 6211fbeb3a8SPaul E. McKenney } 6221fbeb3a8SPaul E. McKenney 6231fbeb3a8SPaul E. McKenney // Print the results of each reader and return the sum of all their durations. 6241fbeb3a8SPaul E. McKenney static u64 process_durations(int n) 6251fbeb3a8SPaul E. McKenney { 6261fbeb3a8SPaul E. McKenney int i; 6271fbeb3a8SPaul E. McKenney struct reader_task *rt; 6281fbeb3a8SPaul E. McKenney char buf1[64]; 6291fbeb3a8SPaul E. McKenney char *buf; 6301fbeb3a8SPaul E. McKenney u64 sum = 0; 6311fbeb3a8SPaul E. McKenney 6329880eb87SLi Zhijian buf = kmalloc(800 + 64, GFP_KERNEL); 6331fbeb3a8SPaul E. McKenney if (!buf) 6341fbeb3a8SPaul E. McKenney return 0; 6351fbeb3a8SPaul E. McKenney buf[0] = 0; 6361fbeb3a8SPaul E. McKenney sprintf(buf, "Experiment #%d (Format: <THREAD-NUM>:<Total loop time in ns>)", 6371fbeb3a8SPaul E. McKenney exp_idx); 6381fbeb3a8SPaul E. McKenney 6391fbeb3a8SPaul E. McKenney for (i = 0; i < n && !torture_must_stop(); i++) { 6401fbeb3a8SPaul E. McKenney rt = &(reader_tasks[i]); 6411fbeb3a8SPaul E. McKenney sprintf(buf1, "%d: %llu\t", i, rt->last_duration_ns); 6421fbeb3a8SPaul E. McKenney 6431fbeb3a8SPaul E. McKenney if (i % 5 == 0) 6441fbeb3a8SPaul E. McKenney strcat(buf, "\n"); 6459880eb87SLi Zhijian if (strlen(buf) >= 800) { 6469880eb87SLi Zhijian pr_alert("%s", buf); 6479880eb87SLi Zhijian buf[0] = 0; 6489880eb87SLi Zhijian } 6491fbeb3a8SPaul E. McKenney strcat(buf, buf1); 6501fbeb3a8SPaul E. McKenney 6511fbeb3a8SPaul E. McKenney sum += rt->last_duration_ns; 6521fbeb3a8SPaul E. McKenney } 6539880eb87SLi Zhijian pr_alert("%s\n", buf); 6541fbeb3a8SPaul E. McKenney 6551fbeb3a8SPaul E. McKenney kfree(buf); 6561fbeb3a8SPaul E. McKenney return sum; 6571fbeb3a8SPaul E. McKenney } 6581fbeb3a8SPaul E. McKenney 6591fbeb3a8SPaul E. McKenney // The main_func is the main orchestrator, it performs a bunch of 6601fbeb3a8SPaul E. McKenney // experiments. For every experiment, it orders all the readers 6611fbeb3a8SPaul E. McKenney // involved to start and waits for them to finish the experiment. It 6621fbeb3a8SPaul E. McKenney // then reads their timestamps and starts the next experiment. Each 6631fbeb3a8SPaul E. McKenney // experiment progresses from 1 concurrent reader to N of them at which 6641fbeb3a8SPaul E. McKenney // point all the timestamps are printed. 6651fbeb3a8SPaul E. McKenney static int main_func(void *arg) 6661fbeb3a8SPaul E. McKenney { 6671fbeb3a8SPaul E. McKenney int exp, r; 6681fbeb3a8SPaul E. McKenney char buf1[64]; 6691fbeb3a8SPaul E. McKenney char *buf; 6701fbeb3a8SPaul E. McKenney u64 *result_avg; 6711fbeb3a8SPaul E. McKenney 6721fbeb3a8SPaul E. McKenney set_cpus_allowed_ptr(current, cpumask_of(nreaders % nr_cpu_ids)); 6731fbeb3a8SPaul E. McKenney set_user_nice(current, MAX_NICE); 6741fbeb3a8SPaul E. McKenney 6751fbeb3a8SPaul E. McKenney VERBOSE_SCALEOUT("main_func task started"); 6761fbeb3a8SPaul E. McKenney result_avg = kzalloc(nruns * sizeof(*result_avg), GFP_KERNEL); 6779880eb87SLi Zhijian buf = kzalloc(800 + 64, GFP_KERNEL); 6781fbeb3a8SPaul E. McKenney if (!result_avg || !buf) { 6794feeb9d5SLi Zhijian SCALEOUT_ERRSTRING("out of memory"); 680c30c8763SLi Zhijian goto oom_exit; 6811fbeb3a8SPaul E. McKenney } 6821fbeb3a8SPaul E. McKenney if (holdoff) 6831fbeb3a8SPaul E. McKenney schedule_timeout_interruptible(holdoff * HZ); 6841fbeb3a8SPaul E. McKenney 6851fbeb3a8SPaul E. McKenney // Wait for all threads to start. 6861fbeb3a8SPaul E. McKenney atomic_inc(&n_init); 6871fbeb3a8SPaul E. McKenney while (atomic_read(&n_init) < nreaders + 1) 6881fbeb3a8SPaul E. McKenney schedule_timeout_uninterruptible(1); 6891fbeb3a8SPaul E. McKenney 6901fbeb3a8SPaul E. McKenney // Start exp readers up per experiment 6911fbeb3a8SPaul E. McKenney for (exp = 0; exp < nruns && !torture_must_stop(); exp++) { 6921fbeb3a8SPaul E. McKenney if (torture_must_stop()) 6931fbeb3a8SPaul E. McKenney goto end; 6941fbeb3a8SPaul E. McKenney 6951fbeb3a8SPaul E. McKenney reset_readers(); 6961fbeb3a8SPaul E. McKenney atomic_set(&nreaders_exp, nreaders); 6971fbeb3a8SPaul E. McKenney atomic_set(&n_started, nreaders); 6981fbeb3a8SPaul E. McKenney atomic_set(&n_warmedup, nreaders); 6991fbeb3a8SPaul E. McKenney atomic_set(&n_cooleddown, nreaders); 7001fbeb3a8SPaul E. McKenney 7011fbeb3a8SPaul E. McKenney exp_idx = exp; 7021fbeb3a8SPaul E. McKenney 7031fbeb3a8SPaul E. McKenney for (r = 0; r < nreaders; r++) { 7041fbeb3a8SPaul E. McKenney smp_store_release(&reader_tasks[r].start_reader, 1); 7051fbeb3a8SPaul E. McKenney wake_up(&reader_tasks[r].wq); 7061fbeb3a8SPaul E. McKenney } 7071fbeb3a8SPaul E. McKenney 7081fbeb3a8SPaul E. McKenney VERBOSE_SCALEOUT("main_func: experiment started, waiting for %d readers", 7091fbeb3a8SPaul E. McKenney nreaders); 7101fbeb3a8SPaul E. McKenney 7111fbeb3a8SPaul E. McKenney wait_event(main_wq, 7121fbeb3a8SPaul E. McKenney !atomic_read(&nreaders_exp) || torture_must_stop()); 7131fbeb3a8SPaul E. McKenney 7141fbeb3a8SPaul E. McKenney VERBOSE_SCALEOUT("main_func: experiment ended"); 7151fbeb3a8SPaul E. McKenney 7161fbeb3a8SPaul E. McKenney if (torture_must_stop()) 7171fbeb3a8SPaul E. McKenney goto end; 7181fbeb3a8SPaul E. McKenney 7191fbeb3a8SPaul E. McKenney result_avg[exp] = div_u64(1000 * process_durations(nreaders), nreaders * loops); 7201fbeb3a8SPaul E. McKenney } 7211fbeb3a8SPaul E. McKenney 7221fbeb3a8SPaul E. McKenney // Print the average of all experiments 7231fbeb3a8SPaul E. McKenney SCALEOUT("END OF TEST. Calculating average duration per loop (nanoseconds)...\n"); 7241fbeb3a8SPaul E. McKenney 7259880eb87SLi Zhijian pr_alert("Runs\tTime(ns)\n"); 7261fbeb3a8SPaul E. McKenney for (exp = 0; exp < nruns; exp++) { 7271fbeb3a8SPaul E. McKenney u64 avg; 7281fbeb3a8SPaul E. McKenney u32 rem; 7291fbeb3a8SPaul E. McKenney 7301fbeb3a8SPaul E. McKenney avg = div_u64_rem(result_avg[exp], 1000, &rem); 7311fbeb3a8SPaul E. McKenney sprintf(buf1, "%d\t%llu.%03u\n", exp + 1, avg, rem); 7321fbeb3a8SPaul E. McKenney strcat(buf, buf1); 7339880eb87SLi Zhijian if (strlen(buf) >= 800) { 7349880eb87SLi Zhijian pr_alert("%s", buf); 7359880eb87SLi Zhijian buf[0] = 0; 7369880eb87SLi Zhijian } 7371fbeb3a8SPaul E. McKenney } 7381fbeb3a8SPaul E. McKenney 7399880eb87SLi Zhijian pr_alert("%s", buf); 7401fbeb3a8SPaul E. McKenney 741c30c8763SLi Zhijian oom_exit: 7421fbeb3a8SPaul E. McKenney // This will shutdown everything including us. 7431fbeb3a8SPaul E. McKenney if (shutdown) { 7441fbeb3a8SPaul E. McKenney shutdown_start = 1; 7451fbeb3a8SPaul E. McKenney wake_up(&shutdown_wq); 7461fbeb3a8SPaul E. McKenney } 7471fbeb3a8SPaul E. McKenney 7481fbeb3a8SPaul E. McKenney // Wait for torture to stop us 7491fbeb3a8SPaul E. McKenney while (!torture_must_stop()) 7501fbeb3a8SPaul E. McKenney schedule_timeout_uninterruptible(1); 7511fbeb3a8SPaul E. McKenney 7521fbeb3a8SPaul E. McKenney end: 7531fbeb3a8SPaul E. McKenney torture_kthread_stopping("main_func"); 7541fbeb3a8SPaul E. McKenney kfree(result_avg); 7551fbeb3a8SPaul E. McKenney kfree(buf); 7561fbeb3a8SPaul E. McKenney return 0; 7571fbeb3a8SPaul E. McKenney } 7581fbeb3a8SPaul E. McKenney 7591fbeb3a8SPaul E. McKenney static void 7601fbeb3a8SPaul E. McKenney ref_scale_print_module_parms(struct ref_scale_ops *cur_ops, const char *tag) 7611fbeb3a8SPaul E. McKenney { 7621fbeb3a8SPaul E. McKenney pr_alert("%s" SCALE_FLAG 7631fbeb3a8SPaul E. McKenney "--- %s: verbose=%d shutdown=%d holdoff=%d loops=%ld nreaders=%d nruns=%d readdelay=%d\n", scale_type, tag, 7641fbeb3a8SPaul E. McKenney verbose, shutdown, holdoff, loops, nreaders, nruns, readdelay); 7651fbeb3a8SPaul E. McKenney } 7661fbeb3a8SPaul E. McKenney 7671fbeb3a8SPaul E. McKenney static void 7681fbeb3a8SPaul E. McKenney ref_scale_cleanup(void) 7691fbeb3a8SPaul E. McKenney { 7701fbeb3a8SPaul E. McKenney int i; 7711fbeb3a8SPaul E. McKenney 7721fbeb3a8SPaul E. McKenney if (torture_cleanup_begin()) 7731fbeb3a8SPaul E. McKenney return; 7741fbeb3a8SPaul E. McKenney 7751fbeb3a8SPaul E. McKenney if (!cur_ops) { 7761fbeb3a8SPaul E. McKenney torture_cleanup_end(); 7771fbeb3a8SPaul E. McKenney return; 7781fbeb3a8SPaul E. McKenney } 7791fbeb3a8SPaul E. McKenney 7801fbeb3a8SPaul E. McKenney if (reader_tasks) { 7811fbeb3a8SPaul E. McKenney for (i = 0; i < nreaders; i++) 7821fbeb3a8SPaul E. McKenney torture_stop_kthread("ref_scale_reader", 7831fbeb3a8SPaul E. McKenney reader_tasks[i].task); 7841fbeb3a8SPaul E. McKenney } 7851fbeb3a8SPaul E. McKenney kfree(reader_tasks); 7861fbeb3a8SPaul E. McKenney 7871fbeb3a8SPaul E. McKenney torture_stop_kthread("main_task", main_task); 7881fbeb3a8SPaul E. McKenney kfree(main_task); 7891fbeb3a8SPaul E. McKenney 7901fbeb3a8SPaul E. McKenney // Do scale-type-specific cleanup operations. 7911fbeb3a8SPaul E. McKenney if (cur_ops->cleanup != NULL) 7921fbeb3a8SPaul E. McKenney cur_ops->cleanup(); 7931fbeb3a8SPaul E. McKenney 7941fbeb3a8SPaul E. McKenney torture_cleanup_end(); 7951fbeb3a8SPaul E. McKenney } 7961fbeb3a8SPaul E. McKenney 7971fbeb3a8SPaul E. McKenney // Shutdown kthread. Just waits to be awakened, then shuts down system. 7981fbeb3a8SPaul E. McKenney static int 7991fbeb3a8SPaul E. McKenney ref_scale_shutdown(void *arg) 8001fbeb3a8SPaul E. McKenney { 8011fbeb3a8SPaul E. McKenney wait_event(shutdown_wq, shutdown_start); 8021fbeb3a8SPaul E. McKenney 8031fbeb3a8SPaul E. McKenney smp_mb(); // Wake before output. 8041fbeb3a8SPaul E. McKenney ref_scale_cleanup(); 8051fbeb3a8SPaul E. McKenney kernel_power_off(); 8061fbeb3a8SPaul E. McKenney 8071fbeb3a8SPaul E. McKenney return -EINVAL; 8081fbeb3a8SPaul E. McKenney } 8091fbeb3a8SPaul E. McKenney 8101fbeb3a8SPaul E. McKenney static int __init 8111fbeb3a8SPaul E. McKenney ref_scale_init(void) 8121fbeb3a8SPaul E. McKenney { 8131fbeb3a8SPaul E. McKenney long i; 8141fbeb3a8SPaul E. McKenney int firsterr = 0; 8151fbeb3a8SPaul E. McKenney static struct ref_scale_ops *scale_ops[] = { 816dec86781SPaul E. McKenney &rcu_ops, &srcu_ops, RCU_TRACE_OPS RCU_TASKS_OPS &refcnt_ops, &rwlock_ops, 81725f6fa53SPaul E. McKenney &rwsem_ops, &lock_ops, &lock_irq_ops, &acqrel_ops, &clock_ops, 8181fbeb3a8SPaul E. McKenney }; 8191fbeb3a8SPaul E. McKenney 8201fbeb3a8SPaul E. McKenney if (!torture_init_begin(scale_type, verbose)) 8211fbeb3a8SPaul E. McKenney return -EBUSY; 8221fbeb3a8SPaul E. McKenney 8231fbeb3a8SPaul E. McKenney for (i = 0; i < ARRAY_SIZE(scale_ops); i++) { 8241fbeb3a8SPaul E. McKenney cur_ops = scale_ops[i]; 8251fbeb3a8SPaul E. McKenney if (strcmp(scale_type, cur_ops->name) == 0) 8261fbeb3a8SPaul E. McKenney break; 8271fbeb3a8SPaul E. McKenney } 8281fbeb3a8SPaul E. McKenney if (i == ARRAY_SIZE(scale_ops)) { 8291fbeb3a8SPaul E. McKenney pr_alert("rcu-scale: invalid scale type: \"%s\"\n", scale_type); 8301fbeb3a8SPaul E. McKenney pr_alert("rcu-scale types:"); 8311fbeb3a8SPaul E. McKenney for (i = 0; i < ARRAY_SIZE(scale_ops); i++) 8321fbeb3a8SPaul E. McKenney pr_cont(" %s", scale_ops[i]->name); 8331fbeb3a8SPaul E. McKenney pr_cont("\n"); 8341fbeb3a8SPaul E. McKenney firsterr = -EINVAL; 8351fbeb3a8SPaul E. McKenney cur_ops = NULL; 8361fbeb3a8SPaul E. McKenney goto unwind; 8371fbeb3a8SPaul E. McKenney } 8381fbeb3a8SPaul E. McKenney if (cur_ops->init) 839*3c6496c8SPaul E. McKenney if (!cur_ops->init()) { 840*3c6496c8SPaul E. McKenney firsterr = -EUCLEAN; 841*3c6496c8SPaul E. McKenney goto unwind; 842*3c6496c8SPaul E. McKenney } 8431fbeb3a8SPaul E. McKenney 8441fbeb3a8SPaul E. McKenney ref_scale_print_module_parms(cur_ops, "Start of test"); 8451fbeb3a8SPaul E. McKenney 8461fbeb3a8SPaul E. McKenney // Shutdown task 8471fbeb3a8SPaul E. McKenney if (shutdown) { 8481fbeb3a8SPaul E. McKenney init_waitqueue_head(&shutdown_wq); 8491fbeb3a8SPaul E. McKenney firsterr = torture_create_kthread(ref_scale_shutdown, NULL, 8501fbeb3a8SPaul E. McKenney shutdown_task); 851ed60ad73SPaul E. McKenney if (torture_init_error(firsterr)) 8521fbeb3a8SPaul E. McKenney goto unwind; 8531fbeb3a8SPaul E. McKenney schedule_timeout_uninterruptible(1); 8541fbeb3a8SPaul E. McKenney } 8551fbeb3a8SPaul E. McKenney 8561fbeb3a8SPaul E. McKenney // Reader tasks (default to ~75% of online CPUs). 8571fbeb3a8SPaul E. McKenney if (nreaders < 0) 8581fbeb3a8SPaul E. McKenney nreaders = (num_online_cpus() >> 1) + (num_online_cpus() >> 2); 8590c6d18d8SPaul E. McKenney if (WARN_ONCE(loops <= 0, "%s: loops = %ld, adjusted to 1\n", __func__, loops)) 8600c6d18d8SPaul E. McKenney loops = 1; 8610c6d18d8SPaul E. McKenney if (WARN_ONCE(nreaders <= 0, "%s: nreaders = %d, adjusted to 1\n", __func__, nreaders)) 8620c6d18d8SPaul E. McKenney nreaders = 1; 8630c6d18d8SPaul E. McKenney if (WARN_ONCE(nruns <= 0, "%s: nruns = %d, adjusted to 1\n", __func__, nruns)) 8640c6d18d8SPaul E. McKenney nruns = 1; 8651fbeb3a8SPaul E. McKenney reader_tasks = kcalloc(nreaders, sizeof(reader_tasks[0]), 8661fbeb3a8SPaul E. McKenney GFP_KERNEL); 8671fbeb3a8SPaul E. McKenney if (!reader_tasks) { 8684feeb9d5SLi Zhijian SCALEOUT_ERRSTRING("out of memory"); 8691fbeb3a8SPaul E. McKenney firsterr = -ENOMEM; 8701fbeb3a8SPaul E. McKenney goto unwind; 8711fbeb3a8SPaul E. McKenney } 8721fbeb3a8SPaul E. McKenney 873f71f22b6SLi Zhijian VERBOSE_SCALEOUT("Starting %d reader threads", nreaders); 8741fbeb3a8SPaul E. McKenney 8751fbeb3a8SPaul E. McKenney for (i = 0; i < nreaders; i++) { 8761fbeb3a8SPaul E. McKenney firsterr = torture_create_kthread(ref_scale_reader, (void *)i, 8771fbeb3a8SPaul E. McKenney reader_tasks[i].task); 878ed60ad73SPaul E. McKenney if (torture_init_error(firsterr)) 8791fbeb3a8SPaul E. McKenney goto unwind; 8801fbeb3a8SPaul E. McKenney 8811fbeb3a8SPaul E. McKenney init_waitqueue_head(&(reader_tasks[i].wq)); 8821fbeb3a8SPaul E. McKenney } 8831fbeb3a8SPaul E. McKenney 8841fbeb3a8SPaul E. McKenney // Main Task 8851fbeb3a8SPaul E. McKenney init_waitqueue_head(&main_wq); 8861fbeb3a8SPaul E. McKenney firsterr = torture_create_kthread(main_func, NULL, main_task); 887ed60ad73SPaul E. McKenney if (torture_init_error(firsterr)) 8881fbeb3a8SPaul E. McKenney goto unwind; 8891fbeb3a8SPaul E. McKenney 8901fbeb3a8SPaul E. McKenney torture_init_end(); 8911fbeb3a8SPaul E. McKenney return 0; 8921fbeb3a8SPaul E. McKenney 8931fbeb3a8SPaul E. McKenney unwind: 8941fbeb3a8SPaul E. McKenney torture_init_end(); 8951fbeb3a8SPaul E. McKenney ref_scale_cleanup(); 896bc80d353SPaul E. McKenney if (shutdown) { 897bc80d353SPaul E. McKenney WARN_ON(!IS_MODULE(CONFIG_RCU_REF_SCALE_TEST)); 898bc80d353SPaul E. McKenney kernel_power_off(); 899bc80d353SPaul E. McKenney } 9001fbeb3a8SPaul E. McKenney return firsterr; 9011fbeb3a8SPaul E. McKenney } 9021fbeb3a8SPaul E. McKenney 9031fbeb3a8SPaul E. McKenney module_init(ref_scale_init); 9041fbeb3a8SPaul E. McKenney module_exit(ref_scale_cleanup); 905