11fbeb3a8SPaul E. McKenney // SPDX-License-Identifier: GPL-2.0+ 21fbeb3a8SPaul E. McKenney // 31fbeb3a8SPaul E. McKenney // Scalability test comparing RCU vs other mechanisms 41fbeb3a8SPaul E. McKenney // for acquiring references on objects. 51fbeb3a8SPaul E. McKenney // 61fbeb3a8SPaul E. McKenney // Copyright (C) Google, 2020. 71fbeb3a8SPaul E. McKenney // 81fbeb3a8SPaul E. McKenney // Author: Joel Fernandes <joel@joelfernandes.org> 91fbeb3a8SPaul E. McKenney 101fbeb3a8SPaul E. McKenney #define pr_fmt(fmt) fmt 111fbeb3a8SPaul E. McKenney 121fbeb3a8SPaul E. McKenney #include <linux/atomic.h> 131fbeb3a8SPaul E. McKenney #include <linux/bitops.h> 141fbeb3a8SPaul E. McKenney #include <linux/completion.h> 151fbeb3a8SPaul E. McKenney #include <linux/cpu.h> 161fbeb3a8SPaul E. McKenney #include <linux/delay.h> 171fbeb3a8SPaul E. McKenney #include <linux/err.h> 181fbeb3a8SPaul E. McKenney #include <linux/init.h> 191fbeb3a8SPaul E. McKenney #include <linux/interrupt.h> 201fbeb3a8SPaul E. McKenney #include <linux/kthread.h> 211fbeb3a8SPaul E. McKenney #include <linux/kernel.h> 221fbeb3a8SPaul E. McKenney #include <linux/mm.h> 231fbeb3a8SPaul E. McKenney #include <linux/module.h> 241fbeb3a8SPaul E. McKenney #include <linux/moduleparam.h> 251fbeb3a8SPaul E. McKenney #include <linux/notifier.h> 261fbeb3a8SPaul E. McKenney #include <linux/percpu.h> 271fbeb3a8SPaul E. McKenney #include <linux/rcupdate.h> 281fbeb3a8SPaul E. McKenney #include <linux/rcupdate_trace.h> 291fbeb3a8SPaul E. McKenney #include <linux/reboot.h> 301fbeb3a8SPaul E. McKenney #include <linux/sched.h> 311fbeb3a8SPaul E. McKenney #include <linux/spinlock.h> 321fbeb3a8SPaul E. McKenney #include <linux/smp.h> 331fbeb3a8SPaul E. McKenney #include <linux/stat.h> 341fbeb3a8SPaul E. McKenney #include <linux/srcu.h> 351fbeb3a8SPaul E. McKenney #include <linux/slab.h> 361fbeb3a8SPaul E. McKenney #include <linux/torture.h> 371fbeb3a8SPaul E. McKenney #include <linux/types.h> 381fbeb3a8SPaul E. McKenney 391fbeb3a8SPaul E. McKenney #include "rcu.h" 401fbeb3a8SPaul E. McKenney 411fbeb3a8SPaul E. McKenney #define SCALE_FLAG "-ref-scale: " 421fbeb3a8SPaul E. McKenney 431fbeb3a8SPaul E. McKenney #define SCALEOUT(s, x...) \ 441fbeb3a8SPaul E. McKenney pr_alert("%s" SCALE_FLAG s, scale_type, ## x) 451fbeb3a8SPaul E. McKenney 461fbeb3a8SPaul E. McKenney #define VERBOSE_SCALEOUT(s, x...) \ 471fbeb3a8SPaul E. McKenney do { if (verbose) pr_alert("%s" SCALE_FLAG s, scale_type, ## x); } while (0) 481fbeb3a8SPaul E. McKenney 49e76506f0SPaul E. McKenney static atomic_t verbose_batch_ctr; 50e76506f0SPaul E. McKenney 51e76506f0SPaul E. McKenney #define VERBOSE_SCALEOUT_BATCH(s, x...) \ 52e76506f0SPaul E. McKenney do { \ 53e76506f0SPaul E. McKenney if (verbose && \ 54e76506f0SPaul E. McKenney (verbose_batched <= 0 || \ 55414c116eSPaul E. McKenney !(atomic_inc_return(&verbose_batch_ctr) % verbose_batched))) { \ 56414c116eSPaul E. McKenney schedule_timeout_uninterruptible(1); \ 57e76506f0SPaul E. McKenney pr_alert("%s" SCALE_FLAG s, scale_type, ## x); \ 58414c116eSPaul E. McKenney } \ 59e76506f0SPaul E. McKenney } while (0) 60e76506f0SPaul E. McKenney 61*4feeb9d5SLi Zhijian #define SCALEOUT_ERRSTRING(s, x...) pr_alert("%s" SCALE_FLAG "!!! " s, scale_type, ## x) 621fbeb3a8SPaul E. McKenney 631fbeb3a8SPaul E. McKenney MODULE_LICENSE("GPL"); 641fbeb3a8SPaul E. McKenney MODULE_AUTHOR("Joel Fernandes (Google) <joel@joelfernandes.org>"); 651fbeb3a8SPaul E. McKenney 661fbeb3a8SPaul E. McKenney static char *scale_type = "rcu"; 671fbeb3a8SPaul E. McKenney module_param(scale_type, charp, 0444); 681fbeb3a8SPaul E. McKenney MODULE_PARM_DESC(scale_type, "Type of test (rcu, srcu, refcnt, rwsem, rwlock."); 691fbeb3a8SPaul E. McKenney 701fbeb3a8SPaul E. McKenney torture_param(int, verbose, 0, "Enable verbose debugging printk()s"); 71e76506f0SPaul E. McKenney torture_param(int, verbose_batched, 0, "Batch verbose debugging printk()s"); 721fbeb3a8SPaul E. McKenney 731fbeb3a8SPaul E. McKenney // Wait until there are multiple CPUs before starting test. 741fbeb3a8SPaul E. McKenney torture_param(int, holdoff, IS_BUILTIN(CONFIG_RCU_REF_SCALE_TEST) ? 10 : 0, 751fbeb3a8SPaul E. McKenney "Holdoff time before test start (s)"); 761fbeb3a8SPaul E. McKenney // Number of loops per experiment, all readers execute operations concurrently. 771fbeb3a8SPaul E. McKenney torture_param(long, loops, 10000, "Number of loops per experiment."); 781fbeb3a8SPaul E. McKenney // Number of readers, with -1 defaulting to about 75% of the CPUs. 791fbeb3a8SPaul E. McKenney torture_param(int, nreaders, -1, "Number of readers, -1 for 75% of CPUs."); 801fbeb3a8SPaul E. McKenney // Number of runs. 811fbeb3a8SPaul E. McKenney torture_param(int, nruns, 30, "Number of experiments to run."); 821fbeb3a8SPaul E. McKenney // Reader delay in nanoseconds, 0 for no delay. 831fbeb3a8SPaul E. McKenney torture_param(int, readdelay, 0, "Read-side delay in nanoseconds."); 841fbeb3a8SPaul E. McKenney 851fbeb3a8SPaul E. McKenney #ifdef MODULE 861fbeb3a8SPaul E. McKenney # define REFSCALE_SHUTDOWN 0 871fbeb3a8SPaul E. McKenney #else 881fbeb3a8SPaul E. McKenney # define REFSCALE_SHUTDOWN 1 891fbeb3a8SPaul E. McKenney #endif 901fbeb3a8SPaul E. McKenney 911fbeb3a8SPaul E. McKenney torture_param(bool, shutdown, REFSCALE_SHUTDOWN, 921fbeb3a8SPaul E. McKenney "Shutdown at end of scalability tests."); 931fbeb3a8SPaul E. McKenney 941fbeb3a8SPaul E. McKenney struct reader_task { 951fbeb3a8SPaul E. McKenney struct task_struct *task; 961fbeb3a8SPaul E. McKenney int start_reader; 971fbeb3a8SPaul E. McKenney wait_queue_head_t wq; 981fbeb3a8SPaul E. McKenney u64 last_duration_ns; 991fbeb3a8SPaul E. McKenney }; 1001fbeb3a8SPaul E. McKenney 1011fbeb3a8SPaul E. McKenney static struct task_struct *shutdown_task; 1021fbeb3a8SPaul E. McKenney static wait_queue_head_t shutdown_wq; 1031fbeb3a8SPaul E. McKenney 1041fbeb3a8SPaul E. McKenney static struct task_struct *main_task; 1051fbeb3a8SPaul E. McKenney static wait_queue_head_t main_wq; 1061fbeb3a8SPaul E. McKenney static int shutdown_start; 1071fbeb3a8SPaul E. McKenney 1081fbeb3a8SPaul E. McKenney static struct reader_task *reader_tasks; 1091fbeb3a8SPaul E. McKenney 1101fbeb3a8SPaul E. McKenney // Number of readers that are part of the current experiment. 1111fbeb3a8SPaul E. McKenney static atomic_t nreaders_exp; 1121fbeb3a8SPaul E. McKenney 1131fbeb3a8SPaul E. McKenney // Use to wait for all threads to start. 1141fbeb3a8SPaul E. McKenney static atomic_t n_init; 1151fbeb3a8SPaul E. McKenney static atomic_t n_started; 1161fbeb3a8SPaul E. McKenney static atomic_t n_warmedup; 1171fbeb3a8SPaul E. McKenney static atomic_t n_cooleddown; 1181fbeb3a8SPaul E. McKenney 1191fbeb3a8SPaul E. McKenney // Track which experiment is currently running. 1201fbeb3a8SPaul E. McKenney static int exp_idx; 1211fbeb3a8SPaul E. McKenney 1221fbeb3a8SPaul E. McKenney // Operations vector for selecting different types of tests. 1231fbeb3a8SPaul E. McKenney struct ref_scale_ops { 1241fbeb3a8SPaul E. McKenney void (*init)(void); 1251fbeb3a8SPaul E. McKenney void (*cleanup)(void); 1261fbeb3a8SPaul E. McKenney void (*readsection)(const int nloops); 1271fbeb3a8SPaul E. McKenney void (*delaysection)(const int nloops, const int udl, const int ndl); 1281fbeb3a8SPaul E. McKenney const char *name; 1291fbeb3a8SPaul E. McKenney }; 1301fbeb3a8SPaul E. McKenney 1311fbeb3a8SPaul E. McKenney static struct ref_scale_ops *cur_ops; 1321fbeb3a8SPaul E. McKenney 1331fbeb3a8SPaul E. McKenney static void un_delay(const int udl, const int ndl) 1341fbeb3a8SPaul E. McKenney { 1351fbeb3a8SPaul E. McKenney if (udl) 1361fbeb3a8SPaul E. McKenney udelay(udl); 1371fbeb3a8SPaul E. McKenney if (ndl) 1381fbeb3a8SPaul E. McKenney ndelay(ndl); 1391fbeb3a8SPaul E. McKenney } 1401fbeb3a8SPaul E. McKenney 1411fbeb3a8SPaul E. McKenney static void ref_rcu_read_section(const int nloops) 1421fbeb3a8SPaul E. McKenney { 1431fbeb3a8SPaul E. McKenney int i; 1441fbeb3a8SPaul E. McKenney 1451fbeb3a8SPaul E. McKenney for (i = nloops; i >= 0; i--) { 1461fbeb3a8SPaul E. McKenney rcu_read_lock(); 1471fbeb3a8SPaul E. McKenney rcu_read_unlock(); 1481fbeb3a8SPaul E. McKenney } 1491fbeb3a8SPaul E. McKenney } 1501fbeb3a8SPaul E. McKenney 1511fbeb3a8SPaul E. McKenney static void ref_rcu_delay_section(const int nloops, const int udl, const int ndl) 1521fbeb3a8SPaul E. McKenney { 1531fbeb3a8SPaul E. McKenney int i; 1541fbeb3a8SPaul E. McKenney 1551fbeb3a8SPaul E. McKenney for (i = nloops; i >= 0; i--) { 1561fbeb3a8SPaul E. McKenney rcu_read_lock(); 1571fbeb3a8SPaul E. McKenney un_delay(udl, ndl); 1581fbeb3a8SPaul E. McKenney rcu_read_unlock(); 1591fbeb3a8SPaul E. McKenney } 1601fbeb3a8SPaul E. McKenney } 1611fbeb3a8SPaul E. McKenney 1621fbeb3a8SPaul E. McKenney static void rcu_sync_scale_init(void) 1631fbeb3a8SPaul E. McKenney { 1641fbeb3a8SPaul E. McKenney } 1651fbeb3a8SPaul E. McKenney 1661fbeb3a8SPaul E. McKenney static struct ref_scale_ops rcu_ops = { 1671fbeb3a8SPaul E. McKenney .init = rcu_sync_scale_init, 1681fbeb3a8SPaul E. McKenney .readsection = ref_rcu_read_section, 1691fbeb3a8SPaul E. McKenney .delaysection = ref_rcu_delay_section, 1701fbeb3a8SPaul E. McKenney .name = "rcu" 1711fbeb3a8SPaul E. McKenney }; 1721fbeb3a8SPaul E. McKenney 1731fbeb3a8SPaul E. McKenney // Definitions for SRCU ref scale testing. 1741fbeb3a8SPaul E. McKenney DEFINE_STATIC_SRCU(srcu_refctl_scale); 1751fbeb3a8SPaul E. McKenney static struct srcu_struct *srcu_ctlp = &srcu_refctl_scale; 1761fbeb3a8SPaul E. McKenney 1771fbeb3a8SPaul E. McKenney static void srcu_ref_scale_read_section(const int nloops) 1781fbeb3a8SPaul E. McKenney { 1791fbeb3a8SPaul E. McKenney int i; 1801fbeb3a8SPaul E. McKenney int idx; 1811fbeb3a8SPaul E. McKenney 1821fbeb3a8SPaul E. McKenney for (i = nloops; i >= 0; i--) { 1831fbeb3a8SPaul E. McKenney idx = srcu_read_lock(srcu_ctlp); 1841fbeb3a8SPaul E. McKenney srcu_read_unlock(srcu_ctlp, idx); 1851fbeb3a8SPaul E. McKenney } 1861fbeb3a8SPaul E. McKenney } 1871fbeb3a8SPaul E. McKenney 1881fbeb3a8SPaul E. McKenney static void srcu_ref_scale_delay_section(const int nloops, const int udl, const int ndl) 1891fbeb3a8SPaul E. McKenney { 1901fbeb3a8SPaul E. McKenney int i; 1911fbeb3a8SPaul E. McKenney int idx; 1921fbeb3a8SPaul E. McKenney 1931fbeb3a8SPaul E. McKenney for (i = nloops; i >= 0; i--) { 1941fbeb3a8SPaul E. McKenney idx = srcu_read_lock(srcu_ctlp); 1951fbeb3a8SPaul E. McKenney un_delay(udl, ndl); 1961fbeb3a8SPaul E. McKenney srcu_read_unlock(srcu_ctlp, idx); 1971fbeb3a8SPaul E. McKenney } 1981fbeb3a8SPaul E. McKenney } 1991fbeb3a8SPaul E. McKenney 2001fbeb3a8SPaul E. McKenney static struct ref_scale_ops srcu_ops = { 2011fbeb3a8SPaul E. McKenney .init = rcu_sync_scale_init, 2021fbeb3a8SPaul E. McKenney .readsection = srcu_ref_scale_read_section, 2031fbeb3a8SPaul E. McKenney .delaysection = srcu_ref_scale_delay_section, 2041fbeb3a8SPaul E. McKenney .name = "srcu" 2051fbeb3a8SPaul E. McKenney }; 2061fbeb3a8SPaul E. McKenney 2071fbeb3a8SPaul E. McKenney // Definitions for RCU Tasks ref scale testing: Empty read markers. 2081fbeb3a8SPaul E. McKenney // These definitions also work for RCU Rude readers. 2091fbeb3a8SPaul E. McKenney static void rcu_tasks_ref_scale_read_section(const int nloops) 2101fbeb3a8SPaul E. McKenney { 2111fbeb3a8SPaul E. McKenney int i; 2121fbeb3a8SPaul E. McKenney 2131fbeb3a8SPaul E. McKenney for (i = nloops; i >= 0; i--) 2141fbeb3a8SPaul E. McKenney continue; 2151fbeb3a8SPaul E. McKenney } 2161fbeb3a8SPaul E. McKenney 2171fbeb3a8SPaul E. McKenney static void rcu_tasks_ref_scale_delay_section(const int nloops, const int udl, const int ndl) 2181fbeb3a8SPaul E. McKenney { 2191fbeb3a8SPaul E. McKenney int i; 2201fbeb3a8SPaul E. McKenney 2211fbeb3a8SPaul E. McKenney for (i = nloops; i >= 0; i--) 2221fbeb3a8SPaul E. McKenney un_delay(udl, ndl); 2231fbeb3a8SPaul E. McKenney } 2241fbeb3a8SPaul E. McKenney 2251fbeb3a8SPaul E. McKenney static struct ref_scale_ops rcu_tasks_ops = { 2261fbeb3a8SPaul E. McKenney .init = rcu_sync_scale_init, 2271fbeb3a8SPaul E. McKenney .readsection = rcu_tasks_ref_scale_read_section, 2281fbeb3a8SPaul E. McKenney .delaysection = rcu_tasks_ref_scale_delay_section, 2291fbeb3a8SPaul E. McKenney .name = "rcu-tasks" 2301fbeb3a8SPaul E. McKenney }; 2311fbeb3a8SPaul E. McKenney 2321fbeb3a8SPaul E. McKenney // Definitions for RCU Tasks Trace ref scale testing. 2331fbeb3a8SPaul E. McKenney static void rcu_trace_ref_scale_read_section(const int nloops) 2341fbeb3a8SPaul E. McKenney { 2351fbeb3a8SPaul E. McKenney int i; 2361fbeb3a8SPaul E. McKenney 2371fbeb3a8SPaul E. McKenney for (i = nloops; i >= 0; i--) { 2381fbeb3a8SPaul E. McKenney rcu_read_lock_trace(); 2391fbeb3a8SPaul E. McKenney rcu_read_unlock_trace(); 2401fbeb3a8SPaul E. McKenney } 2411fbeb3a8SPaul E. McKenney } 2421fbeb3a8SPaul E. McKenney 2431fbeb3a8SPaul E. McKenney static void rcu_trace_ref_scale_delay_section(const int nloops, const int udl, const int ndl) 2441fbeb3a8SPaul E. McKenney { 2451fbeb3a8SPaul E. McKenney int i; 2461fbeb3a8SPaul E. McKenney 2471fbeb3a8SPaul E. McKenney for (i = nloops; i >= 0; i--) { 2481fbeb3a8SPaul E. McKenney rcu_read_lock_trace(); 2491fbeb3a8SPaul E. McKenney un_delay(udl, ndl); 2501fbeb3a8SPaul E. McKenney rcu_read_unlock_trace(); 2511fbeb3a8SPaul E. McKenney } 2521fbeb3a8SPaul E. McKenney } 2531fbeb3a8SPaul E. McKenney 2541fbeb3a8SPaul E. McKenney static struct ref_scale_ops rcu_trace_ops = { 2551fbeb3a8SPaul E. McKenney .init = rcu_sync_scale_init, 2561fbeb3a8SPaul E. McKenney .readsection = rcu_trace_ref_scale_read_section, 2571fbeb3a8SPaul E. McKenney .delaysection = rcu_trace_ref_scale_delay_section, 2581fbeb3a8SPaul E. McKenney .name = "rcu-trace" 2591fbeb3a8SPaul E. McKenney }; 2601fbeb3a8SPaul E. McKenney 2611fbeb3a8SPaul E. McKenney // Definitions for reference count 2621fbeb3a8SPaul E. McKenney static atomic_t refcnt; 2631fbeb3a8SPaul E. McKenney 2641fbeb3a8SPaul E. McKenney static void ref_refcnt_section(const int nloops) 2651fbeb3a8SPaul E. McKenney { 2661fbeb3a8SPaul E. McKenney int i; 2671fbeb3a8SPaul E. McKenney 2681fbeb3a8SPaul E. McKenney for (i = nloops; i >= 0; i--) { 2691fbeb3a8SPaul E. McKenney atomic_inc(&refcnt); 2701fbeb3a8SPaul E. McKenney atomic_dec(&refcnt); 2711fbeb3a8SPaul E. McKenney } 2721fbeb3a8SPaul E. McKenney } 2731fbeb3a8SPaul E. McKenney 2741fbeb3a8SPaul E. McKenney static void ref_refcnt_delay_section(const int nloops, const int udl, const int ndl) 2751fbeb3a8SPaul E. McKenney { 2761fbeb3a8SPaul E. McKenney int i; 2771fbeb3a8SPaul E. McKenney 2781fbeb3a8SPaul E. McKenney for (i = nloops; i >= 0; i--) { 2791fbeb3a8SPaul E. McKenney atomic_inc(&refcnt); 2801fbeb3a8SPaul E. McKenney un_delay(udl, ndl); 2811fbeb3a8SPaul E. McKenney atomic_dec(&refcnt); 2821fbeb3a8SPaul E. McKenney } 2831fbeb3a8SPaul E. McKenney } 2841fbeb3a8SPaul E. McKenney 2851fbeb3a8SPaul E. McKenney static struct ref_scale_ops refcnt_ops = { 2861fbeb3a8SPaul E. McKenney .init = rcu_sync_scale_init, 2871fbeb3a8SPaul E. McKenney .readsection = ref_refcnt_section, 2881fbeb3a8SPaul E. McKenney .delaysection = ref_refcnt_delay_section, 2891fbeb3a8SPaul E. McKenney .name = "refcnt" 2901fbeb3a8SPaul E. McKenney }; 2911fbeb3a8SPaul E. McKenney 2921fbeb3a8SPaul E. McKenney // Definitions for rwlock 2931fbeb3a8SPaul E. McKenney static rwlock_t test_rwlock; 2941fbeb3a8SPaul E. McKenney 2951fbeb3a8SPaul E. McKenney static void ref_rwlock_init(void) 2961fbeb3a8SPaul E. McKenney { 2971fbeb3a8SPaul E. McKenney rwlock_init(&test_rwlock); 2981fbeb3a8SPaul E. McKenney } 2991fbeb3a8SPaul E. McKenney 3001fbeb3a8SPaul E. McKenney static void ref_rwlock_section(const int nloops) 3011fbeb3a8SPaul E. McKenney { 3021fbeb3a8SPaul E. McKenney int i; 3031fbeb3a8SPaul E. McKenney 3041fbeb3a8SPaul E. McKenney for (i = nloops; i >= 0; i--) { 3051fbeb3a8SPaul E. McKenney read_lock(&test_rwlock); 3061fbeb3a8SPaul E. McKenney read_unlock(&test_rwlock); 3071fbeb3a8SPaul E. McKenney } 3081fbeb3a8SPaul E. McKenney } 3091fbeb3a8SPaul E. McKenney 3101fbeb3a8SPaul E. McKenney static void ref_rwlock_delay_section(const int nloops, const int udl, const int ndl) 3111fbeb3a8SPaul E. McKenney { 3121fbeb3a8SPaul E. McKenney int i; 3131fbeb3a8SPaul E. McKenney 3141fbeb3a8SPaul E. McKenney for (i = nloops; i >= 0; i--) { 3151fbeb3a8SPaul E. McKenney read_lock(&test_rwlock); 3161fbeb3a8SPaul E. McKenney un_delay(udl, ndl); 3171fbeb3a8SPaul E. McKenney read_unlock(&test_rwlock); 3181fbeb3a8SPaul E. McKenney } 3191fbeb3a8SPaul E. McKenney } 3201fbeb3a8SPaul E. McKenney 3211fbeb3a8SPaul E. McKenney static struct ref_scale_ops rwlock_ops = { 3221fbeb3a8SPaul E. McKenney .init = ref_rwlock_init, 3231fbeb3a8SPaul E. McKenney .readsection = ref_rwlock_section, 3241fbeb3a8SPaul E. McKenney .delaysection = ref_rwlock_delay_section, 3251fbeb3a8SPaul E. McKenney .name = "rwlock" 3261fbeb3a8SPaul E. McKenney }; 3271fbeb3a8SPaul E. McKenney 3281fbeb3a8SPaul E. McKenney // Definitions for rwsem 3291fbeb3a8SPaul E. McKenney static struct rw_semaphore test_rwsem; 3301fbeb3a8SPaul E. McKenney 3311fbeb3a8SPaul E. McKenney static void ref_rwsem_init(void) 3321fbeb3a8SPaul E. McKenney { 3331fbeb3a8SPaul E. McKenney init_rwsem(&test_rwsem); 3341fbeb3a8SPaul E. McKenney } 3351fbeb3a8SPaul E. McKenney 3361fbeb3a8SPaul E. McKenney static void ref_rwsem_section(const int nloops) 3371fbeb3a8SPaul E. McKenney { 3381fbeb3a8SPaul E. McKenney int i; 3391fbeb3a8SPaul E. McKenney 3401fbeb3a8SPaul E. McKenney for (i = nloops; i >= 0; i--) { 3411fbeb3a8SPaul E. McKenney down_read(&test_rwsem); 3421fbeb3a8SPaul E. McKenney up_read(&test_rwsem); 3431fbeb3a8SPaul E. McKenney } 3441fbeb3a8SPaul E. McKenney } 3451fbeb3a8SPaul E. McKenney 3461fbeb3a8SPaul E. McKenney static void ref_rwsem_delay_section(const int nloops, const int udl, const int ndl) 3471fbeb3a8SPaul E. McKenney { 3481fbeb3a8SPaul E. McKenney int i; 3491fbeb3a8SPaul E. McKenney 3501fbeb3a8SPaul E. McKenney for (i = nloops; i >= 0; i--) { 3511fbeb3a8SPaul E. McKenney down_read(&test_rwsem); 3521fbeb3a8SPaul E. McKenney un_delay(udl, ndl); 3531fbeb3a8SPaul E. McKenney up_read(&test_rwsem); 3541fbeb3a8SPaul E. McKenney } 3551fbeb3a8SPaul E. McKenney } 3561fbeb3a8SPaul E. McKenney 3571fbeb3a8SPaul E. McKenney static struct ref_scale_ops rwsem_ops = { 3581fbeb3a8SPaul E. McKenney .init = ref_rwsem_init, 3591fbeb3a8SPaul E. McKenney .readsection = ref_rwsem_section, 3601fbeb3a8SPaul E. McKenney .delaysection = ref_rwsem_delay_section, 3611fbeb3a8SPaul E. McKenney .name = "rwsem" 3621fbeb3a8SPaul E. McKenney }; 3631fbeb3a8SPaul E. McKenney 364e9b800dbSPaul E. McKenney // Definitions for global spinlock 365e9b800dbSPaul E. McKenney static DEFINE_SPINLOCK(test_lock); 366e9b800dbSPaul E. McKenney 367e9b800dbSPaul E. McKenney static void ref_lock_section(const int nloops) 368e9b800dbSPaul E. McKenney { 369e9b800dbSPaul E. McKenney int i; 370e9b800dbSPaul E. McKenney 371e9b800dbSPaul E. McKenney preempt_disable(); 372e9b800dbSPaul E. McKenney for (i = nloops; i >= 0; i--) { 373e9b800dbSPaul E. McKenney spin_lock(&test_lock); 374e9b800dbSPaul E. McKenney spin_unlock(&test_lock); 375e9b800dbSPaul E. McKenney } 376e9b800dbSPaul E. McKenney preempt_enable(); 377e9b800dbSPaul E. McKenney } 378e9b800dbSPaul E. McKenney 379e9b800dbSPaul E. McKenney static void ref_lock_delay_section(const int nloops, const int udl, const int ndl) 380e9b800dbSPaul E. McKenney { 381e9b800dbSPaul E. McKenney int i; 382e9b800dbSPaul E. McKenney 383e9b800dbSPaul E. McKenney preempt_disable(); 384e9b800dbSPaul E. McKenney for (i = nloops; i >= 0; i--) { 385e9b800dbSPaul E. McKenney spin_lock(&test_lock); 386e9b800dbSPaul E. McKenney un_delay(udl, ndl); 387e9b800dbSPaul E. McKenney spin_unlock(&test_lock); 388e9b800dbSPaul E. McKenney } 389e9b800dbSPaul E. McKenney preempt_enable(); 390e9b800dbSPaul E. McKenney } 391e9b800dbSPaul E. McKenney 392e9b800dbSPaul E. McKenney static struct ref_scale_ops lock_ops = { 393e9b800dbSPaul E. McKenney .readsection = ref_lock_section, 394e9b800dbSPaul E. McKenney .delaysection = ref_lock_delay_section, 395e9b800dbSPaul E. McKenney .name = "lock" 396e9b800dbSPaul E. McKenney }; 397e9b800dbSPaul E. McKenney 398e9b800dbSPaul E. McKenney // Definitions for global irq-save spinlock 399e9b800dbSPaul E. McKenney 400e9b800dbSPaul E. McKenney static void ref_lock_irq_section(const int nloops) 401e9b800dbSPaul E. McKenney { 402e9b800dbSPaul E. McKenney unsigned long flags; 403e9b800dbSPaul E. McKenney int i; 404e9b800dbSPaul E. McKenney 405e9b800dbSPaul E. McKenney preempt_disable(); 406e9b800dbSPaul E. McKenney for (i = nloops; i >= 0; i--) { 407e9b800dbSPaul E. McKenney spin_lock_irqsave(&test_lock, flags); 408e9b800dbSPaul E. McKenney spin_unlock_irqrestore(&test_lock, flags); 409e9b800dbSPaul E. McKenney } 410e9b800dbSPaul E. McKenney preempt_enable(); 411e9b800dbSPaul E. McKenney } 412e9b800dbSPaul E. McKenney 413e9b800dbSPaul E. McKenney static void ref_lock_irq_delay_section(const int nloops, const int udl, const int ndl) 414e9b800dbSPaul E. McKenney { 415e9b800dbSPaul E. McKenney unsigned long flags; 416e9b800dbSPaul E. McKenney int i; 417e9b800dbSPaul E. McKenney 418e9b800dbSPaul E. McKenney preempt_disable(); 419e9b800dbSPaul E. McKenney for (i = nloops; i >= 0; i--) { 420e9b800dbSPaul E. McKenney spin_lock_irqsave(&test_lock, flags); 421e9b800dbSPaul E. McKenney un_delay(udl, ndl); 422e9b800dbSPaul E. McKenney spin_unlock_irqrestore(&test_lock, flags); 423e9b800dbSPaul E. McKenney } 424e9b800dbSPaul E. McKenney preempt_enable(); 425e9b800dbSPaul E. McKenney } 426e9b800dbSPaul E. McKenney 427e9b800dbSPaul E. McKenney static struct ref_scale_ops lock_irq_ops = { 428e9b800dbSPaul E. McKenney .readsection = ref_lock_irq_section, 429e9b800dbSPaul E. McKenney .delaysection = ref_lock_irq_delay_section, 430e9b800dbSPaul E. McKenney .name = "lock-irq" 431e9b800dbSPaul E. McKenney }; 432e9b800dbSPaul E. McKenney 433e9b800dbSPaul E. McKenney // Definitions acquire-release. 434e9b800dbSPaul E. McKenney static DEFINE_PER_CPU(unsigned long, test_acqrel); 435e9b800dbSPaul E. McKenney 436e9b800dbSPaul E. McKenney static void ref_acqrel_section(const int nloops) 437e9b800dbSPaul E. McKenney { 438e9b800dbSPaul E. McKenney unsigned long x; 439e9b800dbSPaul E. McKenney int i; 440e9b800dbSPaul E. McKenney 441e9b800dbSPaul E. McKenney preempt_disable(); 442e9b800dbSPaul E. McKenney for (i = nloops; i >= 0; i--) { 443e9b800dbSPaul E. McKenney x = smp_load_acquire(this_cpu_ptr(&test_acqrel)); 444e9b800dbSPaul E. McKenney smp_store_release(this_cpu_ptr(&test_acqrel), x + 1); 445e9b800dbSPaul E. McKenney } 446e9b800dbSPaul E. McKenney preempt_enable(); 447e9b800dbSPaul E. McKenney } 448e9b800dbSPaul E. McKenney 449e9b800dbSPaul E. McKenney static void ref_acqrel_delay_section(const int nloops, const int udl, const int ndl) 450e9b800dbSPaul E. McKenney { 451e9b800dbSPaul E. McKenney unsigned long x; 452e9b800dbSPaul E. McKenney int i; 453e9b800dbSPaul E. McKenney 454e9b800dbSPaul E. McKenney preempt_disable(); 455e9b800dbSPaul E. McKenney for (i = nloops; i >= 0; i--) { 456e9b800dbSPaul E. McKenney x = smp_load_acquire(this_cpu_ptr(&test_acqrel)); 457e9b800dbSPaul E. McKenney un_delay(udl, ndl); 458e9b800dbSPaul E. McKenney smp_store_release(this_cpu_ptr(&test_acqrel), x + 1); 459e9b800dbSPaul E. McKenney } 460e9b800dbSPaul E. McKenney preempt_enable(); 461e9b800dbSPaul E. McKenney } 462e9b800dbSPaul E. McKenney 463e9b800dbSPaul E. McKenney static struct ref_scale_ops acqrel_ops = { 464e9b800dbSPaul E. McKenney .readsection = ref_acqrel_section, 465e9b800dbSPaul E. McKenney .delaysection = ref_acqrel_delay_section, 466e9b800dbSPaul E. McKenney .name = "acqrel" 467e9b800dbSPaul E. McKenney }; 468e9b800dbSPaul E. McKenney 46925f6fa53SPaul E. McKenney static volatile u64 stopopts; 47025f6fa53SPaul E. McKenney 47125f6fa53SPaul E. McKenney static void ref_clock_section(const int nloops) 47225f6fa53SPaul E. McKenney { 47325f6fa53SPaul E. McKenney u64 x = 0; 47425f6fa53SPaul E. McKenney int i; 47525f6fa53SPaul E. McKenney 47625f6fa53SPaul E. McKenney preempt_disable(); 47725f6fa53SPaul E. McKenney for (i = nloops; i >= 0; i--) 47825f6fa53SPaul E. McKenney x += ktime_get_real_fast_ns(); 47925f6fa53SPaul E. McKenney preempt_enable(); 48025f6fa53SPaul E. McKenney stopopts = x; 48125f6fa53SPaul E. McKenney } 48225f6fa53SPaul E. McKenney 48325f6fa53SPaul E. McKenney static void ref_clock_delay_section(const int nloops, const int udl, const int ndl) 48425f6fa53SPaul E. McKenney { 48525f6fa53SPaul E. McKenney u64 x = 0; 48625f6fa53SPaul E. McKenney int i; 48725f6fa53SPaul E. McKenney 48825f6fa53SPaul E. McKenney preempt_disable(); 48925f6fa53SPaul E. McKenney for (i = nloops; i >= 0; i--) { 49025f6fa53SPaul E. McKenney x += ktime_get_real_fast_ns(); 49125f6fa53SPaul E. McKenney un_delay(udl, ndl); 49225f6fa53SPaul E. McKenney } 49325f6fa53SPaul E. McKenney preempt_enable(); 49425f6fa53SPaul E. McKenney stopopts = x; 49525f6fa53SPaul E. McKenney } 49625f6fa53SPaul E. McKenney 49725f6fa53SPaul E. McKenney static struct ref_scale_ops clock_ops = { 49825f6fa53SPaul E. McKenney .readsection = ref_clock_section, 49925f6fa53SPaul E. McKenney .delaysection = ref_clock_delay_section, 50025f6fa53SPaul E. McKenney .name = "clock" 50125f6fa53SPaul E. McKenney }; 50225f6fa53SPaul E. McKenney 5031fbeb3a8SPaul E. McKenney static void rcu_scale_one_reader(void) 5041fbeb3a8SPaul E. McKenney { 5051fbeb3a8SPaul E. McKenney if (readdelay <= 0) 5061fbeb3a8SPaul E. McKenney cur_ops->readsection(loops); 5071fbeb3a8SPaul E. McKenney else 5081fbeb3a8SPaul E. McKenney cur_ops->delaysection(loops, readdelay / 1000, readdelay % 1000); 5091fbeb3a8SPaul E. McKenney } 5101fbeb3a8SPaul E. McKenney 5111fbeb3a8SPaul E. McKenney // Reader kthread. Repeatedly does empty RCU read-side 5121fbeb3a8SPaul E. McKenney // critical section, minimizing update-side interference. 5131fbeb3a8SPaul E. McKenney static int 5141fbeb3a8SPaul E. McKenney ref_scale_reader(void *arg) 5151fbeb3a8SPaul E. McKenney { 5161fbeb3a8SPaul E. McKenney unsigned long flags; 5171fbeb3a8SPaul E. McKenney long me = (long)arg; 5181fbeb3a8SPaul E. McKenney struct reader_task *rt = &(reader_tasks[me]); 5191fbeb3a8SPaul E. McKenney u64 start; 5201fbeb3a8SPaul E. McKenney s64 duration; 5211fbeb3a8SPaul E. McKenney 522e76506f0SPaul E. McKenney VERBOSE_SCALEOUT_BATCH("ref_scale_reader %ld: task started", me); 52305bc276cSPaul E. McKenney WARN_ON_ONCE(set_cpus_allowed_ptr(current, cpumask_of(me % nr_cpu_ids))); 5241fbeb3a8SPaul E. McKenney set_user_nice(current, MAX_NICE); 5251fbeb3a8SPaul E. McKenney atomic_inc(&n_init); 5261fbeb3a8SPaul E. McKenney if (holdoff) 5271fbeb3a8SPaul E. McKenney schedule_timeout_interruptible(holdoff * HZ); 5281fbeb3a8SPaul E. McKenney repeat: 52905bc276cSPaul E. McKenney VERBOSE_SCALEOUT_BATCH("ref_scale_reader %ld: waiting to start next experiment on cpu %d", me, raw_smp_processor_id()); 5301fbeb3a8SPaul E. McKenney 5311fbeb3a8SPaul E. McKenney // Wait for signal that this reader can start. 5321fbeb3a8SPaul E. McKenney wait_event(rt->wq, (atomic_read(&nreaders_exp) && smp_load_acquire(&rt->start_reader)) || 5331fbeb3a8SPaul E. McKenney torture_must_stop()); 5341fbeb3a8SPaul E. McKenney 5351fbeb3a8SPaul E. McKenney if (torture_must_stop()) 5361fbeb3a8SPaul E. McKenney goto end; 5371fbeb3a8SPaul E. McKenney 5381fbeb3a8SPaul E. McKenney // Make sure that the CPU is affinitized appropriately during testing. 53905bc276cSPaul E. McKenney WARN_ON_ONCE(raw_smp_processor_id() != me); 5401fbeb3a8SPaul E. McKenney 5411fbeb3a8SPaul E. McKenney WRITE_ONCE(rt->start_reader, 0); 5421fbeb3a8SPaul E. McKenney if (!atomic_dec_return(&n_started)) 5431fbeb3a8SPaul E. McKenney while (atomic_read_acquire(&n_started)) 5441fbeb3a8SPaul E. McKenney cpu_relax(); 5451fbeb3a8SPaul E. McKenney 546e76506f0SPaul E. McKenney VERBOSE_SCALEOUT_BATCH("ref_scale_reader %ld: experiment %d started", me, exp_idx); 5471fbeb3a8SPaul E. McKenney 5481fbeb3a8SPaul E. McKenney 5491fbeb3a8SPaul E. McKenney // To reduce noise, do an initial cache-warming invocation, check 5501fbeb3a8SPaul E. McKenney // in, and then keep warming until everyone has checked in. 5511fbeb3a8SPaul E. McKenney rcu_scale_one_reader(); 5521fbeb3a8SPaul E. McKenney if (!atomic_dec_return(&n_warmedup)) 5531fbeb3a8SPaul E. McKenney while (atomic_read_acquire(&n_warmedup)) 5541fbeb3a8SPaul E. McKenney rcu_scale_one_reader(); 5551fbeb3a8SPaul E. McKenney // Also keep interrupts disabled. This also has the effect 5561fbeb3a8SPaul E. McKenney // of preventing entries into slow path for rcu_read_unlock(). 5571fbeb3a8SPaul E. McKenney local_irq_save(flags); 5581fbeb3a8SPaul E. McKenney start = ktime_get_mono_fast_ns(); 5591fbeb3a8SPaul E. McKenney 5601fbeb3a8SPaul E. McKenney rcu_scale_one_reader(); 5611fbeb3a8SPaul E. McKenney 5621fbeb3a8SPaul E. McKenney duration = ktime_get_mono_fast_ns() - start; 5631fbeb3a8SPaul E. McKenney local_irq_restore(flags); 5641fbeb3a8SPaul E. McKenney 5651fbeb3a8SPaul E. McKenney rt->last_duration_ns = WARN_ON_ONCE(duration < 0) ? 0 : duration; 5661fbeb3a8SPaul E. McKenney // To reduce runtime-skew noise, do maintain-load invocations until 5671fbeb3a8SPaul E. McKenney // everyone is done. 5681fbeb3a8SPaul E. McKenney if (!atomic_dec_return(&n_cooleddown)) 5691fbeb3a8SPaul E. McKenney while (atomic_read_acquire(&n_cooleddown)) 5701fbeb3a8SPaul E. McKenney rcu_scale_one_reader(); 5711fbeb3a8SPaul E. McKenney 5721fbeb3a8SPaul E. McKenney if (atomic_dec_and_test(&nreaders_exp)) 5731fbeb3a8SPaul E. McKenney wake_up(&main_wq); 5741fbeb3a8SPaul E. McKenney 575e76506f0SPaul E. McKenney VERBOSE_SCALEOUT_BATCH("ref_scale_reader %ld: experiment %d ended, (readers remaining=%d)", 5761fbeb3a8SPaul E. McKenney me, exp_idx, atomic_read(&nreaders_exp)); 5771fbeb3a8SPaul E. McKenney 5781fbeb3a8SPaul E. McKenney if (!torture_must_stop()) 5791fbeb3a8SPaul E. McKenney goto repeat; 5801fbeb3a8SPaul E. McKenney end: 5811fbeb3a8SPaul E. McKenney torture_kthread_stopping("ref_scale_reader"); 5821fbeb3a8SPaul E. McKenney return 0; 5831fbeb3a8SPaul E. McKenney } 5841fbeb3a8SPaul E. McKenney 5851fbeb3a8SPaul E. McKenney static void reset_readers(void) 5861fbeb3a8SPaul E. McKenney { 5871fbeb3a8SPaul E. McKenney int i; 5881fbeb3a8SPaul E. McKenney struct reader_task *rt; 5891fbeb3a8SPaul E. McKenney 5901fbeb3a8SPaul E. McKenney for (i = 0; i < nreaders; i++) { 5911fbeb3a8SPaul E. McKenney rt = &(reader_tasks[i]); 5921fbeb3a8SPaul E. McKenney 5931fbeb3a8SPaul E. McKenney rt->last_duration_ns = 0; 5941fbeb3a8SPaul E. McKenney } 5951fbeb3a8SPaul E. McKenney } 5961fbeb3a8SPaul E. McKenney 5971fbeb3a8SPaul E. McKenney // Print the results of each reader and return the sum of all their durations. 5981fbeb3a8SPaul E. McKenney static u64 process_durations(int n) 5991fbeb3a8SPaul E. McKenney { 6001fbeb3a8SPaul E. McKenney int i; 6011fbeb3a8SPaul E. McKenney struct reader_task *rt; 6021fbeb3a8SPaul E. McKenney char buf1[64]; 6031fbeb3a8SPaul E. McKenney char *buf; 6041fbeb3a8SPaul E. McKenney u64 sum = 0; 6051fbeb3a8SPaul E. McKenney 6069880eb87SLi Zhijian buf = kmalloc(800 + 64, GFP_KERNEL); 6071fbeb3a8SPaul E. McKenney if (!buf) 6081fbeb3a8SPaul E. McKenney return 0; 6091fbeb3a8SPaul E. McKenney buf[0] = 0; 6101fbeb3a8SPaul E. McKenney sprintf(buf, "Experiment #%d (Format: <THREAD-NUM>:<Total loop time in ns>)", 6111fbeb3a8SPaul E. McKenney exp_idx); 6121fbeb3a8SPaul E. McKenney 6131fbeb3a8SPaul E. McKenney for (i = 0; i < n && !torture_must_stop(); i++) { 6141fbeb3a8SPaul E. McKenney rt = &(reader_tasks[i]); 6151fbeb3a8SPaul E. McKenney sprintf(buf1, "%d: %llu\t", i, rt->last_duration_ns); 6161fbeb3a8SPaul E. McKenney 6171fbeb3a8SPaul E. McKenney if (i % 5 == 0) 6181fbeb3a8SPaul E. McKenney strcat(buf, "\n"); 6199880eb87SLi Zhijian if (strlen(buf) >= 800) { 6209880eb87SLi Zhijian pr_alert("%s", buf); 6219880eb87SLi Zhijian buf[0] = 0; 6229880eb87SLi Zhijian } 6231fbeb3a8SPaul E. McKenney strcat(buf, buf1); 6241fbeb3a8SPaul E. McKenney 6251fbeb3a8SPaul E. McKenney sum += rt->last_duration_ns; 6261fbeb3a8SPaul E. McKenney } 6279880eb87SLi Zhijian pr_alert("%s\n", buf); 6281fbeb3a8SPaul E. McKenney 6291fbeb3a8SPaul E. McKenney kfree(buf); 6301fbeb3a8SPaul E. McKenney return sum; 6311fbeb3a8SPaul E. McKenney } 6321fbeb3a8SPaul E. McKenney 6331fbeb3a8SPaul E. McKenney // The main_func is the main orchestrator, it performs a bunch of 6341fbeb3a8SPaul E. McKenney // experiments. For every experiment, it orders all the readers 6351fbeb3a8SPaul E. McKenney // involved to start and waits for them to finish the experiment. It 6361fbeb3a8SPaul E. McKenney // then reads their timestamps and starts the next experiment. Each 6371fbeb3a8SPaul E. McKenney // experiment progresses from 1 concurrent reader to N of them at which 6381fbeb3a8SPaul E. McKenney // point all the timestamps are printed. 6391fbeb3a8SPaul E. McKenney static int main_func(void *arg) 6401fbeb3a8SPaul E. McKenney { 6411fbeb3a8SPaul E. McKenney int exp, r; 6421fbeb3a8SPaul E. McKenney char buf1[64]; 6431fbeb3a8SPaul E. McKenney char *buf; 6441fbeb3a8SPaul E. McKenney u64 *result_avg; 6451fbeb3a8SPaul E. McKenney 6461fbeb3a8SPaul E. McKenney set_cpus_allowed_ptr(current, cpumask_of(nreaders % nr_cpu_ids)); 6471fbeb3a8SPaul E. McKenney set_user_nice(current, MAX_NICE); 6481fbeb3a8SPaul E. McKenney 6491fbeb3a8SPaul E. McKenney VERBOSE_SCALEOUT("main_func task started"); 6501fbeb3a8SPaul E. McKenney result_avg = kzalloc(nruns * sizeof(*result_avg), GFP_KERNEL); 6519880eb87SLi Zhijian buf = kzalloc(800 + 64, GFP_KERNEL); 6521fbeb3a8SPaul E. McKenney if (!result_avg || !buf) { 653*4feeb9d5SLi Zhijian SCALEOUT_ERRSTRING("out of memory"); 654c30c8763SLi Zhijian goto oom_exit; 6551fbeb3a8SPaul E. McKenney } 6561fbeb3a8SPaul E. McKenney if (holdoff) 6571fbeb3a8SPaul E. McKenney schedule_timeout_interruptible(holdoff * HZ); 6581fbeb3a8SPaul E. McKenney 6591fbeb3a8SPaul E. McKenney // Wait for all threads to start. 6601fbeb3a8SPaul E. McKenney atomic_inc(&n_init); 6611fbeb3a8SPaul E. McKenney while (atomic_read(&n_init) < nreaders + 1) 6621fbeb3a8SPaul E. McKenney schedule_timeout_uninterruptible(1); 6631fbeb3a8SPaul E. McKenney 6641fbeb3a8SPaul E. McKenney // Start exp readers up per experiment 6651fbeb3a8SPaul E. McKenney for (exp = 0; exp < nruns && !torture_must_stop(); exp++) { 6661fbeb3a8SPaul E. McKenney if (torture_must_stop()) 6671fbeb3a8SPaul E. McKenney goto end; 6681fbeb3a8SPaul E. McKenney 6691fbeb3a8SPaul E. McKenney reset_readers(); 6701fbeb3a8SPaul E. McKenney atomic_set(&nreaders_exp, nreaders); 6711fbeb3a8SPaul E. McKenney atomic_set(&n_started, nreaders); 6721fbeb3a8SPaul E. McKenney atomic_set(&n_warmedup, nreaders); 6731fbeb3a8SPaul E. McKenney atomic_set(&n_cooleddown, nreaders); 6741fbeb3a8SPaul E. McKenney 6751fbeb3a8SPaul E. McKenney exp_idx = exp; 6761fbeb3a8SPaul E. McKenney 6771fbeb3a8SPaul E. McKenney for (r = 0; r < nreaders; r++) { 6781fbeb3a8SPaul E. McKenney smp_store_release(&reader_tasks[r].start_reader, 1); 6791fbeb3a8SPaul E. McKenney wake_up(&reader_tasks[r].wq); 6801fbeb3a8SPaul E. McKenney } 6811fbeb3a8SPaul E. McKenney 6821fbeb3a8SPaul E. McKenney VERBOSE_SCALEOUT("main_func: experiment started, waiting for %d readers", 6831fbeb3a8SPaul E. McKenney nreaders); 6841fbeb3a8SPaul E. McKenney 6851fbeb3a8SPaul E. McKenney wait_event(main_wq, 6861fbeb3a8SPaul E. McKenney !atomic_read(&nreaders_exp) || torture_must_stop()); 6871fbeb3a8SPaul E. McKenney 6881fbeb3a8SPaul E. McKenney VERBOSE_SCALEOUT("main_func: experiment ended"); 6891fbeb3a8SPaul E. McKenney 6901fbeb3a8SPaul E. McKenney if (torture_must_stop()) 6911fbeb3a8SPaul E. McKenney goto end; 6921fbeb3a8SPaul E. McKenney 6931fbeb3a8SPaul E. McKenney result_avg[exp] = div_u64(1000 * process_durations(nreaders), nreaders * loops); 6941fbeb3a8SPaul E. McKenney } 6951fbeb3a8SPaul E. McKenney 6961fbeb3a8SPaul E. McKenney // Print the average of all experiments 6971fbeb3a8SPaul E. McKenney SCALEOUT("END OF TEST. Calculating average duration per loop (nanoseconds)...\n"); 6981fbeb3a8SPaul E. McKenney 6999880eb87SLi Zhijian pr_alert("Runs\tTime(ns)\n"); 7001fbeb3a8SPaul E. McKenney for (exp = 0; exp < nruns; exp++) { 7011fbeb3a8SPaul E. McKenney u64 avg; 7021fbeb3a8SPaul E. McKenney u32 rem; 7031fbeb3a8SPaul E. McKenney 7041fbeb3a8SPaul E. McKenney avg = div_u64_rem(result_avg[exp], 1000, &rem); 7051fbeb3a8SPaul E. McKenney sprintf(buf1, "%d\t%llu.%03u\n", exp + 1, avg, rem); 7061fbeb3a8SPaul E. McKenney strcat(buf, buf1); 7079880eb87SLi Zhijian if (strlen(buf) >= 800) { 7089880eb87SLi Zhijian pr_alert("%s", buf); 7099880eb87SLi Zhijian buf[0] = 0; 7109880eb87SLi Zhijian } 7111fbeb3a8SPaul E. McKenney } 7121fbeb3a8SPaul E. McKenney 7139880eb87SLi Zhijian pr_alert("%s", buf); 7141fbeb3a8SPaul E. McKenney 715c30c8763SLi Zhijian oom_exit: 7161fbeb3a8SPaul E. McKenney // This will shutdown everything including us. 7171fbeb3a8SPaul E. McKenney if (shutdown) { 7181fbeb3a8SPaul E. McKenney shutdown_start = 1; 7191fbeb3a8SPaul E. McKenney wake_up(&shutdown_wq); 7201fbeb3a8SPaul E. McKenney } 7211fbeb3a8SPaul E. McKenney 7221fbeb3a8SPaul E. McKenney // Wait for torture to stop us 7231fbeb3a8SPaul E. McKenney while (!torture_must_stop()) 7241fbeb3a8SPaul E. McKenney schedule_timeout_uninterruptible(1); 7251fbeb3a8SPaul E. McKenney 7261fbeb3a8SPaul E. McKenney end: 7271fbeb3a8SPaul E. McKenney torture_kthread_stopping("main_func"); 7281fbeb3a8SPaul E. McKenney kfree(result_avg); 7291fbeb3a8SPaul E. McKenney kfree(buf); 7301fbeb3a8SPaul E. McKenney return 0; 7311fbeb3a8SPaul E. McKenney } 7321fbeb3a8SPaul E. McKenney 7331fbeb3a8SPaul E. McKenney static void 7341fbeb3a8SPaul E. McKenney ref_scale_print_module_parms(struct ref_scale_ops *cur_ops, const char *tag) 7351fbeb3a8SPaul E. McKenney { 7361fbeb3a8SPaul E. McKenney pr_alert("%s" SCALE_FLAG 7371fbeb3a8SPaul E. McKenney "--- %s: verbose=%d shutdown=%d holdoff=%d loops=%ld nreaders=%d nruns=%d readdelay=%d\n", scale_type, tag, 7381fbeb3a8SPaul E. McKenney verbose, shutdown, holdoff, loops, nreaders, nruns, readdelay); 7391fbeb3a8SPaul E. McKenney } 7401fbeb3a8SPaul E. McKenney 7411fbeb3a8SPaul E. McKenney static void 7421fbeb3a8SPaul E. McKenney ref_scale_cleanup(void) 7431fbeb3a8SPaul E. McKenney { 7441fbeb3a8SPaul E. McKenney int i; 7451fbeb3a8SPaul E. McKenney 7461fbeb3a8SPaul E. McKenney if (torture_cleanup_begin()) 7471fbeb3a8SPaul E. McKenney return; 7481fbeb3a8SPaul E. McKenney 7491fbeb3a8SPaul E. McKenney if (!cur_ops) { 7501fbeb3a8SPaul E. McKenney torture_cleanup_end(); 7511fbeb3a8SPaul E. McKenney return; 7521fbeb3a8SPaul E. McKenney } 7531fbeb3a8SPaul E. McKenney 7541fbeb3a8SPaul E. McKenney if (reader_tasks) { 7551fbeb3a8SPaul E. McKenney for (i = 0; i < nreaders; i++) 7561fbeb3a8SPaul E. McKenney torture_stop_kthread("ref_scale_reader", 7571fbeb3a8SPaul E. McKenney reader_tasks[i].task); 7581fbeb3a8SPaul E. McKenney } 7591fbeb3a8SPaul E. McKenney kfree(reader_tasks); 7601fbeb3a8SPaul E. McKenney 7611fbeb3a8SPaul E. McKenney torture_stop_kthread("main_task", main_task); 7621fbeb3a8SPaul E. McKenney kfree(main_task); 7631fbeb3a8SPaul E. McKenney 7641fbeb3a8SPaul E. McKenney // Do scale-type-specific cleanup operations. 7651fbeb3a8SPaul E. McKenney if (cur_ops->cleanup != NULL) 7661fbeb3a8SPaul E. McKenney cur_ops->cleanup(); 7671fbeb3a8SPaul E. McKenney 7681fbeb3a8SPaul E. McKenney torture_cleanup_end(); 7691fbeb3a8SPaul E. McKenney } 7701fbeb3a8SPaul E. McKenney 7711fbeb3a8SPaul E. McKenney // Shutdown kthread. Just waits to be awakened, then shuts down system. 7721fbeb3a8SPaul E. McKenney static int 7731fbeb3a8SPaul E. McKenney ref_scale_shutdown(void *arg) 7741fbeb3a8SPaul E. McKenney { 7751fbeb3a8SPaul E. McKenney wait_event(shutdown_wq, shutdown_start); 7761fbeb3a8SPaul E. McKenney 7771fbeb3a8SPaul E. McKenney smp_mb(); // Wake before output. 7781fbeb3a8SPaul E. McKenney ref_scale_cleanup(); 7791fbeb3a8SPaul E. McKenney kernel_power_off(); 7801fbeb3a8SPaul E. McKenney 7811fbeb3a8SPaul E. McKenney return -EINVAL; 7821fbeb3a8SPaul E. McKenney } 7831fbeb3a8SPaul E. McKenney 7841fbeb3a8SPaul E. McKenney static int __init 7851fbeb3a8SPaul E. McKenney ref_scale_init(void) 7861fbeb3a8SPaul E. McKenney { 7871fbeb3a8SPaul E. McKenney long i; 7881fbeb3a8SPaul E. McKenney int firsterr = 0; 7891fbeb3a8SPaul E. McKenney static struct ref_scale_ops *scale_ops[] = { 790e9b800dbSPaul E. McKenney &rcu_ops, &srcu_ops, &rcu_trace_ops, &rcu_tasks_ops, &refcnt_ops, &rwlock_ops, 79125f6fa53SPaul E. McKenney &rwsem_ops, &lock_ops, &lock_irq_ops, &acqrel_ops, &clock_ops, 7921fbeb3a8SPaul E. McKenney }; 7931fbeb3a8SPaul E. McKenney 7941fbeb3a8SPaul E. McKenney if (!torture_init_begin(scale_type, verbose)) 7951fbeb3a8SPaul E. McKenney return -EBUSY; 7961fbeb3a8SPaul E. McKenney 7971fbeb3a8SPaul E. McKenney for (i = 0; i < ARRAY_SIZE(scale_ops); i++) { 7981fbeb3a8SPaul E. McKenney cur_ops = scale_ops[i]; 7991fbeb3a8SPaul E. McKenney if (strcmp(scale_type, cur_ops->name) == 0) 8001fbeb3a8SPaul E. McKenney break; 8011fbeb3a8SPaul E. McKenney } 8021fbeb3a8SPaul E. McKenney if (i == ARRAY_SIZE(scale_ops)) { 8031fbeb3a8SPaul E. McKenney pr_alert("rcu-scale: invalid scale type: \"%s\"\n", scale_type); 8041fbeb3a8SPaul E. McKenney pr_alert("rcu-scale types:"); 8051fbeb3a8SPaul E. McKenney for (i = 0; i < ARRAY_SIZE(scale_ops); i++) 8061fbeb3a8SPaul E. McKenney pr_cont(" %s", scale_ops[i]->name); 8071fbeb3a8SPaul E. McKenney pr_cont("\n"); 8081fbeb3a8SPaul E. McKenney firsterr = -EINVAL; 8091fbeb3a8SPaul E. McKenney cur_ops = NULL; 8101fbeb3a8SPaul E. McKenney goto unwind; 8111fbeb3a8SPaul E. McKenney } 8121fbeb3a8SPaul E. McKenney if (cur_ops->init) 8131fbeb3a8SPaul E. McKenney cur_ops->init(); 8141fbeb3a8SPaul E. McKenney 8151fbeb3a8SPaul E. McKenney ref_scale_print_module_parms(cur_ops, "Start of test"); 8161fbeb3a8SPaul E. McKenney 8171fbeb3a8SPaul E. McKenney // Shutdown task 8181fbeb3a8SPaul E. McKenney if (shutdown) { 8191fbeb3a8SPaul E. McKenney init_waitqueue_head(&shutdown_wq); 8201fbeb3a8SPaul E. McKenney firsterr = torture_create_kthread(ref_scale_shutdown, NULL, 8211fbeb3a8SPaul E. McKenney shutdown_task); 822ed60ad73SPaul E. McKenney if (torture_init_error(firsterr)) 8231fbeb3a8SPaul E. McKenney goto unwind; 8241fbeb3a8SPaul E. McKenney schedule_timeout_uninterruptible(1); 8251fbeb3a8SPaul E. McKenney } 8261fbeb3a8SPaul E. McKenney 8271fbeb3a8SPaul E. McKenney // Reader tasks (default to ~75% of online CPUs). 8281fbeb3a8SPaul E. McKenney if (nreaders < 0) 8291fbeb3a8SPaul E. McKenney nreaders = (num_online_cpus() >> 1) + (num_online_cpus() >> 2); 8300c6d18d8SPaul E. McKenney if (WARN_ONCE(loops <= 0, "%s: loops = %ld, adjusted to 1\n", __func__, loops)) 8310c6d18d8SPaul E. McKenney loops = 1; 8320c6d18d8SPaul E. McKenney if (WARN_ONCE(nreaders <= 0, "%s: nreaders = %d, adjusted to 1\n", __func__, nreaders)) 8330c6d18d8SPaul E. McKenney nreaders = 1; 8340c6d18d8SPaul E. McKenney if (WARN_ONCE(nruns <= 0, "%s: nruns = %d, adjusted to 1\n", __func__, nruns)) 8350c6d18d8SPaul E. McKenney nruns = 1; 8361fbeb3a8SPaul E. McKenney reader_tasks = kcalloc(nreaders, sizeof(reader_tasks[0]), 8371fbeb3a8SPaul E. McKenney GFP_KERNEL); 8381fbeb3a8SPaul E. McKenney if (!reader_tasks) { 839*4feeb9d5SLi Zhijian SCALEOUT_ERRSTRING("out of memory"); 8401fbeb3a8SPaul E. McKenney firsterr = -ENOMEM; 8411fbeb3a8SPaul E. McKenney goto unwind; 8421fbeb3a8SPaul E. McKenney } 8431fbeb3a8SPaul E. McKenney 8441fbeb3a8SPaul E. McKenney VERBOSE_SCALEOUT("Starting %d reader threads\n", nreaders); 8451fbeb3a8SPaul E. McKenney 8461fbeb3a8SPaul E. McKenney for (i = 0; i < nreaders; i++) { 8471fbeb3a8SPaul E. McKenney firsterr = torture_create_kthread(ref_scale_reader, (void *)i, 8481fbeb3a8SPaul E. McKenney reader_tasks[i].task); 849ed60ad73SPaul E. McKenney if (torture_init_error(firsterr)) 8501fbeb3a8SPaul E. McKenney goto unwind; 8511fbeb3a8SPaul E. McKenney 8521fbeb3a8SPaul E. McKenney init_waitqueue_head(&(reader_tasks[i].wq)); 8531fbeb3a8SPaul E. McKenney } 8541fbeb3a8SPaul E. McKenney 8551fbeb3a8SPaul E. McKenney // Main Task 8561fbeb3a8SPaul E. McKenney init_waitqueue_head(&main_wq); 8571fbeb3a8SPaul E. McKenney firsterr = torture_create_kthread(main_func, NULL, main_task); 858ed60ad73SPaul E. McKenney if (torture_init_error(firsterr)) 8591fbeb3a8SPaul E. McKenney goto unwind; 8601fbeb3a8SPaul E. McKenney 8611fbeb3a8SPaul E. McKenney torture_init_end(); 8621fbeb3a8SPaul E. McKenney return 0; 8631fbeb3a8SPaul E. McKenney 8641fbeb3a8SPaul E. McKenney unwind: 8651fbeb3a8SPaul E. McKenney torture_init_end(); 8661fbeb3a8SPaul E. McKenney ref_scale_cleanup(); 867bc80d353SPaul E. McKenney if (shutdown) { 868bc80d353SPaul E. McKenney WARN_ON(!IS_MODULE(CONFIG_RCU_REF_SCALE_TEST)); 869bc80d353SPaul E. McKenney kernel_power_off(); 870bc80d353SPaul E. McKenney } 8711fbeb3a8SPaul E. McKenney return firsterr; 8721fbeb3a8SPaul E. McKenney } 8731fbeb3a8SPaul E. McKenney 8741fbeb3a8SPaul E. McKenney module_init(ref_scale_init); 8751fbeb3a8SPaul E. McKenney module_exit(ref_scale_cleanup); 876