1#!/bin/bash
2#
3# Copyright (C) 2016-2021 Canonical
4#
5# This program is free software; you can redistribute it and/or
6# modify it under the terms of the GNU General Public License
7# as published by the Free Software Foundation; either version 2
8# of the License, or (at your option) any later version.
9#
10# This program is distributed in the hope that it will be useful,
11# but WITHOUT ANY WARRANTY; without even the implied warranty of
12# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13# GNU General Public License for more details.
14#
15# You should have received a copy of the GNU General Public License
16# along with this program; if not, write to the Free Software
17# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
18#
19
20PERF_PARANOID=/proc/sys/kernel/perf_event_paranoid
21SWAP=/tmp/swap.img
22FSIMAGE=/tmp/fs.img
23MNT=/tmp/mnt
24LOG=stress-ng-$(date '+%Y%m%d-%H%M').log
25echo "Logging to $LOG"
26
27rm -f $LOG
28
29#
30# stress-ng kernel coverage test:
31#  - requires lcov to be installed
32#  - requires a gcov kernel with the following configuration:
33#      CONFIG_DEBUG_FS=y
34#      CONFIG_GCOV_KERNEL=y
35#      CONFIG_GCOV_PROFILE_ALL=y
36#  - for ease of use, ensure kernel is built on the target machine
37#  - no support for this script, if it breaks, you get the pieces
38#
39if [ -z "$STRESS_NG" ]; then
40        STRESS_NG=./stress-ng
41fi
42
43if [ ! -x "$STRESS_NG" ]; then
44	echo "Cannot find executable $STRESS_NG"
45	exit 1
46fi
47STRESSORS=$($STRESS_NG --stressors)
48
49mount_filesystem()
50{
51	rm -f ${FSIMAGE}
52	case $1 in
53		ext2)	MKFS_CMD="mkfs.ext2"
54			MKFS_ARGS="-F ${FSIMAGE}"
55			MNT_CMD="sudo mount -o loop ${FSIMAGE} ${MNT}"
56			dd if=/dev/zero of=${FSIMAGE} bs=1M count=1024
57			;;
58		ext3)	MKFS_CMD="mkfs.ext3"
59			MKFS_ARGS="-F ${FSIMAGE}"
60			MNT_CMD="sudo mount -o loop ${FSIMAGE} ${MNT}"
61			dd if=/dev/zero of=${FSIMAGE} bs=1M count=1024
62			;;
63		ext4)	MKFS_CMD="mkfs.ext4"
64			MKFS_ARGS="-F ${FSIMAGE} -O inline_data,dir_index,metadata_csum,64bit,ea_inode,ext_attr,quota,verity,extent,filetype,huge_file,mmp"
65			MNT_CMD="sudo mount -o loop ${FSIMAGE} ${MNT}"
66			dd if=/dev/zero of=${FSIMAGE} bs=1M count=1024
67			;;
68		xfs)
69			MKFS_CMD="mkfs.xfs"
70			MKFS_ARGS="-m crc=1,bigtime=1,finobt=1,rmapbt=1 -f ${FSIMAGE}"
71			MNT_CMD="sudo mount -o loop ${FSIMAGE} ${MNT}"
72			dd if=/dev/zero of=${FSIMAGE} bs=1M count=1024
73			;;
74		hfs)
75			MKFS_CMD="mkfs.hfs"
76			MKFS_ARGS="${FSIMAGE}"
77			MNT_CMD="sudo mount -o loop ${FSIMAGE} ${MNT}"
78			dd if=/dev/zero of=${FSIMAGE} bs=1M count=1024
79			;;
80		hfsplus)
81			MKFS_CMD="mkfs.hfsplus"
82			MKFS_ARGS="-s ${FSIMAGE}"
83			MNT_CMD="sudo mount -o loop ${FSIMAGE} ${MNT}"
84			dd if=/dev/zero of=${FSIMAGE} bs=1M count=1024
85			;;
86		jfs)	MKFS_CMD="mkfs.jfs"
87			MKFS_ARGS="-q ${FSIMAGE}"
88			MNT_CMD="sudo mount -o loop ${FSIMAGE} ${MNT}"
89			dd if=/dev/zero of=${FSIMAGE} bs=1M count=1024
90			;;
91		minix)	MKFS_CMD="mkfs.minix"
92			MKFS_ARGS="-3 ${FSIMAGE}"
93			MNT_CMD="sudo mount -o loop ${FSIMAGE} ${MNT}"
94			dd if=/dev/zero of=${FSIMAGE} bs=1M count=1024
95			;;
96		nilfs)	MKFS_CMD="mkfs.nilfs2"
97			MKFS_ARGS="-f -O block_count ${FSIMAGE}"
98			MNT_CMD="sudo mount -o loop ${FSIMAGE} ${MNT}"
99			dd if=/dev/zero of=${FSIMAGE} bs=1M count=1024
100			;;
101		fat)	MKFS_CMD="mkfs.fat"
102			MKFS_ARGS="${FSIMAGE}"
103			MNT_CMD="sudo mount -o loop ${FSIMAGE} ${MNT}"
104			dd if=/dev/zero of=${FSIMAGE} bs=1M count=1024
105			;;
106		vfat)	MKFS_CMD="mkfs.vfat"
107			MKFS_ARGS="${FSIMAGE}"
108			MNT_CMD="sudo mount -o loop ${FSIMAGE} ${MNT}"
109			dd if=/dev/zero of=${FSIMAGE} bs=1M count=1024
110			;;
111		ubifs)	sudo modprobe nandsim first_id_byte=0x20 \
112			second_id_byte=0xaa third_id_byte=0x00 \
113			fourth_id_byte=0x15
114			sudo modprobe ubi mtd=0
115			sleep 5
116			MKFS_CMD="ubimkvol"
117			MKFS_ARGS="/dev/ubi0 -N ubifs-vol -s 200MiB"
118			MNT_CMD="sudo mount -t ubifs /dev/ubi0_0 ${MNT}"
119			;;
120		udf)	MKFS_CMD="mkfs.udf"
121			MKFS_ARGS="${FSIMAGE}"
122			MNT_CMD="sudo mount -o loop ${FSIMAGE} ${MNT}"
123			dd if=/dev/zero of=${FSIMAGE} bs=1M count=1024
124			;;
125		ntfs)	MKFS_CMD="mkfs.ntfs"
126			MKFS_ARGS="-F -C -s -v 1024 ${FSIMAGE}"
127			MNT_CMD="sudo mount -o loop ${FSIMAGE} ${MNT}"
128			dd if=/dev/zero of=${FSIMAGE} bs=1M count=1024
129			;;
130		f2fs)	MKFS_CMD="mkfs.f2fs"
131			MKFS_ARGS="-f ${FSIMAGE} -i -O encrypt,extra_attr,inode_checksum,quota,verity,sb_checksum,compression,lost_found"
132			MNT_CMD="sudo mount -o loop ${FSIMAGE} ${MNT}"
133			dd if=/dev/zero of=${FSIMAGE} bs=1M count=1024
134			;;
135		bfs)	MKFS_CMD="mkfs.bfs"
136			MKFS_ARGS="${FSIMAGE}"
137			MNT_CMD="sudo mount -o loop ${FSIMAGE} ${MNT}"
138			dd if=/dev/zero of=${FSIMAGE} bs=1M count=1024
139			;;
140		btrfs)	MKFS_CMD="mkfs.btrfs"
141			MKFS_ARGS="-O extref -R quota,free-space-tree -f ${FSIMAGE}"
142			MNT_CMD="sudo mount -o compress -o loop ${FSIMAGE} ${MNT}"
143			dd if=/dev/zero of=${FSIMAGE} bs=1M count=1024
144			;;
145		tmpfs)	MKFS_CMD="true"
146			MKFS_ARGS=""
147			MNT_CMD="sudo mount -t tmpfs -o size=1G,nr_inodes=10k,mode=777 tmpfs ${MNT}"
148			;;
149		ramfs)	MKFS_CMD="true"
150			MKFS_ARGS=""
151			MNT_CMD="sudo mount -t ramfs -o size=1G ramfs ${MNT}"
152			;;
153		reiserfs)
154			MKFS_CMD="mkfs.reiserfs"
155			MKFS_ARGS="-q -f ${FSIMAGE}"
156			MNT_CMD="sudo mount -o loop ${FSIMAGE} ${MNT}"
157			dd if=/dev/zero of=${FSIMAGE} bs=1M count=1024
158			;;
159		*)
160			echo "unsupported file system $1"
161			return 1
162		;;
163	esac
164
165	if which ${MKFS_CMD} ; then
166		echo ${MKFS_CMD} ${MKFS_ARGS}
167		sudo ${MKFS_CMD} ${MKFS_ARGS}
168		rc=$?
169		if [ $rc -ne 0 ]; then
170			echo "${MKFS_CMD} ${MKFS_ARGS} failed, error: $rc"
171			return 1
172		fi
173		mkdir -p ${MNT}
174		sudo ${MNT_CMD}
175		rc=$?
176		if [ $rc -ne 0 ]; then
177			echo "${MNT_CMD} failed, error: $rc"
178			return 1
179		fi
180
181		sudo chmod 777 ${MNT}
182		own=$(whoami)
183		sudo chown $own:$own ${MNT}
184	else
185		echo "${MKFS_CMD} does not exist"
186		return 1
187	fi
188	return 0
189}
190
191umount_filesystem()
192{
193	sudo umount ${MNT}
194	rmdir ${MNT}
195	rm -f ${FSIMAGE}
196
197	case $1 in
198		ubifs)
199			sudo rmmod ubifs
200			sudo rmmod ubi
201		;;
202	esac
203}
204
205#
206#  The stressors can potentially spam the logs so
207#  keep them truncated as much as possible. Hammer to
208#  crack the nut.
209#
210clear_journal()
211{
212	which journalctl >& /dev/null
213	if [ $? -eq 0 ]; then
214		sudo journalctl --rotate >& /dev/null
215		sudo journalctl --vacuum-time 1s >& /dev/null
216		sudo journalctl --rotate >& /dev/null
217	fi
218}
219
220do_stress()
221{
222	ARGS="-t $DURATION --pathological --timestamp --tz --syslog --perf --no-rand-seed --times --metrics"
223	if grep -q "\-\-oom\-pipe" <<< "$*"; then
224		ARGS="$ARGS --oomable"
225	fi
226	echo "STARTED:  $(date '+%F %X') using $* $ARGS" >> $LOG
227	sync
228	echo running $* $ARGS
229	$STRESS_NG $* $ARGS
230	sudo $STRESS_NG $* $ARGS
231	echo "FINISHED: $(date '+%F %X') using $* $ARGS (return $?)" >> $LOG
232	sync
233	clear_journal
234}
235
236if [ -e $PERF_PARANOID ]; then
237	paranoid_saved=$(cat /proc/sys/kernel/perf_event_paranoid)
238	(echo 0 | sudo tee $PERF_PARANOID) > /dev/null
239fi
240
241core_pattern_saved="$(cat /proc/sys/kernel/core_pattern)"
242echo core | sudo tee /proc/sys/kernel/core_pattern >& /dev/null
243
244#
245#  Try to ensure that this script and parent won't be oom'd
246#
247if [ -e /proc/self/oom_score_adj ]; then
248	echo -900 | sudo tee /proc/self/oom_score_adj >& /dev/null
249	echo -900 | sudo tee /proc/$PPID/oom_score_adj >& /dev/null
250elif [ -e /proc/self/oom_adj ]; then
251	echo -14 | sudo tee /proc/self/oom_adj >& /dev/null
252	echo -14 | sudo tee /proc/$PPID/oom_adj >& /dev/null
253fi
254#
255# Ensure oom killer kills the stressor hogs rather
256# than the wrong random process (e.g. this script)
257#
258if [ -e /proc/sys/vm/oom_kill_allocating_task ]; then
259	echo 0 | sudo tee /proc/sys/vm/oom_kill_allocating_task >& /dev/null
260fi
261
262fallocate -l 8G $SWAP
263chmod 0600 $SWAP
264sudo chown root:root $SWAP
265sudo mkswap $SWAP
266sudo swapon $SWAP
267
268sudo lcov --zerocounters
269
270if [ -f	/sys/kernel/debug/tracing/trace_stat/branch_all ]; then
271	sudo cat  /sys/kernel/debug/tracing/trace_stat/branch_all > branch_all.start
272fi
273DURATION=180
274do_stress --dev 32
275
276for FS in bfs btrfs ext4 f2fs fat hfs hfsplus jfs minix nilfs ntfs ramfs reiserfs tmpfs ubifs udf vfat xfs
277do
278	echo "Filesystem: $FS"
279	if mount_filesystem $FS; then
280		DURATION=10
281		do_stress --hdd -1 --hdd-ops 50000 --hdd-opts direct,utimes  --temp-path $MNT
282		do_stress --hdd -1 --hdd-ops 50000 --hdd-opts dsync --temp-path $MNT
283		do_stress --hdd -1 --hdd-ops 50000 --hdd-opts iovec,noatime --temp-path $MNT
284		do_stress --hdd -1 --hdd-ops 50000 --hdd-opts fsync,syncfs --temp-path $MNT
285		do_stress --hdd -1 --hdd-ops 50000 --hdd-opts fdatasync --temp-path $MNT
286		do_stress --hdd -1 --hdd-ops 50000 --hdd-opts rd-rnd,wr-rnd,fadv-rnd --temp-path $MNT
287		do_stress --hdd -1 --hdd-ops 50000 --hdd-opts rd-seq,wr-seq --temp-path $MNT
288		do_stress --hdd -1 --hdd-ops 50000 --hdd-opts fadv-normal --temp-path $MNT
289		do_stress --hdd -1 --hdd-ops 50000 --hdd-opts fadv-noreuse --temp-path $MNT
290		do_stress --hdd -1 --hdd-ops 50000 --hdd-opts fadv-rnd --temp-path $MNT
291		do_stress --hdd -1 --hdd-ops 50000 --hdd-opts fadv-seq --temp-path $MNT
292		do_stress --hdd -1 --hdd-ops 50000 --hdd-opts fadv-willneed --temp-path $MNT
293		do_stress --hdd -1 --hdd-ops 50000 --hdd-opts fadv-dontneed --temp-path $MNT
294		do_stress --verity -1 --temp-path $MNT
295		DURATION=10
296		sudo $STRESS_NG --class filesystem --ftrace --seq -1 -v --timestamp --syslog -t $DURATION --temp-path $MNT
297		sudo $STRESS_NG --class io --ftrace --seq -1 -v --timestamp --syslog -t $DURATION --temp-path $MNT
298		DURATION=5
299		do_stress --sysinfo -1 --temp-path $MNT
300		umount_filesystem $FS
301	fi
302done
303
304#
305#  Exercise CPU schedulers
306#
307DURATION=20
308scheds=$(${STRESS_NG} --sched which 2>&1 | tail -1 | cut -d':' -f2-)
309for s in ${scheds}
310do
311	sudo ${STRESS_NG} --sched $s --cpu -1 -t 5 --timestamp --tz --syslog --perf --no-rand-seed --times --metrics
312	sudo ${STRESS_NG} --sched $s --cpu -1 -t 5 --sched-reclaim --timestamp --tz --syslog --perf --no-rand-seed --times --metrics
313done
314
315#
316#  Exercise ionice classes
317#
318ionices=$(${STRESS_NG} --ionice-class which 2>&1 | tail -1 | cut -d':' -f2-)
319for i in ${innices}
320do
321	do_stress --ionice-class $i --iomix -1 -t 30 --smart
322done
323
324#
325#  Exercise all stressors, limit to 1 CPU for ones that
326#  can spawn way too many processes
327#
328DURATION=15
329for S in $STRESSORS
330do
331	case $S in
332		clone|fork|vfork)
333			do_stress --${S} 1
334			;;
335		*)
336			do_stress --${S} 8
337			;;
338	esac
339done
340
341DURATION=60
342do_stress --all 1
343
344#
345#  Exercise various stressor options
346#
347do_stress --brk -1 --brk-notouch --vmstat 1
348do_stress --brk -1 --brk-mlock
349
350do_stress --cpu -1 --sched batch --thermalstat 1
351do_stress --cpu -1 --taskset 0,2 --ignite-cpu
352do_stress --cpu -1 --taskset 1,2,3
353do_stress --cpu -1 --taskset 0,1,2 --thrash
354do_stress --cpu -1 --cpu-load-slice 50
355do_stress --cpu -1 --thermalstat 1 --vmstat 1 --tz
356
357do_stress --cyclic -1 --cyclic-policy deadline
358do_stress --cyclic -1 --cyclic-policy fifo
359do_stress --cyclic -1 --cyclic-policy rr
360do_stress --cyclic -1 --cyclic-method clock_ns
361do_stress --cyclic -1 --cyclic-method itimer
362do_stress --cyclic -1 --cyclic-method poll
363do_stress --cyclic -1 --cyclic-method posix_ns
364do_stress --cyclic -1 --cyclic-method pselect
365do_stress --cyclic -1 --cyclic-method usleep
366do_stress --cyclic -1 --cyclic-prio 50
367
368do_stress --dccp -1 --dccp-opts send
369do_stress --dccp -1 --dccp-opts sendmsg
370do_stress --dccp -1 --dccp-opts sendmmsg
371
372do_stress --dccp -1 --dccp-domain ipv4
373do_stress --dccp -1 --dccp-domain ipv6
374
375do_stress --epoll -1 --epoll-domain ipv4
376do_stress --epoll -1 --epoll-domain ipv6
377do_stress --epoll -1 --epoll-domain unix
378
379do_stress --eventfd -1 --eventfd-nonblock
380
381do_stress --fork 1 --fork-vm
382
383do_stress --itimer -1 --itimer-rand
384
385do_stress --lease -1 --lease-breakers 8
386do_stress --lockf -1 --lockf-nonblock
387
388do_stress --malloc -1 --malloc-touch
389do_stress --malloc -1 --malloc-pthreads 4
390
391do_stress --memfd -1 --memfd-fds 4096
392
393do_stress --mincore -1 --mincore-random
394
395do_stress --mmap -1 --mmap-file
396do_stress --mmap -1 --mmap-mprotect
397do_stress --mmap -1 --mmap-async
398do_stress --mmap -1 --mmap-odirect
399do_stress --mmap -1 --mmap-osync
400do_stress --mmap -1 --mmap-mmap2
401
402do_stress --mremap -1 --mremap-mlock
403
404do_stress --msg -1 --msg-types 100
405
406do_stress --open -1 --open-fd
407
408do_stress --pipe -1 --pipe-size 64K
409do_stress --pipe -1 --pipe-size 1M
410
411do_stress --pipeherd 1 --pipeherd-yield
412
413do_stress --poll -1 --poll-fds 8192
414
415do_stress --pthread -1 --pthread-max 512
416do_stress --pthread -1 --pthread-max 1024
417
418do_stress --sctp -1 --sctp-domain ipv4
419do_stress --sctp -1 --sctp-domain ipv6
420
421do_stress --shm -1 --shm-objs 100000
422
423do_stress --seek -1 --seek-punch
424
425do_stress --sem -1 --sem-procs 64
426
427do_stress --shm-sysv -1 --shm-sysv-segs 128
428
429do_stress --sock -1 --sock-nodelay
430do_stress --sock -1 --sock-domain ipv4
431do_stress --sock -1 --sock-domain ipv6
432do_stress --sock -1 --sock-domain unix
433do_stress --sock -1 --sock-type stream
434do_stress --sock -1 --sock-type seqpacket
435do_stress --sock -1 --sock-protocol mptcp
436do_stress --sock -1 --sock-opts random
437do_stress --sock -1 --sock-opts send --sock-zerocopy
438
439do_stress --stack -1 --stack-mlock
440do_stress --stack -1 --stack-fill
441
442do_stress --stream -1 --stream-madvise hugepage
443do_stress --stream -1 --stream-madvise nohugepage
444do_stress --stream -1 --stream-madvise normal
445do_stress --stream -1 --stream-index 3A
446
447do_stress --switch -1 --switch-freq 1000000
448
449do_stress --timer -1 --timer-rand
450do_stress --timer -1 --timer-freq 1000000
451do_stress --timer -1 --timer-freq 100000 --timer-slack 1000
452
453do_stress --timerfd -1 --timerfd-rand
454
455do_stress --tmpfs -1 --tmpfs-mmap-async
456do_stress --tmpfs -1 --tmpfs-mmap-file
457
458do_stress --tun -1
459do_stress --tun -1 --tun-tap
460
461do_stress --udp -1 --udp-domain ipv4
462do_stress --udp -1 --udp-domain ipv6
463do_stress --udp -1 --udp-lite
464
465do_stress --udp-flood -1 --udp-flood-domain ipv4
466do_stress --udp-flood -1 --udp-flood-domain ipv6
467
468do_stress --utime -1 --utime-fsync
469
470do_stress --vfork 1 --vfork-vm
471do_stress --vforkmany 1 --vforkmany-vm
472
473do_stress --vm -1 --vm-keep
474do_stress --vm -1 --vm-hang 1
475do_stress --vm -1 --vm-locked
476do_stress --vm -1 --vm-populate
477do_stress --vm -1 --vm-madvise dontneed
478do_stress --vm -1 --vm-madvise hugepage
479do_stress --vm -1 --vm-madvise mergeable
480do_stress --vm -1 --vm-madvise nohugepage
481do_stress --vm -1 --vm-madvise mergeable
482do_stress --vm -1 --vm-madvise normal
483do_stress --vm -1 --vm-madvise random
484do_stress --vm -1 --vm-madvise sequential
485do_stress --vm -1 --vm-madvise unmergeable
486do_stress --vm -1 --vm-madvise willneed --page-in
487
488do_stress --zombie 1 --zombie-max 1000000
489
490#
491#  Longer duration stress testing to get more
492#  coverage because of the large range of files to
493#  traverse
494#
495
496DURATION=360
497do_stress --sysfs 16
498do_stress --procfs 32
499do_stress --sysinval 8 --pathological
500
501DURATION=120
502do_stress --bad-ioctl -1 --pathological
503
504#
505#  And exercise I/O with plenty of time for file setup
506#  overhead.
507#
508DURATION=60
509sudo $STRESS_NG --class filesystem --ftrace --seq -1 -v --timestamp --syslog -t $DURATION
510sudo $STRESS_NG --class io --ftrace --seq -1 -v --timestamp --syslog -t $DURATION
511
512if [ -f	/sys/kernel/debug/tracing/trace_stat/branch_all ]; then
513	sudo cat  /sys/kernel/debug/tracing/trace_stat/branch_all > branch_all.finish
514fi
515
516sudo swapoff $SWAP
517sudo rm $SWAP
518echo "$core_pattern_saved" | sudo tee /proc/sys/kernel/core_pattern >& /dev/null
519
520if [ -e $PERF_PARANOID ]; then
521	(echo $paranoid_saved | sudo tee $PERF_PARANOID) > /dev/null
522fi
523
524sudo lcov -c -o kernel.info >& /dev/null
525sudo genhtml -o html kernel.info
526