1#!/bin/bash
2#
3# Test script for libdeflate
4#
5#	Usage: ./tools/run_tests.sh [TESTGROUP]... [-TESTGROUP]...
6#
7# By default all tests are run, but it is possible to explicitly include or
8# exclude specific test groups.
9#
10
11set -eu -o pipefail
12cd "$(dirname "$0")/.."
13
14TESTGROUPS=(all)
15
16set_test_groups() {
17	TESTGROUPS=("$@")
18	local have_exclusion=0
19	local have_all=0
20	for group in "${TESTGROUPS[@]}"; do
21		if [[ $group == -* ]]; then
22			have_exclusion=1
23		elif [[ $group == all ]]; then
24			have_all=1
25		fi
26	done
27	if (( have_exclusion && !have_all )); then
28		TESTGROUPS=(all "${TESTGROUPS[@]}")
29	fi
30}
31
32if [ $# -gt 0 ]; then
33	set_test_groups "$@"
34fi
35
36SMOKEDATA="${SMOKEDATA:=$HOME/data/smokedata}"
37if [ ! -e "$SMOKEDATA" ]; then
38	echo "SMOKEDATA (value: $SMOKEDATA) does not exist.  Set the" \
39	      "environmental variable SMOKEDATA to a file to use in" \
40	      "compression/decompression tests." 1>&2
41	exit 1
42fi
43
44NDKDIR="${NDKDIR:=/opt/android-ndk}"
45
46FILES=("$SMOKEDATA" ./tools/exec_tests.sh benchmark test_checksums)
47EXEC_TESTS_CMD="WRAPPER= SMOKEDATA=\"$(basename $SMOKEDATA)\" sh exec_tests.sh"
48NPROC=$(grep -c processor /proc/cpuinfo)
49VALGRIND="valgrind --quiet --error-exitcode=100 --leak-check=full --errors-for-leak-kinds=all"
50SANITIZE_CFLAGS="-fsanitize=undefined -fno-sanitize-recover=undefined,integer"
51
52TMPFILE="$(mktemp)"
53trap "rm -f \"$TMPFILE\"" EXIT
54
55###############################################################################
56
57rm -f run_tests.log
58exec >  >(tee -ia run_tests.log)
59exec 2> >(tee -ia run_tests.log >&2)
60
61TESTS_SKIPPED=0
62log_skip() {
63	log "[WARNING, TEST SKIPPED]: $@"
64	TESTS_SKIPPED=1
65}
66
67log() {
68	echo "[$(date)] $@"
69}
70
71run_cmd() {
72	log "$@"
73	"$@" > /dev/null
74}
75
76test_group_included() {
77	local included=0 group
78	for group in "${TESTGROUPS[@]}"; do
79		if [ "$group" = "$1" ]; then
80			included=1 # explicitly included
81			break
82		fi
83		if [ "$group" = "-$1" ]; then
84			included=0 # explicitly excluded
85			break
86		fi
87		if [ "$group" = "all" ]; then # implicitly included
88			included=1
89		fi
90	done
91	if (( included )); then
92		log "Starting test group: $1"
93	fi
94	(( included ))
95}
96
97have_valgrind() {
98	if ! type -P valgrind > /dev/null; then
99		log_skip "valgrind not found; can't run tests with valgrind"
100		return 1
101	fi
102}
103
104have_ubsan() {
105	if ! type -P clang > /dev/null; then
106		log_skip "clang not found; can't run tests with UBSAN"
107		return 1
108	fi
109}
110
111have_python() {
112	if ! type -P python3 > /dev/null; then
113		log_skip "Python not found"
114		return 1
115	fi
116}
117
118###############################################################################
119
120native_build_and_test() {
121	make "$@" -j$NPROC all test_programs > /dev/null
122	WRAPPER="$WRAPPER" SMOKEDATA="$SMOKEDATA" sh ./tools/exec_tests.sh \
123			> /dev/null
124}
125
126native_tests() {
127	test_group_included native || return 0
128	local compiler compilers_to_try=(gcc)
129	local cflags cflags_to_try=("")
130	shopt -s nullglob
131	compilers_to_try+=(/usr/bin/gcc-[0-9]*)
132	compilers_to_try+=(/usr/bin/clang-[0-9]*)
133	compilers_to_try+=(/opt/gcc*/bin/gcc)
134	compilers_to_try+=(/opt/clang*/bin/clang)
135	shopt -u nullglob
136
137	if [ "$(uname -m)" = "x86_64" ]; then
138		cflags_to_try+=("-march=native")
139		cflags_to_try+=("-m32")
140	fi
141	for compiler in ${compilers_to_try[@]}; do
142		for cflags in "${cflags_to_try[@]}"; do
143			if [ "$cflags" = "-m32" ] && \
144			   $compiler -v |& grep -q -- '--disable-multilib'
145			then
146				continue
147			fi
148			log "Running tests with CC=$compiler," \
149				"CFLAGS=$cflags"
150			WRAPPER= native_build_and_test \
151				CC=$compiler CFLAGS="$cflags -Werror"
152		done
153	done
154
155	if have_valgrind; then
156		log "Running tests with Valgrind"
157		WRAPPER="$VALGRIND" native_build_and_test
158	fi
159
160	if have_ubsan; then
161		log "Running tests with undefined behavior sanitizer"
162		WRAPPER= native_build_and_test CC=clang CFLAGS="$SANITIZE_CFLAGS"
163	fi
164}
165
166###############################################################################
167
168checksum_benchmarks() {
169	test_group_included checksum_benchmarks || return 0
170	./tools/checksum_benchmarks.sh
171}
172
173###############################################################################
174
175android_build_and_test() {
176	run_cmd ./tools/android_build.sh --ndkdir="$NDKDIR" "$@"
177	run_cmd adb push "${FILES[@]}" /data/local/tmp/
178
179	# Note: adb shell always returns 0, even if the shell command fails...
180	log "adb shell \"cd /data/local/tmp && $EXEC_TESTS_CMD\""
181	adb shell "cd /data/local/tmp && $EXEC_TESTS_CMD" > "$TMPFILE"
182	if ! grep -q "exec_tests finished successfully" "$TMPFILE"; then
183		log "Android test failure!  adb shell output:"
184		cat "$TMPFILE"
185		return 1
186	fi
187}
188
189android_tests() {
190	local compiler
191
192	test_group_included android || return 0
193	if [ ! -e $NDKDIR ]; then
194		log_skip "Android NDK was not found in NDKDIR=$NDKDIR!" \
195		         "If you want to run the Android tests, set the" \
196			 "environmental variable NDKDIR to the location of" \
197			 "your Android NDK installation"
198		return 0
199	fi
200
201	if ! type -P adb > /dev/null; then
202		log_skip "adb (android-tools) is not installed"
203		return 0
204	fi
205
206	if ! adb devices | grep -q 'device$'; then
207		log_skip "No Android device is currently attached"
208		return 0
209	fi
210
211	for compiler in gcc clang; do
212		for flags in "" "--enable-neon" "--enable-crypto"; do
213			for arch in arm32 arm64; do
214				android_build_and_test --arch=$arch \
215					--compiler=$compiler $flags
216			done
217		done
218	done
219}
220
221###############################################################################
222
223mips_tests() {
224	test_group_included mips || return 0
225	if [ "$(hostname)" != "zzz" ] || [ "$(uname -m)" != "x86_64" ]; then
226		log_skip "MIPS tests are not supported on this host"
227		return 0
228	fi
229	if ! ping -c 1 dd-wrt > /dev/null; then
230		log_skip "Can't run MIPS tests: dd-wrt system not available"
231		return 0
232	fi
233	run_cmd ./tools/mips_build.sh
234	run_cmd scp "${FILES[@]}" root@dd-wrt:
235	run_cmd ssh root@dd-wrt "$EXEC_TESTS_CMD"
236
237	log "Checking that compression on big endian CPU produces same output"
238	run_cmd scp gzip root@dd-wrt:
239	run_cmd ssh root@dd-wrt \
240		"rm -f big*.gz;
241		 ./gzip -c -6 $(basename $SMOKEDATA) > big6.gz;
242		 ./gzip -c -10 $(basename $SMOKEDATA) > big10.gz"
243	run_cmd scp root@dd-wrt:big*.gz .
244	make -j$NPROC gzip > /dev/null
245	./gzip -c -6 "$SMOKEDATA" > little6.gz
246	./gzip -c -10 "$SMOKEDATA" > little10.gz
247	if ! cmp big6.gz little6.gz || ! cmp big10.gz little10.gz; then
248		echo 1>&2 "Compressed data differed on big endian vs. little endian!"
249		return 1
250	fi
251	rm big*.gz little*.gz
252}
253
254###############################################################################
255
256windows_tests() {
257	local arch
258
259	test_group_included windows || return 0
260
261	# Windows: currently compiled but not run
262	for arch in i686 x86_64; do
263		local compiler=${arch}-w64-mingw32-gcc
264		if ! type -P $compiler > /dev/null; then
265			log_skip "$compiler not found"
266			continue
267		fi
268		run_cmd make CC=$compiler CFLAGS=-Werror -j$NPROC \
269			all test_programs
270	done
271}
272
273###############################################################################
274
275static_analysis_tests() {
276	test_group_included static_analysis || return 0
277	if ! type -P scan-build > /dev/null; then
278		log_skip "clang static analyzer (scan-build) not found"
279		return 0
280	fi
281	run_cmd scan-build --status-bugs make -j$NPROC all test_programs
282}
283
284###############################################################################
285
286gzip_tests() {
287	test_group_included gzip || return 0
288
289	local gzip gunzip
290	run_cmd make -j$NPROC gzip gunzip
291	for gzip in "$PWD/gzip" /bin/gzip; do
292		for gunzip in "$PWD/gunzip" /bin/gunzip; do
293			log "Running gzip program tests with GZIP=$gzip," \
294				"GUNZIP=$gunzip"
295			GZIP="$gzip" GUNZIP="$gunzip" SMOKEDATA="$SMOKEDATA" \
296				./tools/gzip_tests.sh
297		done
298	done
299
300	if have_valgrind; then
301		log "Running gzip program tests with Valgrind"
302		GZIP="$VALGRIND $PWD/gzip" GUNZIP="$VALGRIND $PWD/gunzip" \
303			SMOKEDATA="$SMOKEDATA" ./tools/gzip_tests.sh
304	fi
305
306	if have_ubsan; then
307		log "Running gzip program tests with undefined behavior sanitizer"
308		run_cmd make -j$NPROC CC=clang CFLAGS="$SANITIZE_CFLAGS" gzip gunzip
309		GZIP="$PWD/gzip" GUNZIP="$PWD/gunzip" \
310			SMOKEDATA="$SMOKEDATA" ./tools/gzip_tests.sh
311	fi
312}
313
314###############################################################################
315
316edge_case_tests() {
317	test_group_included edge_case || return 0
318
319	# Regression test for "deflate_compress: fix corruption with long
320	# literal run".  Try to compress a file longer than 65535 bytes where no
321	# 2-byte sequence (3 would be sufficient) is repeated <= 32768 bytes
322	# apart, and the distribution of bytes remains constant throughout, and
323	# yet not all bytes are used so the data is still slightly compressible.
324	# There will be no matches in this data, but the compressor should still
325	# output a compressed block, and this block should contain more than
326	# 65535 consecutive literals, which triggered the bug.
327	#
328	# Note: on random data, this situation is extremely unlikely if the
329	# compressor uses all matches it finds, since random data will on
330	# average have a 3-byte match every (256**3)/32768 = 512 bytes.
331	if have_python; then
332		python3 > "$TMPFILE" << EOF
333import sys
334for i in range(2):
335    for stride in range(1,251):
336        b = bytes(stride*multiple % 251 for multiple in range(251))
337        sys.stdout.buffer.write(b)
338EOF
339		run_cmd make -j$NPROC benchmark
340		run_cmd ./benchmark -3 "$TMPFILE"
341		run_cmd ./benchmark -6 "$TMPFILE"
342		run_cmd ./benchmark -12 "$TMPFILE"
343	fi
344}
345
346###############################################################################
347
348log "Starting libdeflate tests"
349log "	TESTGROUPS=(${TESTGROUPS[@]})"
350log "	SMOKEDATA=$SMOKEDATA"
351log "	NDKDIR=$NDKDIR"
352
353native_tests
354checksum_benchmarks
355android_tests
356mips_tests
357windows_tests
358static_analysis_tests
359gzip_tests
360edge_case_tests
361
362if (( TESTS_SKIPPED )); then
363	log "No tests failed, but some tests were skipped.  See above."
364else
365	log "All tests passed!"
366fi
367