1#!/usr/bin/env bash 2# Copyright (c) the JPEG XL Project Authors. All rights reserved. 3# 4# Use of this source code is governed by a BSD-style 5# license that can be found in the LICENSE file. 6 7# Continuous integration helper module. This module is meant to be called from 8# the .gitlab-ci.yml file during the continuous integration build, as well as 9# from the command line for developers. 10 11set -eu 12 13OS=`uname -s` 14 15MYDIR=$(dirname $(realpath "$0")) 16 17### Environment parameters: 18TEST_STACK_LIMIT="${TEST_STACK_LIMIT:-128}" 19CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE:-RelWithDebInfo} 20CMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH:-} 21CMAKE_C_COMPILER_LAUNCHER=${CMAKE_C_COMPILER_LAUNCHER:-} 22CMAKE_CXX_COMPILER_LAUNCHER=${CMAKE_CXX_COMPILER_LAUNCHER:-} 23CMAKE_MAKE_PROGRAM=${CMAKE_MAKE_PROGRAM:-} 24SKIP_TEST="${SKIP_TEST:-0}" 25BUILD_TARGET="${BUILD_TARGET:-}" 26ENABLE_WASM_SIMD="${ENABLE_WASM_SIMD:-0}" 27if [[ -n "${BUILD_TARGET}" ]]; then 28 BUILD_DIR="${BUILD_DIR:-${MYDIR}/build-${BUILD_TARGET%%-*}}" 29else 30 BUILD_DIR="${BUILD_DIR:-${MYDIR}/build}" 31fi 32# Whether we should post a message in the MR when the build fails. 33POST_MESSAGE_ON_ERROR="${POST_MESSAGE_ON_ERROR:-1}" 34 35# Set default compilers to clang if not already set 36export CC=${CC:-clang} 37export CXX=${CXX:-clang++} 38 39# Time limit for the "fuzz" command in seconds (0 means no limit). 40FUZZER_MAX_TIME="${FUZZER_MAX_TIME:-0}" 41 42SANITIZER="none" 43 44if [[ "${BUILD_TARGET}" == wasm* ]]; then 45 # Check that environment is setup for the WASM build target. 46 if [[ -z "${EMSCRIPTEN}" ]]; then 47 echo "'EMSCRIPTEN' is not defined. Use 'emconfigure' wrapper to setup WASM build environment" >&2 48 return 1 49 fi 50 # Remove the side-effect of "emconfigure" wrapper - it considers NodeJS environment. 51 unset EMMAKEN_JUST_CONFIGURE 52 EMS_TOOLCHAIN_FILE="${EMSCRIPTEN}/cmake/Modules/Platform/Emscripten.cmake" 53 if [[ -f "${EMS_TOOLCHAIN_FILE}" ]]; then 54 CMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE:-${EMS_TOOLCHAIN_FILE}} 55 else 56 echo "Warning: EMSCRIPTEN CMake module not found" >&2 57 fi 58 CMAKE_CROSSCOMPILING_EMULATOR="${MYDIR}/js-wasm-wrapper.sh" 59fi 60 61if [[ "${BUILD_TARGET%%-*}" == "x86_64" || 62 "${BUILD_TARGET%%-*}" == "i686" ]]; then 63 # Default to building all targets, even if compiler baseline is SSE4 64 HWY_BASELINE_TARGETS=${HWY_BASELINE_TARGETS:-HWY_SCALAR} 65else 66 HWY_BASELINE_TARGETS=${HWY_BASELINE_TARGETS:-} 67fi 68 69# Convenience flag to pass both CMAKE_C_FLAGS and CMAKE_CXX_FLAGS 70CMAKE_FLAGS=${CMAKE_FLAGS:-} 71CMAKE_C_FLAGS="${CMAKE_C_FLAGS:-} ${CMAKE_FLAGS}" 72CMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS:-} ${CMAKE_FLAGS}" 73 74CMAKE_CROSSCOMPILING_EMULATOR=${CMAKE_CROSSCOMPILING_EMULATOR:-} 75CMAKE_EXE_LINKER_FLAGS=${CMAKE_EXE_LINKER_FLAGS:-} 76CMAKE_FIND_ROOT_PATH=${CMAKE_FIND_ROOT_PATH:-} 77CMAKE_MODULE_LINKER_FLAGS=${CMAKE_MODULE_LINKER_FLAGS:-} 78CMAKE_SHARED_LINKER_FLAGS=${CMAKE_SHARED_LINKER_FLAGS:-} 79CMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE:-} 80 81if [[ "${ENABLE_WASM_SIMD}" -ne "0" ]]; then 82 CMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS} -msimd128" 83 CMAKE_C_FLAGS="${CMAKE_C_FLAGS} -msimd128" 84 CMAKE_EXE_LINKER_FLAGS="${CMAKE_EXE_LINKER_FLAGS} -msimd128" 85fi 86 87if [[ ! -z "${HWY_BASELINE_TARGETS}" ]]; then 88 CMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS} -DHWY_BASELINE_TARGETS=${HWY_BASELINE_TARGETS}" 89fi 90 91# Version inferred from the CI variables. 92CI_COMMIT_SHA=${CI_COMMIT_SHA:-${GITHUB_SHA:-}} 93JPEGXL_VERSION=${JPEGXL_VERSION:-${CI_COMMIT_SHA:0:8}} 94 95# Benchmark parameters 96STORE_IMAGES=${STORE_IMAGES:-1} 97BENCHMARK_CORPORA="${MYDIR}/third_party/corpora" 98 99# Local flags passed to sanitizers. 100UBSAN_FLAGS=( 101 -fsanitize=alignment 102 -fsanitize=bool 103 -fsanitize=bounds 104 -fsanitize=builtin 105 -fsanitize=enum 106 -fsanitize=float-cast-overflow 107 -fsanitize=float-divide-by-zero 108 -fsanitize=integer-divide-by-zero 109 -fsanitize=null 110 -fsanitize=object-size 111 -fsanitize=pointer-overflow 112 -fsanitize=return 113 -fsanitize=returns-nonnull-attribute 114 -fsanitize=shift-base 115 -fsanitize=shift-exponent 116 -fsanitize=unreachable 117 -fsanitize=vla-bound 118 119 -fno-sanitize-recover=undefined 120 # Brunsli uses unaligned accesses to uint32_t, so alignment is just a warning. 121 -fsanitize-recover=alignment 122) 123# -fsanitize=function doesn't work on aarch64 and arm. 124if [[ "${BUILD_TARGET%%-*}" != "aarch64" && 125 "${BUILD_TARGET%%-*}" != "arm" ]]; then 126 UBSAN_FLAGS+=( 127 -fsanitize=function 128 ) 129fi 130if [[ "${BUILD_TARGET%%-*}" != "arm" ]]; then 131 UBSAN_FLAGS+=( 132 -fsanitize=signed-integer-overflow 133 ) 134fi 135 136CLANG_TIDY_BIN=$(which clang-tidy-6.0 clang-tidy-7 clang-tidy-8 clang-tidy | head -n 1) 137# Default to "cat" if "colordiff" is not installed or if stdout is not a tty. 138if [[ -t 1 ]]; then 139 COLORDIFF_BIN=$(which colordiff cat | head -n 1) 140else 141 COLORDIFF_BIN="cat" 142fi 143FIND_BIN=$(which gfind find | head -n 1) 144# "false" will disable wine64 when not installed. This won't allow 145# cross-compiling. 146WINE_BIN=$(which wine64 false | head -n 1) 147 148CLANG_VERSION="${CLANG_VERSION:-}" 149# Detect the clang version suffix and store it in CLANG_VERSION. For example, 150# "6.0" for clang 6 or "7" for clang 7. 151detect_clang_version() { 152 if [[ -n "${CLANG_VERSION}" ]]; then 153 return 0 154 fi 155 local clang_version=$("${CC:-clang}" --version | head -n1) 156 clang_version=${clang_version#"Debian "} 157 local llvm_tag 158 case "${clang_version}" in 159 "clang version 6."*) 160 CLANG_VERSION="6.0" 161 ;; 162 "clang version "*) 163 # Any other clang version uses just the major version number. 164 local suffix="${clang_version#clang version }" 165 CLANG_VERSION="${suffix%%.*}" 166 ;; 167 "emcc"*) 168 # We can't use asan or msan in the emcc case. 169 ;; 170 *) 171 echo "Unknown clang version: ${clang_version}" >&2 172 return 1 173 esac 174} 175 176# Temporary files cleanup hooks. 177CLEANUP_FILES=() 178cleanup() { 179 if [[ ${#CLEANUP_FILES[@]} -ne 0 ]]; then 180 rm -fr "${CLEANUP_FILES[@]}" 181 fi 182} 183 184# Executed on exit. 185on_exit() { 186 local retcode="$1" 187 # Always cleanup the CLEANUP_FILES. 188 cleanup 189 190 # Post a message in the MR when requested with POST_MESSAGE_ON_ERROR but only 191 # if the run failed and we are not running from a MR pipeline. 192 if [[ ${retcode} -ne 0 && -n "${CI_BUILD_NAME:-}" && 193 -n "${POST_MESSAGE_ON_ERROR}" && -z "${CI_MERGE_REQUEST_ID:-}" && 194 "${CI_BUILD_REF_NAME}" = "master" ]]; then 195 load_mr_vars_from_commit 196 { set +xeu; } 2>/dev/null 197 local message="**Run ${CI_BUILD_NAME} @ ${CI_COMMIT_SHORT_SHA} failed.** 198 199Check the output of the job at ${CI_JOB_URL:-} to see if this was your problem. 200If it was, please rollback this change or fix the problem ASAP, broken builds 201slow down development. Check if the error already existed in the previous build 202as well. 203 204Pipeline: ${CI_PIPELINE_URL} 205 206Previous build commit: ${CI_COMMIT_BEFORE_SHA} 207" 208 cmd_post_mr_comment "${message}" 209 fi 210} 211 212trap 'retcode=$?; { set +x; } 2>/dev/null; on_exit ${retcode}' INT TERM EXIT 213 214 215# These variables are populated when calling merge_request_commits(). 216 217# The current hash at the top of the current branch or merge request branch (if 218# running from a merge request pipeline). 219MR_HEAD_SHA="" 220# The common ancestor between the current commit and the tracked branch, such 221# as master. This includes a list 222MR_ANCESTOR_SHA="" 223 224# Populate MR_HEAD_SHA and MR_ANCESTOR_SHA. 225merge_request_commits() { 226 { set +x; } 2>/dev/null 227 # GITHUB_SHA is the current reference being build in GitHub Actions. 228 if [[ -n "${GITHUB_SHA:-}" ]]; then 229 # GitHub normally does a checkout of a merge commit on a shallow repository 230 # by default. We want to get a bit more of the history to be able to diff 231 # changes on the Pull Request if needed. This fetches 10 more commits which 232 # should be enough given that PR normally should have 1 commit. 233 git -C "${MYDIR}" fetch -q origin "${GITHUB_SHA}" --depth 10 234 MR_HEAD_SHA="$(git rev-parse "FETCH_HEAD^2" 2>/dev/null || 235 echo "${GITHUB_SHA}")" 236 else 237 # CI_BUILD_REF is the reference currently being build in the CI workflow. 238 MR_HEAD_SHA=$(git -C "${MYDIR}" rev-parse -q "${CI_BUILD_REF:-HEAD}") 239 fi 240 241 if [[ -n "${CI_MERGE_REQUEST_IID:-}" ]]; then 242 # Merge request pipeline in CI. In this case the upstream is called "origin" 243 # but it refers to the forked project that's the source of the merge 244 # request. We need to get the target of the merge request, for which we need 245 # to query that repository using our CI_JOB_TOKEN. 246 echo "machine gitlab.com login gitlab-ci-token password ${CI_JOB_TOKEN}" \ 247 >> "${HOME}/.netrc" 248 git -C "${MYDIR}" fetch "${CI_MERGE_REQUEST_PROJECT_URL}" \ 249 "${CI_MERGE_REQUEST_TARGET_BRANCH_NAME}" 250 MR_ANCESTOR_SHA=$(git -C "${MYDIR}" rev-parse -q FETCH_HEAD) 251 elif [[ -n "${GITHUB_BASE_REF:-}" ]]; then 252 # Pull request workflow in GitHub Actions. GitHub checkout action uses 253 # "origin" as the remote for the git checkout. 254 git -C "${MYDIR}" fetch -q origin "${GITHUB_BASE_REF}" 255 MR_ANCESTOR_SHA=$(git -C "${MYDIR}" rev-parse -q FETCH_HEAD) 256 else 257 # We are in a local branch, not a merge request. 258 MR_ANCESTOR_SHA=$(git -C "${MYDIR}" rev-parse -q HEAD@{upstream} || true) 259 fi 260 261 if [[ -z "${MR_ANCESTOR_SHA}" ]]; then 262 echo "Warning, not tracking any branch, using the last commit in HEAD.">&2 263 # This prints the return value with just HEAD. 264 MR_ANCESTOR_SHA=$(git -C "${MYDIR}" rev-parse -q "${MR_HEAD_SHA}^") 265 else 266 # GitHub runs the pipeline on a merge commit, no need to look for the common 267 # ancestor in that case. 268 if [[ -z "${GITHUB_BASE_REF:-}" ]]; then 269 MR_ANCESTOR_SHA=$(git -C "${MYDIR}" merge-base \ 270 "${MR_ANCESTOR_SHA}" "${MR_HEAD_SHA}") 271 fi 272 fi 273 set -x 274} 275 276# Load the MR iid from the landed commit message when running not from a 277# merge request workflow. This is useful to post back results at the merge 278# request when running pipelines from master. 279load_mr_vars_from_commit() { 280 { set +x; } 2>/dev/null 281 if [[ -z "${CI_MERGE_REQUEST_IID:-}" ]]; then 282 local mr_iid=$(git rev-list --format=%B --max-count=1 HEAD | 283 grep -F "${CI_PROJECT_URL}" | grep -F "/merge_requests" | head -n 1) 284 # mr_iid contains a string like this if it matched: 285 # Part-of: <https://gitlab.com/wg1/jpeg-xlm/merge_requests/123456> 286 if [[ -n "${mr_iid}" ]]; then 287 mr_iid=$(echo "${mr_iid}" | 288 sed -E 's,^.*merge_requests/([0-9]+)>.*$,\1,') 289 CI_MERGE_REQUEST_IID="${mr_iid}" 290 CI_MERGE_REQUEST_PROJECT_ID=${CI_PROJECT_ID} 291 fi 292 fi 293 set -x 294} 295 296# Posts a comment to the current merge request. 297cmd_post_mr_comment() { 298 { set +x; } 2>/dev/null 299 local comment="$1" 300 if [[ -n "${BOT_TOKEN:-}" && -n "${CI_MERGE_REQUEST_IID:-}" ]]; then 301 local url="${CI_API_V4_URL}/projects/${CI_MERGE_REQUEST_PROJECT_ID}/merge_requests/${CI_MERGE_REQUEST_IID}/notes" 302 curl -X POST -g \ 303 -H "PRIVATE-TOKEN: ${BOT_TOKEN}" \ 304 --data-urlencode "body=${comment}" \ 305 --output /dev/null \ 306 "${url}" 307 fi 308 set -x 309} 310 311# Set up and export the environment variables needed by the child processes. 312export_env() { 313 if [[ "${BUILD_TARGET}" == *mingw32 ]]; then 314 # Wine needs to know the paths to the mingw dlls. These should be 315 # separated by ';'. 316 WINEPATH=$("${CC:-clang}" -print-search-dirs --target="${BUILD_TARGET}" \ 317 | grep -F 'libraries: =' | cut -f 2- -d '=' | tr ':' ';') 318 # We also need our own libraries in the wine path. 319 local real_build_dir=$(realpath "${BUILD_DIR}") 320 # Some library .dll dependencies are installed in /bin: 321 export WINEPATH="${WINEPATH};${real_build_dir};${real_build_dir}/third_party/brotli;/usr/${BUILD_TARGET}/bin" 322 323 local prefix="${BUILD_DIR}/wineprefix" 324 mkdir -p "${prefix}" 325 export WINEPREFIX=$(realpath "${prefix}") 326 fi 327 # Sanitizers need these variables to print and properly format the stack 328 # traces: 329 LLVM_SYMBOLIZER=$("${CC:-clang}" -print-prog-name=llvm-symbolizer || true) 330 if [[ -n "${LLVM_SYMBOLIZER}" ]]; then 331 export ASAN_SYMBOLIZER_PATH="${LLVM_SYMBOLIZER}" 332 export MSAN_SYMBOLIZER_PATH="${LLVM_SYMBOLIZER}" 333 export UBSAN_SYMBOLIZER_PATH="${LLVM_SYMBOLIZER}" 334 fi 335} 336 337cmake_configure() { 338 export_env 339 340 if [[ "${STACK_SIZE:-0}" == 1 ]]; then 341 # Dump the stack size of each function in the .stack_sizes section for 342 # analysis. 343 CMAKE_C_FLAGS+=" -fstack-size-section" 344 CMAKE_CXX_FLAGS+=" -fstack-size-section" 345 fi 346 347 local args=( 348 -B"${BUILD_DIR}" -H"${MYDIR}" 349 -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" 350 -G Ninja 351 -DCMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS}" 352 -DCMAKE_C_FLAGS="${CMAKE_C_FLAGS}" 353 -DCMAKE_TOOLCHAIN_FILE="${CMAKE_TOOLCHAIN_FILE}" 354 -DCMAKE_EXE_LINKER_FLAGS="${CMAKE_EXE_LINKER_FLAGS}" 355 -DCMAKE_MODULE_LINKER_FLAGS="${CMAKE_MODULE_LINKER_FLAGS}" 356 -DCMAKE_SHARED_LINKER_FLAGS="${CMAKE_SHARED_LINKER_FLAGS}" 357 -DJPEGXL_VERSION="${JPEGXL_VERSION}" 358 -DSANITIZER="${SANITIZER}" 359 # These are not enabled by default in cmake. 360 -DJPEGXL_ENABLE_VIEWERS=ON 361 -DJPEGXL_ENABLE_PLUGINS=ON 362 -DJPEGXL_ENABLE_DEVTOOLS=ON 363 # We always use libfuzzer in the ci.sh wrapper. 364 -DJPEGXL_FUZZER_LINK_FLAGS="-fsanitize=fuzzer" 365 ) 366 if [[ "${BUILD_TARGET}" != *mingw32 ]]; then 367 args+=( 368 -DJPEGXL_WARNINGS_AS_ERRORS=ON 369 ) 370 fi 371 if [[ -n "${BUILD_TARGET}" ]]; then 372 local system_name="Linux" 373 if [[ "${BUILD_TARGET}" == *mingw32 ]]; then 374 # When cross-compiling with mingw the target must be set to Windows and 375 # run programs with wine. 376 system_name="Windows" 377 args+=( 378 -DCMAKE_CROSSCOMPILING_EMULATOR="${WINE_BIN}" 379 # Normally CMake automatically defines MINGW=1 when building with the 380 # mingw compiler (x86_64-w64-mingw32-gcc) but we are normally compiling 381 # with clang. 382 -DMINGW=1 383 ) 384 fi 385 # EMSCRIPTEN toolchain sets the right values itself 386 if [[ "${BUILD_TARGET}" != wasm* ]]; then 387 # If set, BUILD_TARGET must be the target triplet such as 388 # x86_64-unknown-linux-gnu. 389 args+=( 390 -DCMAKE_C_COMPILER_TARGET="${BUILD_TARGET}" 391 -DCMAKE_CXX_COMPILER_TARGET="${BUILD_TARGET}" 392 # Only the first element of the target triplet. 393 -DCMAKE_SYSTEM_PROCESSOR="${BUILD_TARGET%%-*}" 394 -DCMAKE_SYSTEM_NAME="${system_name}" 395 ) 396 else 397 # sjpeg confuses WASM SIMD with SSE. 398 args+=( 399 -DSJPEG_ENABLE_SIMD=OFF 400 ) 401 fi 402 args+=( 403 # These are needed to make googletest work when cross-compiling. 404 -DCMAKE_CROSSCOMPILING=1 405 -DHAVE_STD_REGEX=0 406 -DHAVE_POSIX_REGEX=0 407 -DHAVE_GNU_POSIX_REGEX=0 408 -DHAVE_STEADY_CLOCK=0 409 -DHAVE_THREAD_SAFETY_ATTRIBUTES=0 410 ) 411 if [[ -z "${CMAKE_FIND_ROOT_PATH}" ]]; then 412 # find_package() will look in this prefix for libraries. 413 CMAKE_FIND_ROOT_PATH="/usr/${BUILD_TARGET}" 414 fi 415 if [[ -z "${CMAKE_PREFIX_PATH}" ]]; then 416 CMAKE_PREFIX_PATH="/usr/${BUILD_TARGET}" 417 fi 418 # Use pkg-config for the target. If there's no pkg-config available for the 419 # target we can set the PKG_CONFIG_PATH to the appropriate path in most 420 # linux distributions. 421 local pkg_config=$(which "${BUILD_TARGET}-pkg-config" || true) 422 if [[ -z "${pkg_config}" ]]; then 423 pkg_config=$(which pkg-config) 424 export PKG_CONFIG_LIBDIR="/usr/${BUILD_TARGET}/lib/pkgconfig" 425 fi 426 if [[ -n "${pkg_config}" ]]; then 427 args+=(-DPKG_CONFIG_EXECUTABLE="${pkg_config}") 428 fi 429 fi 430 if [[ -n "${CMAKE_CROSSCOMPILING_EMULATOR}" ]]; then 431 args+=( 432 -DCMAKE_CROSSCOMPILING_EMULATOR="${CMAKE_CROSSCOMPILING_EMULATOR}" 433 ) 434 fi 435 if [[ -n "${CMAKE_FIND_ROOT_PATH}" ]]; then 436 args+=( 437 -DCMAKE_FIND_ROOT_PATH="${CMAKE_FIND_ROOT_PATH}" 438 ) 439 fi 440 if [[ -n "${CMAKE_PREFIX_PATH}" ]]; then 441 args+=( 442 -DCMAKE_PREFIX_PATH="${CMAKE_PREFIX_PATH}" 443 ) 444 fi 445 if [[ -n "${CMAKE_C_COMPILER_LAUNCHER}" ]]; then 446 args+=( 447 -DCMAKE_C_COMPILER_LAUNCHER="${CMAKE_C_COMPILER_LAUNCHER}" 448 ) 449 fi 450 if [[ -n "${CMAKE_CXX_COMPILER_LAUNCHER}" ]]; then 451 args+=( 452 -DCMAKE_CXX_COMPILER_LAUNCHER="${CMAKE_CXX_COMPILER_LAUNCHER}" 453 ) 454 fi 455 if [[ -n "${CMAKE_MAKE_PROGRAM}" ]]; then 456 args+=( 457 -DCMAKE_MAKE_PROGRAM="${CMAKE_MAKE_PROGRAM}" 458 ) 459 fi 460 cmake "${args[@]}" "$@" 461} 462 463cmake_build_and_test() { 464 # gtest_discover_tests() runs the test binaries to discover the list of tests 465 # at build time, which fails under qemu. 466 ASAN_OPTIONS=detect_leaks=0 cmake --build "${BUILD_DIR}" -- all doc 467 # Pack test binaries if requested. 468 if [[ "${PACK_TEST:-}" == "1" ]]; then 469 (cd "${BUILD_DIR}" 470 ${FIND_BIN} -name '*.cmake' -a '!' -path '*CMakeFiles*' 471 # gtest / gmock / gtest_main shared libs 472 ${FIND_BIN} lib/ -name 'libg*.so*' 473 ${FIND_BIN} -type d -name tests -a '!' -path '*CMakeFiles*' 474 ) | tar -C "${BUILD_DIR}" -cf "${BUILD_DIR}/tests.tar.xz" -T - \ 475 --use-compress-program="xz --threads=$(nproc --all || echo 1) -6" 476 du -h "${BUILD_DIR}/tests.tar.xz" 477 # Pack coverage data if also available. 478 touch "${BUILD_DIR}/gcno.sentinel" 479 (cd "${BUILD_DIR}"; echo gcno.sentinel; ${FIND_BIN} -name '*gcno') | \ 480 tar -C "${BUILD_DIR}" -cvf "${BUILD_DIR}/gcno.tar.xz" -T - \ 481 --use-compress-program="xz --threads=$(nproc --all || echo 1) -6" 482 fi 483 484 if [[ "${SKIP_TEST}" -ne "1" ]]; then 485 (cd "${BUILD_DIR}" 486 export UBSAN_OPTIONS=print_stacktrace=1 487 [[ "${TEST_STACK_LIMIT}" == "none" ]] || ulimit -s "${TEST_STACK_LIMIT}" 488 ctest -j $(nproc --all || echo 1) --output-on-failure) 489 fi 490} 491 492# Configure the build to strip unused functions. This considerably reduces the 493# output size, specially for tests which only use a small part of the whole 494# library. 495strip_dead_code() { 496 # Emscripten does tree shaking without any extra flags. 497 if [[ "${CMAKE_TOOLCHAIN_FILE##*/}" == "Emscripten.cmake" ]]; then 498 return 0 499 fi 500 # -ffunction-sections, -fdata-sections and -Wl,--gc-sections effectively 501 # discard all unreachable code, reducing the code size. For this to work, we 502 # need to also pass --no-export-dynamic to prevent it from exporting all the 503 # internal symbols (like functions) making them all reachable and thus not a 504 # candidate for removal. 505 CMAKE_CXX_FLAGS+=" -ffunction-sections -fdata-sections" 506 CMAKE_C_FLAGS+=" -ffunction-sections -fdata-sections" 507 if [[ "${OS}" == "Darwin" ]]; then 508 CMAKE_EXE_LINKER_FLAGS+=" -dead_strip" 509 CMAKE_SHARED_LINKER_FLAGS+=" -dead_strip" 510 else 511 CMAKE_EXE_LINKER_FLAGS+=" -Wl,--gc-sections -Wl,--no-export-dynamic" 512 CMAKE_SHARED_LINKER_FLAGS+=" -Wl,--gc-sections -Wl,--no-export-dynamic" 513 fi 514} 515 516### Externally visible commands 517 518cmd_debug() { 519 CMAKE_BUILD_TYPE="Debug" 520 cmake_configure "$@" 521 cmake_build_and_test 522} 523 524cmd_release() { 525 CMAKE_BUILD_TYPE="Release" 526 strip_dead_code 527 cmake_configure "$@" 528 cmake_build_and_test 529} 530 531cmd_opt() { 532 CMAKE_BUILD_TYPE="RelWithDebInfo" 533 CMAKE_CXX_FLAGS+=" -DJXL_DEBUG_WARNING -DJXL_DEBUG_ON_ERROR" 534 cmake_configure "$@" 535 cmake_build_and_test 536} 537 538cmd_coverage() { 539 # -O0 prohibits stack space reuse -> causes stack-overflow on dozens of tests. 540 TEST_STACK_LIMIT="none" 541 542 cmd_release -DJPEGXL_ENABLE_COVERAGE=ON "$@" 543 544 if [[ "${SKIP_TEST}" -ne "1" ]]; then 545 # If we didn't run the test we also don't print a coverage report. 546 cmd_coverage_report 547 fi 548} 549 550cmd_coverage_report() { 551 LLVM_COV=$("${CC:-clang}" -print-prog-name=llvm-cov) 552 local real_build_dir=$(realpath "${BUILD_DIR}") 553 local gcovr_args=( 554 -r "${real_build_dir}" 555 --gcov-executable "${LLVM_COV} gcov" 556 # Only print coverage information for the libjxl directories. The rest 557 # is not part of the code under test. 558 --filter '.*jxl/.*' 559 --exclude '.*_test.cc' 560 --exclude '.*_testonly..*' 561 --exclude '.*_debug.*' 562 --exclude '.*test_utils..*' 563 --object-directory "${real_build_dir}" 564 ) 565 566 ( 567 cd "${real_build_dir}" 568 gcovr "${gcovr_args[@]}" --html --html-details \ 569 --output="${real_build_dir}/coverage.html" 570 gcovr "${gcovr_args[@]}" --print-summary | 571 tee "${real_build_dir}/coverage.txt" 572 gcovr "${gcovr_args[@]}" --xml --output="${real_build_dir}/coverage.xml" 573 ) 574} 575 576cmd_test() { 577 export_env 578 # Unpack tests if needed. 579 if [[ -e "${BUILD_DIR}/tests.tar.xz" && ! -d "${BUILD_DIR}/tests" ]]; then 580 tar -C "${BUILD_DIR}" -Jxvf "${BUILD_DIR}/tests.tar.xz" 581 fi 582 if [[ -e "${BUILD_DIR}/gcno.tar.xz" && ! -d "${BUILD_DIR}/gcno.sentinel" ]]; then 583 tar -C "${BUILD_DIR}" -Jxvf "${BUILD_DIR}/gcno.tar.xz" 584 fi 585 (cd "${BUILD_DIR}" 586 export UBSAN_OPTIONS=print_stacktrace=1 587 [[ "${TEST_STACK_LIMIT}" == "none" ]] || ulimit -s "${TEST_STACK_LIMIT}" 588 ctest -j $(nproc --all || echo 1) --output-on-failure "$@") 589} 590 591cmd_gbench() { 592 export_env 593 (cd "${BUILD_DIR}" 594 export UBSAN_OPTIONS=print_stacktrace=1 595 lib/jxl_gbench \ 596 --benchmark_counters_tabular=true \ 597 --benchmark_out_format=json \ 598 --benchmark_out=gbench.json "$@" 599 ) 600} 601 602cmd_asanfuzz() { 603 CMAKE_CXX_FLAGS+=" -fsanitize=fuzzer-no-link -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION=1" 604 CMAKE_C_FLAGS+=" -fsanitize=fuzzer-no-link -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION=1" 605 cmd_asan -DJPEGXL_ENABLE_FUZZERS=ON "$@" 606} 607 608cmd_msanfuzz() { 609 # Install msan if needed before changing the flags. 610 detect_clang_version 611 local msan_prefix="${HOME}/.msan/${CLANG_VERSION}" 612 if [[ ! -d "${msan_prefix}" || -e "${msan_prefix}/lib/libc++abi.a" ]]; then 613 # Install msan libraries for this version if needed or if an older version 614 # with libc++abi was installed. 615 cmd_msan_install 616 fi 617 618 CMAKE_CXX_FLAGS+=" -fsanitize=fuzzer-no-link -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION=1" 619 CMAKE_C_FLAGS+=" -fsanitize=fuzzer-no-link -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION=1" 620 cmd_msan -DJPEGXL_ENABLE_FUZZERS=ON "$@" 621} 622 623cmd_asan() { 624 SANITIZER="asan" 625 CMAKE_C_FLAGS+=" -DJXL_ENABLE_ASSERT=1 -g -DADDRESS_SANITIZER \ 626 -fsanitize=address ${UBSAN_FLAGS[@]}" 627 CMAKE_CXX_FLAGS+=" -DJXL_ENABLE_ASSERT=1 -g -DADDRESS_SANITIZER \ 628 -fsanitize=address ${UBSAN_FLAGS[@]}" 629 strip_dead_code 630 cmake_configure "$@" -DJPEGXL_ENABLE_TCMALLOC=OFF 631 cmake_build_and_test 632} 633 634cmd_tsan() { 635 SANITIZER="tsan" 636 local tsan_args=( 637 -DJXL_ENABLE_ASSERT=1 638 -g 639 -DTHREAD_SANITIZER 640 ${UBSAN_FLAGS[@]} 641 -fsanitize=thread 642 ) 643 CMAKE_C_FLAGS+=" ${tsan_args[@]}" 644 CMAKE_CXX_FLAGS+=" ${tsan_args[@]}" 645 646 CMAKE_BUILD_TYPE="RelWithDebInfo" 647 cmake_configure "$@" -DJPEGXL_ENABLE_TCMALLOC=OFF 648 cmake_build_and_test 649} 650 651cmd_msan() { 652 SANITIZER="msan" 653 detect_clang_version 654 local msan_prefix="${HOME}/.msan/${CLANG_VERSION}" 655 if [[ ! -d "${msan_prefix}" || -e "${msan_prefix}/lib/libc++abi.a" ]]; then 656 # Install msan libraries for this version if needed or if an older version 657 # with libc++abi was installed. 658 cmd_msan_install 659 fi 660 661 local msan_c_flags=( 662 -fsanitize=memory 663 -fno-omit-frame-pointer 664 -fsanitize-memory-track-origins 665 666 -DJXL_ENABLE_ASSERT=1 667 -g 668 -DMEMORY_SANITIZER 669 670 # Force gtest to not use the cxxbai. 671 -DGTEST_HAS_CXXABI_H_=0 672 ) 673 local msan_cxx_flags=( 674 "${msan_c_flags[@]}" 675 676 # Some C++ sources don't use the std at all, so the -stdlib=libc++ is unused 677 # in those cases. Ignore the warning. 678 -Wno-unused-command-line-argument 679 -stdlib=libc++ 680 681 # We include the libc++ from the msan directory instead, so we don't want 682 # the std includes. 683 -nostdinc++ 684 -cxx-isystem"${msan_prefix}/include/c++/v1" 685 ) 686 687 local msan_linker_flags=( 688 -L"${msan_prefix}"/lib 689 -Wl,-rpath -Wl,"${msan_prefix}"/lib/ 690 ) 691 692 CMAKE_C_FLAGS+=" ${msan_c_flags[@]} ${UBSAN_FLAGS[@]}" 693 CMAKE_CXX_FLAGS+=" ${msan_cxx_flags[@]} ${UBSAN_FLAGS[@]}" 694 CMAKE_EXE_LINKER_FLAGS+=" ${msan_linker_flags[@]}" 695 CMAKE_MODULE_LINKER_FLAGS+=" ${msan_linker_flags[@]}" 696 CMAKE_SHARED_LINKER_FLAGS+=" ${msan_linker_flags[@]}" 697 strip_dead_code 698 cmake_configure "$@" \ 699 -DCMAKE_CROSSCOMPILING=1 -DRUN_HAVE_STD_REGEX=0 -DRUN_HAVE_POSIX_REGEX=0 \ 700 -DJPEGXL_ENABLE_TCMALLOC=OFF -DJPEGXL_WARNINGS_AS_ERRORS=OFF \ 701 -DCMAKE_REQUIRED_LINK_OPTIONS="${msan_linker_flags[@]}" 702 cmake_build_and_test 703} 704 705# Install libc++ libraries compiled with msan in the msan_prefix for the current 706# compiler version. 707cmd_msan_install() { 708 local tmpdir=$(mktemp -d) 709 CLEANUP_FILES+=("${tmpdir}") 710 # Detect the llvm to install: 711 export CC="${CC:-clang}" 712 export CXX="${CXX:-clang++}" 713 detect_clang_version 714 local llvm_tag="llvmorg-${CLANG_VERSION}.0.0" 715 case "${CLANG_VERSION}" in 716 "6.0") 717 llvm_tag="llvmorg-6.0.1" 718 ;; 719 "7") 720 llvm_tag="llvmorg-7.0.1" 721 ;; 722 esac 723 local llvm_targz="${tmpdir}/${llvm_tag}.tar.gz" 724 curl -L --show-error -o "${llvm_targz}" \ 725 "https://github.com/llvm/llvm-project/archive/${llvm_tag}.tar.gz" 726 tar -C "${tmpdir}" -zxf "${llvm_targz}" 727 local llvm_root="${tmpdir}/llvm-project-${llvm_tag}" 728 729 local msan_prefix="${HOME}/.msan/${CLANG_VERSION}" 730 rm -rf "${msan_prefix}" 731 732 declare -A CMAKE_EXTRAS 733 CMAKE_EXTRAS[libcxx]="\ 734 -DLIBCXX_CXX_ABI=libstdc++ \ 735 -DLIBCXX_INSTALL_EXPERIMENTAL_LIBRARY=ON" 736 737 for project in libcxx; do 738 local proj_build="${tmpdir}/build-${project}" 739 local proj_dir="${llvm_root}/${project}" 740 mkdir -p "${proj_build}" 741 cmake -B"${proj_build}" -H"${proj_dir}" \ 742 -G Ninja \ 743 -DCMAKE_BUILD_TYPE=Release \ 744 -DLLVM_USE_SANITIZER=Memory \ 745 -DLLVM_PATH="${llvm_root}/llvm" \ 746 -DLLVM_CONFIG_PATH="$(which llvm-config llvm-config-7 llvm-config-6.0 | \ 747 head -n1)" \ 748 -DCMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS}" \ 749 -DCMAKE_C_FLAGS="${CMAKE_C_FLAGS}" \ 750 -DCMAKE_EXE_LINKER_FLAGS="${CMAKE_EXE_LINKER_FLAGS}" \ 751 -DCMAKE_SHARED_LINKER_FLAGS="${CMAKE_SHARED_LINKER_FLAGS}" \ 752 -DCMAKE_INSTALL_PREFIX="${msan_prefix}" \ 753 ${CMAKE_EXTRAS[${project}]} 754 cmake --build "${proj_build}" 755 ninja -C "${proj_build}" install 756 done 757} 758 759# Internal build step shared between all cmd_ossfuzz_* commands. 760_cmd_ossfuzz() { 761 local sanitizer="$1" 762 shift 763 mkdir -p "${BUILD_DIR}" 764 local real_build_dir=$(realpath "${BUILD_DIR}") 765 766 # oss-fuzz defines three directories: 767 # * /work, with the working directory to do re-builds 768 # * /src, with the source code to build 769 # * /out, with the output directory where to copy over the built files. 770 # We use $BUILD_DIR as the /work and the script directory as the /src. The 771 # /out directory is ignored as developers are used to look for the fuzzers in 772 # $BUILD_DIR/tools/ directly. 773 774 if [[ "${sanitizer}" = "memory" && ! -d "${BUILD_DIR}/msan" ]]; then 775 sudo docker run --rm -i \ 776 --user $(id -u):$(id -g) \ 777 -v "${real_build_dir}":/work \ 778 gcr.io/oss-fuzz-base/msan-libs-builder \ 779 bash -c "cp -r /msan /work" 780 fi 781 782 # Args passed to ninja. These will be evaluated as a string separated by 783 # spaces. 784 local jpegxl_extra_args="$@" 785 786 sudo docker run --rm -i \ 787 -e JPEGXL_UID=$(id -u) \ 788 -e JPEGXL_GID=$(id -g) \ 789 -e FUZZING_ENGINE="${FUZZING_ENGINE:-libfuzzer}" \ 790 -e SANITIZER="${sanitizer}" \ 791 -e ARCHITECTURE=x86_64 \ 792 -e FUZZING_LANGUAGE=c++ \ 793 -e MSAN_LIBS_PATH="/work/msan" \ 794 -e JPEGXL_EXTRA_ARGS="${jpegxl_extra_args}" \ 795 -v "${MYDIR}":/src/libjxl \ 796 -v "${MYDIR}/tools/ossfuzz-build.sh":/src/build.sh \ 797 -v "${real_build_dir}":/work \ 798 gcr.io/oss-fuzz/libjxl 799} 800 801cmd_ossfuzz_asan() { 802 _cmd_ossfuzz address "$@" 803} 804cmd_ossfuzz_msan() { 805 _cmd_ossfuzz memory "$@" 806} 807cmd_ossfuzz_ubsan() { 808 _cmd_ossfuzz undefined "$@" 809} 810 811cmd_ossfuzz_ninja() { 812 [[ -e "${BUILD_DIR}/build.ninja" ]] 813 local real_build_dir=$(realpath "${BUILD_DIR}") 814 815 if [[ -e "${BUILD_DIR}/msan" ]]; then 816 echo "ossfuzz_ninja doesn't work with msan builds. Use ossfuzz_msan." >&2 817 exit 1 818 fi 819 820 sudo docker run --rm -i \ 821 --user $(id -u):$(id -g) \ 822 -v "${MYDIR}":/src/libjxl \ 823 -v "${real_build_dir}":/work \ 824 gcr.io/oss-fuzz/libjxl \ 825 ninja -C /work "$@" 826} 827 828cmd_fast_benchmark() { 829 local small_corpus_tar="${BENCHMARK_CORPORA}/jyrki-full.tar" 830 mkdir -p "${BENCHMARK_CORPORA}" 831 curl --show-error -o "${small_corpus_tar}" -z "${small_corpus_tar}" \ 832 "https://storage.googleapis.com/artifacts.jpegxl.appspot.com/corpora/jyrki-full.tar" 833 834 local tmpdir=$(mktemp -d) 835 CLEANUP_FILES+=("${tmpdir}") 836 tar -xf "${small_corpus_tar}" -C "${tmpdir}" 837 838 run_benchmark "${tmpdir}" 1048576 839} 840 841cmd_benchmark() { 842 local nikon_corpus_tar="${BENCHMARK_CORPORA}/nikon-subset.tar" 843 mkdir -p "${BENCHMARK_CORPORA}" 844 curl --show-error -o "${nikon_corpus_tar}" -z "${nikon_corpus_tar}" \ 845 "https://storage.googleapis.com/artifacts.jpegxl.appspot.com/corpora/nikon-subset.tar" 846 847 local tmpdir=$(mktemp -d) 848 CLEANUP_FILES+=("${tmpdir}") 849 tar -xvf "${nikon_corpus_tar}" -C "${tmpdir}" 850 851 local sem_id="jpegxl_benchmark-$$" 852 local nprocs=$(nproc --all || echo 1) 853 images=() 854 local filename 855 while IFS= read -r filename; do 856 # This removes the './' 857 filename="${filename:2}" 858 local mode 859 if [[ "${filename:0:4}" == "srgb" ]]; then 860 mode="RGB_D65_SRG_Rel_SRG" 861 elif [[ "${filename:0:5}" == "adobe" ]]; then 862 mode="RGB_D65_Ado_Rel_Ado" 863 else 864 echo "Unknown image colorspace: ${filename}" >&2 865 exit 1 866 fi 867 png_filename="${filename%.ppm}.png" 868 png_filename=$(echo "${png_filename}" | tr '/' '_') 869 sem --bg --id "${sem_id}" -j"${nprocs}" -- \ 870 "${BUILD_DIR}/tools/decode_and_encode" \ 871 "${tmpdir}/${filename}" "${mode}" "${tmpdir}/${png_filename}" 872 images+=( "${png_filename}" ) 873 done < <(cd "${tmpdir}"; ${FIND_BIN} . -name '*.ppm' -type f) 874 sem --id "${sem_id}" --wait 875 876 # We need about 10 GiB per thread on these images. 877 run_benchmark "${tmpdir}" 10485760 878} 879 880get_mem_available() { 881 if [[ "${OS}" == "Darwin" ]]; then 882 echo $(vm_stat | grep -F 'Pages free:' | awk '{print $3 * 4}') 883 else 884 echo $(grep -F MemAvailable: /proc/meminfo | awk '{print $2}') 885 fi 886} 887 888run_benchmark() { 889 local src_img_dir="$1" 890 local mem_per_thread="${2:-10485760}" 891 892 local output_dir="${BUILD_DIR}/benchmark_results" 893 mkdir -p "${output_dir}" 894 895 # The memory available at the beginning of the benchmark run in kB. The number 896 # of threads depends on the available memory, and the passed memory per 897 # thread. We also add a 2 GiB of constant memory. 898 local mem_available="$(get_mem_available)" 899 # Check that we actually have a MemAvailable value. 900 [[ -n "${mem_available}" ]] 901 local num_threads=$(( (${mem_available} - 1048576) / ${mem_per_thread} )) 902 if [[ ${num_threads} -le 0 ]]; then 903 num_threads=1 904 fi 905 906 local benchmark_args=( 907 --input "${src_img_dir}/*.png" 908 --codec=jpeg:yuv420:q85,webp:q80,jxl:fast:d1,jxl:fast:d1:downsampling=8,jxl:fast:d4,jxl:fast:d4:downsampling=8,jxl:cheetah:m,jxl:m:cheetah:P6,jxl:m:falcon:q80 909 --output_dir "${output_dir}" 910 --noprofiler --show_progress 911 --num_threads="${num_threads}" 912 ) 913 if [[ "${STORE_IMAGES}" == "1" ]]; then 914 benchmark_args+=(--save_decompressed --save_compressed) 915 fi 916 ( 917 [[ "${TEST_STACK_LIMIT}" == "none" ]] || ulimit -s "${TEST_STACK_LIMIT}" 918 "${BUILD_DIR}/tools/benchmark_xl" "${benchmark_args[@]}" | \ 919 tee "${output_dir}/results.txt" 920 921 # Check error code for benckmark_xl command. This will exit if not. 922 return ${PIPESTATUS[0]} 923 ) 924 925 if [[ -n "${CI_BUILD_NAME:-}" ]]; then 926 { set +x; } 2>/dev/null 927 local message="Results for ${CI_BUILD_NAME} @ ${CI_COMMIT_SHORT_SHA} (job ${CI_JOB_URL:-}): 928 929$(cat "${output_dir}/results.txt") 930" 931 cmd_post_mr_comment "${message}" 932 set -x 933 fi 934} 935 936# Helper function to wait for the CPU temperature to cool down on ARM. 937wait_for_temp() { 938 { set +x; } 2>/dev/null 939 local temp_limit=${1:-38000} 940 if [[ -z "${THERMAL_FILE:-}" ]]; then 941 echo "Must define the THERMAL_FILE with the thermal_zoneX/temp file" \ 942 "to read the temperature from. This is normally set in the runner." >&2 943 exit 1 944 fi 945 local org_temp=$(cat "${THERMAL_FILE}") 946 if [[ "${org_temp}" -ge "${temp_limit}" ]]; then 947 echo -n "Waiting for temp to get down from ${org_temp}... " 948 fi 949 local temp="${org_temp}" 950 local secs=0 951 while [[ "${temp}" -ge "${temp_limit}" ]]; do 952 sleep 1 953 temp=$(cat "${THERMAL_FILE}") 954 echo -n "${temp} " 955 secs=$((secs + 1)) 956 if [[ ${secs} -ge 5 ]]; then 957 break 958 fi 959 done 960 if [[ "${org_temp}" -ge "${temp_limit}" ]]; then 961 echo "Done, temp=${temp}" 962 fi 963 set -x 964} 965 966# Helper function to set the cpuset restriction of the current process. 967cmd_cpuset() { 968 [[ "${SKIP_CPUSET:-}" != "1" ]] || return 0 969 local newset="$1" 970 local mycpuset=$(cat /proc/self/cpuset) 971 mycpuset="/dev/cpuset${mycpuset}" 972 # Check that the directory exists: 973 [[ -d "${mycpuset}" ]] 974 if [[ -e "${mycpuset}/cpuset.cpus" ]]; then 975 echo "${newset}" >"${mycpuset}/cpuset.cpus" 976 else 977 echo "${newset}" >"${mycpuset}/cpus" 978 fi 979} 980 981# Return the encoding/decoding speed from the Stats output. 982_speed_from_output() { 983 local speed="$1" 984 local unit="${2:-MP/s}" 985 if [[ "${speed}" == *"${unit}"* ]]; then 986 speed="${speed%% ${unit}*}" 987 speed="${speed##* }" 988 echo "${speed}" 989 fi 990} 991 992 993# Run benchmarks on ARM for the big and little CPUs. 994cmd_arm_benchmark() { 995 # Flags used for cjxl encoder with .png inputs 996 local jxl_png_benchmarks=( 997 # Lossy options: 998 "--epf=0 --distance=1.0 --speed=cheetah" 999 "--epf=2 --distance=1.0 --speed=cheetah" 1000 "--epf=0 --distance=8.0 --speed=cheetah" 1001 "--epf=1 --distance=8.0 --speed=cheetah" 1002 "--epf=2 --distance=8.0 --speed=cheetah" 1003 "--epf=3 --distance=8.0 --speed=cheetah" 1004 "--modular -Q 90" 1005 "--modular -Q 50" 1006 # Lossless options: 1007 "--modular" 1008 "--modular -E 0 -I 0" 1009 "--modular -P 5" 1010 "--modular --responsive=1" 1011 # Near-lossless options: 1012 "--epf=0 --distance=0.3 --speed=fast" 1013 "--modular -Q 97" 1014 ) 1015 1016 # Flags used for cjxl encoder with .jpg inputs. These should do lossless 1017 # JPEG recompression (of pixels or full jpeg). 1018 local jxl_jpeg_benchmarks=( 1019 "--num_reps=3" 1020 ) 1021 1022 local images=( 1023 "third_party/testdata/imagecompression.info/flower_foveon.png" 1024 ) 1025 1026 local jpg_images=( 1027 "third_party/testdata/imagecompression.info/flower_foveon.png.im_q85_420.jpg" 1028 ) 1029 1030 if [[ "${SKIP_CPUSET:-}" == "1" ]]; then 1031 # Use a single cpu config in this case. 1032 local cpu_confs=("?") 1033 else 1034 # Otherwise the CPU config comes from the environment: 1035 local cpu_confs=( 1036 "${RUNNER_CPU_LITTLE}" 1037 "${RUNNER_CPU_BIG}" 1038 # The CPU description is something like 3-7, so these configurations only 1039 # take the first CPU of the group. 1040 "${RUNNER_CPU_LITTLE%%-*}" 1041 "${RUNNER_CPU_BIG%%-*}" 1042 ) 1043 # Check that RUNNER_CPU_ALL is defined. In the SKIP_CPUSET=1 case this will 1044 # be ignored but still evaluated when calling cmd_cpuset. 1045 [[ -n "${RUNNER_CPU_ALL}" ]] 1046 fi 1047 1048 local jpg_dirname="third_party/corpora/jpeg" 1049 mkdir -p "${jpg_dirname}" 1050 local jpg_qualities=( 50 80 95 ) 1051 for src_img in "${images[@]}"; do 1052 for q in "${jpg_qualities[@]}"; do 1053 local jpeg_name="${jpg_dirname}/"$(basename "${src_img}" .png)"-q${q}.jpg" 1054 convert -sampling-factor 1x1 -quality "${q}" \ 1055 "${src_img}" "${jpeg_name}" 1056 jpg_images+=("${jpeg_name}") 1057 done 1058 done 1059 1060 local output_dir="${BUILD_DIR}/benchmark_results" 1061 mkdir -p "${output_dir}" 1062 local runs_file="${output_dir}/runs.txt" 1063 1064 if [[ ! -e "${runs_file}" ]]; then 1065 echo -e "binary\tflags\tsrc_img\tsrc size\tsrc pixels\tcpuset\tenc size (B)\tenc speed (MP/s)\tdec speed (MP/s)\tJPG dec speed (MP/s)\tJPG dec speed (MB/s)" | 1066 tee -a "${runs_file}" 1067 fi 1068 1069 mkdir -p "${BUILD_DIR}/arm_benchmark" 1070 local flags 1071 local src_img 1072 for src_img in "${jpg_images[@]}" "${images[@]}"; do 1073 local src_img_hash=$(sha1sum "${src_img}" | cut -f 1 -d ' ') 1074 local enc_binaries=("${BUILD_DIR}/tools/cjxl") 1075 local src_ext="${src_img##*.}" 1076 for enc_binary in "${enc_binaries[@]}"; do 1077 local enc_binary_base=$(basename "${enc_binary}") 1078 1079 # Select the list of flags to use for the current encoder/image pair. 1080 local img_benchmarks 1081 if [[ "${src_ext}" == "jpg" ]]; then 1082 img_benchmarks=("${jxl_jpeg_benchmarks[@]}") 1083 else 1084 img_benchmarks=("${jxl_png_benchmarks[@]}") 1085 fi 1086 1087 for flags in "${img_benchmarks[@]}"; do 1088 # Encoding step. 1089 local enc_file_hash="${enc_binary_base} || $flags || ${src_img} || ${src_img_hash}" 1090 enc_file_hash=$(echo "${enc_file_hash}" | sha1sum | cut -f 1 -d ' ') 1091 local enc_file="${BUILD_DIR}/arm_benchmark/${enc_file_hash}.jxl" 1092 1093 for cpu_conf in "${cpu_confs[@]}"; do 1094 cmd_cpuset "${cpu_conf}" 1095 # nproc returns the number of active CPUs, which is given by the cpuset 1096 # mask. 1097 local num_threads="$(nproc)" 1098 1099 echo "Encoding with: ${enc_binary_base} img=${src_img} cpus=${cpu_conf} enc_flags=${flags}" 1100 local enc_output 1101 if [[ "${flags}" == *"modular"* ]]; then 1102 # We don't benchmark encoding speed in this case. 1103 if [[ ! -f "${enc_file}" ]]; then 1104 cmd_cpuset "${RUNNER_CPU_ALL:-}" 1105 "${enc_binary}" ${flags} "${src_img}" "${enc_file}.tmp" 1106 mv "${enc_file}.tmp" "${enc_file}" 1107 cmd_cpuset "${cpu_conf}" 1108 fi 1109 enc_output=" ?? MP/s" 1110 else 1111 wait_for_temp 1112 enc_output=$("${enc_binary}" ${flags} "${src_img}" "${enc_file}.tmp" \ 1113 2>&1 | tee /dev/stderr | grep -F "MP/s [") 1114 mv "${enc_file}.tmp" "${enc_file}" 1115 fi 1116 local enc_speed=$(_speed_from_output "${enc_output}") 1117 local enc_size=$(stat -c "%s" "${enc_file}") 1118 1119 echo "Decoding with: img=${src_img} cpus=${cpu_conf} enc_flags=${flags}" 1120 1121 local dec_output 1122 wait_for_temp 1123 dec_output=$("${BUILD_DIR}/tools/djxl" "${enc_file}" \ 1124 --num_reps=5 --num_threads="${num_threads}" 2>&1 | tee /dev/stderr | 1125 grep -E "M[BP]/s \[") 1126 local img_size=$(echo "${dec_output}" | cut -f 1 -d ',') 1127 local img_size_x=$(echo "${img_size}" | cut -f 1 -d ' ') 1128 local img_size_y=$(echo "${img_size}" | cut -f 3 -d ' ') 1129 local img_size_px=$(( ${img_size_x} * ${img_size_y} )) 1130 local dec_speed=$(_speed_from_output "${dec_output}") 1131 1132 # For JPEG lossless recompression modes (where the original is a JPEG) 1133 # decode to JPG as well. 1134 local jpeg_dec_mps_speed="" 1135 local jpeg_dec_mbs_speed="" 1136 if [[ "${src_ext}" == "jpg" ]]; then 1137 wait_for_temp 1138 local dec_file="${BUILD_DIR}/arm_benchmark/${enc_file_hash}.jpg" 1139 dec_output=$("${BUILD_DIR}/tools/djxl" "${enc_file}" \ 1140 "${dec_file}" --num_reps=5 --num_threads="${num_threads}" 2>&1 | \ 1141 tee /dev/stderr | grep -E "M[BP]/s \[") 1142 local jpeg_dec_mps_speed=$(_speed_from_output "${dec_output}") 1143 local jpeg_dec_mbs_speed=$(_speed_from_output "${dec_output}" MB/s) 1144 if ! cmp --quiet "${src_img}" "${dec_file}"; then 1145 # Add a start at the end to signal that the files are different. 1146 jpeg_dec_mbs_speed+="*" 1147 fi 1148 fi 1149 1150 # Record entry in a tab-separated file. 1151 local src_img_base=$(basename "${src_img}") 1152 echo -e "${enc_binary_base}\t${flags}\t${src_img_base}\t${img_size}\t${img_size_px}\t${cpu_conf}\t${enc_size}\t${enc_speed}\t${dec_speed}\t${jpeg_dec_mps_speed}\t${jpeg_dec_mbs_speed}" | 1153 tee -a "${runs_file}" 1154 done 1155 done 1156 done 1157 done 1158 cmd_cpuset "${RUNNER_CPU_ALL:-}" 1159 cat "${runs_file}" 1160 1161 if [[ -n "${CI_BUILD_NAME:-}" ]]; then 1162 load_mr_vars_from_commit 1163 { set +x; } 2>/dev/null 1164 local message="Results for ${CI_BUILD_NAME} @ ${CI_COMMIT_SHORT_SHA} (job ${CI_JOB_URL:-}): 1165 1166\`\`\` 1167$(column -t -s " " "${runs_file}") 1168\`\`\` 1169" 1170 cmd_post_mr_comment "${message}" 1171 set -x 1172 fi 1173} 1174 1175# Generate a corpus and run the fuzzer on that corpus. 1176cmd_fuzz() { 1177 local corpus_dir=$(realpath "${BUILD_DIR}/fuzzer_corpus") 1178 local fuzzer_crash_dir=$(realpath "${BUILD_DIR}/fuzzer_crash") 1179 mkdir -p "${corpus_dir}" "${fuzzer_crash_dir}" 1180 # Generate step. 1181 "${BUILD_DIR}/tools/fuzzer_corpus" "${corpus_dir}" 1182 # Run step: 1183 local nprocs=$(nproc --all || echo 1) 1184 ( 1185 cd "${BUILD_DIR}" 1186 "tools/djxl_fuzzer" "${fuzzer_crash_dir}" "${corpus_dir}" \ 1187 -max_total_time="${FUZZER_MAX_TIME}" -jobs=${nprocs} \ 1188 -artifact_prefix="${fuzzer_crash_dir}/" 1189 ) 1190} 1191 1192# Runs the linter (clang-format) on the pending CLs. 1193cmd_lint() { 1194 merge_request_commits 1195 { set +x; } 2>/dev/null 1196 local versions=(${1:-6.0 7 8 9 10 11}) 1197 local clang_format_bins=("${versions[@]/#/clang-format-}" clang-format) 1198 local tmpdir=$(mktemp -d) 1199 CLEANUP_FILES+=("${tmpdir}") 1200 1201 local ret=0 1202 local build_patch="${tmpdir}/build_cleaner.patch" 1203 if ! "${MYDIR}/tools/build_cleaner.py" >"${build_patch}"; then 1204 ret=1 1205 echo "build_cleaner.py findings:" >&2 1206 "${COLORDIFF_BIN}" <"${build_patch}" 1207 echo "Run \`tools/build_cleaner.py --update\` to apply them" >&2 1208 fi 1209 1210 local installed=() 1211 local clang_patch 1212 local clang_format 1213 for clang_format in "${clang_format_bins[@]}"; do 1214 if ! which "${clang_format}" >/dev/null; then 1215 continue 1216 fi 1217 installed+=("${clang_format}") 1218 local tmppatch="${tmpdir}/${clang_format}.patch" 1219 # We include in this linter all the changes including the uncommitted changes 1220 # to avoid printing changes already applied. 1221 set -x 1222 git -C "${MYDIR}" "${clang_format}" --binary "${clang_format}" \ 1223 --style=file --diff "${MR_ANCESTOR_SHA}" -- >"${tmppatch}" 1224 { set +x; } 2>/dev/null 1225 1226 if grep -E '^--- ' "${tmppatch}">/dev/null; then 1227 if [[ -n "${LINT_OUTPUT:-}" ]]; then 1228 cp "${tmppatch}" "${LINT_OUTPUT}" 1229 fi 1230 clang_patch="${tmppatch}" 1231 else 1232 echo "clang-format check OK" >&2 1233 return ${ret} 1234 fi 1235 done 1236 1237 if [[ ${#installed[@]} -eq 0 ]]; then 1238 echo "You must install clang-format for \"git clang-format\"" >&2 1239 exit 1 1240 fi 1241 1242 # clang-format is installed but found problems. 1243 echo "clang-format findings:" >&2 1244 "${COLORDIFF_BIN}" < "${clang_patch}" 1245 1246 echo "clang-format found issues in your patches from ${MR_ANCESTOR_SHA}" \ 1247 "to the current patch. Run \`./ci.sh lint | patch -p1\` from the base" \ 1248 "directory to apply them." >&2 1249 exit 1 1250} 1251 1252# Runs clang-tidy on the pending CLs. If the "all" argument is passed it runs 1253# clang-tidy over all the source files instead. 1254cmd_tidy() { 1255 local what="${1:-}" 1256 1257 if [[ -z "${CLANG_TIDY_BIN}" ]]; then 1258 echo "ERROR: You must install clang-tidy-7 or newer to use ci.sh tidy" >&2 1259 exit 1 1260 fi 1261 1262 local git_args=() 1263 if [[ "${what}" == "all" ]]; then 1264 git_args=(ls-files) 1265 shift 1266 else 1267 merge_request_commits 1268 git_args=( 1269 diff-tree --no-commit-id --name-only -r "${MR_ANCESTOR_SHA}" 1270 "${MR_HEAD_SHA}" 1271 ) 1272 fi 1273 1274 # Clang-tidy needs the compilation database generated by cmake. 1275 if [[ ! -e "${BUILD_DIR}/compile_commands.json" ]]; then 1276 # Generate the build options in debug mode, since we need the debug asserts 1277 # enabled for the clang-tidy analyzer to use them. 1278 CMAKE_BUILD_TYPE="Debug" 1279 cmake_configure 1280 # Build the autogen targets to generate the .h files from the .ui files. 1281 local autogen_targets=( 1282 $(ninja -C "${BUILD_DIR}" -t targets | grep -F _autogen: | 1283 cut -f 1 -d :) 1284 ) 1285 if [[ ${#autogen_targets[@]} != 0 ]]; then 1286 ninja -C "${BUILD_DIR}" "${autogen_targets[@]}" 1287 fi 1288 fi 1289 1290 cd "${MYDIR}" 1291 local nprocs=$(nproc --all || echo 1) 1292 local ret=0 1293 if ! parallel -j"${nprocs}" --keep-order -- \ 1294 "${CLANG_TIDY_BIN}" -p "${BUILD_DIR}" -format-style=file -quiet "$@" {} \ 1295 < <(git "${git_args[@]}" | grep -E '(\.cc|\.cpp)$') \ 1296 >"${BUILD_DIR}/clang-tidy.txt"; then 1297 ret=1 1298 fi 1299 { set +x; } 2>/dev/null 1300 echo "Findings statistics:" >&2 1301 grep -E ' \[[A-Za-z\.,\-]+\]' -o "${BUILD_DIR}/clang-tidy.txt" | sort \ 1302 | uniq -c >&2 1303 1304 if [[ $ret -ne 0 ]]; then 1305 cat >&2 <<EOF 1306Errors found, see ${BUILD_DIR}/clang-tidy.txt for details. 1307To automatically fix them, run: 1308 1309 SKIP_TEST=1 ./ci.sh debug 1310 ${CLANG_TIDY_BIN} -p ${BUILD_DIR} -fix -format-style=file -quiet $@ \$(git ${git_args[@]} | grep -E '(\.cc|\.cpp)\$') 1311EOF 1312 fi 1313 1314 return ${ret} 1315} 1316 1317# Print stats about all the packages built in ${BUILD_DIR}/debs/. 1318cmd_debian_stats() { 1319 { set +x; } 2>/dev/null 1320 local debsdir="${BUILD_DIR}/debs" 1321 local f 1322 while IFS='' read -r -d '' f; do 1323 echo "=====================================================================" 1324 echo "Package $f:" 1325 dpkg --info $f 1326 dpkg --contents $f 1327 done < <(find "${BUILD_DIR}/debs" -maxdepth 1 -mindepth 1 -type f \ 1328 -name '*.deb' -print0) 1329} 1330 1331build_debian_pkg() { 1332 local srcdir="$1" 1333 local srcpkg="$2" 1334 1335 local debsdir="${BUILD_DIR}/debs" 1336 local builddir="${debsdir}/${srcpkg}" 1337 1338 # debuild doesn't have an easy way to build out of tree, so we make a copy 1339 # of with all symlinks on the first level. 1340 mkdir -p "${builddir}" 1341 for f in $(find "${srcdir}" -mindepth 1 -maxdepth 1 -printf '%P\n'); do 1342 if [[ ! -L "${builddir}/$f" ]]; then 1343 rm -f "${builddir}/$f" 1344 ln -s "${srcdir}/$f" "${builddir}/$f" 1345 fi 1346 done 1347 ( 1348 cd "${builddir}" 1349 debuild -b -uc -us 1350 ) 1351} 1352 1353cmd_debian_build() { 1354 local srcpkg="${1:-}" 1355 1356 case "${srcpkg}" in 1357 jpeg-xl) 1358 build_debian_pkg "${MYDIR}" "jpeg-xl" 1359 ;; 1360 highway) 1361 build_debian_pkg "${MYDIR}/third_party/highway" "highway" 1362 ;; 1363 *) 1364 echo "ERROR: Must pass a valid source package name to build." >&2 1365 ;; 1366 esac 1367} 1368 1369get_version() { 1370 local varname=$1 1371 local line=$(grep -F "set(${varname} " lib/CMakeLists.txt | head -n 1) 1372 [[ -n "${line}" ]] 1373 line="${line#set(${varname} }" 1374 line="${line%)}" 1375 echo "${line}" 1376} 1377 1378cmd_bump_version() { 1379 local newver="${1:-}" 1380 1381 if ! which dch >/dev/null; then 1382 echo "Run:\n sudo apt install debhelper" 1383 exit 1 1384 fi 1385 1386 if [[ -z "${newver}" ]]; then 1387 local major=$(get_version JPEGXL_MAJOR_VERSION) 1388 local minor=$(get_version JPEGXL_MINOR_VERSION) 1389 local patch=0 1390 minor=$(( ${minor} + 1)) 1391 else 1392 local major="${newver%%.*}" 1393 newver="${newver#*.}" 1394 local minor="${newver%%.*}" 1395 newver="${newver#${minor}}" 1396 local patch="${newver#.}" 1397 if [[ -z "${patch}" ]]; then 1398 patch=0 1399 fi 1400 fi 1401 1402 newver="${major}.${minor}" 1403 if [[ "${patch}" != "0" ]]; then 1404 newver="${newver}.${patch}" 1405 fi 1406 echo "Bumping version to ${newver} (${major}.${minor}.${patch})" 1407 sed -E \ 1408 -e "s/(set\\(JPEGXL_MAJOR_VERSION) [0-9]+\\)/\\1 ${major})/" \ 1409 -e "s/(set\\(JPEGXL_MINOR_VERSION) [0-9]+\\)/\\1 ${minor})/" \ 1410 -e "s/(set\\(JPEGXL_PATCH_VERSION) [0-9]+\\)/\\1 ${patch})/" \ 1411 -i lib/CMakeLists.txt 1412 1413 # Update lib.gni 1414 tools/build_cleaner.py --update 1415 1416 # Mark the previous version as "unstable". 1417 DEBCHANGE_RELEASE_HEURISTIC=log dch -M --distribution unstable --release '' 1418 DEBCHANGE_RELEASE_HEURISTIC=log dch -M \ 1419 --newversion "${newver}" \ 1420 "Bump JPEG XL version to ${newver}." 1421} 1422 1423# Check that the AUTHORS file contains the email of the committer. 1424cmd_authors() { 1425 merge_request_commits 1426 # TODO(deymo): Handle multiple commits and check that they are all the same 1427 # author. 1428 local email=$(git log --format='%ae' "${MR_HEAD_SHA}^!") 1429 local name=$(git log --format='%an' "${MR_HEAD_SHA}^!") 1430 "${MYDIR}"/tools/check_author.py "${email}" "${name}" 1431} 1432 1433main() { 1434 local cmd="${1:-}" 1435 if [[ -z "${cmd}" ]]; then 1436 cat >&2 <<EOF 1437Use: $0 CMD 1438 1439Where cmd is one of: 1440 opt Build and test a Release with symbols build. 1441 debug Build and test a Debug build (NDEBUG is not defined). 1442 release Build and test a striped Release binary without debug information. 1443 asan Build and test an ASan (AddressSanitizer) build. 1444 msan Build and test an MSan (MemorySanitizer) build. Needs to have msan 1445 c++ libs installed with msan_install first. 1446 tsan Build and test a TSan (ThreadSanitizer) build. 1447 asanfuzz Build and test an ASan (AddressSanitizer) build for fuzzing. 1448 msanfuzz Build and test an MSan (MemorySanitizer) build for fuzzing. 1449 test Run the tests build by opt, debug, release, asan or msan. Useful when 1450 building with SKIP_TEST=1. 1451 gbench Run the Google benchmark tests. 1452 fuzz Generate the fuzzer corpus and run the fuzzer on it. Useful after 1453 building with asan or msan. 1454 benchmark Run the benchmark over the default corpus. 1455 fast_benchmark Run the benchmark over the small corpus. 1456 1457 coverage Buils and run tests with coverage support. Runs coverage_report as 1458 well. 1459 coverage_report Generate HTML, XML and text coverage report after a coverage 1460 run. 1461 1462 lint Run the linter checks on the current commit or merge request. 1463 tidy Run clang-tidy on the current commit or merge request. 1464 authors Check that the last commit's author is listed in the AUTHORS file. 1465 1466 msan_install Install the libc++ libraries required to build in msan mode. This 1467 needs to be done once. 1468 1469 debian_build <srcpkg> Build the given source package. 1470 debian_stats Print stats about the built packages. 1471 1472oss-fuzz commands: 1473 ossfuzz_asan Build the local source inside oss-fuzz docker with asan. 1474 ossfuzz_msan Build the local source inside oss-fuzz docker with msan. 1475 ossfuzz_ubsan Build the local source inside oss-fuzz docker with ubsan. 1476 ossfuzz_ninja Run ninja on the BUILD_DIR inside the oss-fuzz docker. Extra 1477 parameters are passed to ninja, for example "djxl_fuzzer" will 1478 only build that ninja target. Use for faster build iteration 1479 after one of the ossfuzz_*san commands. 1480 1481You can pass some optional environment variables as well: 1482 - BUILD_DIR: The output build directory (by default "$$repo/build") 1483 - BUILD_TARGET: The target triplet used when cross-compiling. 1484 - CMAKE_FLAGS: Convenience flag to pass both CMAKE_C_FLAGS and CMAKE_CXX_FLAGS. 1485 - CMAKE_PREFIX_PATH: Installation prefixes to be searched by the find_package. 1486 - ENABLE_WASM_SIMD=1: enable experimental SIMD in WASM build (only). 1487 - FUZZER_MAX_TIME: "fuzz" command fuzzer running timeout in seconds. 1488 - LINT_OUTPUT: Path to the output patch from the "lint" command. 1489 - SKIP_CPUSET=1: Skip modifying the cpuset in the arm_benchmark. 1490 - SKIP_TEST=1: Skip the test stage. 1491 - STORE_IMAGES=0: Makes the benchmark discard the computed images. 1492 - TEST_STACK_LIMIT: Stack size limit (ulimit -s) during tests, in KiB. 1493 - STACK_SIZE=1: Generate binaries with the .stack_sizes sections. 1494 1495These optional environment variables are forwarded to the cmake call as 1496parameters: 1497 - CMAKE_BUILD_TYPE 1498 - CMAKE_C_FLAGS 1499 - CMAKE_CXX_FLAGS 1500 - CMAKE_C_COMPILER_LAUNCHER 1501 - CMAKE_CXX_COMPILER_LAUNCHER 1502 - CMAKE_CROSSCOMPILING_EMULATOR 1503 - CMAKE_FIND_ROOT_PATH 1504 - CMAKE_EXE_LINKER_FLAGS 1505 - CMAKE_MAKE_PROGRAM 1506 - CMAKE_MODULE_LINKER_FLAGS 1507 - CMAKE_SHARED_LINKER_FLAGS 1508 - CMAKE_TOOLCHAIN_FILE 1509 1510Example: 1511 BUILD_DIR=/tmp/build $0 opt 1512EOF 1513 exit 1 1514 fi 1515 1516 cmd="cmd_${cmd}" 1517 shift 1518 set -x 1519 "${cmd}" "$@" 1520} 1521 1522main "$@" 1523
