1#!/bin/sh -e 2 3# Upload a created tarball to Coverity Scan, as per 4# https://scan.coverity.com/projects/qemu/builds/new 5 6# This work is licensed under the terms of the GNU GPL version 2, 7# or (at your option) any later version. 8# See the COPYING file in the top-level directory. 9# 10# Copyright (c) 2017-2020 Linaro Limited 11# Written by Peter Maydell 12 13# Note that this script will automatically download and 14# run the (closed-source) coverity build tools, so don't 15# use it if you don't trust them! 16 17# This script assumes that you're running it from a QEMU source 18# tree, and that tree is a fresh clean one, because we do an in-tree 19# build. (This is necessary so that the filenames that the Coverity 20# Scan server sees are relative paths that match up with the component 21# regular expressions it uses; an out-of-tree build won't work for this.) 22# The host machine should have as many of QEMU's dependencies 23# installed as possible, for maximum coverity coverage. 24 25# To do an upload you need to be a maintainer in the Coverity online 26# service, and you will need to know the "Coverity token", which is a 27# secret 8 digit hex string. You can find that from the web UI in the 28# project settings, if you have maintainer access there. 29 30# Command line options: 31# --check-upload-only : return success if upload is possible 32# --dry-run : run the tools, but don't actually do the upload 33# --docker : create and work inside a container 34# --docker-engine : specify the container engine to use (docker/podman/auto); 35# implies --docker 36# --update-tools-only : update the cached copy of the tools, but don't run them 37# --no-update-tools : do not update the cached copy of the tools 38# --tokenfile : file to read Coverity token from 39# --version ver : specify version being analyzed (default: ask git) 40# --description desc : specify description of this version (default: ask git) 41# --srcdir : QEMU source tree to analyze (default: current working dir) 42# --results-tarball : path to copy the results tarball to (default: don't 43# copy it anywhere, just upload it) 44# --src-tarball : tarball to untar into src dir (default: none); this 45# is intended mainly for internal use by the Docker support 46# 47# User-specifiable environment variables: 48# COVERITY_TOKEN -- Coverity token (default: looks at your 49# coverity.token config) 50# COVERITY_EMAIL -- the email address to use for uploads (default: 51# looks at your git coverity.email or user.email config) 52# COVERITY_BUILD_CMD -- make command (default: 'make -jN' where N is 53# number of CPUs as determined by 'nproc') 54# COVERITY_TOOL_BASE -- set to directory to put coverity tools 55# (default: /tmp/coverity-tools) 56# 57# You must specify the token, either by environment variable or by 58# putting it in a file and using --tokenfile. Everything else has 59# a reasonable default if this is run from a git tree. 60 61upload_permitted() { 62 # Check whether we can do an upload to the server; will exit *the script* 63 # with status 99 if the check failed (usually a bad token); 64 # will return from the function with status 1 if the check indicated 65 # that we can't upload yet (ie we are at quota) 66 # Assumes that COVERITY_TOKEN and PROJNAME have been initialized. 67 68 echo "Checking upload permissions..." 69 70 if ! up_perm="$(wget https://scan.coverity.com/api/upload_permitted --post-data "token=$COVERITY_TOKEN&project=$PROJNAME" -q -O -)"; then 71 echo "Coverity Scan API access denied: bad token?" 72 exit 99 73 fi 74 75 # Really up_perm is a JSON response with either 76 # {upload_permitted:true} or {next_upload_permitted_at:<date>} 77 # We do some hacky string parsing instead of properly parsing it. 78 case "$up_perm" in 79 *upload_permitted*true*) 80 return 0 81 ;; 82 *next_upload_permitted_at*) 83 return 1 84 ;; 85 *) 86 echo "Coverity Scan upload check: unexpected result $up_perm" 87 exit 99 88 ;; 89 esac 90} 91 92 93check_upload_permissions() { 94 # Check whether we can do an upload to the server; will exit the script 95 # with status 99 if the check failed (usually a bad token); 96 # will exit the script with status 0 if the check indicated that we 97 # can't upload yet (ie we are at quota) 98 # Assumes that COVERITY_TOKEN, PROJNAME and DRYRUN have been initialized. 99 100 if upload_permitted; then 101 echo "Coverity Scan: upload permitted" 102 else 103 if [ "$DRYRUN" = yes ]; then 104 echo "Coverity Scan: upload quota reached, continuing dry run" 105 else 106 echo "Coverity Scan: upload quota reached; stopping here" 107 # Exit success as this isn't a build error. 108 exit 0 109 fi 110 fi 111} 112 113 114build_docker_image() { 115 # build docker container including the coverity-scan tools 116 echo "Building docker container..." 117 # TODO: This re-unpacks the tools every time, rather than caching 118 # and reusing the image produced by the COPY of the .tgz file. 119 # Not sure why. 120 tests/docker/docker.py --engine ${DOCKER_ENGINE} build \ 121 -t coverity-scanner -f scripts/coverity-scan/coverity-scan.docker \ 122 --extra-files scripts/coverity-scan/run-coverity-scan \ 123 "$COVERITY_TOOL_BASE"/coverity_tool.tgz 124} 125 126update_coverity_tools () { 127 # Check for whether we need to download the Coverity tools 128 # (either because we don't have a copy, or because it's out of date) 129 # Assumes that COVERITY_TOOL_BASE, COVERITY_TOKEN and PROJNAME are set. 130 131 mkdir -p "$COVERITY_TOOL_BASE" 132 cd "$COVERITY_TOOL_BASE" 133 134 echo "Checking for new version of coverity build tools..." 135 wget https://scan.coverity.com/download/cxx/linux64 --post-data "token=$COVERITY_TOKEN&project=$PROJNAME&md5=1" -O coverity_tool.md5.new 136 137 if ! cmp -s coverity_tool.md5 coverity_tool.md5.new; then 138 # out of date md5 or no md5: download new build tool 139 # blow away the old build tool 140 echo "Downloading coverity build tools..." 141 rm -rf coverity_tool coverity_tool.tgz 142 wget https://scan.coverity.com/download/cxx/linux64 --post-data "token=$COVERITY_TOKEN&project=$PROJNAME" -O coverity_tool.tgz 143 if ! (cat coverity_tool.md5.new; echo " coverity_tool.tgz") | md5sum -c --status; then 144 echo "Downloaded tarball didn't match md5sum!" 145 exit 1 146 fi 147 148 if [ "$DOCKER" != yes ]; then 149 # extract the new one, keeping it corralled in a 'coverity_tool' directory 150 echo "Unpacking coverity build tools..." 151 mkdir -p coverity_tool 152 cd coverity_tool 153 tar xf ../coverity_tool.tgz 154 cd .. 155 mv coverity_tool.md5.new coverity_tool.md5 156 fi 157 fi 158 rm -f coverity_tool.md5.new 159 cd "$SRCDIR" 160 161 if [ "$DOCKER" = yes ]; then 162 build_docker_image 163 fi 164} 165 166 167# Check user-provided environment variables and arguments 168DRYRUN=no 169UPDATE=yes 170DOCKER=no 171PROJNAME=QEMU 172 173while [ "$#" -ge 1 ]; do 174 case "$1" in 175 --check-upload-only) 176 shift 177 DRYRUN=check 178 ;; 179 --dry-run) 180 shift 181 DRYRUN=yes 182 ;; 183 --no-update-tools) 184 shift 185 UPDATE=no 186 ;; 187 --update-tools-only) 188 shift 189 UPDATE=only 190 ;; 191 --version) 192 shift 193 if [ $# -eq 0 ]; then 194 echo "--version needs an argument" 195 exit 1 196 fi 197 VERSION="$1" 198 shift 199 ;; 200 --description) 201 shift 202 if [ $# -eq 0 ]; then 203 echo "--description needs an argument" 204 exit 1 205 fi 206 DESCRIPTION="$1" 207 shift 208 ;; 209 --tokenfile) 210 shift 211 if [ $# -eq 0 ]; then 212 echo "--tokenfile needs an argument" 213 exit 1 214 fi 215 COVERITY_TOKEN="$(cat "$1")" 216 shift 217 ;; 218 --srcdir) 219 shift 220 if [ $# -eq 0 ]; then 221 echo "--srcdir needs an argument" 222 exit 1 223 fi 224 SRCDIR="$1" 225 shift 226 ;; 227 --results-tarball) 228 shift 229 if [ $# -eq 0 ]; then 230 echo "--results-tarball needs an argument" 231 exit 1 232 fi 233 RESULTSTARBALL="$1" 234 shift 235 ;; 236 --src-tarball) 237 shift 238 if [ $# -eq 0 ]; then 239 echo "--src-tarball needs an argument" 240 exit 1 241 fi 242 SRCTARBALL="$1" 243 shift 244 ;; 245 --docker) 246 DOCKER=yes 247 DOCKER_ENGINE=auto 248 shift 249 ;; 250 --docker-engine) 251 shift 252 if [ $# -eq 0 ]; then 253 echo "--docker-engine needs an argument" 254 exit 1 255 fi 256 DOCKER=yes 257 DOCKER_ENGINE="$1" 258 shift 259 ;; 260 *) 261 echo "Unexpected argument '$1'" 262 exit 1 263 ;; 264 esac 265done 266 267if [ -z "$COVERITY_TOKEN" ]; then 268 COVERITY_TOKEN="$(git config coverity.token)" 269fi 270if [ -z "$COVERITY_TOKEN" ]; then 271 echo "COVERITY_TOKEN environment variable not set" 272 exit 1 273fi 274 275if [ "$DRYRUN" = check ]; then 276 upload_permitted 277 exit $? 278fi 279 280if [ -z "$COVERITY_BUILD_CMD" ]; then 281 NPROC=$(nproc) 282 COVERITY_BUILD_CMD="make -j$NPROC" 283 echo "COVERITY_BUILD_CMD: using default '$COVERITY_BUILD_CMD'" 284fi 285 286if [ -z "$COVERITY_TOOL_BASE" ]; then 287 echo "COVERITY_TOOL_BASE: using default /tmp/coverity-tools" 288 COVERITY_TOOL_BASE=/tmp/coverity-tools 289fi 290 291if [ -z "$SRCDIR" ]; then 292 SRCDIR="$PWD" 293fi 294 295TARBALL=cov-int.tar.xz 296 297if [ "$UPDATE" = only ]; then 298 # Just do the tools update; we don't need to check whether 299 # we are in a source tree or have upload rights for this, 300 # so do it before some of the command line and source tree checks. 301 302 if [ "$DOCKER" = yes ] && [ ! -z "$SRCTARBALL" ]; then 303 echo --update-tools-only --docker is incompatible with --src-tarball. 304 exit 1 305 fi 306 307 update_coverity_tools 308 exit 0 309fi 310 311if [ ! -e "$SRCDIR" ]; then 312 mkdir "$SRCDIR" 313fi 314 315cd "$SRCDIR" 316 317if [ ! -z "$SRCTARBALL" ]; then 318 echo "Untarring source tarball into $SRCDIR..." 319 tar xvf "$SRCTARBALL" 320fi 321 322echo "Checking this is a QEMU source tree..." 323if ! [ -e "$SRCDIR/VERSION" ]; then 324 echo "Not in a QEMU source tree?" 325 exit 1 326fi 327 328# Fill in defaults used by the non-update-only process 329if [ -z "$VERSION" ]; then 330 VERSION="$(git describe --always HEAD)" 331fi 332 333if [ -z "$DESCRIPTION" ]; then 334 DESCRIPTION="$(git rev-parse HEAD)" 335fi 336 337if [ -z "$COVERITY_EMAIL" ]; then 338 COVERITY_EMAIL="$(git config coverity.email)" 339fi 340if [ -z "$COVERITY_EMAIL" ]; then 341 COVERITY_EMAIL="$(git config user.email)" 342fi 343 344# Otherwise, continue with the full build and upload process. 345 346check_upload_permissions 347 348if [ "$UPDATE" != no ]; then 349 update_coverity_tools 350fi 351 352# Run ourselves inside docker if that's what the user wants 353if [ "$DOCKER" = yes ]; then 354 # Put the Coverity token into a temporary file that only 355 # we have read access to, and then pass it to docker build 356 # using a volume. A volume is enough for the token not to 357 # leak into the Docker image. 358 umask 077 359 SECRETDIR=$(mktemp -d) 360 if [ -z "$SECRETDIR" ]; then 361 echo "Failed to create temporary directory" 362 exit 1 363 fi 364 trap 'rm -rf "$SECRETDIR"' INT TERM EXIT 365 echo "Created temporary directory $SECRETDIR" 366 SECRET="$SECRETDIR/token" 367 echo "$COVERITY_TOKEN" > "$SECRET" 368 echo "Archiving sources to be analyzed..." 369 ./scripts/archive-source.sh "$SECRETDIR/qemu-sources.tgz" 370 ARGS="--no-update-tools" 371 if [ "$DRYRUN" = yes ]; then 372 ARGS="$ARGS --dry-run" 373 fi 374 echo "Running scanner..." 375 # If we need to capture the output tarball, get the inner run to 376 # save it to the secrets directory so we can copy it out before the 377 # directory is cleaned up. 378 if [ ! -z "$RESULTSTARBALL" ]; then 379 ARGS="$ARGS --results-tarball /work/cov-int.tar.xz" 380 fi 381 # Arrange for this docker run to get access to the sources with -v. 382 # We pass through all the configuration from the outer script to the inner. 383 export COVERITY_EMAIL COVERITY_BUILD_CMD 384 tests/docker/docker.py run -it --env COVERITY_EMAIL --env COVERITY_BUILD_CMD \ 385 -v "$SECRETDIR:/work" coverity-scanner \ 386 ./run-coverity-scan --version "$VERSION" \ 387 --description "$DESCRIPTION" $ARGS --tokenfile /work/token \ 388 --srcdir /qemu --src-tarball /work/qemu-sources.tgz 389 if [ ! -z "$RESULTSTARBALL" ]; then 390 echo "Copying results tarball to $RESULTSTARBALL..." 391 cp "$SECRETDIR/cov-int.tar.xz" "$RESULTSTARBALL" 392 fi 393 echo "Docker work complete." 394 exit 0 395fi 396 397TOOLBIN="$(cd "$COVERITY_TOOL_BASE" && echo $PWD/coverity_tool/cov-analysis-*/bin)" 398 399if ! test -x "$TOOLBIN/cov-build"; then 400 echo "Couldn't find cov-build in the coverity build-tool directory??" 401 exit 1 402fi 403 404export PATH="$TOOLBIN:$PATH" 405 406cd "$SRCDIR" 407 408echo "Nuking build directory..." 409rm -rf +build 410mkdir +build 411cd +build 412 413echo "Configuring..." 414# We configure with a fixed set of enables here to ensure that we don't 415# accidentally reduce the scope of the analysis by doing the build on 416# the system that's missing a dependency that we need to build part of 417# the codebase. 418../configure --disable-modules --enable-sdl --enable-gtk \ 419 --enable-opengl --enable-vte --enable-gnutls \ 420 --enable-nettle --enable-curses --enable-curl \ 421 --audio-drv-list=oss,alsa,sdl,pa --enable-virtfs \ 422 --enable-vnc --enable-vnc-sasl --enable-vnc-jpeg --enable-png \ 423 --enable-xen --enable-brlapi \ 424 --enable-linux-aio --enable-attr \ 425 --enable-cap-ng --enable-trace-backends=log --enable-spice --enable-rbd \ 426 --enable-libusb --enable-usb-redir \ 427 --enable-libiscsi --enable-libnfs --enable-seccomp \ 428 --enable-tpm --enable-libssh --enable-lzo --enable-snappy --enable-bzip2 \ 429 --enable-numa --enable-rdma --enable-smartcard --enable-virglrenderer \ 430 --enable-mpath --enable-glusterfs \ 431 --enable-virtfs --enable-zstd 432 433echo "Running cov-build..." 434rm -rf cov-int 435mkdir cov-int 436cov-build --dir cov-int $COVERITY_BUILD_CMD 437 438echo "Creating results tarball..." 439tar cvf - cov-int | xz > "$TARBALL" 440 441if [ ! -z "$RESULTSTARBALL" ]; then 442 echo "Copying results tarball to $RESULTSTARBALL..." 443 cp "$TARBALL" "$RESULTSTARBALL" 444fi 445 446echo "Uploading results tarball..." 447 448if [ "$DRYRUN" = yes ]; then 449 echo "Dry run only, not uploading $TARBALL" 450 exit 0 451fi 452 453curl --form token="$COVERITY_TOKEN" --form email="$COVERITY_EMAIL" \ 454 --form file=@"$TARBALL" --form version="$VERSION" \ 455 --form description="$DESCRIPTION" \ 456 https://scan.coverity.com/builds?project="$PROJNAME" 457 458echo "Done." 459