1#!/bin/sh -e
2
3# Upload a created tarball to Coverity Scan, as per
4# https://scan.coverity.com/projects/qemu/builds/new
5
6# This work is licensed under the terms of the GNU GPL version 2,
7# or (at your option) any later version.
8# See the COPYING file in the top-level directory.
9#
10# Copyright (c) 2017-2020 Linaro Limited
11# Written by Peter Maydell
12
13# Note that this script will automatically download and
14# run the (closed-source) coverity build tools, so don't
15# use it if you don't trust them!
16
17# This script assumes that you're running it from a QEMU source
18# tree, and that tree is a fresh clean one, because we do an in-tree
19# build. (This is necessary so that the filenames that the Coverity
20# Scan server sees are relative paths that match up with the component
21# regular expressions it uses; an out-of-tree build won't work for this.)
22# The host machine should have as many of QEMU's dependencies
23# installed as possible, for maximum coverity coverage.
24
25# To do an upload you need to be a maintainer in the Coverity online
26# service, and you will need to know the "Coverity token", which is a
27# secret 8 digit hex string. You can find that from the web UI in the
28# project settings, if you have maintainer access there.
29
30# Command line options:
31#   --dry-run : run the tools, but don't actually do the upload
32#   --docker : create and work inside a docker container
33#   --update-tools-only : update the cached copy of the tools, but don't run them
34#   --tokenfile : file to read Coverity token from
35#   --version ver : specify version being analyzed (default: ask git)
36#   --description desc : specify description of this version (default: ask git)
37#   --srcdir : QEMU source tree to analyze (default: current working dir)
38#   --results-tarball : path to copy the results tarball to (default: don't
39#                       copy it anywhere, just upload it)
40#   --src-tarball : tarball to untar into src dir (default: none); this
41#                   is intended mainly for internal use by the Docker support
42#
43# User-specifiable environment variables:
44#  COVERITY_TOKEN -- Coverity token
45#  COVERITY_EMAIL -- the email address to use for uploads (default:
46#                    looks at your git user.email config)
47#  COVERITY_BUILD_CMD -- make command (default: 'make -jN' where N is
48#                    number of CPUs as determined by 'nproc')
49#  COVERITY_TOOL_BASE -- set to directory to put coverity tools
50#                        (default: /tmp/coverity-tools)
51#
52# You must specify the token, either by environment variable or by
53# putting it in a file and using --tokenfile. Everything else has
54# a reasonable default if this is run from a git tree.
55
56check_upload_permissions() {
57    # Check whether we can do an upload to the server; will exit the script
58    # with status 1 if the check failed (usually a bad token);
59    # will exit the script with status 0 if the check indicated that we
60    # can't upload yet (ie we are at quota)
61    # Assumes that PROJTOKEN, PROJNAME and DRYRUN have been initialized.
62
63    echo "Checking upload permissions..."
64
65    if ! up_perm="$(wget https://scan.coverity.com/api/upload_permitted --post-data "token=$PROJTOKEN&project=$PROJNAME" -q -O -)"; then
66        echo "Coverity Scan API access denied: bad token?"
67        exit 1
68    fi
69
70    # Really up_perm is a JSON response with either
71    # {upload_permitted:true} or {next_upload_permitted_at:<date>}
72    # We do some hacky string parsing instead of properly parsing it.
73    case "$up_perm" in
74        *upload_permitted*true*)
75            echo "Coverity Scan: upload permitted"
76            ;;
77        *next_upload_permitted_at*)
78            if [ "$DRYRUN" = yes ]; then
79                echo "Coverity Scan: upload quota reached, continuing dry run"
80            else
81                echo "Coverity Scan: upload quota reached; stopping here"
82                # Exit success as this isn't a build error.
83                exit 0
84            fi
85            ;;
86        *)
87            echo "Coverity Scan upload check: unexpected result $up_perm"
88            exit 1
89            ;;
90    esac
91}
92
93
94update_coverity_tools () {
95    # Check for whether we need to download the Coverity tools
96    # (either because we don't have a copy, or because it's out of date)
97    # Assumes that COVERITY_TOOL_BASE, PROJTOKEN and PROJNAME are set.
98
99    mkdir -p "$COVERITY_TOOL_BASE"
100    cd "$COVERITY_TOOL_BASE"
101
102    echo "Checking for new version of coverity build tools..."
103    wget https://scan.coverity.com/download/linux64 --post-data "token=$PROJTOKEN&project=$PROJNAME&md5=1" -O coverity_tool.md5.new
104
105    if ! cmp -s coverity_tool.md5 coverity_tool.md5.new; then
106        # out of date md5 or no md5: download new build tool
107        # blow away the old build tool
108        echo "Downloading coverity build tools..."
109        rm -rf coverity_tool coverity_tool.tgz
110        wget https://scan.coverity.com/download/linux64 --post-data "token=$PROJTOKEN&project=$PROJNAME" -O coverity_tool.tgz
111        if ! (cat coverity_tool.md5.new; echo "  coverity_tool.tgz") | md5sum -c --status; then
112            echo "Downloaded tarball didn't match md5sum!"
113            exit 1
114        fi
115        # extract the new one, keeping it corralled in a 'coverity_tool' directory
116        echo "Unpacking coverity build tools..."
117        mkdir -p coverity_tool
118        cd coverity_tool
119        tar xf ../coverity_tool.tgz
120        cd ..
121        mv coverity_tool.md5.new coverity_tool.md5
122    fi
123
124    rm -f coverity_tool.md5.new
125}
126
127
128# Check user-provided environment variables and arguments
129DRYRUN=no
130UPDATE_ONLY=no
131DOCKER=no
132
133while [ "$#" -ge 1 ]; do
134    case "$1" in
135        --dry-run)
136            shift
137            DRYRUN=yes
138            ;;
139        --update-tools-only)
140            shift
141            UPDATE_ONLY=yes
142            ;;
143        --version)
144            shift
145            if [ $# -eq 0 ]; then
146                echo "--version needs an argument"
147                exit 1
148            fi
149            VERSION="$1"
150            shift
151            ;;
152        --description)
153            shift
154            if [ $# -eq 0 ]; then
155                echo "--description needs an argument"
156                exit 1
157            fi
158            DESCRIPTION="$1"
159            shift
160            ;;
161        --tokenfile)
162            shift
163            if [ $# -eq 0 ]; then
164                echo "--tokenfile needs an argument"
165                exit 1
166            fi
167            COVERITY_TOKEN="$(cat "$1")"
168            shift
169            ;;
170        --srcdir)
171            shift
172            if [ $# -eq 0 ]; then
173                echo "--srcdir needs an argument"
174                exit 1
175            fi
176            SRCDIR="$1"
177            shift
178            ;;
179        --results-tarball)
180            shift
181            if [ $# -eq 0 ]; then
182                echo "--results-tarball needs an argument"
183                exit 1
184            fi
185            RESULTSTARBALL="$1"
186            shift
187            ;;
188        --src-tarball)
189            shift
190            if [ $# -eq 0 ]; then
191                echo "--src-tarball needs an argument"
192                exit 1
193            fi
194            SRCTARBALL="$1"
195            shift
196            ;;
197        --docker)
198            DOCKER=yes
199            shift
200            ;;
201        *)
202            echo "Unexpected argument '$1'"
203            exit 1
204            ;;
205    esac
206done
207
208if [ -z "$COVERITY_TOKEN" ]; then
209    echo "COVERITY_TOKEN environment variable not set"
210    exit 1
211fi
212
213if [ -z "$COVERITY_BUILD_CMD" ]; then
214    NPROC=$(nproc)
215    COVERITY_BUILD_CMD="make -j$NPROC"
216    echo "COVERITY_BUILD_CMD: using default '$COVERITY_BUILD_CMD'"
217fi
218
219if [ -z "$COVERITY_TOOL_BASE" ]; then
220    echo "COVERITY_TOOL_BASE: using default /tmp/coverity-tools"
221    COVERITY_TOOL_BASE=/tmp/coverity-tools
222fi
223
224if [ -z "$SRCDIR" ]; then
225    SRCDIR="$PWD"
226fi
227
228PROJTOKEN="$COVERITY_TOKEN"
229PROJNAME=QEMU
230TARBALL=cov-int.tar.xz
231
232if [ "$UPDATE_ONLY" = yes ] && [ "$DOCKER" = yes ]; then
233    echo "Combining --docker and --update-only is not supported"
234    exit 1
235fi
236
237if [ "$UPDATE_ONLY" = yes ]; then
238    # Just do the tools update; we don't need to check whether
239    # we are in a source tree or have upload rights for this,
240    # so do it before some of the command line and source tree checks.
241    update_coverity_tools
242    exit 0
243fi
244
245if [ ! -e "$SRCDIR" ]; then
246    mkdir "$SRCDIR"
247fi
248
249cd "$SRCDIR"
250
251if [ ! -z "$SRCTARBALL" ]; then
252    echo "Untarring source tarball into $SRCDIR..."
253    tar xvf "$SRCTARBALL"
254fi
255
256echo "Checking this is a QEMU source tree..."
257if ! [ -e "$SRCDIR/VERSION" ]; then
258    echo "Not in a QEMU source tree?"
259    exit 1
260fi
261
262# Fill in defaults used by the non-update-only process
263if [ -z "$VERSION" ]; then
264    VERSION="$(git describe --always HEAD)"
265fi
266
267if [ -z "$DESCRIPTION" ]; then
268    DESCRIPTION="$(git rev-parse HEAD)"
269fi
270
271if [ -z "$COVERITY_EMAIL" ]; then
272    COVERITY_EMAIL="$(git config user.email)"
273fi
274
275# Run ourselves inside docker if that's what the user wants
276if [ "$DOCKER" = yes ]; then
277    # build docker container including the coverity-scan tools
278    # Put the Coverity token into a temporary file that only
279    # we have read access to, and then pass it to docker build
280    # using --secret. This requires at least Docker 18.09.
281    # Mostly what we are trying to do here is ensure we don't leak
282    # the token into the Docker image.
283    umask 077
284    SECRETDIR=$(mktemp -d)
285    if [ -z "$SECRETDIR" ]; then
286        echo "Failed to create temporary directory"
287        exit 1
288    fi
289    trap 'rm -rf "$SECRETDIR"' INT TERM EXIT
290    echo "Created temporary directory $SECRETDIR"
291    SECRET="$SECRETDIR/token"
292    echo "$COVERITY_TOKEN" > "$SECRET"
293    echo "Building docker container..."
294    # TODO: This re-downloads the tools every time, rather than
295    # caching and reusing the image produced with the downloaded tools.
296    # Not sure why.
297    # TODO: how do you get 'docker build' to print the output of the
298    # commands it is running to its stdout? This would be useful for debug.
299    DOCKER_BUILDKIT=1 docker build -t coverity-scanner \
300                   --secret id=coverity.token,src="$SECRET" \
301                   -f scripts/coverity-scan/coverity-scan.docker \
302                   scripts/coverity-scan
303    echo "Archiving sources to be analyzed..."
304    ./scripts/archive-source.sh "$SECRETDIR/qemu-sources.tgz"
305    if [ "$DRYRUN" = yes ]; then
306        DRYRUNARG=--dry-run
307    fi
308    echo "Running scanner..."
309    # If we need to capture the output tarball, get the inner run to
310    # save it to the secrets directory so we can copy it out before the
311    # directory is cleaned up.
312    if [ ! -z "$RESULTSTARBALL" ]; then
313        RTARGS="--results-tarball /work/cov-int.tar.xz"
314    else
315        RTARGS=""
316    fi
317    # Arrange for this docker run to get access to the sources with -v.
318    # We pass through all the configuration from the outer script to the inner.
319    export COVERITY_EMAIL COVERITY_BUILD_CMD
320    docker run -it --env COVERITY_EMAIL --env COVERITY_BUILD_CMD \
321           -v "$SECRETDIR:/work" coverity-scanner \
322           ./run-coverity-scan --version "$VERSION" \
323           --description "$DESCRIPTION" $DRYRUNARG --tokenfile /work/token \
324           --srcdir /qemu --src-tarball /work/qemu-sources.tgz $RTARGS
325    if [ ! -z "$RESULTSTARBALL" ]; then
326        echo "Copying results tarball to $RESULTSTARBALL..."
327        cp "$SECRETDIR/cov-int.tar.xz" "$RESULTSTARBALL"
328    fi
329    echo "Docker work complete."
330    exit 0
331fi
332
333# Otherwise, continue with the full build and upload process.
334
335check_upload_permissions
336
337update_coverity_tools
338
339TOOLBIN="$(cd "$COVERITY_TOOL_BASE" && echo $PWD/coverity_tool/cov-analysis-*/bin)"
340
341if ! test -x "$TOOLBIN/cov-build"; then
342    echo "Couldn't find cov-build in the coverity build-tool directory??"
343    exit 1
344fi
345
346export PATH="$TOOLBIN:$PATH"
347
348cd "$SRCDIR"
349
350echo "Doing make distclean..."
351make distclean
352
353echo "Configuring..."
354# We configure with a fixed set of enables here to ensure that we don't
355# accidentally reduce the scope of the analysis by doing the build on
356# the system that's missing a dependency that we need to build part of
357# the codebase.
358./configure --disable-modules --enable-sdl --enable-gtk \
359    --enable-opengl --enable-vte --enable-gnutls \
360    --enable-nettle --enable-curses --enable-curl \
361    --audio-drv-list=oss,alsa,sdl,pa --enable-virtfs \
362    --enable-vnc --enable-vnc-sasl --enable-vnc-jpeg --enable-vnc-png \
363    --enable-xen --enable-brlapi \
364    --enable-linux-aio --enable-attr \
365    --enable-cap-ng --enable-trace-backends=log --enable-spice --enable-rbd \
366    --enable-xfsctl --enable-libusb --enable-usb-redir \
367    --enable-libiscsi --enable-libnfs --enable-seccomp \
368    --enable-tpm --enable-libssh --enable-lzo --enable-snappy --enable-bzip2 \
369    --enable-numa --enable-rdma --enable-smartcard --enable-virglrenderer \
370    --enable-mpath --enable-libxml2 --enable-glusterfs \
371    --enable-virtfs --enable-zstd
372
373echo "Making libqemustub.a..."
374make libqemustub.a
375
376echo "Running cov-build..."
377rm -rf cov-int
378mkdir cov-int
379cov-build --dir cov-int $COVERITY_BUILD_CMD
380
381echo "Creating results tarball..."
382tar cvf - cov-int | xz > "$TARBALL"
383
384if [ ! -z "$RESULTSTARBALL" ]; then
385    echo "Copying results tarball to $RESULTSTARBALL..."
386    cp "$TARBALL" "$RESULTSTARBALL"
387fi
388
389echo "Uploading results tarball..."
390
391if [ "$DRYRUN" = yes ]; then
392    echo "Dry run only, not uploading $TARBALL"
393    exit 0
394fi
395
396curl --form token="$PROJTOKEN" --form email="$COVERITY_EMAIL" \
397     --form file=@"$TARBALL" --form version="$VERSION" \
398     --form description="$DESCRIPTION" \
399     https://scan.coverity.com/builds?project="$PROJNAME"
400
401echo "Done."
402