1#!/bin/bash
2# This file is part of cloud-init. See LICENSE file for license information.
3#
4# shellcheck disable=2015,2016,2039,2162,2166
5
6set -u
7
8VERBOSITY=0
9KEEP=false
10CONTAINER=""
11DEFAULT_WAIT_MAX=30
12
13error() { echo "$@" 1>&2; }
14fail() { [ $# -eq 0 ] || error "$@"; exit 1; }
15errorrc() { local r=$?; error "$@" "ret=$r"; return $r; }
16
17Usage() {
18    cat <<EOF
19Usage: ${0##*/} [ options ] [images:]image-ref
20
21    This utility can makes it easier to run tests, build rpm and source rpm
22        generation inside a LXC of the specified version of CentOS.
23
24    To see images available, run 'lxc image list images:'
25    Example input:
26       centos/7
27       opensuse/42.3
28       debian/10
29
30    options:
31      -a | --artifacts DIR   copy build artifacts out to DIR.
32                             by default artifacts are not copied out.
33           --dirty           apply local changes before running tests.
34                             If not provided, a clean checkout of branch is
35                             tested.  Inside container, changes are in
36                             local-changes.diff.
37      -k | --keep            keep container after tests
38      -p | --package         build a binary package (.deb or .rpm)
39      -s | --source-package  build source package (debuild -S or srpm)
40      -u | --unittest        run unit tests
41
42    Example:
43      * ${0##*/} --package --source-package --unittest centos/6
44EOF
45}
46
47bad_Usage() { Usage 1>&2; [ $# -eq 0 ] || error "$@"; return 1; }
48cleanup() {
49    if [ -n "$CONTAINER" ]; then
50        if [ "$KEEP" = "true" ]; then
51            error "not deleting container '$CONTAINER' due to --keep"
52        else
53            delete_container "$CONTAINER"
54        fi
55    fi
56}
57
58debug() {
59    local level=${1}; shift;
60    [ "${level}" -gt "${VERBOSITY}" ] && return
61    error "${@}"
62}
63
64
65inside_as() {
66    # inside_as(container_name, user, cmd[, args])
67    # executes cmd with args inside container as user in users home dir.
68    local name="$1" user="$2"
69    shift 2
70    if [ "$user" = "root" ]; then
71        inside "$name" "$@"
72        return
73    fi
74    local stuffed="" b64=""
75    stuffed=$(getopt --shell sh --options "" -- -- "$@")
76    stuffed=${stuffed# -- }
77    b64=$(printf "%s\n" "$stuffed" | base64 --wrap=0)
78    inside "$name" su "$user" -c \
79        'cd; eval set -- "$(echo '"$b64"' | base64 --decode)" && exec "$@"';
80}
81
82inside_as_cd() {
83    local name="$1" user="$2" dir="$3"
84    shift 3
85    inside_as "$name" "$user" sh -c 'cd "$0" && exec "$@"' "$dir" "$@"
86}
87
88inside() {
89    local name="$1"
90    shift
91    lxc exec "$name" -- "$@"
92}
93
94inject_cloud_init(){
95    # take current cloud-init git dir and put it inside $name at
96    # ~$user/cloud-init.
97    local name="$1" user="$2" dirty="$3"
98    local dname="cloud-init" gitdir="" commitish=""
99    gitdir=$(git rev-parse --git-dir) || {
100        errorrc "Failed to get git dir in $PWD";
101        return
102    }
103    local t=${gitdir%/*}
104    case "$t" in
105        */worktrees)
106            if [ -f "${t%worktrees}/config" ]; then
107                gitdir="${t%worktrees}"
108            fi
109    esac
110
111    # attempt to get branch name.
112    commitish=$(git rev-parse --abbrev-ref HEAD) || {
113        errorrc "Failed git rev-parse --abbrev-ref HEAD"
114        return
115    }
116    if [ "$commitish" = "HEAD" ]; then
117        # detached head
118        commitish=$(git rev-parse HEAD) || {
119            errorrc "failed git rev-parse HEAD"
120            return
121        }
122    fi
123
124    local local_changes=false
125    if ! git diff --quiet "$commitish"; then
126        # there are local changes not committed.
127        local_changes=true
128        if [ "$dirty" = "false" ]; then
129            error "WARNING: You had uncommitted changes.  Those changes will "
130            error "be put into 'local-changes.diff' inside the container. "
131            error "To test these changes you must pass --dirty."
132        fi
133    fi
134
135    debug 1 "collecting ${gitdir} ($dname) into user $user in $name."
136    tar -C "${gitdir}" -cpf - . |
137        inside_as "$name" "$user" sh -ec '
138            dname=$1
139            commitish=$2
140            rm -Rf "$dname"
141            mkdir -p $dname/.git
142            cd $dname/.git
143            tar -xpf -
144            cd ..
145            git config core.bare false
146            out=$(git checkout $commitish 2>&1) ||
147                { echo "failed git checkout $commitish: $out" 1>&2; exit 1; }
148            out=$(git checkout . 2>&1) ||
149                { echo "failed git checkout .: $out" 1>&2; exit 1; }
150            ' extract "$dname" "$commitish"
151    [ "${PIPESTATUS[*]}" = "0 0" ] || {
152        error "Failed to push tarball of '$gitdir' into $name" \
153            " for user $user (dname=$dname)"
154        return 1
155    }
156
157    echo "local_changes=$local_changes dirty=$dirty"
158    if [ "$local_changes" = "true" ]; then
159        git diff "$commitish" |
160            inside_as "$name" "$user" sh -exc '
161                cd "$1"
162                if [ "$2" = "true" ]; then
163                    git apply
164                else
165                    cat > local-changes.diff
166                fi
167                ' insert_changes "$dname" "$dirty"
168        [ "${PIPESTATUS[*]}" = "0 0" ] || {
169            error "Failed to apply local changes."
170            return 1
171        }
172    fi
173
174    return 0
175}
176
177get_os_info_in() {
178    # prep the container (install very basic dependencies)
179    [ -n "${OS_VERSION:-}" -a -n "${OS_NAME:-}" ] && return 0
180    data=$(run_self_inside "$name" os_info) ||
181        { errorrc "Failed to get os-info in container $name"; return; }
182    eval "$data" && [ -n "${OS_VERSION:-}" -a -n "${OS_NAME:-}" ] || return
183    debug 1 "determined $name is $OS_NAME/$OS_VERSION"
184}
185
186os_info() {
187    get_os_info || return
188    echo "OS_NAME=$OS_NAME"
189    echo "OS_VERSION=$OS_VERSION"
190}
191
192get_os_info() {
193    # run inside container, set OS_NAME, OS_VERSION
194    # example OS_NAME are centos, debian, opensuse, rockylinux
195    [ -n "${OS_NAME:-}" -a -n "${OS_VERSION:-}" ] && return 0
196    if [ -f /etc/os-release ]; then
197        OS_NAME=$(sh -c '. /etc/os-release; echo $ID')
198        OS_VERSION=$(sh -c '. /etc/os-release; echo $VERSION_ID')
199        if [ -z "$OS_VERSION" ]; then
200            local pname=""
201            pname=$(sh -c '. /etc/os-release; echo $PRETTY_NAME')
202            case "$pname" in
203                *buster*) OS_VERSION=10;;
204                *sid*) OS_VERSION="sid";;
205            esac
206        fi
207    elif [ -f /etc/centos-release ]; then
208        local line=""
209        read line < /etc/centos-release
210        case "$line" in
211            CentOS\ *\ 6.*) OS_VERSION="6"; OS_NAME="centos";;
212        esac
213    fi
214    [ -n "${OS_NAME:-}" -a -n "${OS_VERSION:-}" ] ||
215        { error "Unable to determine OS_NAME/OS_VERSION"; return 1; }
216}
217
218yum_install() {
219    local n=0 max=10 ret
220    bcmd="yum install --downloadonly --assumeyes --setopt=keepcache=1"
221    while n=$((n+1)); do
222       error ":: running $bcmd $* [$n/$max]"
223       $bcmd "$@"
224       ret=$?
225       [ $ret -eq 0 ] && break
226       [ $n -ge $max ] && { error "gave up on $bcmd"; exit $ret; }
227       nap=$((n*5))
228       error ":: failed [$ret] ($n/$max). sleeping $nap."
229       sleep $nap
230    done
231    error ":: running yum install --cacheonly --assumeyes $*"
232    yum install --cacheonly --assumeyes "$@"
233}
234
235zypper_install() {
236    local pkgs="$*"
237    set -- zypper --non-interactive --gpg-auto-import-keys install \
238        --auto-agree-with-licenses "$@"
239    debug 1 ":: installing $pkgs with zypper: $*"
240    "$@"
241}
242
243apt_install() {
244    apt-get update -q && apt-get install --no-install-recommends "$@"
245}
246
247install_packages() {
248    get_os_info || return
249    case "$OS_NAME" in
250        centos|rocky*) yum_install "$@";;
251        opensuse) zypper_install "$@";;
252        debian|ubuntu) apt_install "$@";;
253        *) error "Do not know how to install packages on ${OS_NAME}";
254           return 1;;
255    esac
256}
257
258prep() {
259    # we need some very basic things not present in the container.
260    #  - git
261    #  - tar (CentOS 6 lxc container does not have it)
262    #  - python3
263    local needed="" pair="" pkg="" cmd="" needed=""
264    local pairs="tar:tar git:git"
265    get_os_info
266    local py3pkg="python3"
267    case "$OS_NAME" in
268        opensuse)
269            py3pkg="python3-base";;
270    esac
271
272    pairs="$pairs python3:$py3pkg"
273
274    for pair in $pairs; do
275        pkg=${pair#*:}
276        cmd=${pair%%:*}
277        command -v "$cmd" >/dev/null 2>&1 || needed="${needed} $pkg"
278    done
279    needed=${needed# }
280    if [ -z "$needed" ]; then
281        error "No prep packages needed"
282        return 0
283    fi
284    error "Installing prep packages: ${needed}"
285    # shellcheck disable=SC2086
286    set -- $needed
287    install_packages "$@"
288}
289
290pytest() {
291    python3 -m pytest "$@"
292}
293
294is_done_cloudinit() {
295    [ -e "/run/cloud-init/result.json" ]
296    _RET=""
297}
298
299is_done_systemd() {
300    local s="" num="$1"
301    s=$(systemctl is-system-running 2>&1);
302    _RET="$? $s"
303    case "$s" in
304        initializing|starting) return 1;;
305        *[Ff]ailed*connect*bus*)
306            # warn if not the first run.
307            [ "$num" -lt 5 ] ||
308                error "Failed to connect to systemd bus [${_RET%% *}]";
309            return 1;;
310    esac
311    return 0
312}
313
314is_done_other() {
315    local out=""
316    out=$(getent hosts ubuntu.com 2>&1)
317    return
318}
319
320wait_inside() {
321    local name="$1" max="${2:-${DEFAULT_WAIT_MAX}}" debug=${3:-0}
322    local i=0 check="is_done_other";
323    if [ -e /run/systemd ]; then
324        check=is_done_systemd
325    elif [ -x /usr/bin/cloud-init ]; then
326        check=is_done_cloudinit
327    fi
328    [ "$debug" != "0" ] && debug 1 "check=$check"
329    while ! $check $i && i=$((i+1)); do
330        [ "$i" -ge "$max" ] && exit 1
331        [ "$debug" = "0" ] || echo -n .
332        sleep 1
333    done
334    if [ "$debug" != "0" ]; then
335        read up _ </proc/uptime
336        debug 1 "[$name ${i:+done after $i }up=$up${_RET:+ ${_RET}}]"
337    fi
338}
339
340wait_for_boot() {
341    local name="$1"
342    local out="" ret="" wtime=$DEFAULT_WAIT_MAX
343    get_os_info_in "$name"
344    [ "$OS_NAME" = "debian" ] && wtime=300 &&
345        debug 1 "on debian we wait for ${wtime}s"
346    debug 1 "waiting for boot of $name"
347    run_self_inside "$name" wait_inside "$name" "$wtime" "$VERBOSITY" ||
348        { errorrc "wait inside $name failed."; return; }
349
350    if [ -n "${http_proxy-}" ]; then
351        if [ "$OS_NAME" = "centos" ]; then
352            debug 1 "configuring proxy ${http_proxy}"
353            inside "$name" sh -c "echo proxy=$http_proxy >> /etc/yum.conf"
354            inside "$name" sh -c "sed -i --regexp-extended '/^#baseurl=/s/#// ; /^(mirrorlist|metalink)=/s/^/#/' /etc/yum.repos.d/*.repo"
355            inside "$name" sh -c "sed -i 's/download\.fedoraproject\.org/dl.fedoraproject.org/g' /etc/yum.repos.d/*.repo"
356            inside "$name" sh -c "sed -i 's/download\.example/dl.fedoraproject.org/g' /etc/yum.repos.d/*.repo"
357        else
358            debug 1 "do not know how to configure proxy on $OS_NAME"
359        fi
360    fi
361}
362
363start_container() {
364    local src="$1" name="$2"
365    debug 1 "starting container $name from '$src'"
366    lxc launch "$src" "$name" || {
367        errorrc "Failed to start container '$name' from '$src'";
368        return
369    }
370    CONTAINER=$name
371    wait_for_boot "$name"
372}
373
374delete_container() {
375    debug 1 "removing container $1 [--keep to keep]"
376    lxc delete --force "$1"
377}
378
379run_self_inside() {
380    # run_self_inside(container, args)
381    local name="$1"
382    shift
383    inside "$name" bash -s "$@" <"$0"
384}
385
386run_self_inside_as_cd() {
387    local name="$1" user="$2" dir="$3"
388    shift 3
389    inside_as_cd "$name" "$user" "$dir" bash -s "$@" <"$0"
390}
391
392main() {
393    local short_opts="a:hknpsuv"
394    local long_opts="artifacts:,dirty,help,keep,name:,package,source-package,unittest,verbose"
395    local getopt_out=""
396    getopt_out=$(getopt --name "${0##*/}" \
397        --options "${short_opts}" --long "${long_opts}" -- "$@") &&
398        eval set -- "${getopt_out}" ||
399        { bad_Usage; return; }
400
401    local cur="" next=""
402    local package=false srcpackage=false unittest="" name=""
403    local dirty=false artifact_d="."
404
405    while [ $# -ne 0 ]; do
406        cur="${1:-}"; next="${2:-}";
407        case "$cur" in
408            -a|--artifacts) artifact_d="$next";;
409               --dirty) dirty=true;;
410            -h|--help) Usage ; exit 0;;
411            -k|--keep) KEEP=true;;
412            -n|--name) name="$next"; shift;;
413            -p|--package) package=true;;
414            -s|--source-package) srcpackage=true;;
415            -u|--unittest) unittest=1;;
416            -v|--verbose) VERBOSITY=$((VERBOSITY+1));;
417            --) shift; break;;
418        esac
419        shift;
420    done
421
422    [ $# -eq 1 ] || { bad_Usage "Expected 1 arg, got $# ($*)"; return; }
423    local img_ref_in="$1"
424    case "${img_ref_in}" in
425        *:*) img_ref="${img_ref_in}";;
426        *) img_ref="images:${img_ref_in}";;
427    esac
428
429    # program starts here
430    local out="" user="ci-test" cdir="" home=""
431    home="/home/$user"
432    cdir="$home/cloud-init"
433    if [ -z "$name" ]; then
434        if out=$(petname 2>&1); then
435            name="ci-${out}"
436        elif out=$(uuidgen -t 2>&1); then
437            name="ci-${out%%-*}"
438        else
439            error "Must provide name or have petname or uuidgen"
440            return 1
441        fi
442    fi
443
444    trap cleanup EXIT
445
446    start_container "$img_ref" "$name" ||
447        { errorrc "Failed to start container for $img_ref"; return; }
448
449    get_os_info_in "$name" ||
450        { errorrc "failed to get os_info in $name"; return; }
451
452    # prep the container (install very basic dependencies)
453    run_self_inside "$name" prep ||
454        { errorrc "Failed to prep container $name"; return; }
455
456    # add the user
457    inside "$name" useradd "$user" --create-home "--home-dir=$home" ||
458        { errorrc "Failed to add user '$user' in '$name'"; return 1; }
459
460    debug 1 "inserting cloud-init"
461    inject_cloud_init "$name" "$user" "$dirty" || {
462        errorrc "FAIL: injecting cloud-init into $name failed."
463        return
464    }
465
466    local rdcmd=(python3 tools/read-dependencies "--distro=${OS_NAME}" --install --test-distro)
467    inside_as_cd "$name" root "$cdir" "${rdcmd[@]}" || {
468        errorrc "FAIL: failed to install dependencies with read-dependencies"
469        return
470    }
471
472    local errors=( )
473    inside_as_cd "$name" "$user" "$cdir" git status || {
474        errorrc "git checkout failed."
475        errors[${#errors[@]}]="git checkout";
476    }
477
478    if [ -n "$unittest" ]; then
479        debug 1 "running unit tests."
480        run_self_inside_as_cd "$name" "$user" "$cdir" pytest \
481            tests/unittests cloudinit/ || {
482                errorrc "pytest failed.";
483                errors[${#errors[@]}]="pytest"
484            }
485    fi
486
487    local build_pkg="" build_srcpkg="" pkg_ext="" distflag=""
488    case "$OS_NAME" in
489        centos|rocky) distflag="--distro=redhat";;
490        opensuse) distflag="--distro=suse";;
491    esac
492
493    case "$OS_NAME" in
494        debian|ubuntu)
495            build_pkg="./packages/bddeb -d"
496            build_srcpkg="./packages/bddeb -S -d"
497            pkg_ext=".deb";;
498        centos|opensuse|rocky)
499            build_pkg="./packages/brpm $distflag"
500            build_srcpkg="./packages/brpm $distflag --srpm"
501            pkg_ext=".rpm";;
502    esac
503    if [ "$srcpackage" = "true" ]; then
504        [ -n "$build_srcpkg" ] || {
505            error "Unknown package command for $OS_NAME"
506            return 1
507        }
508        debug 1 "building source package with $build_srcpkg."
509        # shellcheck disable=SC2086
510        inside_as_cd "$name" "$user" "$cdir" python3 $build_srcpkg || {
511            errorrc "failed: $build_srcpkg";
512            errors[${#errors[@]}]="source package"
513        }
514    fi
515
516    if [ "$package" = "true" ]; then
517        [ -n "$build_pkg" ] || {
518            error "Unknown build source command for $OS_NAME"
519            return 1
520        }
521        debug 1 "building binary package with $build_pkg."
522        # shellcheck disable=SC2086
523        inside_as_cd "$name" "$user" "$cdir" python3 $build_pkg || {
524            errorrc "failed: $build_pkg";
525            errors[${#errors[@]}]="binary package"
526        }
527    fi
528
529    if [ -n "$artifact_d" ] &&
530        [ "$package" = "true" -o "$srcpackage" = "true" ]; then
531        local art=""
532        artifact_d="${artifact_d%/}/"
533        [ -d "${artifact_d}" ] || mkdir -p "$artifact_d" || {
534            errorrc "failed to create artifact dir '$artifact_d'"
535            return
536        }
537
538        for art in $(inside "$name" sh -c "echo $cdir/*${pkg_ext}"); do
539            lxc file pull "$name/$art" "$artifact_d" || {
540                errorrc "Failed to pull '$name/$art' to ${artifact_d}"
541                errors[${#errors[@]}]="artifact copy: $art"
542            }
543            debug 1 "wrote ${artifact_d}${art##*/}"
544        done
545    fi
546
547    if [ "${#errors[@]}" != "0" ]; then
548        local e=""
549        error "there were ${#errors[@]} errors."
550        for e in "${errors[@]}"; do
551            error "  $e"
552        done
553        return 1
554    fi
555    return 0
556}
557
558case "${1:-}" in
559    prep|os_info|wait_inside|pytest) _n=$1; shift; "$_n" "$@";;
560    *) main "$@";;
561esac
562
563# vi: ts=4 expandtab
564