1#!/usr/bin/env bash
2#
3# Run all etcd tests
4# ./test
5# ./test -v
6#
7#
8# Run specified test pass
9#
10# $ PASSES=unit ./test
11# $ PASSES=integration ./test
12#
13#
14# Run tests for one package
15# Each pass has different default timeout, if you just run tests in one package or 1 test case then you can set TIMEOUT
16# flag for different expectation
17#
18# $ PASSES=unit PKG=./wal TIMEOUT=1m ./test
19# $ PASSES=integration PKG=./clientv3 TIMEOUT=1m ./test
20#
21# Run specified unit tests in one package
22# To run all the tests with prefix of "TestNew", set "TESTCASE=TestNew ";
23# to run only "TestNew", set "TESTCASE="\bTestNew\b""
24#
25# $ PASSES=unit PKG=./wal TESTCASE=TestNew TIMEOUT=1m ./test
26# $ PASSES=unit PKG=./wal TESTCASE="\bTestNew\b" TIMEOUT=1m ./test
27# $ PASSES=integration PKG=./client/integration TESTCASE="\bTestV2NoRetryEOF\b" TIMEOUT=1m ./test
28#
29#
30# Run code coverage
31# COVERDIR must either be a absolute path or a relative path to the etcd root
32# $ COVERDIR=coverage PASSES="build build_cov cov" ./test
33# $ go tool cover -html ./coverage/cover.out
34set -e
35set -o pipefail
36
37
38# Consider command as failed when any component of the pipe fails:
39# https://stackoverflow.com/questions/1221833/pipe-output-and-capture-exit-status-in-bash
40set -o pipefail
41
42# The test script is not supposed to make any changes to the files
43# e.g. add/update missing dependencies. Such divergences should be
44# detected and trigger a failure that needs explicit developer's action.
45export GOFLAGS=-mod=readonly
46
47source ./scripts/test_lib.sh
48source ./build.sh
49
50PASSES=${PASSES:-"fmt bom dep build unit"}
51PKG=${PKG:-}
52
53if [ -z "$GOARCH" ]; then
54  GOARCH=$(go env GOARCH);
55fi
56
57# determine whether target supports race detection
58if [ -z "${RACE}" ] ; then
59  if [ "$GOARCH" == "amd64" ]; then
60    RACE="--race"
61  else
62    RACE="--race=false"
63  fi
64else
65  RACE="--race=${RACE:-true}"
66fi
67
68# This options make sense for cases where SUT (System Under Test) is compiled by test.
69COMMON_TEST_FLAGS=("${RACE}")
70if [[ -n "${CPU}" ]]; then
71  COMMON_TEST_FLAGS+=("--cpu=${CPU}")
72fi
73
74log_callout "Running with ${COMMON_TEST_FLAGS[*]}"
75
76RUN_ARG=()
77if [ -n "${TESTCASE}" ]; then
78  RUN_ARG=("-run=${TESTCASE}")
79fi
80
81function build_pass {
82  log_callout "Building etcd"
83  run_for_modules run go build "${@}" || return 2
84  GO_BUILD_FLAGS="-v" etcd_build "${@}"
85  GO_BUILD_FLAGS="-v" tools_build "${@}"
86}
87
88################# REGULAR TESTS ################################################
89
90# run_unit_tests [pkgs] runs unit tests for a current module and givesn set of [pkgs]
91function run_unit_tests {
92  local pkgs="${1:-./...}"
93  shift 1
94  # shellcheck disable=SC2086
95  GOLANG_TEST_SHORT=true go_test "${pkgs}" "parallel" : -short -timeout="${TIMEOUT:-3m}" "${COMMON_TEST_FLAGS[@]}" "${RUN_ARG[@]}" "$@"
96}
97
98function unit_pass {
99  run_for_modules run_unit_tests "$@"
100}
101
102function integration_extra {
103  if [ -z "${PKG}" ] ; then
104    run_for_module "."  go_test "./contrib/raftexample" "keep_going" :  -timeout="${TIMEOUT:-5m}" "${RUN_ARG[@]}" "${COMMON_TEST_FLAGS[@]}" "$@" || return $?
105    run_for_module "tests"  go_test "./integration/v2store/..." "keep_going" : -tags v2v3 -timeout="${TIMEOUT:-5m}" "${RUN_ARG[@]}" "${COMMON_TEST_FLAGS[@]}" "$@" || return $?
106  else
107    log_warning "integration_extra ignored when PKG is specified"
108  fi
109}
110
111function integration_pass {
112  local pkgs=${USERPKG:-"./integration/..."}
113  run_for_module "tests" go_test "${pkgs}" "parallel" : -timeout="${TIMEOUT:-30m}" "${COMMON_TEST_FLAGS[@]}" "${RUN_ARG[@]}" "$@" || return $?
114  integration_extra "$@"
115}
116
117function e2e_pass {
118  # e2e tests are running pre-build binary. Settings like --race,-cover,-cpu does not have any impact.
119  run_for_module "tests" go_test "./e2e/..." "keep_going" : -timeout="${TIMEOUT:-30m}" "${RUN_ARG[@]}" "$@"
120}
121
122function integration_e2e_pass {
123  run_pass "integration" "${@}"
124  run_pass "e2e" "${@}"
125}
126
127# generic_checker [cmd...]
128# executes given command in the current module, and clearly fails if it
129# failed or returned output.
130function generic_checker {
131  local cmd=("$@")
132  if ! output=$("${cmd[@]}"); then
133    echo "${output}"
134    log_error -e "FAIL: '${cmd[*]}' checking failed (!=0 return code)"
135    return 255
136  fi
137  if [ -n "${output}" ]; then
138    echo "${output}"
139    log_error -e "FAIL: '${cmd[*]}' checking failed (printed output)"
140    return 255
141  fi
142}
143
144function functional_pass {
145  run ./tests/functional/build
146
147  # Clean up any data and logs from previous runs
148  rm -rf /tmp/etcd-functional-* /tmp/etcd-functional-*.backup
149
150  # TODO: These ports should be dynamically allocated instead of hard-coded.
151  for a in 1 2 3; do
152    ./bin/etcd-agent --network tcp --address 127.0.0.1:${a}9027 < /dev/null &
153    pid="$!"
154    agent_pids="${agent_pids} $pid"
155  done
156
157  for a in 1 2 3; do
158    log_callout "Waiting for 'etcd-agent' on ${a}9027..."
159    while ! nc -z localhost ${a}9027; do
160      sleep 1
161    done
162  done
163
164  log_callout "functional test START!"
165  run ./bin/etcd-tester --config ./tests/functional/functional.yaml && log_success "'etcd-tester' succeeded"
166  local etcd_tester_exit_code=$?
167
168  if [[ "${etcd_tester_exit_code}" -ne "0" ]]; then
169    log_error "ETCD_TESTER_EXIT_CODE:" ${etcd_tester_exit_code}
170  fi
171
172  # shellcheck disable=SC2206
173  agent_pids=($agent_pids)
174  kill -s TERM "${agent_pids[@]}" || true
175
176  if [[ "${etcd_tester_exit_code}" -ne "0" ]]; then
177    log_error -e "\\nFAILED! 'tail -1000 /tmp/etcd-functional-1/etcd.log'"
178    tail -1000 /tmp/etcd-functional-1/etcd.log
179
180    log_error -e "\\nFAILED! 'tail -1000 /tmp/etcd-functional-2/etcd.log'"
181    tail -1000 /tmp/etcd-functional-2/etcd.log
182
183    log_error -e "\\nFAILED! 'tail -1000 /tmp/etcd-functional-3/etcd.log'"
184    tail -1000 /tmp/etcd-functional-3/etcd.log
185
186    log_error "--- FAIL: exit code" ${etcd_tester_exit_code}
187    return ${etcd_tester_exit_code}
188  fi
189  log_success "functional test PASS!"
190}
191
192function grpcproxy_pass {
193  run_for_module "tests" go_test "./integration/... ./e2e" "fail_fast" : \
194      -timeout=30m -tags cluster_proxy "${COMMON_TEST_FLAGS[@]}" "$@"
195}
196
197################# COVERAGE #####################################################
198
199# Builds artifacts used by tests/e2e in coverage mode.
200function build_cov_pass {
201  run_for_module "server" run go test -tags cov -c -covermode=set -coverpkg="./..." -o "../bin/etcd_test"
202  run_for_module "etcdctl" run go test -tags cov -c -covermode=set -coverpkg="./..." -o "../bin/etcdctl_test"
203}
204
205# pkg_to_coverflag [prefix] [pkgs]
206# produces name of .coverprofile file to be used for tests of this package
207function pkg_to_coverprofileflag {
208  local prefix="${1}"
209  local pkgs="${2}"
210  local pkgs_normalized
211  prefix_normalized=$(echo "${prefix}" | tr "./ " "__+")
212  if [ "${pkgs}" == "./..." ]; then
213    pkgs_normalized="all"
214  else
215    pkgs_normalized=$(echo "${pkgs}" | tr "./ " "__+")
216  fi
217  mkdir -p "${coverdir}/${prefix_normalized}"
218  echo -n "-coverprofile=${coverdir}/${prefix_normalized}/${pkgs_normalized}.coverprofile"
219}
220
221function not_test_packages {
222  for m in $(modules); do
223    if [[ $m =~ .*/etcd/tests/v3 ]]; then continue; fi
224    if [[ $m =~ .*/etcd/v3 ]]; then continue; fi
225    echo "${m}/..."
226  done
227}
228
229# split_dir [dir] [num]
230function split_dir {
231  local d="${1}"
232  local num="${2}"
233  local i=0
234  for f in "${d}/"*; do
235    local g=$(( "${i}" % "${num}" ))
236    mkdir -p "${d}_${g}"
237    mv "${f}" "${d}_${g}/"
238    (( i++ ))
239  done
240}
241
242function split_dir_pass {
243  split_dir ./covdir/integration 4
244}
245
246
247# merge_cov_files [coverdir] [outfile]
248# merges all coverprofile files into a single file in the given directory.
249function merge_cov_files {
250  local coverdir="${1}"
251  local cover_out_file="${2}"
252  log_callout "Merging coverage results in: ${coverdir}"
253  # gocovmerge requires not-empty test to start with:
254  echo "mode: set" > "${cover_out_file}"
255
256  local i=0
257  local count
258  count=$(find "${coverdir}"/*.coverprofile | wc -l)
259  for f in "${coverdir}"/*.coverprofile; do
260    # print once per 20 files
261    if ! (( "${i}" % 20 )); then
262      log_callout "${i} of ${count}: Merging file: ${f}"
263    fi
264    run_go_tool "github.com/gyuho/gocovmerge" "${f}" "${cover_out_file}"  > "${coverdir}/cover.tmp" 2>/dev/null
265    if [ -s "${coverdir}"/cover.tmp ]; then
266      mv "${coverdir}/cover.tmp" "${cover_out_file}"
267    fi
268    (( i++ ))
269  done
270}
271
272# merge_cov [coverdir]
273function merge_cov {
274  log_callout "[$(date)] Merging coverage files ..."
275  coverdir="${1}"
276  for d in "${coverdir}"/*/; do
277    d=${d%*/}  # remove the trailing "/"
278    merge_cov_files "${d}" "${d}.coverprofile" &
279  done
280  wait
281  merge_cov_files "${coverdir}" "${coverdir}/all.coverprofile"
282}
283
284function cov_pass {
285  # shellcheck disable=SC2153
286  if [ -z "$COVERDIR" ]; then
287    log_error "COVERDIR undeclared"
288    return 255
289  fi
290
291  if [ ! -f "bin/etcd_test" ]; then
292    log_error "etcd_test binary not found. Call: PASSES='build_cov' ./test"
293    return 255
294  fi
295
296  local coverdir
297  coverdir=$(readlink -f "${COVERDIR}")
298  mkdir -p "${coverdir}"
299  find "${coverdir}" -print0 -name '*.coverprofile' | xargs -0 rm
300
301  local covpkgs
302  covpkgs=$(not_test_packages)
303  local coverpkg_comma
304  coverpkg_comma=$(echo "${covpkgs[@]}" | xargs | tr ' ' ',')
305  local gocov_build_flags=("-covermode=set" "-coverpkg=$coverpkg_comma")
306
307  local failed=""
308
309  log_callout "[$(date)] Collecting coverage from unit tests ..."
310  for m in $(module_dirs); do
311    GOLANG_TEST_SHORT=true run_for_module "${m}" go_test "./..." "parallel" "pkg_to_coverprofileflag unit_${m}" -short -timeout=30m \
312       "${gocov_build_flags[@]}" "$@" || failed="$failed unit"
313  done
314
315  log_callout "[$(date)] Collecting coverage from integration tests ..."
316  run_for_module "tests" go_test "./integration/..." "parallel" "pkg_to_coverprofileflag integration" \
317      -timeout=30m "${gocov_build_flags[@]}" "$@" || failed="$failed integration"
318  # integration-store-v2
319  run_for_module "tests" go_test "./integration/v2store/..." "keep_going" "pkg_to_coverprofileflag store_v2" \
320      -tags v2v3 -timeout=5m "${gocov_build_flags[@]}" "$@" || failed="$failed integration_v2v3"
321  # integration_cluster_proxy
322  run_for_module "tests" go_test "./integration/..." "parallel" "pkg_to_coverprofileflag integration_cluster_proxy" \
323      -tags cluster_proxy -timeout=5m "${gocov_build_flags[@]}" || failed="$failed integration_cluster_proxy"
324
325  log_callout "[$(date)] Collecting coverage from e2e tests ..."
326  # We don't pass 'gocov_build_flags' nor 'pkg_to_coverprofileflag' here,
327  # as the coverage is collected from the ./bin/etcd_test & ./bin/etcdctl_test internally spawned.
328  mkdir -p "${COVERDIR}/e2e"
329  COVERDIR="${COVERDIR}/e2e" run_for_module "tests" go_test "./e2e/..." "keep_going" : -tags=cov -timeout 30m "$@" || failed="$failed tests_e2e"
330  split_dir "${COVERDIR}/e2e" 10
331
332  log_callout "[$(date)] Collecting coverage from e2e tests with proxy ..."
333  mkdir -p "${COVERDIR}/e2e_proxy"
334  COVERDIR="${COVERDIR}/e2e_proxy" run_for_module "tests" go_test "./e2e/..." "keep_going" : -tags="cov cluster_proxy" -timeout 30m "$@" || failed="$failed tests_e2e_proxy"
335  split_dir "${COVERDIR}/e2e_proxy" 10
336
337  local cover_out_file="${coverdir}/all.coverprofile"
338  merge_cov "${coverdir}"
339
340  # strip out generated files (using GNU-style sed)
341  sed --in-place -E "/[.]pb[.](gw[.])?go/d" "${cover_out_file}" || true
342
343  sed --in-place -E "s|go.etcd.io/etcd/api/v3/|api/|g" "${cover_out_file}" || true
344  sed --in-place -E "s|go.etcd.io/etcd/client/v3/|client/v3/|g" "${cover_out_file}" || true
345  sed --in-place -E "s|go.etcd.io/etcd/client/v2/|client/v2/|g" "${cover_out_file}" || true
346  sed --in-place -E "s|go.etcd.io/etcd/etcdctl/v3/|etcdctl/|g" "${cover_out_file}" || true
347  sed --in-place -E "s|go.etcd.io/etcd/pkg/v3/|pkg/|g" "${cover_out_file}" || true
348  sed --in-place -E "s|go.etcd.io/etcd/raft/v3/|raft/|g" "${cover_out_file}" || true
349  sed --in-place -E "s|go.etcd.io/etcd/server/v3/|server/|g" "${cover_out_file}" || true
350
351  # held failures to generate the full coverage file, now fail
352  if [ -n "$failed" ]; then
353    for f in $failed; do
354      log_error "--- FAIL:" "$f"
355    done
356    log_warning "Despite failures, you can see partial report:"
357    log_warning "  go tool cover -html ${cover_out_file}"
358    return 255
359  fi
360
361  log_success "done :) [see report: go tool cover -html ${cover_out_file}]"
362}
363
364######### Code formatting checkers #############################################
365
366function fmt_pass {
367  toggle_failpoints disable
368
369  # TODO: add "unparam","staticcheck", "unconvert", "ineffasign","nakedret"
370  # after resolving ore-existing errors.
371  for p in shellcheck \
372      markdown_you \
373      goword \
374      gofmt \
375      govet \
376      revive \
377      license_header \
378      receiver_name \
379      mod_tidy \
380      dep \
381      shellcheck \
382      shellws \
383      ; do
384    run_pass "${p}" "${@}"
385  done
386}
387
388function shellcheck_pass {
389  if tool_exists "shellcheck" "https://github.com/koalaman/shellcheck#installing"; then
390    generic_checker run shellcheck -fgcc build test scripts/*.sh ./*.sh
391  fi
392}
393
394function shellws_pass {
395  TAB=$'\t'
396  log_callout "Ensuring no tab-based indention in shell scripts"
397  local files
398  files=$(find ./ -name '*.sh' -print0 | xargs -0 )
399  # shellcheck disable=SC2206
400  files=( ${files[@]} "./scripts/build-binary" "./scripts/build-docker" "./scripts/release" )
401  log_cmd "grep -E -n $'^ *${TAB}' ${files[*]}"
402  # shellcheck disable=SC2086
403  if grep -E -n $'^ *${TAB}' "${files[@]}" | sed $'s|${TAB}|[\\\\tab]|g'; then
404    log_error "FAIL: found tab-based indention in bash scripts. Use '  ' (double space)."
405    local files_with_tabs
406    files_with_tabs=$(grep -E -l $'^ *\\t' "${files[@]}")
407    log_warning "Try: sed -i 's|\\t|  |g' $files_with_tabs"
408    return 1
409  else
410    log_success "SUCCESS: no tabulators found."
411    return 0
412  fi
413}
414
415function markdown_you_find_eschew_you {
416  local find_you_cmd="find . -name \\*.md ! -path '*/vendor/*' ! -path './Documentation/*' ! -path './gopath.proto/*' ! -path './release/*' -exec grep -E --color '[Yy]ou[r]?[ '\\''.,;]' {} + || true"
417  run eval "${find_you_cmd}"
418}
419
420function markdown_you_pass {
421  generic_checker markdown_you_find_eschew_you
422}
423
424function markdown_marker_pass {
425  # TODO: check other markdown files when marker handles headers with '[]'
426  if tool_exists "marker" "https://crates.io/crates/marker"; then
427    generic_checker run marker --skip-http --root ./Documentation 2>&1
428  fi
429}
430
431function govet_pass {
432  run_for_modules generic_checker run go vet
433}
434
435function govet_shadow_pass {
436  local shadow
437  shadow=$(tool_get_bin "golang.org/x/tools/go/analysis/passes/shadow/cmd/shadow")
438  run_for_modules generic_checker run go vet -all -vettool="${shadow}"
439}
440
441function unparam_pass {
442  run_for_modules generic_checker run_go_tool "mvdan.cc/unparam"
443}
444
445function staticcheck_pass {
446  run_for_modules generic_checker run_go_tool "honnef.co/go/tools/cmd/staticcheck"
447}
448
449function revive_pass {
450  run_for_modules generic_checker run_go_tool "github.com/mgechev/revive" -config "${ETCD_ROOT_DIR}/tests/revive.toml" -exclude "vendor/..."
451}
452
453function unconvert_pass {
454  run_for_modules generic_checker run_go_tool "github.com/mdempsky/unconvert" unconvert -v
455}
456
457function ineffassign_per_package {
458  # bash 3.x compatible replacement of: mapfile -t gofiles < <(go_srcs_in_module "$1")
459  local gofiles=()
460  while IFS= read -r line; do gofiles+=("$line"); done < <(go_srcs_in_module "$1")
461  run_go_tool github.com/gordonklaus/ineffassign "${gofiles[@]}"
462}
463
464function ineffassign_pass {
465  run_for_modules generic_checker ineffassign_per_package
466}
467
468function nakedret_pass {
469  run_for_modules generic_checker run_go_tool "github.com/alexkohler/nakedret"
470}
471
472function license_header_pass {
473  # bash 3.x compatible replacement of: mapfile -t gofiles < <(go_srcs_in_module "$1")
474  local gofiles=()
475  while IFS= read -r line; do gofiles+=("$line"); done < <(go_srcs_in_module "$1")
476
477  for file in "${gofiles[@]}"; do
478    if ! head -n3 "${file}" | grep -Eq "(Copyright|generated|GENERATED)" ; then
479      licRes="${licRes}"$(echo -e "  ${file}")
480    fi
481  done
482  if [ -n "${licRes}" ]; then
483    log_error -e "license header checking failed:\\n${licRes}"
484    return 255
485  fi
486}
487
488function receiver_name_for_package {
489  # bash 3.x compatible replacement of: mapfile -t gofiles < <(go_srcs_in_module "$1")
490  local gofiles=()
491  while IFS= read -r line; do gofiles+=("$line"); done < <(go_srcs_in_module "$1")
492
493  recvs=$(grep 'func ([^*]' "${gofiles[@]}"  | tr  ':' ' ' |  \
494    awk ' { print $2" "$3" "$4" "$1 }' | sed "s/[a-zA-Z\\.]*go//g" |  sort  | uniq  | \
495    grep -Ev  "(Descriptor|Proto|_)"  | awk ' { print $3" "$4 } ' | sort | uniq -c | grep -v ' 1 ' | awk ' { print $2 } ')
496  if [ -n "${recvs}" ]; then
497    # shellcheck disable=SC2206
498    recvs=($recvs)
499    for recv in "${recvs[@]}"; do
500      log_error "Mismatched receiver for $recv..."
501      grep "$recv" "${gofiles[@]}" | grep 'func ('
502    done
503    return 255
504  fi
505}
506
507function receiver_name_pass {
508  run_for_modules receiver_name_for_package
509}
510
511# goword_for_package package
512# checks spelling and comments in the 'package' in the current module
513#
514function goword_for_package {
515  # bash 3.x compatible replacement of: mapfile -t gofiles < <(go_srcs_in_module "$1")
516  local gofiles=()
517  while IFS= read -r line; do gofiles+=("$line"); done < <(go_srcs_in_module "$1")
518
519  local gowordRes
520
521  # spellchecking can be enabled with GOBINARGS="--tags=spell"
522  # but it requires heavy dependencies installation, like:
523  # apt-get install libaspell-dev libhunspell-dev hunspell-en-us aspell-en
524
525  # only check for broke exported godocs
526  if gowordRes=$(run_go_tool "github.com/chzchzchz/goword" -use-spell=false "${gofiles[@]}" | grep godoc-export | sort); then
527    log_error -e "goword checking failed:\\n${gowordRes}"
528    return 255
529  fi
530  if [ -n "$gowordRes" ]; then
531    log_error -e "goword checking returned output:\\n${gowordRes}"
532    return 255
533  fi
534}
535
536
537function goword_pass {
538  run_for_modules goword_for_package || return 255
539}
540
541function go_fmt_for_package {
542  # We utilize 'go fmt' to find all files suitable for formatting,
543  # but reuse full power gofmt to perform just RO check.
544  go fmt -n "$1" | sed 's| -w | -d |g' | sh
545}
546
547function gofmt_pass {
548  run_for_modules generic_checker go_fmt_for_package
549}
550
551function bom_pass {
552  log_callout "Checking bill of materials..."
553  # https://github.com/golang/go/commit/7c388cc89c76bc7167287fb488afcaf5a4aa12bf
554  # shellcheck disable=SC2207
555  modules=($(modules_exp))
556
557  # Internally license-bill-of-materials tends to modify go.sum
558  run cp go.sum go.sum.tmp || return 2
559  run cp go.mod go.mod.tmp || return 2
560
561  output=$(GOFLAGS=-mod=mod run_go_tool github.com/coreos/license-bill-of-materials \
562    --override-file ./bill-of-materials.override.json \
563    "${modules[@]}")
564  code="$?"
565
566  run cp go.sum.tmp go.sum || return 2
567  run cp go.mod.tmp go.mod || return 2
568
569  if [ "${code}" -ne 0 ] ; then
570    log_error -e "license-bill-of-materials (code: ${code}) failed with:\\n${output}"
571    return 255
572  else
573    echo "${output}" > "bom-now.json.tmp"
574  fi
575  if ! diff ./bill-of-materials.json bom-now.json.tmp; then
576    log_error "modularized licenses do not match given bill of materials"
577    return 255
578  fi
579  rm bom-now.json.tmp
580}
581
582######## VARIOUS CHECKERS ######################################################
583
584function dump_deps_of_module() {
585  local module
586  if ! module=$(run go list -m); then
587    return 255
588  fi
589  run go list -f "{{if not .Indirect}}{{if .Version}}{{.Path}},{{.Version}},${module}{{end}}{{end}}" -m all
590}
591
592# Checks whether dependencies are consistent across modules
593function dep_pass {
594  local all_dependencies
595  all_dependencies=$(run_for_modules dump_deps_of_module | sort) || return 2
596
597  local duplicates
598  duplicates=$(echo "${all_dependencies}" | cut -d ',' -f 1,2 | sort | uniq | cut -d ',' -f 1 | sort | uniq -d) || return 2
599
600  for dup in ${duplicates}; do
601    log_error "FAIL: inconsistent versions for depencency: ${dup}"
602    echo "${all_dependencies}" | grep "${dup}" | sed "s|\\([^,]*\\),\\([^,]*\\),\\([^,]*\\)|  - \\1@\\2 from: \\3|g"
603  done
604  if [[ -n "${duplicates}" ]]; then
605    log_error "FAIL: inconsistent dependencies"
606    return 2
607  else
608    log_success "SUCCESS: dependencies are consistent across modules"
609  fi
610}
611
612function release_pass {
613  rm -f ./bin/etcd-last-release
614  # to grab latest patch release; bump this up for every minor release
615  UPGRADE_VER=$(git tag -l --sort=-version:refname "v3.3.*" | head -1)
616  if [ -n "$MANUAL_VER" ]; then
617    # in case, we need to test against different version
618    UPGRADE_VER=$MANUAL_VER
619  fi
620  if [[ -z ${UPGRADE_VER} ]]; then
621    UPGRADE_VER="v3.3.0"
622    log_warning "fallback to" ${UPGRADE_VER}
623  fi
624
625  local file="etcd-$UPGRADE_VER-linux-$GOARCH.tar.gz"
626  log_callout "Downloading $file"
627
628  set +e
629  curl --fail -L "https://github.com/etcd-io/etcd/releases/download/$UPGRADE_VER/$file" -o "/tmp/$file"
630  local result=$?
631  set -e
632  case $result in
633    0)  ;;
634    *)  log_error "--- FAIL:" ${result}
635      return $result
636      ;;
637  esac
638
639  tar xzvf "/tmp/$file" -C /tmp/ --strip-components=1
640  mkdir -p ./bin
641  mv /tmp/etcd ./bin/etcd-last-release
642}
643
644function mod_tidy_for_module {
645  # Watch for upstream solution: https://github.com/golang/go/issues/27005
646  local tmpModDir
647  tmpModDir=$(mktemp -d -t 'tmpModDir.XXXXXX')
648  run cp "./go.mod" "./go.sum" "${tmpModDir}" || return 2
649
650  # Guarantees keeping go.sum minimal
651  # If this is causing too much problems, we should
652  # stop controlling go.sum at all.
653  rm go.sum
654  run go mod tidy || return 2
655
656  set +e
657  local tmpFileGoModInSync
658  diff -C 5 "${tmpModDir}/go.mod" "./go.mod"
659  tmpFileGoModInSync="$?"
660
661  local tmpFileGoSumInSync
662  diff -C 5 "${tmpModDir}/go.sum" "./go.sum"
663  tmpFileGoSumInSync="$?"
664  set -e
665
666  # Bring back initial state
667  mv "${tmpModDir}/go.mod" "./go.mod"
668  mv "${tmpModDir}/go.sum" "./go.sum"
669
670  if [ "${tmpFileGoModInSync}" -ne 0 ]; then
671    log_error "${PWD}/go.mod is not in sync with 'go mod tidy'"
672    return 255
673  fi
674  if [ "${tmpFileGoSumInSync}" -ne 0 ]; then
675    log_error "${PWD}/go.sum is not in sync with 'rm go.sum; go mod tidy'"
676    return 255
677  fi
678}
679
680function mod_tidy_pass {
681  run_for_modules mod_tidy_for_module
682}
683
684########### MAIN ###############################################################
685
686function run_pass {
687  local pass="${1}"
688  shift 1
689  log_callout -e "\\n'${pass}' started at $(date)"
690  if "${pass}_pass" "$@" ; then
691    log_success "'${pass}' completed at $(date)"
692  else
693    log_error "FAIL: '${pass}' failed at $(date)"
694    exit 255
695  fi
696}
697
698log_callout "Starting at: $(date)"
699for pass in $PASSES; do
700  run_pass "${pass}" "${@}"
701done
702
703log_success "SUCCESS"
704