1#!/bin/sh
2
3# Copyright (C) 2020-2021 Internet Systems Consortium, Inc. ("ISC")
4#
5# This Source Code Form is subject to the terms of the Mozilla Public
6# License, v. 2.0. If a copy of the MPL was not distributed with this
7# file, You can obtain one at http://mozilla.org/MPL/2.0/.
8
9# shellcheck disable=SC2039
10# SC2039: In POSIX sh, 'local' is undefined.
11
12# Exit with error if commands exit with non-zero and if undefined variables are
13# used.
14set -eu
15
16############################### Public functions ###############################
17
18# Add an entry to the XML test report.
19report_test_result_in_xml() {
20    # If GTEST_OUTPUT is not defined...
21    if ! test -n "${GTEST_OUTPUT+x}"; then
22        # There is nowhere to report.
23        return
24    fi
25
26    # Declarations
27    local test_name="${1}"; shift
28    local exit_code="${1}"; shift
29    local duration="${1}"; shift # milliseconds
30    local now
31    local test_case
32    local test_suite
33    local xml
34    now=$(date '+%FT%H:%M:%S')
35    test_suite=$(printf '%s' "${test_name}" | cut -d '.' -f 1)
36    test_case=$(printf '%s' "${test_name}" | cut -d '.' -f 2-)
37
38    # Strip the 'xml:' at the start of GTEST_OUTPUT if it is there.
39    xml="${GTEST_OUTPUT}"
40    if test "$(printf '%s' "${xml}" | cut -c 1-4)" = 'xml:'; then
41        xml=$(printf '%s' "${xml}" | cut -c 5-)
42    fi
43    xml="${xml}/${test_suite}.sh.xml"
44
45    # Convert to seconds, but keep the millisecond precision.
46    duration=$(_calculate "${duration} / 1000.0")
47
48    # For test suites that have a single test case and no name for the test
49    # case, name the test case after the test suite.
50    if test -z "${test_case}"; then
51        test_case="${test_suite}"
52    fi
53
54    # Determine result based on exit code. Googletest seems to omit the failed
55    # tests, instead we are explicitly adding them with a 'failed' result.
56    local result
57    if test "${exit_code}" -eq 0; then
58        result='success'
59    else
60        result='failed'
61    fi
62
63    _create_xml "${xml}" "${now}"
64
65    _add_test_suite "${test_suite}" "${xml}" "${now}"
66
67    _add_test_case "${test_suite}" "${test_case}" "${result}" "${duration}" \
68        "${xml}" "${now}"
69}
70
71############################## Private functions ###############################
72
73# Add ${string} after ${reference} in ${file}.
74_add_after() {
75    local string="${1}"; shift
76    local reference="${1}"; shift
77    local file="${1}"; shift
78
79    # Escape all slashes.
80    string=$(printf '%s' "${string}" | sed 's#\/#\\\/#g')
81    reference=$(printf '%s' "${reference}" | sed 's#\/#\\\/#g')
82
83    # Escape all spaces. Only trailing spaces need escaped, but that's harder
84    # and this still empirically works.
85    string=$(printf '%s' "${string}" | sed 's#\ #\\\ #g')
86    reference=$(printf '%s' "${reference}" | sed 's#\ #\\\ #g')
87
88    # Linearize. To avoid this change, add one line at a time.
89    string=$(printf '%s' "${string}" | tr '\n' ' ')
90
91    # Add ${string} after ${reference} in ${file}.
92    # The "\\" followed by newline is for BSD support.
93    sed "/${reference}/a\\
94${string}
95" "${file}" > "${file}.tmp"
96    mv "${file}.tmp" "${file}"
97}
98
99# Add ${string} before ${reference} in ${file}.
100_add_before() {
101    local string="${1}"; shift
102    local reference="${1}"; shift
103    local file="${1}"; shift
104
105    # Get the line number of the reference line.
106    local line_number
107    line_number=$(grep -Fn "${reference}" "${file}" | cut -d ':' -f 1)
108
109    # Escape all slashes.
110    string=$(printf '%s' "${string}" | sed 's#\/#\\\/#g')
111    reference=$(printf '%s' "${reference}" | sed 's#\/#\\\/#g')
112
113    # Escape all spaces. Only trailing spaces need escaped, but that's harder
114    # and this still empirically works.
115    string=$(printf '%s' "${string}" | sed 's#\ #\\\ #g')
116    reference=$(printf '%s' "${reference}" | sed 's#\ #\\\ #g')
117
118    # Linearize. To avoid this change, add one line at a time.
119    string=$(printf '%s' "${string}" | tr '\n' ' ')
120
121    # Add ${string} before ${reference} in ${file}.
122    # The "\\" followed by newline is for BSD support.
123    sed "${line_number}i\\
124${string}
125" "${file}" > "${file}.tmp"
126    mv "${file}.tmp" "${file}"
127}
128
129_add_failure_tag() {
130    local test_case_tag="${1}"; shift
131    local xml="${1}"; shift
132
133    local closing_tag='    </testcase>'
134    local failure_tag
135    local failure_text
136    local linearized_failure_text
137    # Remove characters which are suspected to not be allowed in:
138    # * sed
139    # * XML attribute values
140    # * XML CDATA
141    failure_text=$(printf '%s\n%s' "${ERROR}" "${OUTPUT}" | \
142        sed 's/"/ /g' | sed 's/\[/ /g' | sed 's/\]/ /g')
143    linearized_failure_text=$(printf '%s' "${failure_text}" | tr '\n' ' ')
144    failure_tag=$(printf '      <failure message="%s" type=""><![CDATA[%s]]></failure>' \
145        "${linearized_failure_text}" "${failure_text}")
146
147    # Add.
148    _add_after "${closing_tag}" "${test_case_tag}" "${xml}"
149    _add_after "${failure_tag}" "${test_case_tag}" "${xml}"
150}
151
152# Add test result if not in file.
153_add_test_case() {
154    local test_suite="${1}"; shift
155    local test_case="${1}"; shift
156    local result="${1}"; shift
157    local duration="${1}"; shift
158    local xml="${1}"; shift
159    local now="${1}"; shift
160
161    # Determine the test case tag.
162    local closing_backslash
163    local closing_tag
164    if test "${result}" = 'success'; then
165        closing_backslash=' /'
166    else
167        closing_backslash=
168    fi
169
170    # Create the test XML tag.
171    local test_case_line
172    test_case_line=$(printf '    <testcase name="%s" status="run" result="completed" time="%s" timestamp="%s" classname="%s"%s>' \
173        "${test_case}" "${duration}" "${now}" "${test_suite}" \
174        "${closing_backslash}")
175
176    # Add this test case to all the other test cases.
177    local all_test_cases
178    all_test_cases=$(_print_lines_between_matching_patterns \
179        "  <testsuite name=\"${test_suite}\"" '  </testsuite>' "${xml}")
180    all_test_cases=$(printf '%s\n%s' "${all_test_cases}" "${test_case_line}")
181
182    # Find the test following this one.
183    local following_line
184    following_line=$(printf '%s' "${all_test_cases}" | \
185        grep -A1 -F "${test_case_line}" | \
186        grep -Fv "${test_case_line}" || true)
187    if test -n "${following_line}"; then
188        # If found, add it before.
189        _add_before "${test_case_line}" "${following_line}" "${xml}"
190    else
191        # Find the test before this one.
192        local previous_line
193        previous_line=$(printf '%s' "${all_test_cases}" | \
194            grep -B1 -F "${test_case_line}" | \
195            grep -Fv "${test_case_line}" || true)
196        if test -n "${previous_line}"; then
197            # If found, add it after.
198            _add_after "${test_case_line}" "${previous_line}" "${xml}"
199        else
200            # If neither were found, add it as the first test case following the test
201            # suite line.
202            _add_after "${test_case_line}" "  <testsuite name=\"${test_suite}\"" "${xml}"
203        fi
204    fi
205
206    # Add the failure tag if it is the case.
207    if test "${result}" != 'success'; then
208        _add_failure_tag "${test_case_line}" "${xml}"
209    fi
210
211    # Retrieve again to include the failure tag that may have just been added
212    # among other tags or lines.
213    all_test_cases=$(_print_lines_between_matching_patterns \
214        "  <testsuite name=\"${test_suite}\"" '  </testsuite>' "${xml}")
215
216    # Update attributes for the parent <testsuite> and the global <testsuites>.
217    _update_test_suite_metrics "${test_suite}" "${all_test_cases}" "${xml}" "${now}"
218}
219
220# Add a set of test suite tags if not already present in the XML.
221_add_test_suite() {
222    local test_suite="${1}"; shift
223    local xml="${1}"; shift
224    local now="${1}"; shift
225    local test_suite_line
226    local all_test_suites
227
228    # If test suite tag is already there, then there is nothing to do.
229    if grep -F "<testsuite name=\"${test_suite}\"" "${xml}" \
230        > /dev/null 2>&1; then
231        return
232    fi
233
234    # Create the test suite XML tag.
235    local test_suite_line
236    test_suite_line=$(printf '  <testsuite name="%s" tests="0" failures="0" disabled="0" errors="0" time="0" timestamp="%s">' \
237        "${test_suite}" "${now}")
238
239    # Add this test suite to all the other test suites and sort them.
240    local all_test_suites
241    all_test_suites=$(printf '%s\n%s' "  ${test_suite_line}" \
242        "$(grep -E '  <testsuite name=|</testsuites>' "${xml}")")
243
244    # Find the test suite following this one.
245    local following_line
246    following_line=$(printf '%s' "${all_test_suites}" | \
247        grep -A1 -F "${test_suite_line}" | \
248        grep -Fv "${test_suite_line}" || true)
249
250    # Add the test suite tag to the XML.
251    _add_before "${test_suite_line}" "${following_line}" "${xml}"
252    _add_after '  </testsuite>' "${test_suite_line}" "${xml}"
253}
254
255# Calculate the given mathematical expression and print it in a format that
256# matches googletest's time in the XML attribute time="..." which is seconds
257# rounded to 3 decimals.
258_calculate() {
259    awk "BEGIN{print ${*}}";
260}
261
262# Create XML with header and top-level tags if the file doesn't exist.
263_create_xml() {
264    # If file exists and we have set GTEST_OUTPUT_CREATED previously, then there
265    # is nothing to do.
266    if test -f "${xml}" && test -n "${GTEST_OUTPUT_CREATED+x}"; then
267        return;
268    fi
269
270    local xml="${1}"; shift
271    local now="${1}"; shift
272
273    mkdir -p "$(dirname "${xml}")"
274    printf \
275'<?xml version="1.0" encoding="UTF-8"?>
276<testsuites tests="0" failures="0" disabled="0" errors="0" time="0" timestamp="%s" name="AllTests">
277</testsuites>
278' "${now}" > "${xml}"
279
280    # GTEST_OUTPUT_CREATED is not a googletest variable, but our way of allowing
281    # to overwrite XMLs created in a previous test run. The lifetime of
282    # GTEST_OUTPUT_CREATED is extended to the oldest ancestor file who has
283    # sourced this script i.e. the *_test.sh file. So it gets lost from one
284    # *_test.sh to another. The consensus that need to be kept so that this
285    # works correctly are:
286    #   * Needless to say, don't set this variable on your own.
287    #   * Always call these scripts directly or through `make check`.
288    # Never source test files e.g. `source memfile_tests.sh` or
289    # `. memfile_tests.sh`.
290    #   * The ${xml} passed here must be deterministically and uniquely
291    # attributed to the *_test.sh. At the time of this writing, ${xml} is the
292    # part of the name before the dot. So for example, for memfile, all tests
293    # should start with the same thing e.g. `memfile.*`.
294    export GTEST_OUTPUT_CREATED=true
295}
296
297# Print the lines between two matching regex patterns from a file. Excludes the
298# lines that contain the patterns themselves. Matches only the first occurrence.
299_print_lines_between_matching_patterns() {
300    local start_pattern="${1}"; shift
301    local end_pattern="${1}"; shift
302    local file="${1}"; shift
303
304    # Escape all slashes.
305    start_pattern=$(printf '%s' "${start_pattern}" | sed 's#\/#\\\/#g')
306    end_pattern=$(printf '%s' "${end_pattern}" | sed 's#\/#\\\/#g')
307
308    # Print with sed.
309    sed -n "/${start_pattern}/,/${end_pattern}/p;/${end_pattern}/q" "${file}" \
310        | sed '$d' | tail -n +2
311}
312
313# Update the test suite XML attributes with metrics collected from the child
314# test cases.
315_update_test_suite_metrics() {
316    local test_suite="${1}"; shift
317    local all_test_cases="${1}"; shift
318    local xml="${1}"; shift
319    local now="${1}"; shift
320
321    # Get the metrics on the parent test suite.
322    local duration
323    local durations_summed
324    local failures
325    local tests
326    tests=$(printf '%s' "${all_test_cases}" | \
327        grep -Fc '<testcase' || true)
328    failures=$(printf '%s' "${all_test_cases}" | \
329        grep -Fc '<failure' || true)
330    durations_summed=$(printf '%s' "${all_test_cases}" | \
331        grep -Eo 'time="[0-9.]+"' | cut -d '"' -f 2 | xargs | sed 's/ / + /g')
332    duration=$(_calculate "${durations_summed}")
333
334    # Create the test suite XML tag.
335    local test_suite_line
336    test_suite_line=$(printf '  <testsuite name="%s" tests="%s" failures="%s" disabled="0" errors="0" time="%s" timestamp="%s">' \
337        "${test_suite}" "${tests}" "${failures}" "${duration}" "${now}")
338
339    # Update the test suite with the collected metrics.
340    sed "s#  <testsuite name=\"${test_suite}\".*>#${test_suite_line}#g" \
341        "${xml}" > "${xml}.tmp"
342    mv "${xml}.tmp" "${xml}"
343
344    # Create the test suites XML tag.
345    local test_suites_line
346    test_suites_line=$(printf '<testsuites tests="%s" failures="%s" disabled="0" errors="0" time="%s" timestamp="%s" name="AllTests">' \
347        "${tests}" "${failures}" "${duration}" "${now}")
348
349    # Update the test suites with the collected metrics.
350    sed "s#<testsuites .*>#${test_suites_line}#g" \
351        "${xml}" > "${xml}.tmp"
352    mv "${xml}.tmp" "${xml}"
353}
354