1#!/bin/sh 2 3# Copyright (C) 2020-2021 Internet Systems Consortium, Inc. ("ISC") 4# 5# This Source Code Form is subject to the terms of the Mozilla Public 6# License, v. 2.0. If a copy of the MPL was not distributed with this 7# file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 9# shellcheck disable=SC2039 10# SC2039: In POSIX sh, 'local' is undefined. 11 12# Exit with error if commands exit with non-zero and if undefined variables are 13# used. 14set -eu 15 16############################### Public functions ############################### 17 18# Add an entry to the XML test report. 19report_test_result_in_xml() { 20 # If GTEST_OUTPUT is not defined... 21 if ! test -n "${GTEST_OUTPUT+x}"; then 22 # There is nowhere to report. 23 return 24 fi 25 26 # Declarations 27 local test_name="${1}"; shift 28 local exit_code="${1}"; shift 29 local duration="${1}"; shift # milliseconds 30 local now 31 local test_case 32 local test_suite 33 local xml 34 now=$(date '+%FT%H:%M:%S') 35 test_suite=$(printf '%s' "${test_name}" | cut -d '.' -f 1) 36 test_case=$(printf '%s' "${test_name}" | cut -d '.' -f 2-) 37 38 # Strip the 'xml:' at the start of GTEST_OUTPUT if it is there. 39 xml="${GTEST_OUTPUT}" 40 if test "$(printf '%s' "${xml}" | cut -c 1-4)" = 'xml:'; then 41 xml=$(printf '%s' "${xml}" | cut -c 5-) 42 fi 43 xml="${xml}/${test_suite}.sh.xml" 44 45 # Convert to seconds, but keep the millisecond precision. 46 duration=$(_calculate "${duration} / 1000.0") 47 48 # For test suites that have a single test case and no name for the test 49 # case, name the test case after the test suite. 50 if test -z "${test_case}"; then 51 test_case="${test_suite}" 52 fi 53 54 # Determine result based on exit code. Googletest seems to omit the failed 55 # tests, instead we are explicitly adding them with a 'failed' result. 56 local result 57 if test "${exit_code}" -eq 0; then 58 result='success' 59 else 60 result='failed' 61 fi 62 63 _create_xml "${xml}" "${now}" 64 65 _add_test_suite "${test_suite}" "${xml}" "${now}" 66 67 _add_test_case "${test_suite}" "${test_case}" "${result}" "${duration}" \ 68 "${xml}" "${now}" 69} 70 71############################## Private functions ############################### 72 73# Add ${string} after ${reference} in ${file}. 74_add_after() { 75 local string="${1}"; shift 76 local reference="${1}"; shift 77 local file="${1}"; shift 78 79 # Escape all slashes. 80 string=$(printf '%s' "${string}" | sed 's#\/#\\\/#g') 81 reference=$(printf '%s' "${reference}" | sed 's#\/#\\\/#g') 82 83 # Escape all spaces. Only trailing spaces need escaped, but that's harder 84 # and this still empirically works. 85 string=$(printf '%s' "${string}" | sed 's#\ #\\\ #g') 86 reference=$(printf '%s' "${reference}" | sed 's#\ #\\\ #g') 87 88 # Linearize. To avoid this change, add one line at a time. 89 string=$(printf '%s' "${string}" | tr '\n' ' ') 90 91 # Add ${string} after ${reference} in ${file}. 92 # The "\\" followed by newline is for BSD support. 93 sed "/${reference}/a\\ 94${string} 95" "${file}" > "${file}.tmp" 96 mv "${file}.tmp" "${file}" 97} 98 99# Add ${string} before ${reference} in ${file}. 100_add_before() { 101 local string="${1}"; shift 102 local reference="${1}"; shift 103 local file="${1}"; shift 104 105 # Get the line number of the reference line. 106 local line_number 107 line_number=$(grep -Fn "${reference}" "${file}" | cut -d ':' -f 1) 108 109 # Escape all slashes. 110 string=$(printf '%s' "${string}" | sed 's#\/#\\\/#g') 111 reference=$(printf '%s' "${reference}" | sed 's#\/#\\\/#g') 112 113 # Escape all spaces. Only trailing spaces need escaped, but that's harder 114 # and this still empirically works. 115 string=$(printf '%s' "${string}" | sed 's#\ #\\\ #g') 116 reference=$(printf '%s' "${reference}" | sed 's#\ #\\\ #g') 117 118 # Linearize. To avoid this change, add one line at a time. 119 string=$(printf '%s' "${string}" | tr '\n' ' ') 120 121 # Add ${string} before ${reference} in ${file}. 122 # The "\\" followed by newline is for BSD support. 123 sed "${line_number}i\\ 124${string} 125" "${file}" > "${file}.tmp" 126 mv "${file}.tmp" "${file}" 127} 128 129_add_failure_tag() { 130 local test_case_tag="${1}"; shift 131 local xml="${1}"; shift 132 133 local closing_tag=' </testcase>' 134 local failure_tag 135 local failure_text 136 local linearized_failure_text 137 # Remove characters which are suspected to not be allowed in: 138 # * sed 139 # * XML attribute values 140 # * XML CDATA 141 failure_text=$(printf '%s\n%s' "${ERROR}" "${OUTPUT}" | \ 142 sed 's/"/ /g' | sed 's/\[/ /g' | sed 's/\]/ /g') 143 linearized_failure_text=$(printf '%s' "${failure_text}" | tr '\n' ' ') 144 failure_tag=$(printf ' <failure message="%s" type=""><![CDATA[%s]]></failure>' \ 145 "${linearized_failure_text}" "${failure_text}") 146 147 # Add. 148 _add_after "${closing_tag}" "${test_case_tag}" "${xml}" 149 _add_after "${failure_tag}" "${test_case_tag}" "${xml}" 150} 151 152# Add test result if not in file. 153_add_test_case() { 154 local test_suite="${1}"; shift 155 local test_case="${1}"; shift 156 local result="${1}"; shift 157 local duration="${1}"; shift 158 local xml="${1}"; shift 159 local now="${1}"; shift 160 161 # Determine the test case tag. 162 local closing_backslash 163 local closing_tag 164 if test "${result}" = 'success'; then 165 closing_backslash=' /' 166 else 167 closing_backslash= 168 fi 169 170 # Create the test XML tag. 171 local test_case_line 172 test_case_line=$(printf ' <testcase name="%s" status="run" result="completed" time="%s" timestamp="%s" classname="%s"%s>' \ 173 "${test_case}" "${duration}" "${now}" "${test_suite}" \ 174 "${closing_backslash}") 175 176 # Add this test case to all the other test cases. 177 local all_test_cases 178 all_test_cases=$(_print_lines_between_matching_patterns \ 179 " <testsuite name=\"${test_suite}\"" ' </testsuite>' "${xml}") 180 all_test_cases=$(printf '%s\n%s' "${all_test_cases}" "${test_case_line}") 181 182 # Find the test following this one. 183 local following_line 184 following_line=$(printf '%s' "${all_test_cases}" | \ 185 grep -A1 -F "${test_case_line}" | \ 186 grep -Fv "${test_case_line}" || true) 187 if test -n "${following_line}"; then 188 # If found, add it before. 189 _add_before "${test_case_line}" "${following_line}" "${xml}" 190 else 191 # Find the test before this one. 192 local previous_line 193 previous_line=$(printf '%s' "${all_test_cases}" | \ 194 grep -B1 -F "${test_case_line}" | \ 195 grep -Fv "${test_case_line}" || true) 196 if test -n "${previous_line}"; then 197 # If found, add it after. 198 _add_after "${test_case_line}" "${previous_line}" "${xml}" 199 else 200 # If neither were found, add it as the first test case following the test 201 # suite line. 202 _add_after "${test_case_line}" " <testsuite name=\"${test_suite}\"" "${xml}" 203 fi 204 fi 205 206 # Add the failure tag if it is the case. 207 if test "${result}" != 'success'; then 208 _add_failure_tag "${test_case_line}" "${xml}" 209 fi 210 211 # Retrieve again to include the failure tag that may have just been added 212 # among other tags or lines. 213 all_test_cases=$(_print_lines_between_matching_patterns \ 214 " <testsuite name=\"${test_suite}\"" ' </testsuite>' "${xml}") 215 216 # Update attributes for the parent <testsuite> and the global <testsuites>. 217 _update_test_suite_metrics "${test_suite}" "${all_test_cases}" "${xml}" "${now}" 218} 219 220# Add a set of test suite tags if not already present in the XML. 221_add_test_suite() { 222 local test_suite="${1}"; shift 223 local xml="${1}"; shift 224 local now="${1}"; shift 225 local test_suite_line 226 local all_test_suites 227 228 # If test suite tag is already there, then there is nothing to do. 229 if grep -F "<testsuite name=\"${test_suite}\"" "${xml}" \ 230 > /dev/null 2>&1; then 231 return 232 fi 233 234 # Create the test suite XML tag. 235 local test_suite_line 236 test_suite_line=$(printf ' <testsuite name="%s" tests="0" failures="0" disabled="0" errors="0" time="0" timestamp="%s">' \ 237 "${test_suite}" "${now}") 238 239 # Add this test suite to all the other test suites and sort them. 240 local all_test_suites 241 all_test_suites=$(printf '%s\n%s' " ${test_suite_line}" \ 242 "$(grep -E ' <testsuite name=|</testsuites>' "${xml}")") 243 244 # Find the test suite following this one. 245 local following_line 246 following_line=$(printf '%s' "${all_test_suites}" | \ 247 grep -A1 -F "${test_suite_line}" | \ 248 grep -Fv "${test_suite_line}" || true) 249 250 # Add the test suite tag to the XML. 251 _add_before "${test_suite_line}" "${following_line}" "${xml}" 252 _add_after ' </testsuite>' "${test_suite_line}" "${xml}" 253} 254 255# Calculate the given mathematical expression and print it in a format that 256# matches googletest's time in the XML attribute time="..." which is seconds 257# rounded to 3 decimals. 258_calculate() { 259 awk "BEGIN{print ${*}}"; 260} 261 262# Create XML with header and top-level tags if the file doesn't exist. 263_create_xml() { 264 # If file exists and we have set GTEST_OUTPUT_CREATED previously, then there 265 # is nothing to do. 266 if test -f "${xml}" && test -n "${GTEST_OUTPUT_CREATED+x}"; then 267 return; 268 fi 269 270 local xml="${1}"; shift 271 local now="${1}"; shift 272 273 mkdir -p "$(dirname "${xml}")" 274 printf \ 275'<?xml version="1.0" encoding="UTF-8"?> 276<testsuites tests="0" failures="0" disabled="0" errors="0" time="0" timestamp="%s" name="AllTests"> 277</testsuites> 278' "${now}" > "${xml}" 279 280 # GTEST_OUTPUT_CREATED is not a googletest variable, but our way of allowing 281 # to overwrite XMLs created in a previous test run. The lifetime of 282 # GTEST_OUTPUT_CREATED is extended to the oldest ancestor file who has 283 # sourced this script i.e. the *_test.sh file. So it gets lost from one 284 # *_test.sh to another. The consensus that need to be kept so that this 285 # works correctly are: 286 # * Needless to say, don't set this variable on your own. 287 # * Always call these scripts directly or through `make check`. 288 # Never source test files e.g. `source memfile_tests.sh` or 289 # `. memfile_tests.sh`. 290 # * The ${xml} passed here must be deterministically and uniquely 291 # attributed to the *_test.sh. At the time of this writing, ${xml} is the 292 # part of the name before the dot. So for example, for memfile, all tests 293 # should start with the same thing e.g. `memfile.*`. 294 export GTEST_OUTPUT_CREATED=true 295} 296 297# Print the lines between two matching regex patterns from a file. Excludes the 298# lines that contain the patterns themselves. Matches only the first occurrence. 299_print_lines_between_matching_patterns() { 300 local start_pattern="${1}"; shift 301 local end_pattern="${1}"; shift 302 local file="${1}"; shift 303 304 # Escape all slashes. 305 start_pattern=$(printf '%s' "${start_pattern}" | sed 's#\/#\\\/#g') 306 end_pattern=$(printf '%s' "${end_pattern}" | sed 's#\/#\\\/#g') 307 308 # Print with sed. 309 sed -n "/${start_pattern}/,/${end_pattern}/p;/${end_pattern}/q" "${file}" \ 310 | sed '$d' | tail -n +2 311} 312 313# Update the test suite XML attributes with metrics collected from the child 314# test cases. 315_update_test_suite_metrics() { 316 local test_suite="${1}"; shift 317 local all_test_cases="${1}"; shift 318 local xml="${1}"; shift 319 local now="${1}"; shift 320 321 # Get the metrics on the parent test suite. 322 local duration 323 local durations_summed 324 local failures 325 local tests 326 tests=$(printf '%s' "${all_test_cases}" | \ 327 grep -Fc '<testcase' || true) 328 failures=$(printf '%s' "${all_test_cases}" | \ 329 grep -Fc '<failure' || true) 330 durations_summed=$(printf '%s' "${all_test_cases}" | \ 331 grep -Eo 'time="[0-9.]+"' | cut -d '"' -f 2 | xargs | sed 's/ / + /g') 332 duration=$(_calculate "${durations_summed}") 333 334 # Create the test suite XML tag. 335 local test_suite_line 336 test_suite_line=$(printf ' <testsuite name="%s" tests="%s" failures="%s" disabled="0" errors="0" time="%s" timestamp="%s">' \ 337 "${test_suite}" "${tests}" "${failures}" "${duration}" "${now}") 338 339 # Update the test suite with the collected metrics. 340 sed "s# <testsuite name=\"${test_suite}\".*>#${test_suite_line}#g" \ 341 "${xml}" > "${xml}.tmp" 342 mv "${xml}.tmp" "${xml}" 343 344 # Create the test suites XML tag. 345 local test_suites_line 346 test_suites_line=$(printf '<testsuites tests="%s" failures="%s" disabled="0" errors="0" time="%s" timestamp="%s" name="AllTests">' \ 347 "${tests}" "${failures}" "${duration}" "${now}") 348 349 # Update the test suites with the collected metrics. 350 sed "s#<testsuites .*>#${test_suites_line}#g" \ 351 "${xml}" > "${xml}.tmp" 352 mv "${xml}.tmp" "${xml}" 353} 354