1#===----------------------------------------------------------------------===## 2# 3# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4# See https://llvm.org/LICENSE.txt for license information. 5# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6# 7#===----------------------------------------------------------------------===## 8 9import lit 10import lit.formats 11import os 12import pipes 13import re 14import shutil 15 16def _getTempPaths(test): 17 """ 18 Return the values to use for the %T and %t substitutions, respectively. 19 20 The difference between this and Lit's default behavior is that we guarantee 21 that %T is a path unique to the test being run. 22 """ 23 tmpDir, _ = lit.TestRunner.getTempPaths(test) 24 _, testName = os.path.split(test.getExecPath()) 25 tmpDir = os.path.join(tmpDir, testName + '.dir') 26 tmpBase = os.path.join(tmpDir, 't') 27 return tmpDir, tmpBase 28 29def _checkBaseSubstitutions(substitutions): 30 substitutions = [s for (s, _) in substitutions] 31 for s in ['%{cxx}', '%{compile_flags}', '%{link_flags}', '%{flags}', '%{exec}']: 32 assert s in substitutions, "Required substitution {} was not provided".format(s) 33 34def parseScript(test, preamble): 35 """ 36 Extract the script from a test, with substitutions applied. 37 38 Returns a list of commands ready to be executed. 39 40 - test 41 The lit.Test to parse. 42 43 - preamble 44 A list of commands to perform before any command in the test. 45 These commands can contain unexpanded substitutions, but they 46 must not be of the form 'RUN:' -- they must be proper commands 47 once substituted. 48 """ 49 # Get the default substitutions 50 tmpDir, tmpBase = _getTempPaths(test) 51 substitutions = lit.TestRunner.getDefaultSubstitutions(test, tmpDir, tmpBase) 52 53 # Check base substitutions and add the %{build} and %{run} convenience substitutions 54 _checkBaseSubstitutions(substitutions) 55 substitutions.append(('%{build}', '%{cxx} %s %{flags} %{compile_flags} %{link_flags} -o %t.exe')) 56 substitutions.append(('%{run}', '%{exec} %t.exe')) 57 58 # Parse the test file, including custom directives 59 additionalCompileFlags = [] 60 fileDependencies = [] 61 parsers = [ 62 lit.TestRunner.IntegratedTestKeywordParser('FILE_DEPENDENCIES:', 63 lit.TestRunner.ParserKind.LIST, 64 initial_value=fileDependencies), 65 lit.TestRunner.IntegratedTestKeywordParser('ADDITIONAL_COMPILE_FLAGS:', 66 lit.TestRunner.ParserKind.LIST, 67 initial_value=additionalCompileFlags) 68 ] 69 70 # Add conditional parsers for ADDITIONAL_COMPILE_FLAGS. This should be replaced by first 71 # class support for conditional keywords in Lit, which would allow evaluating arbitrary 72 # Lit boolean expressions instead. 73 for feature in test.config.available_features: 74 parser = lit.TestRunner.IntegratedTestKeywordParser('ADDITIONAL_COMPILE_FLAGS({}):'.format(feature), 75 lit.TestRunner.ParserKind.LIST, 76 initial_value=additionalCompileFlags) 77 parsers.append(parser) 78 79 scriptInTest = lit.TestRunner.parseIntegratedTestScript(test, additional_parsers=parsers, 80 require_script=not preamble) 81 if isinstance(scriptInTest, lit.Test.Result): 82 return scriptInTest 83 84 script = [] 85 86 # For each file dependency in FILE_DEPENDENCIES, inject a command to copy 87 # that file to the execution directory. Execute the copy from %S to allow 88 # relative paths from the test directory. 89 for dep in fileDependencies: 90 script += ['%dbg(SETUP) cd %S && cp {} %T'.format(dep)] 91 script += preamble 92 script += scriptInTest 93 94 # Add compile flags specified with ADDITIONAL_COMPILE_FLAGS. 95 substitutions = [(s, x + ' ' + ' '.join(additionalCompileFlags)) if s == '%{compile_flags}' 96 else (s, x) for (s, x) in substitutions] 97 98 # Perform substitutions in the script itself. 99 script = lit.TestRunner.applySubstitutions(script, substitutions, 100 recursion_limit=test.config.recursiveExpansionLimit) 101 102 return script 103 104 105class CxxStandardLibraryTest(lit.formats.TestFormat): 106 """ 107 Lit test format for the C++ Standard Library conformance test suite. 108 109 This test format is based on top of the ShTest format -- it basically 110 creates a shell script performing the right operations (compile/link/run) 111 based on the extension of the test file it encounters. It supports files 112 with the following extensions: 113 114 FOO.pass.cpp - Compiles, links and runs successfully 115 FOO.pass.mm - Same as .pass.cpp, but for Objective-C++ 116 117 FOO.compile.pass.cpp - Compiles successfully, link and run not attempted 118 FOO.compile.fail.cpp - Does not compile successfully 119 120 FOO.link.pass.cpp - Compiles and links successfully, run not attempted 121 FOO.link.fail.cpp - Compiles successfully, but fails to link 122 123 FOO.sh.<anything> - A builtin Lit Shell test 124 125 FOO.verify.cpp - Compiles with clang-verify. This type of test is 126 automatically marked as UNSUPPORTED if the compiler 127 does not support Clang-verify. 128 129 FOO.fail.cpp - Compiled with clang-verify if clang-verify is 130 supported, and equivalent to a .compile.fail.cpp 131 test otherwise. This is supported only for backwards 132 compatibility with the test suite. 133 134 135 Substitution requirements 136 =============================== 137 The test format operates by assuming that each test's configuration provides 138 the following substitutions, which it will reuse in the shell scripts it 139 constructs: 140 %{cxx} - A command that can be used to invoke the compiler 141 %{compile_flags} - Flags to use when compiling a test case 142 %{link_flags} - Flags to use when linking a test case 143 %{flags} - Flags to use either when compiling or linking a test case 144 %{exec} - A command to prefix the execution of executables 145 146 Note that when building an executable (as opposed to only compiling a source 147 file), all three of %{flags}, %{compile_flags} and %{link_flags} will be used 148 in the same command line. In other words, the test format doesn't perform 149 separate compilation and linking steps in this case. 150 151 152 Additional supported directives 153 =============================== 154 In addition to everything that's supported in Lit ShTests, this test format 155 also understands the following directives inside test files: 156 157 // FILE_DEPENDENCIES: file, directory, /path/to/file 158 159 This directive expresses that the test requires the provided files 160 or directories in order to run. An example is a test that requires 161 some test input stored in a data file. When a test file contains 162 such a directive, this test format will collect them and copy them 163 to the directory represented by %T. The intent is that %T contains 164 all the inputs necessary to run the test, such that e.g. execution 165 on a remote host can be done by simply copying %T to the host. 166 167 // ADDITIONAL_COMPILE_FLAGS: flag1, flag2, flag3 168 169 This directive will cause the provided flags to be added to the 170 %{compile_flags} substitution for the test that contains it. This 171 allows adding special compilation flags without having to use a 172 .sh.cpp test, which would be more powerful but perhaps overkill. 173 174 175 Additional provided substitutions and features 176 ============================================== 177 The test format will define the following substitutions for use inside tests: 178 179 %{build} 180 Expands to a command-line that builds the current source 181 file with the %{flags}, %{compile_flags} and %{link_flags} 182 substitutions, and that produces an executable named %t.exe. 183 184 %{run} 185 Equivalent to `%{exec} %t.exe`. This is intended to be used 186 in conjunction with the %{build} substitution. 187 """ 188 def getTestsInDirectory(self, testSuite, pathInSuite, litConfig, localConfig): 189 SUPPORTED_SUFFIXES = ['[.]pass[.]cpp$', '[.]pass[.]mm$', 190 '[.]compile[.]pass[.]cpp$', '[.]compile[.]fail[.]cpp$', 191 '[.]link[.]pass[.]cpp$', '[.]link[.]fail[.]cpp$', 192 '[.]sh[.][^.]+$', 193 '[.]verify[.]cpp$', 194 '[.]fail[.]cpp$'] 195 sourcePath = testSuite.getSourcePath(pathInSuite) 196 for filename in os.listdir(sourcePath): 197 # Ignore dot files and excluded tests. 198 if filename.startswith('.') or filename in localConfig.excludes: 199 continue 200 201 filepath = os.path.join(sourcePath, filename) 202 if not os.path.isdir(filepath): 203 if any([re.search(ext, filename) for ext in SUPPORTED_SUFFIXES]): 204 yield lit.Test.Test(testSuite, pathInSuite + (filename,), localConfig) 205 206 def execute(self, test, litConfig): 207 VERIFY_FLAGS = '-Xclang -verify -Xclang -verify-ignore-unexpected=note -ferror-limit=0' 208 supportsVerify = 'verify-support' in test.config.available_features 209 filename = test.path_in_suite[-1] 210 211 if re.search('[.]sh[.][^.]+$', filename): 212 steps = [ ] # The steps are already in the script 213 return self._executeShTest(test, litConfig, steps) 214 elif filename.endswith('.compile.pass.cpp'): 215 steps = [ 216 "%dbg(COMPILED WITH) %{cxx} %s %{flags} %{compile_flags} -fsyntax-only" 217 ] 218 return self._executeShTest(test, litConfig, steps) 219 elif filename.endswith('.compile.fail.cpp'): 220 steps = [ 221 "%dbg(COMPILED WITH) ! %{cxx} %s %{flags} %{compile_flags} -fsyntax-only" 222 ] 223 return self._executeShTest(test, litConfig, steps) 224 elif filename.endswith('.link.pass.cpp'): 225 steps = [ 226 "%dbg(COMPILED WITH) %{cxx} %s %{flags} %{compile_flags} %{link_flags} -o %t.exe" 227 ] 228 return self._executeShTest(test, litConfig, steps) 229 elif filename.endswith('.link.fail.cpp'): 230 steps = [ 231 "%dbg(COMPILED WITH) %{cxx} %s %{flags} %{compile_flags} -c -o %t.o", 232 "%dbg(LINKED WITH) ! %{cxx} %t.o %{flags} %{link_flags} -o %t.exe" 233 ] 234 return self._executeShTest(test, litConfig, steps) 235 elif filename.endswith('.verify.cpp'): 236 if not supportsVerify: 237 return lit.Test.Result(lit.Test.UNSUPPORTED, 238 "Test {} requires support for Clang-verify, which isn't supported by the compiler".format(test.getFullName())) 239 steps = [ 240 # Note: Use -Wno-error to make sure all diagnostics are not treated as errors, 241 # which doesn't make sense for clang-verify tests. 242 "%dbg(COMPILED WITH) %{{cxx}} %s %{{flags}} %{{compile_flags}} -fsyntax-only -Wno-error {}".format(VERIFY_FLAGS) 243 ] 244 return self._executeShTest(test, litConfig, steps) 245 # Make sure to check these ones last, since they will match other 246 # suffixes above too. 247 elif filename.endswith('.pass.cpp') or filename.endswith('.pass.mm'): 248 steps = [ 249 "%dbg(COMPILED WITH) %{cxx} %s %{flags} %{compile_flags} %{link_flags} -o %t.exe", 250 "%dbg(EXECUTED AS) %{exec} %t.exe" 251 ] 252 return self._executeShTest(test, litConfig, steps) 253 # This is like a .verify.cpp test when clang-verify is supported, 254 # otherwise it's like a .compile.fail.cpp test. This is only provided 255 # for backwards compatibility with the test suite. 256 elif filename.endswith('.fail.cpp'): 257 if supportsVerify: 258 steps = [ 259 "%dbg(COMPILED WITH) %{{cxx}} %s %{{flags}} %{{compile_flags}} -fsyntax-only -Wno-error {}".format(VERIFY_FLAGS) 260 ] 261 else: 262 steps = [ 263 "%dbg(COMPILED WITH) ! %{cxx} %s %{flags} %{compile_flags} -fsyntax-only" 264 ] 265 return self._executeShTest(test, litConfig, steps) 266 else: 267 return lit.Test.Result(lit.Test.UNRESOLVED, "Unknown test suffix for '{}'".format(filename)) 268 269 # Utility function to add compile flags in lit.local.cfg files. 270 def addCompileFlags(self, config, *flags): 271 string = ' '.join(flags) 272 config.substitutions = [(s, x + ' ' + string) if s == '%{compile_flags}' else (s, x) for (s, x) in config.substitutions] 273 274 def _executeShTest(self, test, litConfig, steps): 275 if test.config.unsupported: 276 return lit.Test.Result(lit.Test.UNSUPPORTED, 'Test is unsupported') 277 278 script = parseScript(test, steps) 279 if isinstance(script, lit.Test.Result): 280 return script 281 282 if litConfig.noExecute: 283 return lit.Test.Result(lit.Test.XFAIL if test.isExpectedToFail() else lit.Test.PASS) 284 else: 285 _, tmpBase = _getTempPaths(test) 286 useExternalSh = False 287 return lit.TestRunner._runShTest(test, litConfig, useExternalSh, script, tmpBase) 288