1# Status: ported, except for --out-xml 2# Base revision: 64488 3# 4# Copyright 2005 Dave Abrahams 5# Copyright 2002, 2003, 2004, 2005, 2010 Vladimir Prus 6# Distributed under the Boost Software License, Version 1.0. 7# (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt) 8 9# This module implements regression testing framework. It declares a number of 10# main target rules which perform some action and, if the results are OK, 11# creates an output file. 12# 13# The exact list of rules is: 14# 'compile' -- creates .test file if compilation of sources was 15# successful. 16# 'compile-fail' -- creates .test file if compilation of sources failed. 17# 'run' -- creates .test file is running of executable produced from 18# sources was successful. Also leaves behind .output file 19# with the output from program run. 20# 'run-fail' -- same as above, but .test file is created if running fails. 21# 22# In all cases, presence of .test file is an indication that the test passed. 23# For more convenient reporting, you might want to use C++ Boost regression 24# testing utilities (see http://www.boost.org/more/regression.html). 25# 26# For historical reason, a 'unit-test' rule is available which has the same 27# syntax as 'exe' and behaves just like 'run'. 28 29# Things to do: 30# - Teach compiler_status handle Jamfile.v2. 31# Notes: 32# - <no-warn> is not implemented, since it is Como-specific, and it is not 33# clear how to implement it 34# - std::locale-support is not implemented (it is used in one test). 35 36import b2.build.feature as feature 37import b2.build.type as type 38import b2.build.targets as targets 39import b2.build.generators as generators 40import b2.build.toolset as toolset 41import b2.tools.common as common 42import b2.util.option as option 43import b2.build_system as build_system 44 45 46 47from b2.manager import get_manager 48from b2.util import stem, bjam_signature, is_iterable_typed 49from b2.util.sequence import unique 50 51import bjam 52 53import re 54import os.path 55import sys 56 57def init(): 58 pass 59 60# Feature controlling the command used to lanch test programs. 61feature.feature("testing.launcher", [], ["free", "optional"]) 62 63feature.feature("test-info", [], ["free", "incidental"]) 64feature.feature("testing.arg", [], ["free", "incidental"]) 65feature.feature("testing.input-file", [], ["free", "dependency"]) 66 67feature.feature("preserve-test-targets", ["on", "off"], ["incidental", "propagated"]) 68 69# Register target types. 70type.register("TEST", ["test"]) 71type.register("COMPILE", [], "TEST") 72type.register("COMPILE_FAIL", [], "TEST") 73 74type.register("RUN_OUTPUT", ["run"]) 75type.register("RUN", [], "TEST") 76type.register("RUN_FAIL", [], "TEST") 77 78type.register("LINK", [], "TEST") 79type.register("LINK_FAIL", [], "TEST") 80type.register("UNIT_TEST", ["passed"], "TEST") 81 82__all_tests = [] 83 84# Declare the rules which create main targets. While the 'type' module already 85# creates rules with the same names for us, we need extra convenience: default 86# name of main target, so write our own versions. 87 88# Helper rule. Create a test target, using basename of first source if no target 89# name is explicitly passed. Remembers the created target in a global variable. 90def make_test(target_type, sources, requirements, target_name=None): 91 assert isinstance(target_type, basestring) 92 assert is_iterable_typed(sources, basestring) 93 assert is_iterable_typed(requirements, basestring) 94 assert isinstance(target_type, basestring) or target_type is None 95 if not target_name: 96 target_name = stem(os.path.basename(sources[0])) 97 98 # Having periods (".") in the target name is problematic because the typed 99 # generator will strip the suffix and use the bare name for the file 100 # targets. Even though the location-prefix averts problems most times it 101 # does not prevent ambiguity issues when referring to the test targets. For 102 # example when using the XML log output. So we rename the target to remove 103 # the periods, and provide an alias for users. 104 real_name = target_name.replace(".", "~") 105 106 project = get_manager().projects().current() 107 # The <location-prefix> forces the build system for generate paths in the 108 # form '$build_dir/array1.test/gcc/debug'. This is necessary to allow 109 # post-processing tools to work. 110 t = get_manager().targets().create_typed_target( 111 type.type_from_rule_name(target_type), project, real_name, sources, 112 requirements + ["<location-prefix>" + real_name + ".test"], [], []) 113 114 # The alias to the real target, per period replacement above. 115 if real_name != target_name: 116 get_manager().projects().project_rules().rules["alias"]( 117 target_name, [t]) 118 119 # Remember the test (for --dump-tests). A good way would be to collect all 120 # given a project. This has some technical problems: e.g. we can not call 121 # this dump from a Jamfile since projects referred by 'build-project' are 122 # not available until the whole Jamfile has been loaded. 123 __all_tests.append(t) 124 return t 125 126 127# Note: passing more that one cpp file here is known to fail. Passing a cpp file 128# and a library target works. 129# 130@bjam_signature((["sources", "*"], ["requirements", "*"], ["target_name", "?"])) 131def compile(sources, requirements, target_name=None): 132 return make_test("compile", sources, requirements, target_name) 133 134@bjam_signature((["sources", "*"], ["requirements", "*"], ["target_name", "?"])) 135def compile_fail(sources, requirements, target_name=None): 136 return make_test("compile-fail", sources, requirements, target_name) 137 138@bjam_signature((["sources", "*"], ["requirements", "*"], ["target_name", "?"])) 139def link(sources, requirements, target_name=None): 140 return make_test("link", sources, requirements, target_name) 141 142@bjam_signature((["sources", "*"], ["requirements", "*"], ["target_name", "?"])) 143def link_fail(sources, requirements, target_name=None): 144 return make_test("link-fail", sources, requirements, target_name) 145 146def handle_input_files(input_files): 147 if len(input_files) > 1: 148 # Check that sorting made when creating property-set instance will not 149 # change the ordering. 150 if sorted(input_files) != input_files: 151 get_manager().errors()("Names of input files must be sorted alphabetically\n" + 152 "due to internal limitations") 153 return ["<testing.input-file>" + f for f in input_files] 154 155@bjam_signature((["sources", "*"], ["args", "*"], ["input_files", "*"], 156 ["requirements", "*"], ["target_name", "?"], 157 ["default_build", "*"])) 158def run(sources, args, input_files, requirements, target_name=None, default_build=[]): 159 if args: 160 requirements.append("<testing.arg>" + " ".join(args)) 161 requirements.extend(handle_input_files(input_files)) 162 return make_test("run", sources, requirements, target_name) 163 164@bjam_signature((["sources", "*"], ["args", "*"], ["input_files", "*"], 165 ["requirements", "*"], ["target_name", "?"], 166 ["default_build", "*"])) 167def run_fail(sources, args, input_files, requirements, target_name=None, default_build=[]): 168 if args: 169 requirements.append("<testing.arg>" + " ".join(args)) 170 requirements.extend(handle_input_files(input_files)) 171 return make_test("run-fail", sources, requirements, target_name) 172 173# Register all the rules 174for name in ["compile", "compile-fail", "link", "link-fail", "run", "run-fail"]: 175 get_manager().projects().add_rule(name, getattr(sys.modules[__name__], name.replace("-", "_"))) 176 177# Use 'test-suite' as a synonym for 'alias', for backward compatibility. 178from b2.build.alias import alias 179get_manager().projects().add_rule("test-suite", alias) 180 181# For all main targets in 'project-module', which are typed targets with type 182# derived from 'TEST', produce some interesting information. 183# 184def dump_tests(): 185 for t in __all_tests: 186 dump_test(t) 187 188# Given a project location in normalized form (slashes are forward), compute the 189# name of the Boost library. 190# 191__ln1 = re.compile("/(tools|libs)/(.*)/(test|example)") 192__ln2 = re.compile("/(tools|libs)/(.*)$") 193__ln3 = re.compile("(/status$)") 194def get_library_name(path): 195 assert isinstance(path, basestring) 196 197 path = path.replace("\\", "/") 198 match1 = __ln1.match(path) 199 match2 = __ln2.match(path) 200 match3 = __ln3.match(path) 201 202 if match1: 203 return match1.group(2) 204 elif match2: 205 return match2.group(2) 206 elif match3: 207 return "" 208 elif option.get("dump-tests", False, True): 209 # The 'run' rule and others might be used outside boost. In that case, 210 # just return the path, since the 'library name' makes no sense. 211 return path 212 213# Was an XML dump requested? 214__out_xml = option.get("out-xml", False, True) 215 216# Takes a target (instance of 'basic-target') and prints 217# - its type 218# - its name 219# - comments specified via the <test-info> property 220# - relative location of all source from the project root. 221# 222def dump_test(target): 223 assert isinstance(target, targets.AbstractTarget) 224 type = target.type() 225 name = target.name() 226 project = target.project() 227 228 project_root = project.get('project-root') 229 library = get_library_name(os.path.abspath(project.get('location'))) 230 if library: 231 name = library + "/" + name 232 233 sources = target.sources() 234 source_files = [] 235 for s in sources: 236 if isinstance(s, targets.FileReference): 237 location = os.path.abspath(os.path.join(s.location(), s.name())) 238 source_files.append(os.path.relpath(location, os.path.abspath(project_root))) 239 240 target_name = project.get('location') + "//" + target.name() + ".test" 241 242 test_info = target.requirements().get('test-info') 243 test_info = " ".join('"' + ti + '"' for ti in test_info) 244 245 # If the user requested XML output on the command-line, add the test info to 246 # that XML file rather than dumping them to stdout. 247 #if $(.out-xml) 248 #{ 249# local nl = " 250#" ; 251# .contents on $(.out-xml) += 252# "$(nl) <test type=\"$(type)\" name=\"$(name)\">" 253# "$(nl) <target><![CDATA[$(target-name)]]></target>" 254# "$(nl) <info><![CDATA[$(test-info)]]></info>" 255# "$(nl) <source><![CDATA[$(source-files)]]></source>" 256# "$(nl) </test>" 257# ; 258# } 259# else 260 261 source_files = " ".join('"' + s + '"' for s in source_files) 262 if test_info: 263 print 'boost-test(%s) "%s" [%s] : %s' % (type, name, test_info, source_files) 264 else: 265 print 'boost-test(%s) "%s" : %s' % (type, name, source_files) 266 267# Register generators. Depending on target type, either 'expect-success' or 268# 'expect-failure' rule will be used. 269generators.register_standard("testing.expect-success", ["OBJ"], ["COMPILE"]) 270generators.register_standard("testing.expect-failure", ["OBJ"], ["COMPILE_FAIL"]) 271generators.register_standard("testing.expect-success", ["RUN_OUTPUT"], ["RUN"]) 272generators.register_standard("testing.expect-failure", ["RUN_OUTPUT"], ["RUN_FAIL"]) 273generators.register_standard("testing.expect-success", ["EXE"], ["LINK"]) 274generators.register_standard("testing.expect-failure", ["EXE"], ["LINK_FAIL"]) 275 276# Generator which runs an EXE and captures output. 277generators.register_standard("testing.capture-output", ["EXE"], ["RUN_OUTPUT"]) 278 279# Generator which creates a target if sources run successfully. Differs from RUN 280# in that run output is not captured. The reason why it exists is that the 'run' 281# rule is much better for automated testing, but is not user-friendly (see 282# http://article.gmane.org/gmane.comp.lib.boost.build/6353). 283generators.register_standard("testing.unit-test", ["EXE"], ["UNIT_TEST"]) 284 285# FIXME: if those calls are after bjam.call, then bjam will crash 286# when toolset.flags calls bjam.caller. 287toolset.flags("testing.capture-output", "ARGS", [], ["<testing.arg>"]) 288toolset.flags("testing.capture-output", "INPUT_FILES", [], ["<testing.input-file>"]) 289toolset.flags("testing.capture-output", "LAUNCHER", [], ["<testing.launcher>"]) 290 291toolset.flags("testing.unit-test", "LAUNCHER", [], ["<testing.launcher>"]) 292toolset.flags("testing.unit-test", "ARGS", [], ["<testing.arg>"]) 293 294# This is a composing generator to support cases where a generator for the 295# specified target constructs other targets as well. One such example is msvc's 296# exe generator that constructs both EXE and PDB targets. 297type.register("TIME", ["time"]) 298generators.register_composing("testing.time", [], ["TIME"]) 299 300 301# The following code sets up actions for this module. It's pretty convoluted, 302# but the basic points is that we most of actions are defined by Jam code 303# contained in testing-aux.jam, which we load into Jam module named 'testing' 304 305def run_path_setup(target, sources, ps): 306 if __debug__: 307 from ..build.property_set import PropertySet 308 assert is_iterable_typed(target, basestring) or isinstance(target, basestring) 309 assert is_iterable_typed(sources, basestring) 310 assert isinstance(ps, PropertySet) 311 # For testing, we need to make sure that all dynamic libraries needed by the 312 # test are found. So, we collect all paths from dependency libraries (via 313 # xdll-path property) and add whatever explicit dll-path user has specified. 314 # The resulting paths are added to the environment on each test invocation. 315 dll_paths = ps.get('dll-path') 316 dll_paths.extend(ps.get('xdll-path')) 317 dll_paths.extend(bjam.call("get-target-variable", sources, "RUN_PATH")) 318 dll_paths = unique(dll_paths) 319 if dll_paths: 320 bjam.call("set-target-variable", target, "PATH_SETUP", 321 common.prepend_path_variable_command( 322 common.shared_library_path_variable(), dll_paths)) 323 324def capture_output_setup(target, sources, ps): 325 if __debug__: 326 from ..build.property_set import PropertySet 327 assert is_iterable_typed(target, basestring) 328 assert is_iterable_typed(sources, basestring) 329 assert isinstance(ps, PropertySet) 330 run_path_setup(target[0], sources, ps) 331 332 if ps.get('preserve-test-targets') == ['off']: 333 bjam.call("set-target-variable", target, "REMOVE_TEST_TARGETS", "1") 334 335get_manager().engine().register_bjam_action("testing.capture-output", 336 capture_output_setup) 337 338 339path = os.path.dirname(__file__) 340import b2.util.os_j 341get_manager().projects().project_rules()._import_rule("testing", "os.name", 342 b2.util.os_j.name) 343import b2.tools.common 344get_manager().projects().project_rules()._import_rule("testing", "common.rm-command", 345 b2.tools.common.rm_command) 346get_manager().projects().project_rules()._import_rule("testing", "common.file-creation-command", 347 b2.tools.common.file_creation_command) 348 349bjam.call("load", "testing", os.path.join(path, "testing-aux.jam")) 350 351 352for name in ["expect-success", "expect-failure", "time"]: 353 get_manager().engine().register_bjam_action("testing." + name) 354 355get_manager().engine().register_bjam_action("testing.unit-test", 356 run_path_setup) 357 358if option.get("dump-tests", False, True): 359 build_system.add_pre_build_hook(dump_tests) 360