1# Status: ported, except for --out-xml
2# Base revision: 64488
3#
4# Copyright 2005 Dave Abrahams
5# Copyright 2002, 2003, 2004, 2005, 2010 Vladimir Prus
6# Distributed under the Boost Software License, Version 1.0.
7# (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
8
9# This module implements regression testing framework. It declares a number of
10# main target rules which perform some action and, if the results are OK,
11# creates an output file.
12#
13# The exact list of rules is:
14# 'compile'       -- creates .test file if compilation of sources was
15#                    successful.
16# 'compile-fail'  -- creates .test file if compilation of sources failed.
17# 'run'           -- creates .test file is running of executable produced from
18#                    sources was successful. Also leaves behind .output file
19#                    with the output from program run.
20# 'run-fail'      -- same as above, but .test file is created if running fails.
21#
22# In all cases, presence of .test file is an indication that the test passed.
23# For more convenient reporting, you might want to use C++ Boost regression
24# testing utilities (see http://www.boost.org/more/regression.html).
25#
26# For historical reason, a 'unit-test' rule is available which has the same
27# syntax as 'exe' and behaves just like 'run'.
28
29# Things to do:
30#  - Teach compiler_status handle Jamfile.v2.
31# Notes:
32#  - <no-warn> is not implemented, since it is Como-specific, and it is not
33#    clear how to implement it
34#  - std::locale-support is not implemented (it is used in one test).
35
36import b2.build.feature as feature
37import b2.build.type as type
38import b2.build.targets as targets
39import b2.build.generators as generators
40import b2.build.toolset as toolset
41import b2.tools.common as common
42import b2.util.option as option
43import b2.build_system as build_system
44
45
46
47from b2.manager import get_manager
48from b2.util import stem, bjam_signature
49from b2.util.sequence import unique
50
51import bjam
52
53import re
54import os.path
55import sys
56
57def init():
58    pass
59
60# Feature controling the command used to lanch test programs.
61feature.feature("testing.launcher", [], ["free", "optional"])
62
63feature.feature("test-info", [], ["free", "incidental"])
64feature.feature("testing.arg", [], ["free", "incidental"])
65feature.feature("testing.input-file", [], ["free", "dependency"])
66
67feature.feature("preserve-test-targets", ["on", "off"], ["incidental", "propagated"])
68
69# Register target types.
70type.register("TEST", ["test"])
71type.register("COMPILE", [], "TEST")
72type.register("COMPILE_FAIL", [], "TEST")
73
74type.register("RUN_OUTPUT", ["run"])
75type.register("RUN", [], "TEST")
76type.register("RUN_FAIL", [], "TEST")
77
78type.register("LINK", [], "TEST")
79type.register("LINK_FAIL", [], "TEST")
80type.register("UNIT_TEST", ["passed"], "TEST")
81
82__all_tests = []
83
84# Declare the rules which create main targets. While the 'type' module already
85# creates rules with the same names for us, we need extra convenience: default
86# name of main target, so write our own versions.
87
88# Helper rule. Create a test target, using basename of first source if no target
89# name is explicitly passed. Remembers the created target in a global variable.
90def make_test(target_type, sources, requirements, target_name=None):
91
92    if not target_name:
93        target_name = stem(os.path.basename(sources[0]))
94
95    # Having periods (".") in the target name is problematic because the typed
96    # generator will strip the suffix and use the bare name for the file
97    # targets. Even though the location-prefix averts problems most times it
98    # does not prevent ambiguity issues when referring to the test targets. For
99    # example when using the XML log output. So we rename the target to remove
100    # the periods, and provide an alias for users.
101    real_name = target_name.replace(".", "~")
102
103    project = get_manager().projects().current()
104    # The <location-prefix> forces the build system for generate paths in the
105    # form '$build_dir/array1.test/gcc/debug'. This is necessary to allow
106    # post-processing tools to work.
107    t = get_manager().targets().create_typed_target(
108        type.type_from_rule_name(target_type), project, real_name, sources,
109        requirements + ["<location-prefix>" + real_name + ".test"], [], [])
110
111    # The alias to the real target, per period replacement above.
112    if real_name != target_name:
113        get_manager().projects().project_rules().all_names_["alias"](
114            target_name, [t])
115
116    # Remember the test (for --dump-tests). A good way would be to collect all
117    # given a project. This has some technical problems: e.g. we can not call
118    # this dump from a Jamfile since projects referred by 'build-project' are
119    # not available until the whole Jamfile has been loaded.
120    __all_tests.append(t)
121    return t
122
123
124# Note: passing more that one cpp file here is known to fail. Passing a cpp file
125# and a library target works.
126#
127@bjam_signature((["sources", "*"], ["requirements", "*"], ["target_name", "?"]))
128def compile(sources, requirements, target_name=None):
129    return make_test("compile", sources, requirements, target_name)
130
131@bjam_signature((["sources", "*"], ["requirements", "*"], ["target_name", "?"]))
132def compile_fail(sources, requirements, target_name=None):
133    return make_test("compile-fail", sources, requirements, target_name)
134
135@bjam_signature((["sources", "*"], ["requirements", "*"], ["target_name", "?"]))
136def link(sources, requirements, target_name=None):
137    return make_test("link", sources, requirements, target_name)
138
139@bjam_signature((["sources", "*"], ["requirements", "*"], ["target_name", "?"]))
140def link_fail(sources, requirements, target_name=None):
141    return make_test("link-fail", sources, requirements, target_name)
142
143def handle_input_files(input_files):
144    if len(input_files) > 1:
145        # Check that sorting made when creating property-set instance will not
146        # change the ordering.
147        if sorted(input_files) != input_files:
148            get_manager().errors()("Names of input files must be sorted alphabetically\n" +
149                                   "due to internal limitations")
150    return ["<testing.input-file>" + f for f in input_files]
151
152@bjam_signature((["sources", "*"], ["args", "*"], ["input_files", "*"],
153                 ["requirements", "*"], ["target_name", "?"],
154                 ["default_build", "*"]))
155def run(sources, args, input_files, requirements, target_name=None, default_build=[]):
156    if args:
157        requirements.append("<testing.arg>" + " ".join(args))
158    requirements.extend(handle_input_files(input_files))
159    return make_test("run", sources, requirements, target_name)
160
161@bjam_signature((["sources", "*"], ["args", "*"], ["input_files", "*"],
162                 ["requirements", "*"], ["target_name", "?"],
163                 ["default_build", "*"]))
164def run_fail(sources, args, input_files, requirements, target_name=None, default_build=[]):
165    if args:
166        requirements.append("<testing.arg>" + " ".join(args))
167    requirements.extend(handle_input_files(input_files))
168    return make_test("run-fail", sources, requirements, target_name)
169
170# Register all the rules
171for name in ["compile", "compile-fail", "link", "link-fail", "run", "run-fail"]:
172    get_manager().projects().add_rule(name, getattr(sys.modules[__name__], name.replace("-", "_")))
173
174# Use 'test-suite' as a synonym for 'alias', for backward compatibility.
175from b2.build.alias import alias
176get_manager().projects().add_rule("test-suite", alias)
177
178# For all main targets in 'project-module', which are typed targets with type
179# derived from 'TEST', produce some interesting information.
180#
181def dump_tests():
182    for t in __all_tests:
183        dump_test(t)
184
185# Given a project location in normalized form (slashes are forward), compute the
186# name of the Boost library.
187#
188__ln1 = re.compile("/(tools|libs)/(.*)/(test|example)")
189__ln2 = re.compile("/(tools|libs)/(.*)$")
190__ln3 = re.compile("(/status$)")
191def get_library_name(path):
192
193    path = path.replace("\\", "/")
194    match1 = __ln1.match(path)
195    match2 = __ln2.match(path)
196    match3 = __ln3.match(path)
197
198    if match1:
199        return match1.group(2)
200    elif match2:
201        return match2.group(2)
202    elif match3:
203        return ""
204    elif option.get("dump-tests", False, True):
205        # The 'run' rule and others might be used outside boost. In that case,
206        # just return the path, since the 'library name' makes no sense.
207        return path
208
209# Was an XML dump requested?
210__out_xml = option.get("out-xml", False, True)
211
212# Takes a target (instance of 'basic-target') and prints
213#   - its type
214#   - its name
215#   - comments specified via the <test-info> property
216#   - relative location of all source from the project root.
217#
218def dump_test(target):
219    type = target.type()
220    name = target.name()
221    project = target.project()
222
223    project_root = project.get('project-root')
224    library = get_library_name(os.path.abspath(project.get('location')))
225    if library:
226        name = library + "/" + name
227
228    sources = target.sources()
229    source_files = []
230    for s in sources:
231        if isinstance(s, targets.FileReference):
232            location = os.path.abspath(os.path.join(s.location(), s.name()))
233            source_files.append(os.path.relpath(location, os.path.abspath(project_root)))
234
235    target_name = project.get('location') + "//" + target.name() + ".test"
236
237    test_info = target.requirements().get('test-info')
238    test_info = " ".join('"' + ti + '"' for ti in test_info)
239
240    # If the user requested XML output on the command-line, add the test info to
241    # that XML file rather than dumping them to stdout.
242    #if $(.out-xml)
243    #{
244#        local nl = "
245#" ;
246#        .contents on $(.out-xml) +=
247#            "$(nl)  <test type=\"$(type)\" name=\"$(name)\">"
248#            "$(nl)    <target><![CDATA[$(target-name)]]></target>"
249#            "$(nl)    <info><![CDATA[$(test-info)]]></info>"
250#            "$(nl)    <source><![CDATA[$(source-files)]]></source>"
251#            "$(nl)  </test>"
252#            ;
253#    }
254#    else
255
256    source_files = " ".join('"' + s + '"' for s in source_files)
257    if test_info:
258        print 'boost-test(%s) "%s" [%s] : %s' % (type, name, test_info, source_files)
259    else:
260        print 'boost-test(%s) "%s" : %s' % (type, name, source_files)
261
262# Register generators. Depending on target type, either 'expect-success' or
263# 'expect-failure' rule will be used.
264generators.register_standard("testing.expect-success", ["OBJ"], ["COMPILE"])
265generators.register_standard("testing.expect-failure", ["OBJ"], ["COMPILE_FAIL"])
266generators.register_standard("testing.expect-success", ["RUN_OUTPUT"], ["RUN"])
267generators.register_standard("testing.expect-failure", ["RUN_OUTPUT"], ["RUN_FAIL"])
268generators.register_standard("testing.expect-success", ["EXE"], ["LINK"])
269generators.register_standard("testing.expect-failure", ["EXE"], ["LINK_FAIL"])
270
271# Generator which runs an EXE and captures output.
272generators.register_standard("testing.capture-output", ["EXE"], ["RUN_OUTPUT"])
273
274# Generator which creates a target if sources run successfully. Differs from RUN
275# in that run output is not captured. The reason why it exists is that the 'run'
276# rule is much better for automated testing, but is not user-friendly (see
277# http://article.gmane.org/gmane.comp.lib.boost.build/6353).
278generators.register_standard("testing.unit-test", ["EXE"], ["UNIT_TEST"])
279
280# FIXME: if those calls are after bjam.call, then bjam will crash
281# when toolset.flags calls bjam.caller.
282toolset.flags("testing.capture-output", "ARGS", [], ["<testing.arg>"])
283toolset.flags("testing.capture-output", "INPUT_FILES", [], ["<testing.input-file>"])
284toolset.flags("testing.capture-output", "LAUNCHER", [], ["<testing.launcher>"])
285
286toolset.flags("testing.unit-test", "LAUNCHER", [], ["<testing.launcher>"])
287toolset.flags("testing.unit-test", "ARGS", [], ["<testing.arg>"])
288
289# This is a composing generator to support cases where a generator for the
290# specified target constructs other targets as well. One such example is msvc's
291# exe generator that constructs both EXE and PDB targets.
292type.register("TIME", ["time"])
293generators.register_composing("testing.time", [], ["TIME"])
294
295
296# The following code sets up actions for this module. It's pretty convoluted,
297# but the basic points is that we most of actions are defined by Jam code
298# contained in testing-aux.jam, which we load into Jam module named 'testing'
299
300def run_path_setup(target, sources, ps):
301
302    # For testing, we need to make sure that all dynamic libraries needed by the
303    # test are found. So, we collect all paths from dependency libraries (via
304    # xdll-path property) and add whatever explicit dll-path user has specified.
305    # The resulting paths are added to the environment on each test invocation.
306    dll_paths = ps.get('dll-path')
307    dll_paths.extend(ps.get('xdll-path'))
308    dll_paths.extend(bjam.call("get-target-variable", sources, "RUN_PATH"))
309    dll_paths = unique(dll_paths)
310    if dll_paths:
311        bjam.call("set-target-variable", target, "PATH_SETUP",
312                  common.prepend_path_variable_command(
313                     common.shared_library_path_variable(), dll_paths))
314
315def capture_output_setup(target, sources, ps):
316    run_path_setup(target, sources, ps)
317
318    if ps.get('preserve-test-targets') == ['off']:
319        bjam.call("set-target-variable", target, "REMOVE_TEST_TARGETS", "1")
320
321get_manager().engine().register_bjam_action("testing.capture-output",
322                                            capture_output_setup)
323
324
325path = os.path.dirname(__file__)
326import b2.util.os_j
327get_manager().projects().project_rules()._import_rule("testing", "os.name",
328                                                      b2.util.os_j.name)
329import b2.tools.common
330get_manager().projects().project_rules()._import_rule("testing", "common.rm-command",
331                                                      b2.tools.common.rm_command)
332get_manager().projects().project_rules()._import_rule("testing", "common.file-creation-command",
333                                                      b2.tools.common.file_creation_command)
334
335bjam.call("load", "testing", os.path.join(path, "testing-aux.jam"))
336
337
338for name in ["expect-success", "expect-failure", "time"]:
339    get_manager().engine().register_bjam_action("testing." + name)
340
341get_manager().engine().register_bjam_action("testing.unit-test",
342                                            run_path_setup)
343
344if option.get("dump-tests", False, True):
345    build_system.add_pre_build_hook(dump_tests)
346