1 /*
2 * Created by Phil on 13/5/2013.
3 * Copyright 2014 Two Blue Cubes Ltd. All rights reserved.
4 *
5 * Distributed under the Boost Software License, Version 1.0. (See accompanying
6 * file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
7 */
8
9 #include "catch.hpp"
10 #include "internal/catch_test_spec_parser.h"
11 #include "internal/catch_config.hpp"
12 #include "internal/catch_commandline.h"
13
14 #ifdef __clang__
15 # pragma clang diagnostic ignored "-Wc++98-compat"
16 #endif
17
fakeTestCase(const char * name,const char * desc="")18 inline Catch::TestCase fakeTestCase(const char* name, const char* desc = "") { return Catch::makeTestCase(nullptr, "", { name, desc }, CATCH_INTERNAL_LINEINFO); }
19
20 TEST_CASE( "Parse test names and tags", "[command-line][test-spec]" ) {
21
22 using Catch::parseTestSpec;
23 using Catch::TestSpec;
24
25 Catch::TestCase tcA = fakeTestCase( "a" );
26 Catch::TestCase tcB = fakeTestCase( "b", "[one][x]" );
27 Catch::TestCase tcC = fakeTestCase( "longer name with spaces", "[two][three][.][x]" );
28 Catch::TestCase tcD = fakeTestCase( "zlonger name with spacesz" );
29
30 SECTION( "Empty test spec should have no filters" ) {
31 TestSpec spec;
32 CHECK( spec.hasFilters() == false );
33 CHECK( spec.matches( tcA ) == false );
34 CHECK( spec.matches( tcB ) == false );
35 }
36
37 SECTION( "Test spec from empty string should have no filters" ) {
38 TestSpec spec = parseTestSpec( "" );
39 CHECK( spec.hasFilters() == false );
40 CHECK( spec.matches(tcA ) == false );
41 CHECK( spec.matches( tcB ) == false );
42 }
43
44 SECTION( "Test spec from just a comma should have no filters" ) {
45 TestSpec spec = parseTestSpec( "," );
46 CHECK( spec.hasFilters() == false );
47 CHECK( spec.matches( tcA ) == false );
48 CHECK( spec.matches( tcB ) == false );
49 }
50
51 SECTION( "Test spec from name should have one filter" ) {
52 TestSpec spec = parseTestSpec( "b" );
53 CHECK( spec.hasFilters() == true );
54 CHECK( spec.matches( tcA ) == false );
55 CHECK( spec.matches( tcB ) == true );
56 }
57
58 SECTION( "Test spec from quoted name should have one filter" ) {
59 TestSpec spec = parseTestSpec( "\"b\"" );
60 CHECK( spec.hasFilters() == true );
61 CHECK( spec.matches( tcA ) == false );
62 CHECK( spec.matches( tcB ) == true );
63 }
64
65 SECTION( "Test spec from name should have one filter" ) {
66 TestSpec spec = parseTestSpec( "b" );
67 CHECK( spec.hasFilters() == true );
68 CHECK( spec.matches( tcA ) == false );
69 CHECK( spec.matches( tcB ) == true );
70 CHECK( spec.matches( tcC ) == false );
71 }
72
73 SECTION( "Wildcard at the start" ) {
74 TestSpec spec = parseTestSpec( "*spaces" );
75 CHECK( spec.hasFilters() == true );
76 CHECK( spec.matches( tcA ) == false );
77 CHECK( spec.matches( tcB ) == false );
78 CHECK( spec.matches( tcC ) == true );
79 CHECK( spec.matches( tcD ) == false );
80 CHECK( parseTestSpec( "*a" ).matches( tcA ) == true );
81 }
82 SECTION( "Wildcard at the end" ) {
83 TestSpec spec = parseTestSpec( "long*" );
84 CHECK( spec.hasFilters() == true );
85 CHECK( spec.matches( tcA ) == false );
86 CHECK( spec.matches( tcB ) == false );
87 CHECK( spec.matches( tcC ) == true );
88 CHECK( spec.matches( tcD ) == false );
89 CHECK( parseTestSpec( "a*" ).matches( tcA ) == true );
90 }
91 SECTION( "Wildcard at both ends" ) {
92 TestSpec spec = parseTestSpec( "*name*" );
93 CHECK( spec.hasFilters() == true );
94 CHECK( spec.matches( tcA ) == false );
95 CHECK( spec.matches( tcB ) == false );
96 CHECK( spec.matches( tcC ) == true );
97 CHECK( spec.matches( tcD ) == true );
98 CHECK( parseTestSpec( "*a*" ).matches( tcA ) == true );
99 }
100 SECTION( "Redundant wildcard at the start" ) {
101 TestSpec spec = parseTestSpec( "*a" );
102 CHECK( spec.hasFilters() == true );
103 CHECK( spec.matches( tcA ) == true );
104 CHECK( spec.matches( tcB ) == false );
105 }
106 SECTION( "Redundant wildcard at the end" ) {
107 TestSpec spec = parseTestSpec( "a*" );
108 CHECK( spec.hasFilters() == true );
109 CHECK( spec.matches( tcA ) == true );
110 CHECK( spec.matches( tcB ) == false );
111 }
112 SECTION( "Redundant wildcard at both ends" ) {
113 TestSpec spec = parseTestSpec( "*a*" );
114 CHECK( spec.hasFilters() == true );
115 CHECK( spec.matches( tcA ) == true );
116 CHECK( spec.matches( tcB ) == false );
117 }
118 SECTION( "Wildcard at both ends, redundant at start" ) {
119 TestSpec spec = parseTestSpec( "*longer*" );
120 CHECK( spec.hasFilters() == true );
121 CHECK( spec.matches( tcA ) == false );
122 CHECK( spec.matches( tcB ) == false );
123 CHECK( spec.matches( tcC ) == true );
124 CHECK( spec.matches( tcD ) == true );
125 }
126 SECTION( "Just wildcard" ) {
127 TestSpec spec = parseTestSpec( "*" );
128 CHECK( spec.hasFilters() == true );
129 CHECK( spec.matches( tcA ) == true );
130 CHECK( spec.matches( tcB ) == true );
131 CHECK( spec.matches( tcC ) == true );
132 CHECK( spec.matches( tcD ) == true );
133 }
134
135 SECTION( "Single tag" ) {
136 TestSpec spec = parseTestSpec( "[one]" );
137 CHECK( spec.hasFilters() == true );
138 CHECK( spec.matches( tcA ) == false );
139 CHECK( spec.matches( tcB ) == true );
140 CHECK( spec.matches( tcC ) == false );
141 }
142 SECTION( "Single tag, two matches" ) {
143 TestSpec spec = parseTestSpec( "[x]" );
144 CHECK( spec.hasFilters() == true );
145 CHECK( spec.matches( tcA ) == false );
146 CHECK( spec.matches( tcB ) == true );
147 CHECK( spec.matches( tcC ) == true );
148 }
149 SECTION( "Two tags" ) {
150 TestSpec spec = parseTestSpec( "[two][x]" );
151 CHECK( spec.hasFilters() == true );
152 CHECK( spec.matches( tcA ) == false );
153 CHECK( spec.matches( tcB ) == false );
154 CHECK( spec.matches( tcC ) == true );
155 }
156 SECTION( "Two tags, spare separated" ) {
157 TestSpec spec = parseTestSpec( "[two] [x]" );
158 CHECK( spec.hasFilters() == true );
159 CHECK( spec.matches( tcA ) == false );
160 CHECK( spec.matches( tcB ) == false );
161 CHECK( spec.matches( tcC ) == true );
162 }
163 SECTION( "Wildcarded name and tag" ) {
164 TestSpec spec = parseTestSpec( "*name*[x]" );
165 CHECK( spec.hasFilters() == true );
166 CHECK( spec.matches( tcA ) == false );
167 CHECK( spec.matches( tcB ) == false );
168 CHECK( spec.matches( tcC ) == true );
169 CHECK( spec.matches( tcD ) == false );
170 }
171 SECTION( "Single tag exclusion" ) {
172 TestSpec spec = parseTestSpec( "~[one]" );
173 CHECK( spec.hasFilters() == true );
174 CHECK( spec.matches( tcA ) == true );
175 CHECK( spec.matches( tcB ) == false );
176 CHECK( spec.matches( tcC ) == true );
177 }
178 SECTION( "One tag exclusion and one tag inclusion" ) {
179 TestSpec spec = parseTestSpec( "~[two][x]" );
180 CHECK( spec.hasFilters() == true );
181 CHECK( spec.matches( tcA ) == false );
182 CHECK( spec.matches( tcB ) == true );
183 CHECK( spec.matches( tcC ) == false );
184 }
185 SECTION( "One tag exclusion and one wldcarded name inclusion" ) {
186 TestSpec spec = parseTestSpec( "~[two]*name*" );
187 CHECK( spec.hasFilters() == true );
188 CHECK( spec.matches( tcA ) == false );
189 CHECK( spec.matches( tcB ) == false );
190 CHECK( spec.matches( tcC ) == false );
191 CHECK( spec.matches( tcD ) == true );
192 }
193 SECTION( "One tag exclusion, using exclude:, and one wldcarded name inclusion" ) {
194 TestSpec spec = parseTestSpec( "exclude:[two]*name*" );
195 CHECK( spec.hasFilters() == true );
196 CHECK( spec.matches( tcA ) == false );
197 CHECK( spec.matches( tcB ) == false );
198 CHECK( spec.matches( tcC ) == false );
199 CHECK( spec.matches( tcD ) == true );
200 }
201 SECTION( "name exclusion" ) {
202 TestSpec spec = parseTestSpec( "~b" );
203 CHECK( spec.hasFilters() == true );
204 CHECK( spec.matches( tcA ) == true );
205 CHECK( spec.matches( tcB ) == false );
206 CHECK( spec.matches( tcC ) == true );
207 CHECK( spec.matches( tcD ) == true );
208 }
209 SECTION( "wildcarded name exclusion" ) {
210 TestSpec spec = parseTestSpec( "~*name*" );
211 CHECK( spec.hasFilters() == true );
212 CHECK( spec.matches( tcA ) == true );
213 CHECK( spec.matches( tcB ) == true );
214 CHECK( spec.matches( tcC ) == false );
215 CHECK( spec.matches( tcD ) == false );
216 }
217 SECTION( "wildcarded name exclusion with tag inclusion" ) {
218 TestSpec spec = parseTestSpec( "~*name*,[three]" );
219 CHECK( spec.hasFilters() == true );
220 CHECK( spec.matches( tcA ) == true );
221 CHECK( spec.matches( tcB ) == true );
222 CHECK( spec.matches( tcC ) == true );
223 CHECK( spec.matches( tcD ) == false );
224 }
225 SECTION( "wildcarded name exclusion, using exclude:, with tag inclusion" ) {
226 TestSpec spec = parseTestSpec( "exclude:*name*,[three]" );
227 CHECK( spec.hasFilters() == true );
228 CHECK( spec.matches( tcA ) == true );
229 CHECK( spec.matches( tcB ) == true );
230 CHECK( spec.matches( tcC ) == true );
231 CHECK( spec.matches( tcD ) == false );
232 }
233 SECTION( "two wildcarded names" ) {
234 TestSpec spec = parseTestSpec( "\"longer*\"\"*spaces\"" );
235 CHECK( spec.hasFilters() == true );
236 CHECK( spec.matches( tcA ) == false );
237 CHECK( spec.matches( tcB ) == false );
238 CHECK( spec.matches( tcC ) == true );
239 CHECK( spec.matches( tcD ) == false );
240 }
241 SECTION( "empty tag" ) {
242 TestSpec spec = parseTestSpec( "[]" );
243 CHECK( spec.hasFilters() == false );
244 CHECK( spec.matches( tcA ) == false );
245 CHECK( spec.matches( tcB ) == false );
246 CHECK( spec.matches( tcC ) == false );
247 CHECK( spec.matches( tcD ) == false );
248 }
249 SECTION( "empty quoted name" ) {
250 TestSpec spec = parseTestSpec( "\"\"" );
251 CHECK( spec.hasFilters() == false );
252 CHECK( spec.matches( tcA ) == false );
253 CHECK( spec.matches( tcB ) == false );
254 CHECK( spec.matches( tcC ) == false );
255 CHECK( spec.matches( tcD ) == false );
256 }
257 SECTION( "quoted string followed by tag exclusion" ) {
258 TestSpec spec = parseTestSpec( "\"*name*\"~[.]" );
259 CHECK( spec.hasFilters() == true );
260 CHECK( spec.matches( tcA ) == false );
261 CHECK( spec.matches( tcB ) == false );
262 CHECK( spec.matches( tcC ) == false );
263 CHECK( spec.matches( tcD ) == true );
264 }
265 SECTION( "Leading and trailing spaces in test spec" ) {
266 TestSpec spec = parseTestSpec( "\" aardvark \"" );
267 CHECK( spec.matches( fakeTestCase( " aardvark " ) ) );
268 CHECK( spec.matches( fakeTestCase( " aardvark" ) ) );
269 CHECK( spec.matches( fakeTestCase( " aardvark " ) ) );
270 CHECK( spec.matches( fakeTestCase( "aardvark " ) ) );
271 CHECK( spec.matches( fakeTestCase( "aardvark" ) ) );
272 }
273 SECTION( "Leading and trailing spaces in test name" ) {
274 TestSpec spec = parseTestSpec( "aardvark" );
275 CHECK( spec.matches( fakeTestCase( " aardvark " ) ) );
276 CHECK( spec.matches( fakeTestCase( " aardvark" ) ) );
277 CHECK( spec.matches( fakeTestCase( " aardvark " ) ) );
278 CHECK( spec.matches( fakeTestCase( "aardvark " ) ) );
279 CHECK( spec.matches( fakeTestCase( "aardvark" ) ) );
280 }
281 SECTION("Shortened hide tags are split apart when parsing") {
282 TestSpec spec = parseTestSpec("[.foo]");
283 CHECK(spec.matches(fakeTestCase("hidden and foo", "[.][foo]")));
284 CHECK_FALSE(spec.matches(fakeTestCase("only foo", "[foo]")));
285 }
286 SECTION("Shortened hide tags also properly handle exclusion") {
287 TestSpec spec = parseTestSpec("~[.foo]");
288 CHECK_FALSE(spec.matches(fakeTestCase("hidden and foo", "[.][foo]")));
289 CHECK_FALSE(spec.matches(fakeTestCase("only foo", "[foo]")));
290 CHECK_FALSE(spec.matches(fakeTestCase("only hidden", "[.]")));
291 CHECK(spec.matches(fakeTestCase("neither foo nor hidden", "[bar]")));
292 }
293 }
294
295 TEST_CASE( "Process can be configured on command line", "[config][command-line]" ) {
296
297 #ifndef CATCH_CONFIG_DISABLE_MATCHERS
298 using namespace Catch::Matchers;
299 #endif
300
301 Catch::ConfigData config;
302 auto cli = Catch::makeCommandLineParser(config);
303
304 SECTION("empty args don't cause a crash") {
305 auto result = cli.parse({""});
306 CHECK(result);
307 CHECK(config.processName == "");
308 }
309
310 SECTION("default - no arguments") {
311 auto result = cli.parse({"test"});
312 CHECK(result);
313 CHECK(config.processName == "test");
314 CHECK(config.shouldDebugBreak == false);
315 CHECK(config.abortAfter == -1);
316 CHECK(config.noThrow == false);
317 CHECK(config.reporterName == "console");
318
319 Catch::Config cfg(config);
320 CHECK_FALSE(cfg.hasTestFilters());
321 }
322
323 SECTION("test lists") {
324 SECTION("Specify one test case using") {
325 auto result = cli.parse({"test", "test1"});
326 CHECK(result);
327
328 Catch::Config cfg(config);
329 REQUIRE(cfg.hasTestFilters());
330 REQUIRE(cfg.testSpec().matches(fakeTestCase("notIncluded")) == false);
331 REQUIRE(cfg.testSpec().matches(fakeTestCase("test1")));
332 }
333 SECTION("Specify one test case exclusion using exclude:") {
334 auto result = cli.parse({"test", "exclude:test1"});
335 CHECK(result);
336
337 Catch::Config cfg(config);
338 REQUIRE(cfg.hasTestFilters());
339 REQUIRE(cfg.testSpec().matches(fakeTestCase("test1")) == false);
340 REQUIRE(cfg.testSpec().matches(fakeTestCase("alwaysIncluded")));
341 }
342
343 SECTION("Specify one test case exclusion using ~") {
344 auto result = cli.parse({"test", "~test1"});
345 CHECK(result);
346
347 Catch::Config cfg(config);
348 REQUIRE(cfg.hasTestFilters());
349 REQUIRE(cfg.testSpec().matches(fakeTestCase("test1")) == false);
350 REQUIRE(cfg.testSpec().matches(fakeTestCase("alwaysIncluded")));
351 }
352
353 }
354
355 SECTION("reporter") {
356 SECTION("-r/console") {
357 CHECK(cli.parse({"test", "-r", "console"}));
358
359 REQUIRE(config.reporterName == "console");
360 }
361 SECTION("-r/xml") {
362 CHECK(cli.parse({"test", "-r", "xml"}));
363
364 REQUIRE(config.reporterName == "xml");
365 }
366 SECTION("--reporter/junit") {
367 CHECK(cli.parse({"test", "--reporter", "junit"}));
368
369 REQUIRE(config.reporterName == "junit");
370 }
371 SECTION("Only one reporter is accepted") {
372 REQUIRE_FALSE(cli.parse({ "test", "-r", "xml", "-r", "junit" }));
373 }
374 SECTION("must match one of the available ones") {
375 auto result = cli.parse({"test", "--reporter", "unsupported"});
376 CHECK(!result);
377
378 #ifndef CATCH_CONFIG_DISABLE_MATCHERS
379 REQUIRE_THAT(result.errorMessage(), Contains("Unrecognized reporter"));
380 #endif
381 }
382 }
383
384 SECTION("debugger") {
385 SECTION("-b") {
386 CHECK(cli.parse({"test", "-b"}));
387
388 REQUIRE(config.shouldDebugBreak == true);
389 }
390 SECTION("--break") {
391 CHECK(cli.parse({"test", "--break"}));
392
393 REQUIRE(config.shouldDebugBreak);
394 }
395 }
396
397
398 SECTION("abort") {
399 SECTION("-a aborts after first failure") {
400 CHECK(cli.parse({"test", "-a"}));
401
402 REQUIRE(config.abortAfter == 1);
403 }
404 SECTION("-x 2 aborts after two failures") {
405 CHECK(cli.parse({"test", "-x", "2"}));
406
407 REQUIRE(config.abortAfter == 2);
408 }
409 SECTION("-x must be numeric") {
410 auto result = cli.parse({"test", "-x", "oops"});
411 CHECK(!result);
412
413 #ifndef CATCH_CONFIG_DISABLE_MATCHERS
414 REQUIRE_THAT(result.errorMessage(), Contains("convert") && Contains("oops"));
415 #endif
416 }
417 }
418
419 SECTION("nothrow") {
420 SECTION("-e") {
421 CHECK(cli.parse({"test", "-e"}));
422
423 REQUIRE(config.noThrow);
424 }
425 SECTION("--nothrow") {
426 CHECK(cli.parse({"test", "--nothrow"}));
427
428 REQUIRE(config.noThrow);
429 }
430 }
431
432 SECTION("output filename") {
433 SECTION("-o filename") {
434 CHECK(cli.parse({"test", "-o", "filename.ext"}));
435
436 REQUIRE(config.outputFilename == "filename.ext");
437 }
438 SECTION("--out") {
439 CHECK(cli.parse({"test", "--out", "filename.ext"}));
440
441 REQUIRE(config.outputFilename == "filename.ext");
442 }
443 }
444
445 SECTION("combinations") {
446 SECTION("Single character flags can be combined") {
447 CHECK(cli.parse({"test", "-abe"}));
448
449 CHECK(config.abortAfter == 1);
450 CHECK(config.shouldDebugBreak);
451 CHECK(config.noThrow == true);
452 }
453 }
454
455
456 SECTION( "use-colour") {
457
458 using Catch::UseColour;
459
460 SECTION( "without option" ) {
461 CHECK(cli.parse({"test"}));
462
463 REQUIRE( config.useColour == UseColour::Auto );
464 }
465
466 SECTION( "auto" ) {
467 CHECK(cli.parse({"test", "--use-colour", "auto"}));
468
469 REQUIRE( config.useColour == UseColour::Auto );
470 }
471
472 SECTION( "yes" ) {
473 CHECK(cli.parse({"test", "--use-colour", "yes"}));
474
475 REQUIRE( config.useColour == UseColour::Yes );
476 }
477
478 SECTION( "no" ) {
479 CHECK(cli.parse({"test", "--use-colour", "no"}));
480
481 REQUIRE( config.useColour == UseColour::No );
482 }
483
484 SECTION( "error" ) {
485 auto result = cli.parse({"test", "--use-colour", "wrong"});
486 CHECK( !result );
487 #ifndef CATCH_CONFIG_DISABLE_MATCHERS
488 CHECK_THAT( result.errorMessage(), Contains( "colour mode must be one of" ) );
489 #endif
490 }
491 }
492
493 SECTION("Benchmark options") {
494 SECTION("samples") {
495 CHECK(cli.parse({ "test", "--benchmark-samples=200" }));
496
497 REQUIRE(config.benchmarkSamples == 200);
498 }
499
500 SECTION("resamples") {
501 CHECK(cli.parse({ "test", "--benchmark-resamples=20000" }));
502
503 REQUIRE(config.benchmarkResamples == 20000);
504 }
505
506 SECTION("resamples") {
507 CHECK(cli.parse({ "test", "--benchmark-confidence-interval=0.99" }));
508
509 REQUIRE(config.benchmarkConfidenceInterval == Catch::Detail::Approx(0.99));
510 }
511
512 SECTION("resamples") {
513 CHECK(cli.parse({ "test", "--benchmark-no-analysis" }));
514
515 REQUIRE(config.benchmarkNoAnalysis);
516 }
517 }
518 }
519
520 TEST_CASE("Test with special, characters \"in name", "[cli][regression]") {
521 // This test case succeeds if we can invoke it from the CLI
522 SUCCEED();
523 }
524