1 use super::*;
2
3 use crate::{
4 bench::Bencher,
5 console::OutputLocation,
6 formatters::PrettyFormatter,
7 options::OutputFormat,
8 test::{
9 filter_tests,
10 parse_opts,
11 run_test,
12 DynTestFn,
13 DynTestName,
14 MetricMap,
15 RunIgnored,
16 RunStrategy,
17 ShouldPanic,
18 StaticTestName,
19 TestDesc,
20 TestDescAndFn,
21 TestOpts,
22 TrIgnored,
23 TrOk,
24 // FIXME (introduced by #65251)
25 // ShouldPanic, StaticTestName, TestDesc, TestDescAndFn, TestOpts, TestTimeOptions,
26 // TestType, TrFailedMsg, TrIgnored, TrOk,
27 },
28 time::{TestTimeOptions, TimeThreshold},
29 };
30 use std::sync::mpsc::channel;
31 use std::time::Duration;
32
33 impl TestOpts {
new() -> TestOpts34 fn new() -> TestOpts {
35 TestOpts {
36 list: false,
37 filters: vec![],
38 filter_exact: false,
39 force_run_in_process: false,
40 exclude_should_panic: false,
41 run_ignored: RunIgnored::No,
42 run_tests: false,
43 bench_benchmarks: false,
44 logfile: None,
45 nocapture: false,
46 color: AutoColor,
47 format: OutputFormat::Pretty,
48 shuffle: false,
49 shuffle_seed: None,
50 test_threads: None,
51 skip: vec![],
52 time_options: None,
53 options: Options::new(),
54 }
55 }
56 }
57
one_ignored_one_unignored_test() -> Vec<TestDescAndFn>58 fn one_ignored_one_unignored_test() -> Vec<TestDescAndFn> {
59 vec![
60 TestDescAndFn {
61 desc: TestDesc {
62 name: StaticTestName("1"),
63 ignore: true,
64 should_panic: ShouldPanic::No,
65 allow_fail: false,
66 compile_fail: false,
67 no_run: false,
68 test_type: TestType::Unknown,
69 },
70 testfn: DynTestFn(Box::new(move || {})),
71 },
72 TestDescAndFn {
73 desc: TestDesc {
74 name: StaticTestName("2"),
75 ignore: false,
76 should_panic: ShouldPanic::No,
77 allow_fail: false,
78 compile_fail: false,
79 no_run: false,
80 test_type: TestType::Unknown,
81 },
82 testfn: DynTestFn(Box::new(move || {})),
83 },
84 ]
85 }
86
87 #[test]
do_not_run_ignored_tests()88 pub fn do_not_run_ignored_tests() {
89 fn f() {
90 panic!();
91 }
92 let desc = TestDescAndFn {
93 desc: TestDesc {
94 name: StaticTestName("whatever"),
95 ignore: true,
96 should_panic: ShouldPanic::No,
97 allow_fail: false,
98 compile_fail: false,
99 no_run: false,
100 test_type: TestType::Unknown,
101 },
102 testfn: DynTestFn(Box::new(f)),
103 };
104 let (tx, rx) = channel();
105 run_test(&TestOpts::new(), false, TestId(0), desc, RunStrategy::InProcess, tx, Concurrent::No);
106 let result = rx.recv().unwrap().result;
107 assert_ne!(result, TrOk);
108 }
109
110 #[test]
ignored_tests_result_in_ignored()111 pub fn ignored_tests_result_in_ignored() {
112 fn f() {}
113 let desc = TestDescAndFn {
114 desc: TestDesc {
115 name: StaticTestName("whatever"),
116 ignore: true,
117 should_panic: ShouldPanic::No,
118 allow_fail: false,
119 compile_fail: false,
120 no_run: false,
121 test_type: TestType::Unknown,
122 },
123 testfn: DynTestFn(Box::new(f)),
124 };
125 let (tx, rx) = channel();
126 run_test(&TestOpts::new(), false, TestId(0), desc, RunStrategy::InProcess, tx, Concurrent::No);
127 let result = rx.recv().unwrap().result;
128 assert_eq!(result, TrIgnored);
129 }
130
131 // FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251)
132 #[test]
133 #[cfg(not(target_os = "emscripten"))]
test_should_panic()134 fn test_should_panic() {
135 fn f() {
136 panic!();
137 }
138 let desc = TestDescAndFn {
139 desc: TestDesc {
140 name: StaticTestName("whatever"),
141 ignore: false,
142 should_panic: ShouldPanic::Yes,
143 allow_fail: false,
144 compile_fail: false,
145 no_run: false,
146 test_type: TestType::Unknown,
147 },
148 testfn: DynTestFn(Box::new(f)),
149 };
150 let (tx, rx) = channel();
151 run_test(&TestOpts::new(), false, TestId(0), desc, RunStrategy::InProcess, tx, Concurrent::No);
152 let result = rx.recv().unwrap().result;
153 assert_eq!(result, TrOk);
154 }
155
156 // FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251)
157 #[test]
158 #[cfg(not(target_os = "emscripten"))]
test_should_panic_good_message()159 fn test_should_panic_good_message() {
160 fn f() {
161 panic!("an error message");
162 }
163 let desc = TestDescAndFn {
164 desc: TestDesc {
165 name: StaticTestName("whatever"),
166 ignore: false,
167 should_panic: ShouldPanic::YesWithMessage("error message"),
168 allow_fail: false,
169 compile_fail: false,
170 no_run: false,
171 test_type: TestType::Unknown,
172 },
173 testfn: DynTestFn(Box::new(f)),
174 };
175 let (tx, rx) = channel();
176 run_test(&TestOpts::new(), false, TestId(0), desc, RunStrategy::InProcess, tx, Concurrent::No);
177 let result = rx.recv().unwrap().result;
178 assert_eq!(result, TrOk);
179 }
180
181 // FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251)
182 #[test]
183 #[cfg(not(target_os = "emscripten"))]
test_should_panic_bad_message()184 fn test_should_panic_bad_message() {
185 use crate::tests::TrFailedMsg;
186 fn f() {
187 panic!("an error message");
188 }
189 let expected = "foobar";
190 let failed_msg = r#"panic did not contain expected string
191 panic message: `"an error message"`,
192 expected substring: `"foobar"`"#;
193 let desc = TestDescAndFn {
194 desc: TestDesc {
195 name: StaticTestName("whatever"),
196 ignore: false,
197 should_panic: ShouldPanic::YesWithMessage(expected),
198 allow_fail: false,
199 compile_fail: false,
200 no_run: false,
201 test_type: TestType::Unknown,
202 },
203 testfn: DynTestFn(Box::new(f)),
204 };
205 let (tx, rx) = channel();
206 run_test(&TestOpts::new(), false, TestId(0), desc, RunStrategy::InProcess, tx, Concurrent::No);
207 let result = rx.recv().unwrap().result;
208 assert_eq!(result, TrFailedMsg(failed_msg.to_string()));
209 }
210
211 // FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251)
212 #[test]
213 #[cfg(not(target_os = "emscripten"))]
test_should_panic_non_string_message_type()214 fn test_should_panic_non_string_message_type() {
215 use crate::tests::TrFailedMsg;
216 use std::any::TypeId;
217 fn f() {
218 std::panic::panic_any(1i32);
219 }
220 let expected = "foobar";
221 let failed_msg = format!(
222 r#"expected panic with string value,
223 found non-string value: `{:?}`
224 expected substring: `"foobar"`"#,
225 TypeId::of::<i32>()
226 );
227 let desc = TestDescAndFn {
228 desc: TestDesc {
229 name: StaticTestName("whatever"),
230 ignore: false,
231 should_panic: ShouldPanic::YesWithMessage(expected),
232 allow_fail: false,
233 compile_fail: false,
234 no_run: false,
235 test_type: TestType::Unknown,
236 },
237 testfn: DynTestFn(Box::new(f)),
238 };
239 let (tx, rx) = channel();
240 run_test(&TestOpts::new(), false, TestId(0), desc, RunStrategy::InProcess, tx, Concurrent::No);
241 let result = rx.recv().unwrap().result;
242 assert_eq!(result, TrFailedMsg(failed_msg));
243 }
244
245 // FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251)
246 #[test]
247 #[cfg(not(target_os = "emscripten"))]
test_should_panic_but_succeeds()248 fn test_should_panic_but_succeeds() {
249 let should_panic_variants = [ShouldPanic::Yes, ShouldPanic::YesWithMessage("error message")];
250
251 for &should_panic in should_panic_variants.iter() {
252 fn f() {}
253 let desc = TestDescAndFn {
254 desc: TestDesc {
255 name: StaticTestName("whatever"),
256 ignore: false,
257 should_panic,
258 allow_fail: false,
259 compile_fail: false,
260 no_run: false,
261 test_type: TestType::Unknown,
262 },
263 testfn: DynTestFn(Box::new(f)),
264 };
265 let (tx, rx) = channel();
266 run_test(
267 &TestOpts::new(),
268 false,
269 TestId(0),
270 desc,
271 RunStrategy::InProcess,
272 tx,
273 Concurrent::No,
274 );
275 let result = rx.recv().unwrap().result;
276 assert_eq!(
277 result,
278 TrFailedMsg("test did not panic as expected".to_string()),
279 "should_panic == {:?}",
280 should_panic
281 );
282 }
283 }
284
report_time_test_template(report_time: bool) -> Option<TestExecTime>285 fn report_time_test_template(report_time: bool) -> Option<TestExecTime> {
286 fn f() {}
287 let desc = TestDescAndFn {
288 desc: TestDesc {
289 name: StaticTestName("whatever"),
290 ignore: false,
291 should_panic: ShouldPanic::No,
292 allow_fail: false,
293 compile_fail: false,
294 no_run: false,
295 test_type: TestType::Unknown,
296 },
297 testfn: DynTestFn(Box::new(f)),
298 };
299 let time_options = if report_time { Some(TestTimeOptions::default()) } else { None };
300
301 let test_opts = TestOpts { time_options, ..TestOpts::new() };
302 let (tx, rx) = channel();
303 run_test(&test_opts, false, TestId(0), desc, RunStrategy::InProcess, tx, Concurrent::No);
304 let exec_time = rx.recv().unwrap().exec_time;
305 exec_time
306 }
307
308 #[test]
test_should_not_report_time()309 fn test_should_not_report_time() {
310 let exec_time = report_time_test_template(false);
311 assert!(exec_time.is_none());
312 }
313
314 #[test]
test_should_report_time()315 fn test_should_report_time() {
316 let exec_time = report_time_test_template(true);
317 assert!(exec_time.is_some());
318 }
319
time_test_failure_template(test_type: TestType) -> TestResult320 fn time_test_failure_template(test_type: TestType) -> TestResult {
321 fn f() {}
322 let desc = TestDescAndFn {
323 desc: TestDesc {
324 name: StaticTestName("whatever"),
325 ignore: false,
326 should_panic: ShouldPanic::No,
327 allow_fail: false,
328 compile_fail: false,
329 no_run: false,
330 test_type,
331 },
332 testfn: DynTestFn(Box::new(f)),
333 };
334 // `Default` will initialize all the thresholds to 0 milliseconds.
335 let mut time_options = TestTimeOptions::default();
336 time_options.error_on_excess = true;
337
338 let test_opts = TestOpts { time_options: Some(time_options), ..TestOpts::new() };
339 let (tx, rx) = channel();
340 run_test(&test_opts, false, TestId(0), desc, RunStrategy::InProcess, tx, Concurrent::No);
341 let result = rx.recv().unwrap().result;
342
343 result
344 }
345
346 #[test]
test_error_on_exceed()347 fn test_error_on_exceed() {
348 let types = [TestType::UnitTest, TestType::IntegrationTest, TestType::DocTest];
349
350 for test_type in types.iter() {
351 let result = time_test_failure_template(*test_type);
352
353 assert_eq!(result, TestResult::TrTimedFail);
354 }
355
356 // Check that for unknown tests thresholds aren't applied.
357 let result = time_test_failure_template(TestType::Unknown);
358 assert_eq!(result, TestResult::TrOk);
359 }
360
typed_test_desc(test_type: TestType) -> TestDesc361 fn typed_test_desc(test_type: TestType) -> TestDesc {
362 TestDesc {
363 name: StaticTestName("whatever"),
364 ignore: false,
365 should_panic: ShouldPanic::No,
366 allow_fail: false,
367 compile_fail: false,
368 no_run: false,
369 test_type,
370 }
371 }
372
test_exec_time(millis: u64) -> TestExecTime373 fn test_exec_time(millis: u64) -> TestExecTime {
374 TestExecTime(Duration::from_millis(millis))
375 }
376
377 #[test]
test_time_options_threshold()378 fn test_time_options_threshold() {
379 let unit = TimeThreshold::new(Duration::from_millis(50), Duration::from_millis(100));
380 let integration = TimeThreshold::new(Duration::from_millis(500), Duration::from_millis(1000));
381 let doc = TimeThreshold::new(Duration::from_millis(5000), Duration::from_millis(10000));
382
383 let options = TestTimeOptions {
384 error_on_excess: false,
385 colored: false,
386 unit_threshold: unit.clone(),
387 integration_threshold: integration.clone(),
388 doctest_threshold: doc.clone(),
389 };
390
391 let test_vector = [
392 (TestType::UnitTest, unit.warn.as_millis() - 1, false, false),
393 (TestType::UnitTest, unit.warn.as_millis(), true, false),
394 (TestType::UnitTest, unit.critical.as_millis(), true, true),
395 (TestType::IntegrationTest, integration.warn.as_millis() - 1, false, false),
396 (TestType::IntegrationTest, integration.warn.as_millis(), true, false),
397 (TestType::IntegrationTest, integration.critical.as_millis(), true, true),
398 (TestType::DocTest, doc.warn.as_millis() - 1, false, false),
399 (TestType::DocTest, doc.warn.as_millis(), true, false),
400 (TestType::DocTest, doc.critical.as_millis(), true, true),
401 ];
402
403 for (test_type, time, expected_warn, expected_critical) in test_vector.iter() {
404 let test_desc = typed_test_desc(*test_type);
405 let exec_time = test_exec_time(*time as u64);
406
407 assert_eq!(options.is_warn(&test_desc, &exec_time), *expected_warn);
408 assert_eq!(options.is_critical(&test_desc, &exec_time), *expected_critical);
409 }
410 }
411
412 #[test]
parse_ignored_flag()413 fn parse_ignored_flag() {
414 let args = vec!["progname".to_string(), "filter".to_string(), "--ignored".to_string()];
415 let opts = parse_opts(&args).unwrap().unwrap();
416 assert_eq!(opts.run_ignored, RunIgnored::Only);
417 }
418
419 #[test]
parse_show_output_flag()420 fn parse_show_output_flag() {
421 let args = vec!["progname".to_string(), "filter".to_string(), "--show-output".to_string()];
422 let opts = parse_opts(&args).unwrap().unwrap();
423 assert!(opts.options.display_output);
424 }
425
426 #[test]
parse_include_ignored_flag()427 fn parse_include_ignored_flag() {
428 let args = vec!["progname".to_string(), "filter".to_string(), "--include-ignored".to_string()];
429 let opts = parse_opts(&args).unwrap().unwrap();
430 assert_eq!(opts.run_ignored, RunIgnored::Yes);
431 }
432
433 #[test]
filter_for_ignored_option()434 pub fn filter_for_ignored_option() {
435 // When we run ignored tests the test filter should filter out all the
436 // unignored tests and flip the ignore flag on the rest to false
437
438 let mut opts = TestOpts::new();
439 opts.run_tests = true;
440 opts.run_ignored = RunIgnored::Only;
441
442 let tests = one_ignored_one_unignored_test();
443 let filtered = filter_tests(&opts, tests);
444
445 assert_eq!(filtered.len(), 1);
446 assert_eq!(filtered[0].desc.name.to_string(), "1");
447 assert!(!filtered[0].desc.ignore);
448 }
449
450 #[test]
run_include_ignored_option()451 pub fn run_include_ignored_option() {
452 // When we "--include-ignored" tests, the ignore flag should be set to false on
453 // all tests and no test filtered out
454
455 let mut opts = TestOpts::new();
456 opts.run_tests = true;
457 opts.run_ignored = RunIgnored::Yes;
458
459 let tests = one_ignored_one_unignored_test();
460 let filtered = filter_tests(&opts, tests);
461
462 assert_eq!(filtered.len(), 2);
463 assert!(!filtered[0].desc.ignore);
464 assert!(!filtered[1].desc.ignore);
465 }
466
467 #[test]
exclude_should_panic_option()468 pub fn exclude_should_panic_option() {
469 let mut opts = TestOpts::new();
470 opts.run_tests = true;
471 opts.exclude_should_panic = true;
472
473 let mut tests = one_ignored_one_unignored_test();
474 tests.push(TestDescAndFn {
475 desc: TestDesc {
476 name: StaticTestName("3"),
477 ignore: false,
478 should_panic: ShouldPanic::Yes,
479 allow_fail: false,
480 compile_fail: false,
481 no_run: false,
482 test_type: TestType::Unknown,
483 },
484 testfn: DynTestFn(Box::new(move || {})),
485 });
486
487 let filtered = filter_tests(&opts, tests);
488
489 assert_eq!(filtered.len(), 2);
490 assert!(filtered.iter().all(|test| test.desc.should_panic == ShouldPanic::No));
491 }
492
493 #[test]
exact_filter_match()494 pub fn exact_filter_match() {
495 fn tests() -> Vec<TestDescAndFn> {
496 vec!["base", "base::test", "base::test1", "base::test2"]
497 .into_iter()
498 .map(|name| TestDescAndFn {
499 desc: TestDesc {
500 name: StaticTestName(name),
501 ignore: false,
502 should_panic: ShouldPanic::No,
503 allow_fail: false,
504 compile_fail: false,
505 no_run: false,
506 test_type: TestType::Unknown,
507 },
508 testfn: DynTestFn(Box::new(move || {})),
509 })
510 .collect()
511 }
512
513 let substr =
514 filter_tests(&TestOpts { filters: vec!["base".into()], ..TestOpts::new() }, tests());
515 assert_eq!(substr.len(), 4);
516
517 let substr =
518 filter_tests(&TestOpts { filters: vec!["bas".into()], ..TestOpts::new() }, tests());
519 assert_eq!(substr.len(), 4);
520
521 let substr =
522 filter_tests(&TestOpts { filters: vec!["::test".into()], ..TestOpts::new() }, tests());
523 assert_eq!(substr.len(), 3);
524
525 let substr =
526 filter_tests(&TestOpts { filters: vec!["base::test".into()], ..TestOpts::new() }, tests());
527 assert_eq!(substr.len(), 3);
528
529 let substr = filter_tests(
530 &TestOpts { filters: vec!["test1".into(), "test2".into()], ..TestOpts::new() },
531 tests(),
532 );
533 assert_eq!(substr.len(), 2);
534
535 let exact = filter_tests(
536 &TestOpts { filters: vec!["base".into()], filter_exact: true, ..TestOpts::new() },
537 tests(),
538 );
539 assert_eq!(exact.len(), 1);
540
541 let exact = filter_tests(
542 &TestOpts { filters: vec!["bas".into()], filter_exact: true, ..TestOpts::new() },
543 tests(),
544 );
545 assert_eq!(exact.len(), 0);
546
547 let exact = filter_tests(
548 &TestOpts { filters: vec!["::test".into()], filter_exact: true, ..TestOpts::new() },
549 tests(),
550 );
551 assert_eq!(exact.len(), 0);
552
553 let exact = filter_tests(
554 &TestOpts { filters: vec!["base::test".into()], filter_exact: true, ..TestOpts::new() },
555 tests(),
556 );
557 assert_eq!(exact.len(), 1);
558
559 let exact = filter_tests(
560 &TestOpts {
561 filters: vec!["base".into(), "base::test".into()],
562 filter_exact: true,
563 ..TestOpts::new()
564 },
565 tests(),
566 );
567 assert_eq!(exact.len(), 2);
568 }
569
sample_tests() -> Vec<TestDescAndFn>570 fn sample_tests() -> Vec<TestDescAndFn> {
571 let names = vec![
572 "sha1::test".to_string(),
573 "isize::test_to_str".to_string(),
574 "isize::test_pow".to_string(),
575 "test::do_not_run_ignored_tests".to_string(),
576 "test::ignored_tests_result_in_ignored".to_string(),
577 "test::first_free_arg_should_be_a_filter".to_string(),
578 "test::parse_ignored_flag".to_string(),
579 "test::parse_include_ignored_flag".to_string(),
580 "test::filter_for_ignored_option".to_string(),
581 "test::run_include_ignored_option".to_string(),
582 "test::sort_tests".to_string(),
583 ];
584 fn testfn() {}
585 let mut tests = Vec::new();
586 for name in &names {
587 let test = TestDescAndFn {
588 desc: TestDesc {
589 name: DynTestName((*name).clone()),
590 ignore: false,
591 should_panic: ShouldPanic::No,
592 allow_fail: false,
593 compile_fail: false,
594 no_run: false,
595 test_type: TestType::Unknown,
596 },
597 testfn: DynTestFn(Box::new(testfn)),
598 };
599 tests.push(test);
600 }
601 tests
602 }
603
604 #[test]
sort_tests()605 pub fn sort_tests() {
606 let mut opts = TestOpts::new();
607 opts.run_tests = true;
608
609 let tests = sample_tests();
610 let filtered = filter_tests(&opts, tests);
611
612 let expected = vec![
613 "isize::test_pow".to_string(),
614 "isize::test_to_str".to_string(),
615 "sha1::test".to_string(),
616 "test::do_not_run_ignored_tests".to_string(),
617 "test::filter_for_ignored_option".to_string(),
618 "test::first_free_arg_should_be_a_filter".to_string(),
619 "test::ignored_tests_result_in_ignored".to_string(),
620 "test::parse_ignored_flag".to_string(),
621 "test::parse_include_ignored_flag".to_string(),
622 "test::run_include_ignored_option".to_string(),
623 "test::sort_tests".to_string(),
624 ];
625
626 for (a, b) in expected.iter().zip(filtered) {
627 assert_eq!(*a, b.desc.name.to_string());
628 }
629 }
630
631 #[test]
shuffle_tests()632 pub fn shuffle_tests() {
633 let mut opts = TestOpts::new();
634 opts.shuffle = true;
635
636 let shuffle_seed = get_shuffle_seed(&opts).unwrap();
637
638 let left =
639 sample_tests().into_iter().enumerate().map(|(i, e)| (TestId(i), e)).collect::<Vec<_>>();
640 let mut right =
641 sample_tests().into_iter().enumerate().map(|(i, e)| (TestId(i), e)).collect::<Vec<_>>();
642
643 assert!(left.iter().zip(&right).all(|(a, b)| a.1.desc.name == b.1.desc.name));
644
645 helpers::shuffle::shuffle_tests(shuffle_seed, right.as_mut_slice());
646
647 assert!(left.iter().zip(right).any(|(a, b)| a.1.desc.name != b.1.desc.name));
648 }
649
650 #[test]
shuffle_tests_with_seed()651 pub fn shuffle_tests_with_seed() {
652 let mut opts = TestOpts::new();
653 opts.shuffle = true;
654
655 let shuffle_seed = get_shuffle_seed(&opts).unwrap();
656
657 let mut left =
658 sample_tests().into_iter().enumerate().map(|(i, e)| (TestId(i), e)).collect::<Vec<_>>();
659 let mut right =
660 sample_tests().into_iter().enumerate().map(|(i, e)| (TestId(i), e)).collect::<Vec<_>>();
661
662 helpers::shuffle::shuffle_tests(shuffle_seed, left.as_mut_slice());
663 helpers::shuffle::shuffle_tests(shuffle_seed, right.as_mut_slice());
664
665 assert!(left.iter().zip(right).all(|(a, b)| a.1.desc.name == b.1.desc.name));
666 }
667
668 #[test]
order_depends_on_more_than_seed()669 pub fn order_depends_on_more_than_seed() {
670 let mut opts = TestOpts::new();
671 opts.shuffle = true;
672
673 let shuffle_seed = get_shuffle_seed(&opts).unwrap();
674
675 let mut left_tests = sample_tests();
676 let mut right_tests = sample_tests();
677
678 left_tests.pop();
679 right_tests.remove(0);
680
681 let mut left =
682 left_tests.into_iter().enumerate().map(|(i, e)| (TestId(i), e)).collect::<Vec<_>>();
683 let mut right =
684 right_tests.into_iter().enumerate().map(|(i, e)| (TestId(i), e)).collect::<Vec<_>>();
685
686 assert_eq!(left.len(), right.len());
687
688 assert!(left.iter().zip(&right).all(|(a, b)| a.0 == b.0));
689
690 helpers::shuffle::shuffle_tests(shuffle_seed, left.as_mut_slice());
691 helpers::shuffle::shuffle_tests(shuffle_seed, right.as_mut_slice());
692
693 assert!(left.iter().zip(right).any(|(a, b)| a.0 != b.0));
694 }
695
696 #[test]
test_metricmap_compare()697 pub fn test_metricmap_compare() {
698 let mut m1 = MetricMap::new();
699 let mut m2 = MetricMap::new();
700 m1.insert_metric("in-both-noise", 1000.0, 200.0);
701 m2.insert_metric("in-both-noise", 1100.0, 200.0);
702
703 m1.insert_metric("in-first-noise", 1000.0, 2.0);
704 m2.insert_metric("in-second-noise", 1000.0, 2.0);
705
706 m1.insert_metric("in-both-want-downwards-but-regressed", 1000.0, 10.0);
707 m2.insert_metric("in-both-want-downwards-but-regressed", 2000.0, 10.0);
708
709 m1.insert_metric("in-both-want-downwards-and-improved", 2000.0, 10.0);
710 m2.insert_metric("in-both-want-downwards-and-improved", 1000.0, 10.0);
711
712 m1.insert_metric("in-both-want-upwards-but-regressed", 2000.0, -10.0);
713 m2.insert_metric("in-both-want-upwards-but-regressed", 1000.0, -10.0);
714
715 m1.insert_metric("in-both-want-upwards-and-improved", 1000.0, -10.0);
716 m2.insert_metric("in-both-want-upwards-and-improved", 2000.0, -10.0);
717 }
718
719 #[test]
test_bench_once_no_iter()720 pub fn test_bench_once_no_iter() {
721 fn f(_: &mut Bencher) {}
722 bench::run_once(f);
723 }
724
725 #[test]
test_bench_once_iter()726 pub fn test_bench_once_iter() {
727 fn f(b: &mut Bencher) {
728 b.iter(|| {})
729 }
730 bench::run_once(f);
731 }
732
733 #[test]
test_bench_no_iter()734 pub fn test_bench_no_iter() {
735 fn f(_: &mut Bencher) {}
736
737 let (tx, rx) = channel();
738
739 let desc = TestDesc {
740 name: StaticTestName("f"),
741 ignore: false,
742 should_panic: ShouldPanic::No,
743 allow_fail: false,
744 compile_fail: false,
745 no_run: false,
746 test_type: TestType::Unknown,
747 };
748
749 crate::bench::benchmark(TestId(0), desc, tx, true, f);
750 rx.recv().unwrap();
751 }
752
753 #[test]
test_bench_iter()754 pub fn test_bench_iter() {
755 fn f(b: &mut Bencher) {
756 b.iter(|| {})
757 }
758
759 let (tx, rx) = channel();
760
761 let desc = TestDesc {
762 name: StaticTestName("f"),
763 ignore: false,
764 should_panic: ShouldPanic::No,
765 allow_fail: false,
766 compile_fail: false,
767 no_run: false,
768 test_type: TestType::Unknown,
769 };
770
771 crate::bench::benchmark(TestId(0), desc, tx, true, f);
772 rx.recv().unwrap();
773 }
774
775 #[test]
should_sort_failures_before_printing_them()776 fn should_sort_failures_before_printing_them() {
777 let test_a = TestDesc {
778 name: StaticTestName("a"),
779 ignore: false,
780 should_panic: ShouldPanic::No,
781 allow_fail: false,
782 compile_fail: false,
783 no_run: false,
784 test_type: TestType::Unknown,
785 };
786
787 let test_b = TestDesc {
788 name: StaticTestName("b"),
789 ignore: false,
790 should_panic: ShouldPanic::No,
791 allow_fail: false,
792 compile_fail: false,
793 no_run: false,
794 test_type: TestType::Unknown,
795 };
796
797 let mut out = PrettyFormatter::new(OutputLocation::Raw(Vec::new()), false, 10, false, None);
798
799 let st = console::ConsoleTestState {
800 log_out: None,
801 total: 0,
802 passed: 0,
803 failed: 0,
804 ignored: 0,
805 allowed_fail: 0,
806 filtered_out: 0,
807 measured: 0,
808 exec_time: None,
809 metrics: MetricMap::new(),
810 failures: vec![(test_b, Vec::new()), (test_a, Vec::new())],
811 options: Options::new(),
812 not_failures: Vec::new(),
813 time_failures: Vec::new(),
814 };
815
816 out.write_failures(&st).unwrap();
817 let s = match out.output_location() {
818 &OutputLocation::Raw(ref m) => String::from_utf8_lossy(&m[..]),
819 &OutputLocation::Pretty(_) => unreachable!(),
820 };
821
822 let apos = s.find("a").unwrap();
823 let bpos = s.find("b").unwrap();
824 assert!(apos < bpos);
825 }
826