tester/src/tests.rs

use super::*;

use crate::{
    bench::Bencher,
    console::OutputLocation,
    formatters::PrettyFormatter,
    options::OutputFormat,
    test::{
        filter_tests,
        parse_opts,
        run_test,
        DynTestFn,
        DynTestName,
        MetricMap,
        RunIgnored,
        RunStrategy,
        ShouldPanic,
        StaticTestName,
        TestDesc,
        TestDescAndFn,
        TestOpts,
        TrIgnored,
        TrOk,
        // FIXME (introduced by #65251)
        // ShouldPanic, StaticTestName, TestDesc, TestDescAndFn, TestOpts, TestTimeOptions,
        // TestType, TrFailedMsg, TrIgnored, TrOk,
    },
    time::{TestTimeOptions, TimeThreshold},
};
use std::any::TypeId;
use std::sync::mpsc::channel;
use std::time::Duration;

impl TestOpts {
    fn new() -> TestOpts {
        TestOpts {
            list: false,
            filters: vec![],
            filter_exact: false,
            force_run_in_process: false,
            exclude_should_panic: false,
            run_ignored: RunIgnored::No,
            run_tests: false,
            bench_benchmarks: false,
            logfile: None,
            nocapture: false,
            color: AutoColor,
            format: OutputFormat::Pretty,
            test_threads: None,
            skip: vec![],
            time_options: None,
            options: Options::new(),
        }
    }
}

fn one_ignored_one_unignored_test() -> Vec<TestDescAndFn> {
    vec![
        TestDescAndFn {
            desc: TestDesc {
                name: StaticTestName("1"),
                ignore: true,
                should_panic: ShouldPanic::No,
                allow_fail: false,
                test_type: TestType::Unknown,
            },
            testfn: DynTestFn(Box::new(move || {})),
        },
        TestDescAndFn {
            desc: TestDesc {
                name: StaticTestName("2"),
                ignore: false,
                should_panic: ShouldPanic::No,
                allow_fail: false,
                test_type: TestType::Unknown,
            },
            testfn: DynTestFn(Box::new(move || {})),
        },
    ]
}

#[test]
pub fn do_not_run_ignored_tests() {
    fn f() {
        panic!();
    }
    let desc = TestDescAndFn {
        desc: TestDesc {
            name: StaticTestName("whatever"),
            ignore: true,
            should_panic: ShouldPanic::No,
            allow_fail: false,
            test_type: TestType::Unknown,
        },
        testfn: DynTestFn(Box::new(f)),
    };
    let (tx, rx) = channel();
    run_test(&TestOpts::new(), false, desc, RunStrategy::InProcess, tx, Concurrent::No);
    let result = rx.recv().unwrap().result;
    assert_ne!(result, TrOk);
}

#[test]
pub fn ignored_tests_result_in_ignored() {
    fn f() {}
    let desc = TestDescAndFn {
        desc: TestDesc {
            name: StaticTestName("whatever"),
            ignore: true,
            should_panic: ShouldPanic::No,
            allow_fail: false,
            test_type: TestType::Unknown,
        },
        testfn: DynTestFn(Box::new(f)),
    };
    let (tx, rx) = channel();
    run_test(&TestOpts::new(), false, desc, RunStrategy::InProcess, tx, Concurrent::No);
    let result = rx.recv().unwrap().result;
    assert_eq!(result, TrIgnored);
}

// FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251)
#[test]
#[cfg(not(target_os = "emscripten"))]
fn test_should_panic() {
    fn f() {
        panic!();
    }
    let desc = TestDescAndFn {
        desc: TestDesc {
            name: StaticTestName("whatever"),
            ignore: false,
            should_panic: ShouldPanic::Yes,
            allow_fail: false,
            test_type: TestType::Unknown,
        },
        testfn: DynTestFn(Box::new(f)),
    };
    let (tx, rx) = channel();
    run_test(&TestOpts::new(), false, desc, RunStrategy::InProcess, tx, Concurrent::No);
    let result = rx.recv().unwrap().result;
    assert_eq!(result, TrOk);
}

// FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251)
#[test]
#[cfg(not(target_os = "emscripten"))]
fn test_should_panic_good_message() {
    fn f() {
        panic!("an error message");
    }
    let desc = TestDescAndFn {
        desc: TestDesc {
            name: StaticTestName("whatever"),
            ignore: false,
            should_panic: ShouldPanic::YesWithMessage("error message"),
            allow_fail: false,
            test_type: TestType::Unknown,
        },
        testfn: DynTestFn(Box::new(f)),
    };
    let (tx, rx) = channel();
    run_test(&TestOpts::new(), false, desc, RunStrategy::InProcess, tx, Concurrent::No);
    let result = rx.recv().unwrap().result;
    assert_eq!(result, TrOk);
}

// FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251)
#[test]
#[cfg(not(target_os = "emscripten"))]
fn test_should_panic_bad_message() {
    use crate::tests::TrFailedMsg;
    fn f() {
        panic!("an error message");
    }
    let expected = "foobar";
    let failed_msg = r#"panic did not contain expected string
      panic message: `"an error message"`,
 expected substring: `"foobar"`"#;
    let desc = TestDescAndFn {
        desc: TestDesc {
            name: StaticTestName("whatever"),
            ignore: false,
            should_panic: ShouldPanic::YesWithMessage(expected),
            allow_fail: false,
            test_type: TestType::Unknown,
        },
        testfn: DynTestFn(Box::new(f)),
    };
    let (tx, rx) = channel();
    run_test(&TestOpts::new(), false, desc, RunStrategy::InProcess, tx, Concurrent::No);
    let result = rx.recv().unwrap().result;
    assert_eq!(result, TrFailedMsg(failed_msg.to_string()));
}

// FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251)
#[test]
#[cfg(not(target_os = "emscripten"))]
fn test_should_panic_non_string_message_type() {
    use crate::tests::TrFailedMsg;
    fn f() {
        panic!(1i32);
    }
    let expected = "foobar";
    let failed_msg = format!(
        r#"expected panic with string value,
 found non-string value: `{:?}`
     expected substring: `"foobar"`"#,
        TypeId::of::<i32>()
    );
    let desc = TestDescAndFn {
        desc: TestDesc {
            name: StaticTestName("whatever"),
            ignore: false,
            should_panic: ShouldPanic::YesWithMessage(expected),
            allow_fail: false,
            test_type: TestType::Unknown,
        },
        testfn: DynTestFn(Box::new(f)),
    };
    let (tx, rx) = channel();
    run_test(&TestOpts::new(), false, desc, RunStrategy::InProcess, tx, Concurrent::No);
    let result = rx.recv().unwrap().result;
    assert_eq!(result, TrFailedMsg(failed_msg));
}

// FIXME: Re-enable emscripten once it can catch panics again (introduced by #65251)
#[test]
#[cfg(not(target_os = "emscripten"))]
fn test_should_panic_but_succeeds() {
    fn f() {}
    let desc = TestDescAndFn {
        desc: TestDesc {
            name: StaticTestName("whatever"),
            ignore: false,
            should_panic: ShouldPanic::Yes,
            allow_fail: false,
            test_type: TestType::Unknown,
        },
        testfn: DynTestFn(Box::new(f)),
    };
    let (tx, rx) = channel();
    run_test(&TestOpts::new(), false, desc, RunStrategy::InProcess, tx, Concurrent::No);
    let result = rx.recv().unwrap().result;
    assert_eq!(result, TrFailedMsg("test did not panic as expected".to_string()));
}

fn report_time_test_template(report_time: bool) -> Option<TestExecTime> {
    fn f() {}
    let desc = TestDescAndFn {
        desc: TestDesc {
            name: StaticTestName("whatever"),
            ignore: false,
            should_panic: ShouldPanic::No,
            allow_fail: false,
            test_type: TestType::Unknown,
        },
        testfn: DynTestFn(Box::new(f)),
    };
    let time_options = if report_time { Some(TestTimeOptions::default()) } else { None };

    let test_opts = TestOpts { time_options, ..TestOpts::new() };
    let (tx, rx) = channel();
    run_test(&test_opts, false, desc, RunStrategy::InProcess, tx, Concurrent::No);
    let exec_time = rx.recv().unwrap().exec_time;
    exec_time
}

#[test]
fn test_should_not_report_time() {
    let exec_time = report_time_test_template(false);
    assert!(exec_time.is_none());
}

#[test]
fn test_should_report_time() {
    let exec_time = report_time_test_template(true);
    assert!(exec_time.is_some());
}

fn time_test_failure_template(test_type: TestType) -> TestResult {
    fn f() {}
    let desc = TestDescAndFn {
        desc: TestDesc {
            name: StaticTestName("whatever"),
            ignore: false,
            should_panic: ShouldPanic::No,
            allow_fail: false,
            test_type,
        },
        testfn: DynTestFn(Box::new(f)),
    };
    // `Default` will initialize all the thresholds to 0 milliseconds.
    let mut time_options = TestTimeOptions::default();
    time_options.error_on_excess = true;

    let test_opts = TestOpts { time_options: Some(time_options), ..TestOpts::new() };
    let (tx, rx) = channel();
    run_test(&test_opts, false, desc, RunStrategy::InProcess, tx, Concurrent::No);
    let result = rx.recv().unwrap().result;

    result
}

#[test]
fn test_error_on_exceed() {
    let types = [TestType::UnitTest, TestType::IntegrationTest, TestType::DocTest];

    for test_type in types.iter() {
        let result = time_test_failure_template(*test_type);

        assert_eq!(result, TestResult::TrTimedFail);
    }

    // Check that for unknown tests thresholds aren't applied.
    let result = time_test_failure_template(TestType::Unknown);
    assert_eq!(result, TestResult::TrOk);
}

fn typed_test_desc(test_type: TestType) -> TestDesc {
    TestDesc {
        name: StaticTestName("whatever"),
        ignore: false,
        should_panic: ShouldPanic::No,
        allow_fail: false,
        test_type,
    }
}

fn test_exec_time(millis: u64) -> TestExecTime {
    TestExecTime(Duration::from_millis(millis))
}

#[test]
fn test_time_options_threshold() {
    let unit = TimeThreshold::new(Duration::from_millis(50), Duration::from_millis(100));
    let integration = TimeThreshold::new(Duration::from_millis(500), Duration::from_millis(1000));
    let doc = TimeThreshold::new(Duration::from_millis(5000), Duration::from_millis(10000));

    let options = TestTimeOptions {
        error_on_excess: false,
        colored: false,
        unit_threshold: unit.clone(),
        integration_threshold: integration.clone(),
        doctest_threshold: doc.clone(),
    };

    let test_vector = [
        (TestType::UnitTest, unit.warn.as_millis() - 1, false, false),
        (TestType::UnitTest, unit.warn.as_millis(), true, false),
        (TestType::UnitTest, unit.critical.as_millis(), true, true),
        (TestType::IntegrationTest, integration.warn.as_millis() - 1, false, false),
        (TestType::IntegrationTest, integration.warn.as_millis(), true, false),
        (TestType::IntegrationTest, integration.critical.as_millis(), true, true),
        (TestType::DocTest, doc.warn.as_millis() - 1, false, false),
        (TestType::DocTest, doc.warn.as_millis(), true, false),
        (TestType::DocTest, doc.critical.as_millis(), true, true),
    ];

    for (test_type, time, expected_warn, expected_critical) in test_vector.iter() {
        let test_desc = typed_test_desc(*test_type);
        let exec_time = test_exec_time(*time as u64);

        assert_eq!(options.is_warn(&test_desc, &exec_time), *expected_warn);
        assert_eq!(options.is_critical(&test_desc, &exec_time), *expected_critical);
    }
}

#[test]
fn parse_ignored_flag() {
    let args = vec!["progname".to_string(), "filter".to_string(), "--ignored".to_string()];
    let opts = parse_opts(&args).unwrap().unwrap();
    assert_eq!(opts.run_ignored, RunIgnored::Only);
}

#[test]
fn parse_show_output_flag() {
    let args = vec!["progname".to_string(), "filter".to_string(), "--show-output".to_string()];
    let opts = parse_opts(&args).unwrap().unwrap();
    assert!(opts.options.display_output);
}

#[test]
fn parse_include_ignored_flag() {
    let args = vec![
        "progname".to_string(),
        "filter".to_string(),
        "-Zunstable-options".to_string(),
        "--include-ignored".to_string(),
    ];
    let opts = parse_opts(&args).unwrap().unwrap();
    assert_eq!(opts.run_ignored, RunIgnored::Yes);
}

#[test]
pub fn filter_for_ignored_option() {
    // When we run ignored tests the test filter should filter out all the
    // unignored tests and flip the ignore flag on the rest to false

    let mut opts = TestOpts::new();
    opts.run_tests = true;
    opts.run_ignored = RunIgnored::Only;

    let tests = one_ignored_one_unignored_test();
    let filtered = filter_tests(&opts, tests);

    assert_eq!(filtered.len(), 1);
    assert_eq!(filtered[0].desc.name.to_string(), "1");
    assert!(!filtered[0].desc.ignore);
}

#[test]
pub fn run_include_ignored_option() {
    // When we "--include-ignored" tests, the ignore flag should be set to false on
    // all tests and no test filtered out

    let mut opts = TestOpts::new();
    opts.run_tests = true;
    opts.run_ignored = RunIgnored::Yes;

    let tests = one_ignored_one_unignored_test();
    let filtered = filter_tests(&opts, tests);

    assert_eq!(filtered.len(), 2);
    assert!(!filtered[0].desc.ignore);
    assert!(!filtered[1].desc.ignore);
}

#[test]
pub fn exclude_should_panic_option() {
    let mut opts = TestOpts::new();
    opts.run_tests = true;
    opts.exclude_should_panic = true;

    let mut tests = one_ignored_one_unignored_test();
    tests.push(TestDescAndFn {
        desc: TestDesc {
            name: StaticTestName("3"),
            ignore: false,
            should_panic: ShouldPanic::Yes,
            allow_fail: false,
            test_type: TestType::Unknown,
        },
        testfn: DynTestFn(Box::new(move || {})),
    });

    let filtered = filter_tests(&opts, tests);

    assert_eq!(filtered.len(), 2);
    assert!(filtered.iter().all(|test| test.desc.should_panic == ShouldPanic::No));
}

#[test]
pub fn exact_filter_match() {
    fn tests() -> Vec<TestDescAndFn> {
        vec!["base", "base::test", "base::test1", "base::test2"]
            .into_iter()
            .map(|name| TestDescAndFn {
                desc: TestDesc {
                    name: StaticTestName(name),
                    ignore: false,
                    should_panic: ShouldPanic::No,
                    allow_fail: false,
                    test_type: TestType::Unknown,
                },
                testfn: DynTestFn(Box::new(move || {})),
            })
            .collect()
    }

    let substr =
        filter_tests(&TestOpts { filters: vec!["base".into()], ..TestOpts::new() }, tests());
    assert_eq!(substr.len(), 4);

    let substr = filter_tests(&TestOpts { filters: vec!["bas".into()], ..TestOpts::new() }, tests());
    assert_eq!(substr.len(), 4);

    let substr =
        filter_tests(&TestOpts { filters: vec!["::test".into()], ..TestOpts::new() }, tests());
    assert_eq!(substr.len(), 3);

    let substr =
        filter_tests(&TestOpts { filters: vec!["base::test".into()], ..TestOpts::new() }, tests());
    assert_eq!(substr.len(), 3);

    let substr = filter_tests(
        &TestOpts { filters: vec!["test1".into(), "test2".into()], ..TestOpts::new() },
        tests(),
    );
    assert_eq!(substr.len(), 2);

    let exact = filter_tests(
        &TestOpts { filters: vec!["base".into()], filter_exact: true, ..TestOpts::new() },
        tests(),
    );
    assert_eq!(exact.len(), 1);

    let exact = filter_tests(
        &TestOpts { filters: vec!["bas".into()], filter_exact: true, ..TestOpts::new() },
        tests(),
    );
    assert_eq!(exact.len(), 0);

    let exact = filter_tests(
        &TestOpts { filters: vec!["::test".into()], filter_exact: true, ..TestOpts::new() },
        tests(),
    );
    assert_eq!(exact.len(), 0);

    let exact = filter_tests(
        &TestOpts { filters: vec!["base::test".into()], filter_exact: true, ..TestOpts::new() },
        tests(),
    );
    assert_eq!(exact.len(), 1);

    let exact = filter_tests(
        &TestOpts {
            filters: vec!["base".into(), "base::test".into()],
            filter_exact: true,
            ..TestOpts::new()
        },
        tests(),
    );
    assert_eq!(exact.len(), 2);
}

#[test]
pub fn sort_tests() {
    let mut opts = TestOpts::new();
    opts.run_tests = true;

    let names = vec![
        "sha1::test".to_string(),
        "isize::test_to_str".to_string(),
        "isize::test_pow".to_string(),
        "test::do_not_run_ignored_tests".to_string(),
        "test::ignored_tests_result_in_ignored".to_string(),
        "test::first_free_arg_should_be_a_filter".to_string(),
        "test::parse_ignored_flag".to_string(),
        "test::parse_include_ignored_flag".to_string(),
        "test::filter_for_ignored_option".to_string(),
        "test::run_include_ignored_option".to_string(),
        "test::sort_tests".to_string(),
    ];
    let tests = {
        fn testfn() {}
        let mut tests = Vec::new();
        for name in &names {
            let test = TestDescAndFn {
                desc: TestDesc {
                    name: DynTestName((*name).clone()),
                    ignore: false,
                    should_panic: ShouldPanic::No,
                    allow_fail: false,
                    test_type: TestType::Unknown,
                },
                testfn: DynTestFn(Box::new(testfn)),
            };
            tests.push(test);
        }
        tests
    };
    let filtered = filter_tests(&opts, tests);

    let expected = vec![
        "isize::test_pow".to_string(),
        "isize::test_to_str".to_string(),
        "sha1::test".to_string(),
        "test::do_not_run_ignored_tests".to_string(),
        "test::filter_for_ignored_option".to_string(),
        "test::first_free_arg_should_be_a_filter".to_string(),
        "test::ignored_tests_result_in_ignored".to_string(),
        "test::parse_ignored_flag".to_string(),
        "test::parse_include_ignored_flag".to_string(),
        "test::run_include_ignored_option".to_string(),
        "test::sort_tests".to_string(),
    ];

    for (a, b) in expected.iter().zip(filtered) {
        assert_eq!(*a, b.desc.name.to_string());
    }
}

#[test]
pub fn test_metricmap_compare() {
    let mut m1 = MetricMap::new();
    let mut m2 = MetricMap::new();
    m1.insert_metric("in-both-noise", 1000.0, 200.0);
    m2.insert_metric("in-both-noise", 1100.0, 200.0);

    m1.insert_metric("in-first-noise", 1000.0, 2.0);
    m2.insert_metric("in-second-noise", 1000.0, 2.0);

    m1.insert_metric("in-both-want-downwards-but-regressed", 1000.0, 10.0);
    m2.insert_metric("in-both-want-downwards-but-regressed", 2000.0, 10.0);

    m1.insert_metric("in-both-want-downwards-and-improved", 2000.0, 10.0);
    m2.insert_metric("in-both-want-downwards-and-improved", 1000.0, 10.0);

    m1.insert_metric("in-both-want-upwards-but-regressed", 2000.0, -10.0);
    m2.insert_metric("in-both-want-upwards-but-regressed", 1000.0, -10.0);

    m1.insert_metric("in-both-want-upwards-and-improved", 1000.0, -10.0);
    m2.insert_metric("in-both-want-upwards-and-improved", 2000.0, -10.0);
}

#[test]
pub fn test_bench_once_no_iter() {
    fn f(_: &mut Bencher) {}
    bench::run_once(f);
}

#[test]
pub fn test_bench_once_iter() {
    fn f(b: &mut Bencher) {
        b.iter(|| {})
    }
    bench::run_once(f);
}

#[test]
pub fn test_bench_no_iter() {
    fn f(_: &mut Bencher) {}

    let (tx, rx) = channel();

    let desc = TestDesc {
        name: StaticTestName("f"),
        ignore: false,
        should_panic: ShouldPanic::No,
        allow_fail: false,
        test_type: TestType::Unknown,
    };

    crate::bench::benchmark(desc, tx, true, f);
    rx.recv().unwrap();
}

#[test]
pub fn test_bench_iter() {
    fn f(b: &mut Bencher) {
        b.iter(|| {})
    }

    let (tx, rx) = channel();

    let desc = TestDesc {
        name: StaticTestName("f"),
        ignore: false,
        should_panic: ShouldPanic::No,
        allow_fail: false,
        test_type: TestType::Unknown,
    };

    crate::bench::benchmark(desc, tx, true, f);
    rx.recv().unwrap();
}

#[test]
fn should_sort_failures_before_printing_them() {
    let test_a = TestDesc {
        name: StaticTestName("a"),
        ignore: false,
        should_panic: ShouldPanic::No,
        allow_fail: false,
        test_type: TestType::Unknown,
    };

    let test_b = TestDesc {
        name: StaticTestName("b"),
        ignore: false,
        should_panic: ShouldPanic::No,
        allow_fail: false,
        test_type: TestType::Unknown,
    };

    let mut out = PrettyFormatter::new(OutputLocation::Raw(Vec::new()), false, 10, false, None);

    let st = console::ConsoleTestState {
        log_out: None,
        total: 0,
        passed: 0,
        failed: 0,
        ignored: 0,
        allowed_fail: 0,
        filtered_out: 0,
        measured: 0,
        exec_time: None,
        metrics: MetricMap::new(),
        failures: vec![(test_b, Vec::new()), (test_a, Vec::new())],
        options: Options::new(),
        not_failures: Vec::new(),
        time_failures: Vec::new(),
    };

    out.write_failures(&st).unwrap();
    let s = match out.output_location() {
        &OutputLocation::Raw(ref m) => String::from_utf8_lossy(&m[..]),
        &OutputLocation::Pretty(_) => unreachable!(),
    };

    let apos = s.find("a").unwrap();
    let bpos = s.find("b").unwrap();
    assert!(apos < bpos);
}