trial/test/test_script.py

# Copyright (c) Twisted Matrix Laboratories.
# See LICENSE for details.

import StringIO
import gc
import re
import sys
import textwrap
import types

from twisted.trial import unittest
from twisted.trial.runner import (
    TrialRunner, TestSuite, DestructiveTestSuite, TestLoader)
from twisted.trial._dist.disttrial import DistTrialRunner
from twisted.scripts import trial
from twisted.python import util
from twisted.python.usage import UsageError
from twisted.python.filepath import FilePath

from twisted.trial.test.test_loader import testNames

pyunit = __import__('unittest')


def sibpath(filename):
    """
    For finding files in twisted/trial/test
    """
    return util.sibpath(__file__, filename)


class ForceGarbageCollection(unittest.SynchronousTestCase):
    """
    Tests for the --force-gc option.
    """

    def setUp(self):
        self.config = trial.Options()
        self.log = []
        self.patch(gc, 'collect', self.collect)
        test = pyunit.FunctionTestCase(self.simpleTest)
        self.test = TestSuite([test, test])


    def simpleTest(self):
        """
        A simple test method that records that it was run.
        """
        self.log.append('test')


    def collect(self):
        """
        A replacement for gc.collect that logs calls to itself.
        """
        self.log.append('collect')


    def makeRunner(self):
        """
        Return a L{TrialRunner} object that is safe to use in tests.
        """
        runner = trial._makeRunner(self.config)
        runner.stream = StringIO.StringIO()
        return runner


    def test_forceGc(self):
        """
        Passing the --force-gc option to the trial script forces the garbage
        collector to run before and after each test.
        """
        self.config['force-gc'] = True
        self.config.postOptions()
        runner = self.makeRunner()
        runner.run(self.test)
        self.assertEqual(self.log, ['collect', 'test', 'collect',
                                    'collect', 'test', 'collect'])


    def test_unforceGc(self):
        """
        By default, no garbage collection is forced.
        """
        self.config.postOptions()
        runner = self.makeRunner()
        runner.run(self.test)
        self.assertEqual(self.log, ['test', 'test'])


class TestSuiteUsed(unittest.SynchronousTestCase):
    """
    Check the category of tests suite used by the loader.
    """

    def setUp(self):
        """
        Create a trial configuration object.
        """
        self.config = trial.Options()


    def test_defaultSuite(self):
        """
        By default, the loader should use L{DestructiveTestSuite}
        """
        loader = trial._getLoader(self.config)
        self.assertEqual(loader.suiteFactory, DestructiveTestSuite)


    def test_untilFailureSuite(self):
        """
        The C{until-failure} configuration uses the L{TestSuite} to keep
        instances alive across runs.
        """
        self.config['until-failure'] = True
        loader = trial._getLoader(self.config)
        self.assertEqual(loader.suiteFactory, TestSuite)


class TestModuleTest(unittest.SynchronousTestCase):
    def setUp(self):
        self.config = trial.Options()

    def tearDown(self):
        self.config = None

    def test_testNames(self):
        """
        Check that the testNames helper method accurately collects the
        names of tests in suite.
        """
        self.assertEqual(testNames(self), [self.id()])

    def assertSuitesEqual(self, test1, names):
        loader = TestLoader()
        names1 = testNames(test1)
        names2 = testNames(TestSuite(map(loader.loadByName, names)))
        names1.sort()
        names2.sort()
        self.assertEqual(names1, names2)

    def test_baseState(self):
        self.assertEqual(0, len(self.config['tests']))

    def test_testmoduleOnModule(self):
        """
        Check that --testmodule loads a suite which contains the tests
        referred to in test-case-name inside its parameter.
        """
        self.config.opt_testmodule(sibpath('moduletest.py'))
        self.assertSuitesEqual(trial._getSuite(self.config),
                               ['twisted.trial.test.test_log'])

    def test_testmoduleTwice(self):
        """
        When the same module is specified with two --testmodule flags, it
        should only appear once in the suite.
        """
        self.config.opt_testmodule(sibpath('moduletest.py'))
        self.config.opt_testmodule(sibpath('moduletest.py'))
        self.assertSuitesEqual(trial._getSuite(self.config),
                               ['twisted.trial.test.test_log'])

    def test_testmoduleOnSourceAndTarget(self):
        """
        If --testmodule is specified twice, once for module A and once for
        a module which refers to module A, then make sure module A is only
        added once.
        """
        self.config.opt_testmodule(sibpath('moduletest.py'))
        self.config.opt_testmodule(sibpath('test_log.py'))
        self.assertSuitesEqual(trial._getSuite(self.config),
                               ['twisted.trial.test.test_log'])

    def test_testmoduleOnSelfModule(self):
        """
        When given a module that refers to *itself* in the test-case-name
        variable, check that --testmodule only adds the tests once.
        """
        self.config.opt_testmodule(sibpath('moduleself.py'))
        self.assertSuitesEqual(trial._getSuite(self.config),
                               ['twisted.trial.test.moduleself'])

    def test_testmoduleOnScript(self):
        """
        Check that --testmodule loads tests referred to in test-case-name
        buffer variables.
        """
        self.config.opt_testmodule(sibpath('scripttest.py'))
        self.assertSuitesEqual(trial._getSuite(self.config),
                               ['twisted.trial.test.test_log',
                                'twisted.trial.test.test_class'])

    def test_testmoduleOnNonexistentFile(self):
        """
        Check that --testmodule displays a meaningful error message when
        passed a non-existent filename.
        """
        buffy = StringIO.StringIO()
        stderr, sys.stderr = sys.stderr, buffy
        filename = 'test_thisbetternoteverexist.py'
        try:
            self.config.opt_testmodule(filename)
            self.assertEqual(0, len(self.config['tests']))
            self.assertEqual("File %r doesn't exist\n" % (filename,),
                                 buffy.getvalue())
        finally:
            sys.stderr = stderr

    def test_testmoduleOnEmptyVars(self):
        """
        Check that --testmodule adds no tests to the suite for modules
        which lack test-case-name buffer variables.
        """
        self.config.opt_testmodule(sibpath('novars.py'))
        self.assertEqual(0, len(self.config['tests']))

    def test_testmoduleOnModuleName(self):
        """
        Check that --testmodule does *not* support module names as arguments
        and that it displays a meaningful error message.
        """
        buffy = StringIO.StringIO()
        stderr, sys.stderr = sys.stderr, buffy
        moduleName = 'twisted.trial.test.test_script'
        try:
            self.config.opt_testmodule(moduleName)
            self.assertEqual(0, len(self.config['tests']))
            self.assertEqual("File %r doesn't exist\n" % (moduleName,),
                                 buffy.getvalue())
        finally:
            sys.stderr = stderr

    def test_parseLocalVariable(self):
        declaration = '-*- test-case-name: twisted.trial.test.test_tests -*-'
        localVars = trial._parseLocalVariables(declaration)
        self.assertEqual({'test-case-name':
                              'twisted.trial.test.test_tests'},
                             localVars)

    def test_trailingSemicolon(self):
        declaration = '-*- test-case-name: twisted.trial.test.test_tests; -*-'
        localVars = trial._parseLocalVariables(declaration)
        self.assertEqual({'test-case-name':
                              'twisted.trial.test.test_tests'},
                             localVars)

    def test_parseLocalVariables(self):
        declaration = ('-*- test-case-name: twisted.trial.test.test_tests; '
                       'foo: bar -*-')
        localVars = trial._parseLocalVariables(declaration)
        self.assertEqual({'test-case-name':
                              'twisted.trial.test.test_tests',
                              'foo': 'bar'},
                             localVars)

    def test_surroundingGuff(self):
        declaration = ('## -*- test-case-name: '
                       'twisted.trial.test.test_tests -*- #')
        localVars = trial._parseLocalVariables(declaration)
        self.assertEqual({'test-case-name':
                              'twisted.trial.test.test_tests'},
                             localVars)

    def test_invalidLine(self):
        self.failUnlessRaises(ValueError, trial._parseLocalVariables,
                              'foo')

    def test_invalidDeclaration(self):
        self.failUnlessRaises(ValueError, trial._parseLocalVariables,
                              '-*- foo -*-')
        self.failUnlessRaises(ValueError, trial._parseLocalVariables,
                              '-*- foo: bar; qux -*-')
        self.failUnlessRaises(ValueError, trial._parseLocalVariables,
                              '-*- foo: bar: baz; qux: qax -*-')

    def test_variablesFromFile(self):
        localVars = trial.loadLocalVariables(sibpath('moduletest.py'))
        self.assertEqual({'test-case-name':
                              'twisted.trial.test.test_log'},
                             localVars)

    def test_noVariablesInFile(self):
        localVars = trial.loadLocalVariables(sibpath('novars.py'))
        self.assertEqual({}, localVars)

    def test_variablesFromScript(self):
        localVars = trial.loadLocalVariables(sibpath('scripttest.py'))
        self.assertEqual(
            {'test-case-name': ('twisted.trial.test.test_log,'
                                'twisted.trial.test.test_class')},
            localVars)

    def test_getTestModules(self):
        modules = trial.getTestModules(sibpath('moduletest.py'))
        self.assertEqual(modules, ['twisted.trial.test.test_log'])

    def test_getTestModules_noVars(self):
        modules = trial.getTestModules(sibpath('novars.py'))
        self.assertEqual(len(modules), 0)

    def test_getTestModules_multiple(self):
        modules = trial.getTestModules(sibpath('scripttest.py'))
        self.assertEqual(set(modules),
                             set(['twisted.trial.test.test_log',
                                  'twisted.trial.test.test_class']))

    def test_looksLikeTestModule(self):
        for filename in ['test_script.py', 'twisted/trial/test/test_script.py']:
            self.failUnless(trial.isTestFile(filename),
                            "%r should be a test file" % (filename,))
        for filename in ['twisted/trial/test/moduletest.py',
                         sibpath('scripttest.py'), sibpath('test_foo.bat')]:
            self.failIf(trial.isTestFile(filename),
                        "%r should *not* be a test file" % (filename,))


class WithoutModuleTests(unittest.SynchronousTestCase):
    """
    Test the C{without-module} flag.
    """

    def setUp(self):
        """
        Create a L{trial.Options} object to be used in the tests, and save
        C{sys.modules}.
        """
        self.config = trial.Options()
        self.savedModules = dict(sys.modules)


    def tearDown(self):
        """
        Restore C{sys.modules}.
        """
        for module in ('imaplib', 'smtplib'):
            if module in self.savedModules:
                sys.modules[module] = self.savedModules[module]
            else:
                sys.modules.pop(module, None)


    def _checkSMTP(self):
        """
        Try to import the C{smtplib} module, and return it.
        """
        import smtplib
        return smtplib


    def _checkIMAP(self):
        """
        Try to import the C{imaplib} module, and return it.
        """
        import imaplib
        return imaplib


    def test_disableOneModule(self):
        """
        Check that after disabling a module, it can't be imported anymore.
        """
        self.config.parseOptions(["--without-module", "smtplib"])
        self.assertRaises(ImportError, self._checkSMTP)
        # Restore sys.modules
        del sys.modules["smtplib"]
        # Then the function should succeed
        self.assertIsInstance(self._checkSMTP(), types.ModuleType)


    def test_disableMultipleModules(self):
        """
        Check that several modules can be disabled at once.
        """
        self.config.parseOptions(["--without-module", "smtplib,imaplib"])
        self.assertRaises(ImportError, self._checkSMTP)
        self.assertRaises(ImportError, self._checkIMAP)
        # Restore sys.modules
        del sys.modules["smtplib"]
        del sys.modules["imaplib"]
        # Then the functions should succeed
        self.assertIsInstance(self._checkSMTP(), types.ModuleType)
        self.assertIsInstance(self._checkIMAP(), types.ModuleType)


    def test_disableAlreadyImportedModule(self):
        """
        Disabling an already imported module should produce a warning.
        """
        self.assertIsInstance(self._checkSMTP(), types.ModuleType)
        self.assertWarns(RuntimeWarning,
                "Module 'smtplib' already imported, disabling anyway.",
                trial.__file__,
                self.config.parseOptions, ["--without-module", "smtplib"])
        self.assertRaises(ImportError, self._checkSMTP)


class CoverageTests(unittest.SynchronousTestCase):
    """
    Tests for the I{coverage} option.
    """
    if getattr(sys, 'gettrace', None) is None:
        skip = (
            "Cannot test trace hook installation without inspection API.")

    def setUp(self):
        """
        Arrange for the current trace hook to be restored when the
        test is complete.
        """
        self.addCleanup(sys.settrace, sys.gettrace())


    def test_tracerInstalled(self):
        """
        L{trial.Options} handles C{"--coverage"} by installing a trace
        hook to record coverage information.
        """
        options = trial.Options()
        options.parseOptions(["--coverage"])
        self.assertEqual(sys.gettrace(), options.tracer.globaltrace)


    def test_coverdirDefault(self):
        """
        L{trial.Options.coverdir} returns a L{FilePath} based on the default
        for the I{temp-directory} option if that option is not specified.
        """
        options = trial.Options()
        self.assertEqual(
            options.coverdir(),
            FilePath(".").descendant([options["temp-directory"], "coverage"]))


    def test_coverdirOverridden(self):
        """
        If a value is specified for the I{temp-directory} option,
        L{trial.Options.coverdir} returns a child of that path.
        """
        path = self.mktemp()
        options = trial.Options()
        options.parseOptions(["--temp-directory", path])
        self.assertEqual(
            options.coverdir(), FilePath(path).child("coverage"))


class OptionsTestCase(unittest.TestCase):
    """
    Tests for L{trial.Options}.
    """

    def setUp(self):
        """
        Build an L{Options} object to be used in the tests.
        """
        self.options = trial.Options()


    def test_getWorkerArguments(self):
        """
        C{_getWorkerArguments} discards options like C{random} as they only
        matter in the manager, and forwards options like C{recursionlimit} or
        C{disablegc}.
        """
        self.addCleanup(sys.setrecursionlimit, sys.getrecursionlimit())
        if gc.isenabled():
            self.addCleanup(gc.enable)

        self.options.parseOptions(["--recursionlimit", "2000", "--random",
                                   "4", "--disablegc"])
        args = self.options._getWorkerArguments()
        self.assertIn("--disablegc", args)
        args.remove("--disablegc")
        self.assertEqual(["--recursionlimit", "2000"], args)


    def test_jobsConflictWithDebug(self):
        """
        C{parseOptions} raises a C{UsageError} when C{--debug} is passed along
        C{--jobs} as it's not supported yet.

        @see: U{http://twistedmatrix.com/trac/ticket/5825}
        """
        error = self.assertRaises(
            UsageError, self.options.parseOptions, ["--jobs", "4", "--debug"])
        self.assertEqual("You can't specify --debug when using --jobs",
                         str(error))


    def test_jobsConflictWithProfile(self):
        """
        C{parseOptions} raises a C{UsageError} when C{--profile} is passed
        along C{--jobs} as it's not supported yet.

        @see: U{http://twistedmatrix.com/trac/ticket/5827}
        """
        error = self.assertRaises(
            UsageError, self.options.parseOptions,
            ["--jobs", "4", "--profile"])
        self.assertEqual("You can't specify --profile when using --jobs",
                         str(error))


    def test_jobsConflictWithDebugStackTraces(self):
        """
        C{parseOptions} raises a C{UsageError} when C{--debug-stacktraces} is
        passed along C{--jobs} as it's not supported yet.

        @see: U{http://twistedmatrix.com/trac/ticket/5826}
        """
        error = self.assertRaises(
            UsageError, self.options.parseOptions,
            ["--jobs", "4", "--debug-stacktraces"])
        self.assertEqual(
            "You can't specify --debug-stacktraces when using --jobs",
            str(error))


    def test_jobsConflictWithExitFirst(self):
        """
        C{parseOptions} raises a C{UsageError} when C{--exitfirst} is passed
        along C{--jobs} as it's not supported yet.

        @see: U{http://twistedmatrix.com/trac/ticket/6436}
        """
        error = self.assertRaises(
            UsageError, self.options.parseOptions,
            ["--jobs", "4", "--exitfirst"])
        self.assertEqual(
            "You can't specify --exitfirst when using --jobs",
            str(error))


    def test_orderConflictWithRandom(self):
        """
        C{parseOptions} raises a C{UsageError} when C{--order} is passed along
        with C{--random}.
        """
        error = self.assertRaises(
            UsageError,
            self.options.parseOptions,
            ["--order", "alphabetical", "--random", "1234"])
        self.assertEqual("You can't specify --random when using --order",
                         str(error))


class MakeRunnerTestCase(unittest.TestCase):
    """
    Tests for the L{_makeRunner} helper.
    """

    def setUp(self):
        self.options = trial.Options()

    def test_jobs(self):
        """
        L{_makeRunner} returns a L{DistTrialRunner} instance when the C{--jobs}
        option is passed, and passes the C{workerNumber} and C{workerArguments}
        parameters to it.
        """
        self.options.parseOptions(["--jobs", "4", "--force-gc"])
        runner = trial._makeRunner(self.options)
        self.assertIsInstance(runner, DistTrialRunner)
        self.assertEqual(4, runner._workerNumber)
        self.assertEqual(["--force-gc"], runner._workerArguments)


    def test_dryRunWithJobs(self):
        """
        L{_makeRunner} returns a L{TrialRunner} instance in C{DRY_RUN} mode
        when the C{--dry-run} option is passed, even if C{--jobs} is set.
        """
        self.options.parseOptions(["--jobs", "4", "--dry-run"])
        runner = trial._makeRunner(self.options)
        self.assertIsInstance(runner, TrialRunner)
        self.assertEqual(TrialRunner.DRY_RUN, runner.mode)


    def test_DebuggerNotFound(self):
        namedAny = trial.reflect.namedAny

        def namedAnyExceptdoNotFind(fqn):
            if fqn == "doNotFind":
                raise trial.reflect.ModuleNotFound(fqn)
            return namedAny(fqn)

        self.patch(trial.reflect, "namedAny", namedAnyExceptdoNotFind)

        options = trial.Options()
        options.parseOptions(["--debug", "--debugger", "doNotFind"])

        self.assertRaises(trial._DebuggerNotFound, trial._makeRunner, options)


    def test_exitfirst(self):
        """
        Passing C{--exitfirst} wraps the reporter with a
        L{reporter._ExitWrapper} that stops on any non-success.
        """
        self.options.parseOptions(["--exitfirst"])
        runner = trial._makeRunner(self.options)
        self.assertTrue(runner._exitFirst)


class TestRun(unittest.TestCase):
    """
    Tests for the L{run} function.
    """

    def setUp(self):
        # don't re-parse cmdline options, because if --reactor was passed to
        # the test run trial will try to restart the (already running) reactor
        self.patch(trial.Options, "parseOptions", lambda self: None)


    def test_debuggerNotFound(self):
        """
        When a debugger is not found, an error message is printed to the user.

        """

        def _makeRunner(*args, **kwargs):
            raise trial._DebuggerNotFound('foo')
        self.patch(trial, "_makeRunner", _makeRunner)

        try:
            trial.run()
        except SystemExit as e:
            self.assertIn("foo", str(e))
        else:
            self.fail("Should have exited due to non-existent debugger!")


class TestArgumentOrderTests(unittest.TestCase):
    """
    Tests for the order-preserving behavior on provided command-line tests.
    """

    def setUp(self):
        self.config = trial.Options()
        self.loader = TestLoader()


    def test_preserveArgumentOrder(self):
        """
        Multiple tests passed on the command line are not reordered.
        """
        tests = [
            "twisted.trial.test.test_tests",
            "twisted.trial.test.test_assertions",
            "twisted.trial.test.test_deferreds",
            ]
        self.config.parseOptions(tests)

        suite = trial._getSuite(self.config)
        names = testNames(suite)

        expectedSuite = TestSuite(map(self.loader.loadByName, tests))
        expectedNames = testNames(expectedSuite)

        self.assertEqual(names, expectedNames)


class OrderTests(unittest.TestCase):
    """
    Tests for the --order option.
    """
    def setUp(self):
        self.config = trial.Options()


    def test_alphabetical(self):
        """
        --order=alphabetical causes trial to run tests alphabetically within
        each test case.
        """
        self.config.parseOptions([
            "--order", "alphabetical",
            "twisted.trial.test.ordertests.FooTest"])

        loader = trial._getLoader(self.config)
        suite = loader.loadByNames(self.config['tests'])

        self.assertEqual(
            testNames(suite), [
            'twisted.trial.test.ordertests.FooTest.test_first',
            'twisted.trial.test.ordertests.FooTest.test_fourth',
            'twisted.trial.test.ordertests.FooTest.test_second',
            'twisted.trial.test.ordertests.FooTest.test_third'])


    def test_alphabeticalModule(self):
        """
        --order=alphabetical causes trial to run test classes within a given
        module alphabetically.
        """
        self.config.parseOptions([
            "--order", "alphabetical", "twisted.trial.test.ordertests"])
        loader = trial._getLoader(self.config)
        suite = loader.loadByNames(self.config['tests'])

        self.assertEqual(
            testNames(suite), [
            'twisted.trial.test.ordertests.BarTest.test_bar',
            'twisted.trial.test.ordertests.BazTest.test_baz',
            'twisted.trial.test.ordertests.FooTest.test_first',
            'twisted.trial.test.ordertests.FooTest.test_fourth',
            'twisted.trial.test.ordertests.FooTest.test_second',
            'twisted.trial.test.ordertests.FooTest.test_third'])


    def test_alphabeticalPackage(self):
        """
        --order=alphabetical causes trial to run test modules within a given
        package alphabetically, with tests within each module alphabetized.
        """
        self.config.parseOptions([
            "--order", "alphabetical", "twisted.trial.test"])
        loader = trial._getLoader(self.config)
        suite = loader.loadByNames(self.config['tests'])

        names = testNames(suite)
        self.assertTrue(names, msg="Failed to load any tests!")
        self.assertEqual(names, sorted(names))


    def test_toptobottom(self):
        """
        --order=toptobottom causes trial to run test methods within a given
        test case from top to bottom as they are defined in the body of the
        class.
        """
        self.config.parseOptions([
            "--order", "toptobottom",
            "twisted.trial.test.ordertests.FooTest"])

        loader = trial._getLoader(self.config)
        suite = loader.loadByNames(self.config['tests'])

        self.assertEqual(
            testNames(suite), [
            'twisted.trial.test.ordertests.FooTest.test_first',
            'twisted.trial.test.ordertests.FooTest.test_second',
            'twisted.trial.test.ordertests.FooTest.test_third',
            'twisted.trial.test.ordertests.FooTest.test_fourth'])


    def test_toptobottomModule(self):
        """
        --order=toptobottom causes trial to run test classes within a given
        module from top to bottom as they are defined in the module's source.
        """
        self.config.parseOptions([
            "--order", "toptobottom", "twisted.trial.test.ordertests"])
        loader = trial._getLoader(self.config)
        suite = loader.loadByNames(self.config['tests'])

        self.assertEqual(
            testNames(suite), [
            'twisted.trial.test.ordertests.FooTest.test_first',
            'twisted.trial.test.ordertests.FooTest.test_second',
            'twisted.trial.test.ordertests.FooTest.test_third',
            'twisted.trial.test.ordertests.FooTest.test_fourth',
            'twisted.trial.test.ordertests.BazTest.test_baz',
            'twisted.trial.test.ordertests.BarTest.test_bar'])


    def test_toptobottomPackage(self):
        """
        --order=toptobottom causes trial to run test modules within a given
        package alphabetically, with tests within each module run top to
        bottom.
        """
        self.config.parseOptions([
            "--order", "toptobottom", "twisted.trial.test"])
        loader = trial._getLoader(self.config)
        suite = loader.loadByNames(self.config['tests'])

        names = testNames(suite)
        # twisted.trial.test.test_module, so split and key on the first 4 to
        # get stable alphabetical sort on those
        self.assertEqual(
            names, sorted(names, key=lambda name : name.split(".")[:4]),
        )


    def test_toptobottomMissingSource(self):
        """
        --order=toptobottom detects the source line of methods from modules
        whose source file is missing.
        """
        tempdir = self.mktemp().encode('utf-8')
        package = FilePath(tempdir).child(b'twisted_toptobottom_temp')
        package.makedirs()
        package.child(b'__init__.py').setContent(b'')
        package.child(b'test_missing.py').setContent(textwrap.dedent(b'''
        from twisted.trial.unittest import TestCase
        class TestMissing(TestCase):
            def test_second(self): pass
            def test_third(self): pass
            def test_fourth(self): pass
            def test_first(self): pass
        '''))
        pathEntry = package.parent().path.decode('utf-8')
        sys.path.insert(0, pathEntry)
        self.addCleanup(sys.path.remove, pathEntry)
        from twisted_toptobottom_temp import test_missing
        self.addCleanup(sys.modules.pop, 'twisted_toptobottom_temp')
        self.addCleanup(sys.modules.pop, test_missing.__name__)
        package.child(b'test_missing.py').remove()

        self.config.parseOptions([
            "--order", "toptobottom", "twisted.trial.test.ordertests"])
        loader = trial._getLoader(self.config)
        suite = loader.loadModule(test_missing)

        self.assertEqual(
            testNames(suite), [
            'twisted_toptobottom_temp.test_missing.TestMissing.test_second',
            'twisted_toptobottom_temp.test_missing.TestMissing.test_third',
            'twisted_toptobottom_temp.test_missing.TestMissing.test_fourth',
            'twisted_toptobottom_temp.test_missing.TestMissing.test_first'])


    def test_unknownOrder(self):
        """
        An unknown order passed to --order raises a L{UsageError}.
        """

        self.assertRaises(
            UsageError, self.config.parseOptions, ["--order", "I don't exist"])


class HelpOrderTests(unittest.TestCase):
    """
    Tests for the --help-orders flag.
    """
    def test_help_ordersPrintsSynopsisAndQuits(self):
        """
        --help-orders prints each of the available orders and then exits.
        """
        self.patch(sys, "stdout", StringIO.StringIO())

        exc = self.assertRaises(
            SystemExit, trial.Options().parseOptions, ["--help-orders"])
        self.assertEqual(exc.code, 0)

        output = sys.stdout.getvalue()

        msg = "%r with its description not properly described in %r"
        for orderName, (orderDesc, _) in trial._runOrders.items():
            match = re.search(
                "%s.*%s" % (re.escape(orderName), re.escape(orderDesc)),
                output,
            )

            self.assertTrue(match, msg=msg % (orderName, output))