1-- | Core types and definitions 2{-# LANGUAGE GeneralizedNewtypeDeriving, FlexibleContexts, 3 ExistentialQuantification, RankNTypes, DeriveDataTypeable, NoMonomorphismRestriction, 4 DeriveGeneric #-} 5module Test.Tasty.Core where 6 7import Control.Exception 8import Test.Tasty.Providers.ConsoleFormat 9import Test.Tasty.Options 10import Test.Tasty.Patterns 11import Test.Tasty.Patterns.Types 12import Data.Foldable 13import qualified Data.Sequence as Seq 14import Data.Monoid 15import Data.Typeable 16import qualified Data.Map as Map 17import Data.Tagged 18import GHC.Generics 19import Prelude -- Silence AMP and FTP import warnings 20import Text.Printf 21 22-- | If a test failed, 'FailureReason' describes why 23data FailureReason 24 = TestFailed 25 -- ^ test provider indicated failure of the code to test, either because 26 -- the tested code returned wrong results, or raised an exception 27 | TestThrewException SomeException 28 -- ^ the test code itself raised an exception. Typical cases include missing 29 -- example input or output files. 30 -- 31 -- Usually, providers do not have to implement this, as their 'run' method 32 -- may simply raise an exception. 33 | TestTimedOut Integer 34 -- ^ test didn't complete in allotted time 35 | TestDepFailed -- See Note [Skipped tests] 36 -- ^ a dependency of this test failed, so this test was skipped. 37 deriving Show 38 39-- | Outcome of a test run 40-- 41-- Note: this is isomorphic to @'Maybe' 'FailureReason'@. You can use the 42-- @generic-maybe@ package to exploit that. 43data Outcome 44 = Success -- ^ test succeeded 45 | Failure FailureReason -- ^ test failed because of the 'FailureReason' 46 deriving (Show, Generic) 47 48-- | Time in seconds. Used to measure how long the tests took to run. 49type Time = Double 50 51-- | A test result 52data Result = Result 53 { resultOutcome :: Outcome 54 -- ^ Did the test fail? If so, why? 55 , resultDescription :: String 56 -- ^ 57 -- 'resultDescription' may contain some details about the test. For 58 -- a passed test it's ok to leave it empty. Providers like SmallCheck and 59 -- QuickCheck use it to provide information about how many tests were 60 -- generated. 61 -- 62 -- For a failed test, 'resultDescription' should typically provide more 63 -- information about the failure. 64 , resultShortDescription :: String 65 -- ^ The short description printed in the test run summary, usually @OK@ or 66 -- @FAIL@. 67 , resultTime :: Time 68 -- ^ How long it took to run the test, in seconds. 69 , resultDetailsPrinter :: ResultDetailsPrinter 70 -- ^ An action that prints additional information about a test. 71 -- 72 -- This is similar to 'resultDescription' except it can produce 73 -- colorful/formatted output; see "Test.Tasty.Providers.ConsoleFormat". 74 -- 75 -- This can be used instead of or in addition to 'resultDescription'. 76 -- 77 -- Usually this is set to 'noResultDetails', which does nothing. 78 -- 79 -- @since 1.3.1 80 } 81 deriving Show 82 83{- Note [Skipped tests] 84 ~~~~~~~~~~~~~~~~~~~~ 85 There are two potential ways to represent the tests that are skipped 86 because of their failed dependencies: 87 1. With Outcome = Failure, and FailureReason giving the specifics (TestDepFailed) 88 2. With a dedicated Outcome = Skipped 89 90 It seems to me that (1) will lead to fewer bugs (esp. in the extension packages), 91 because most of the time skipped tests should be handled in the same way 92 as failed tests. 93 But sometimes it is not obvious what the right behavior should be. E.g. 94 should --hide-successes show or hide the skipped tests? 95 96 Perhaps we should hide them, because they aren't really informative. 97 Or perhaps we shouldn't hide them, because we are not sure that they 98 will pass, and hiding them will imply a false sense of security 99 ("there's at most 2 tests failing", whereas in fact there could be much more). 100 101 So I might change this in the future, but for now treating them as 102 failures seems the easiest yet reasonable approach. 103-} 104 105-- | 'True' for a passed test, 'False' for a failed one. 106resultSuccessful :: Result -> Bool 107resultSuccessful r = 108 case resultOutcome r of 109 Success -> True 110 Failure {} -> False 111 112-- | Shortcut for creating a 'Result' that indicates exception 113exceptionResult :: SomeException -> Result 114exceptionResult e = Result 115 { resultOutcome = Failure $ TestThrewException e 116 , resultDescription = "Exception: " ++ show e 117 , resultShortDescription = "FAIL" 118 , resultTime = 0 119 , resultDetailsPrinter = noResultDetails 120 } 121 122-- | Test progress information. 123-- 124-- This may be used by a runner to provide some feedback to the user while 125-- a long-running test is executing. 126data Progress = Progress 127 { progressText :: String 128 -- ^ textual information about the test's progress 129 , progressPercent :: Float 130 -- ^ 131 -- 'progressPercent' should be a value between 0 and 1. If it's impossible 132 -- to compute the estimate, use 0. 133 } 134 deriving Show 135 136-- | The interface to be implemented by a test provider. 137-- 138-- The type @t@ is the concrete representation of the test which is used by 139-- the provider. 140class Typeable t => IsTest t where 141 -- | Run the test 142 -- 143 -- This method should cleanly catch any exceptions in the code to test, and 144 -- return them as part of the 'Result', see 'FailureReason' for an 145 -- explanation. It is ok for 'run' to raise an exception if there is a 146 -- problem with the test suite code itself (for example, if a file that 147 -- should contain example data or expected output is not found). 148 run 149 :: OptionSet -- ^ options 150 -> t -- ^ the test to run 151 -> (Progress -> IO ()) -- ^ a callback to report progress. 152 -- Note: the callback is a no-op at the moment 153 -- and there are no plans to use it; 154 -- feel free to ignore this argument for now. 155 -> IO Result 156 157 -- | The list of options that affect execution of tests of this type 158 testOptions :: Tagged t [OptionDescription] 159 160-- | The name of a test or a group of tests 161type TestName = String 162 163-- | 'ResourceSpec' describes how to acquire a resource (the first field) 164-- and how to release it (the second field). 165data ResourceSpec a = ResourceSpec (IO a) (a -> IO ()) 166 167-- | A resources-related exception 168data ResourceError 169 = NotRunningTests 170 | UnexpectedState String String 171 | UseOutsideOfTest 172 deriving Typeable 173 174instance Show ResourceError where 175 show NotRunningTests = 176 "Unhandled resource. Probably a bug in the runner you're using." 177 show (UnexpectedState where_ what) = 178 printf "Unexpected state of the resource (%s) in %s. Report as a tasty bug." 179 what where_ 180 show UseOutsideOfTest = 181 "It looks like you're attempting to use a resource outside of its test. Don't do that!" 182 183instance Exception ResourceError 184 185-- | These are the two ways in which one test may depend on the others. 186-- 187-- This is the same distinction as the 188-- <http://testng.org/doc/documentation-main.html#dependent-methods hard vs soft dependencies in TestNG>. 189-- 190-- @since 1.2 191data DependencyType 192 = AllSucceed 193 -- ^ The current test tree will be executed after its dependencies finish, and only 194 -- if all of the dependencies succeed. 195 | AllFinish 196 -- ^ The current test tree will be executed after its dependencies finish, 197 -- regardless of whether they succeed or not. 198 deriving (Eq, Show) 199 200-- | The main data structure defining a test suite. 201-- 202-- It consists of individual test cases and properties, organized in named 203-- groups which form a tree-like hierarchy. 204-- 205-- There is no generic way to create a test case. Instead, every test 206-- provider (tasty-hunit, tasty-smallcheck etc.) provides a function to 207-- turn a test case into a 'TestTree'. 208-- 209-- Groups can be created using 'testGroup'. 210data TestTree 211 = forall t . IsTest t => SingleTest TestName t 212 -- ^ A single test of some particular type 213 | TestGroup TestName [TestTree] 214 -- ^ Assemble a number of tests into a cohesive group 215 | PlusTestOptions (OptionSet -> OptionSet) TestTree 216 -- ^ Add some options to child tests 217 | forall a . WithResource (ResourceSpec a) (IO a -> TestTree) 218 -- ^ Acquire the resource before the tests in the inner tree start and 219 -- release it after they finish. The tree gets an `IO` action which 220 -- yields the resource, although the resource is shared across all the 221 -- tests. 222 | AskOptions (OptionSet -> TestTree) 223 -- ^ Ask for the options and customize the tests based on them 224 | After DependencyType Expr TestTree 225 -- ^ Only run after all tests that match a given pattern finish 226 -- (and, depending on the 'DependencyType', succeed) 227 228-- | Create a named group of test cases or other groups 229testGroup :: TestName -> [TestTree] -> TestTree 230testGroup = TestGroup 231 232-- | Like 'after', but accepts the pattern as a syntax tree instead 233-- of a string. Useful for generating a test tree programmatically. 234-- 235-- ==== __Examples__ 236-- 237-- Only match on the test's own name, ignoring the group names: 238-- 239-- @ 240-- 'after_' 'AllFinish' ('Test.Tasty.Patterns.Types.EQ' ('Field' 'NF') ('StringLit' \"Bar\")) $ 241-- 'testCase' \"A test that depends on Foo.Bar\" $ ... 242-- @ 243-- 244-- @since 1.2 245after_ 246 :: DependencyType -- ^ whether to run the tests even if some of the dependencies fail 247 -> Expr -- ^ the pattern 248 -> TestTree -- ^ the subtree that depends on other tests 249 -> TestTree -- ^ the subtree annotated with dependency information 250after_ = After 251 252-- | The 'after' combinator declares dependencies between tests. 253-- 254-- If a 'TestTree' is wrapped in 'after', the tests in this tree will not run 255-- until certain other tests («dependencies») have finished. These 256-- dependencies are specified using an AWK pattern (see the «Patterns» section 257-- in the README). 258-- 259-- Moreover, if the 'DependencyType' argument is set to 'AllSucceed' and 260-- at least one dependency has failed, this test tree will not run at all. 261-- 262-- Tasty does not check that the pattern matches any tests (let alone the 263-- correct set of tests), so it is on you to supply the right pattern. 264-- 265-- ==== __Examples__ 266-- 267-- The following test will be executed only after all tests that contain 268-- @Foo@ anywhere in their path finish. 269-- 270-- @ 271-- 'after' 'AllFinish' \"Foo\" $ 272-- 'testCase' \"A test that depends on Foo.Bar\" $ ... 273-- @ 274-- 275-- Note, however, that our test also happens to contain @Foo@ as part of its name, 276-- so it also matches the pattern and becomes a dependency of itself. This 277-- will result in a 'DependencyLoop' exception. To avoid this, either 278-- change the test name so that it doesn't mention @Foo@ or make the 279-- pattern more specific. 280-- 281-- You can use AWK patterns, for instance, to specify the full path to the dependency. 282-- 283-- @ 284-- 'after' 'AllFinish' \"$0 == \\\"Tests.Foo.Bar\\\"\" $ 285-- 'testCase' \"A test that depends on Foo.Bar\" $ ... 286-- @ 287-- 288-- Or only specify the dependency's own name, ignoring the group names: 289-- 290-- @ 291-- 'after' 'AllFinish' \"$NF == \\\"Bar\\\"\" $ 292-- 'testCase' \"A test that depends on Foo.Bar\" $ ... 293-- @ 294-- 295-- @since 1.2 296after 297 :: DependencyType -- ^ whether to run the tests even if some of the dependencies fail 298 -> String -- ^ the pattern 299 -> TestTree -- ^ the subtree that depends on other tests 300 -> TestTree -- ^ the subtree annotated with dependency information 301after deptype s = 302 case parseExpr s of 303 Nothing -> error $ "Could not parse pattern " ++ show s 304 Just e -> after_ deptype e 305 306-- | An algebra for folding a `TestTree`. 307-- 308-- Instead of constructing fresh records, build upon `trivialFold` 309-- instead. This way your code won't break when new nodes/fields are 310-- indroduced. 311data TreeFold b = TreeFold 312 { foldSingle :: forall t . IsTest t => OptionSet -> TestName -> t -> b 313 , foldGroup :: OptionSet -> TestName -> b -> b 314 , foldResource :: forall a . OptionSet -> ResourceSpec a -> (IO a -> b) -> b 315 , foldAfter :: OptionSet -> DependencyType -> Expr -> b -> b 316 } 317 318-- | 'trivialFold' can serve as the basis for custom folds. Just override 319-- the fields you need. 320-- 321-- Here's what it does: 322-- 323-- * single tests are mapped to `mempty` (you probably do want to override that) 324-- 325-- * test groups are returned unmodified 326-- 327-- * for a resource, an IO action that throws an exception is passed (you 328-- want to override this for runners/ingredients that execute tests) 329trivialFold :: Monoid b => TreeFold b 330trivialFold = TreeFold 331 { foldSingle = \_ _ _ -> mempty 332 , foldGroup = \_ _ b -> b 333 , foldResource = \_ _ f -> f $ throwIO NotRunningTests 334 , foldAfter = \_ _ _ b -> b 335 } 336 337-- | Fold a test tree into a single value. 338-- 339-- The fold result type should be a monoid. This is used to fold multiple 340-- results in a test group. In particular, empty groups get folded into 'mempty'. 341-- 342-- Apart from pure convenience, this function also does the following 343-- useful things: 344-- 345-- 1. Keeping track of the current options (which may change due to 346-- `PlusTestOptions` nodes) 347-- 348-- 2. Filtering out the tests which do not match the patterns 349-- 350-- Thus, it is preferred to an explicit recursive traversal of the tree. 351-- 352-- Note: right now, the patterns are looked up only once, and won't be 353-- affected by the subsequent option changes. This shouldn't be a problem 354-- in practice; OTOH, this behaviour may be changed later. 355foldTestTree 356 :: forall b . Monoid b 357 => TreeFold b 358 -- ^ the algebra (i.e. how to fold a tree) 359 -> OptionSet 360 -- ^ initial options 361 -> TestTree 362 -- ^ the tree to fold 363 -> b 364foldTestTree (TreeFold fTest fGroup fResource fAfter) opts0 tree0 = 365 go mempty opts0 tree0 366 where 367 go :: (Seq.Seq TestName -> OptionSet -> TestTree -> b) 368 go path opts tree1 = 369 case tree1 of 370 SingleTest name test 371 | testPatternMatches pat (path Seq.|> name) 372 -> fTest opts name test 373 | otherwise -> mempty 374 TestGroup name trees -> 375 fGroup opts name $ foldMap (go (path Seq.|> name) opts) trees 376 PlusTestOptions f tree -> go path (f opts) tree 377 WithResource res0 tree -> fResource opts res0 $ \res -> go path opts (tree res) 378 AskOptions f -> go path opts (f opts) 379 After deptype dep tree -> fAfter opts deptype dep $ go path opts tree 380 where 381 pat = lookupOption opts :: TestPattern 382 383-- | Get the list of options that are relevant for a given test tree 384treeOptions :: TestTree -> [OptionDescription] 385treeOptions = 386 387 Prelude.concat . 388 Map.elems . 389 390 foldTestTree 391 trivialFold { foldSingle = \_ _ -> getTestOptions } 392 mempty 393 394 where 395 getTestOptions 396 :: forall t . IsTest t 397 => t -> Map.Map TypeRep [OptionDescription] 398 getTestOptions t = 399 Map.singleton (typeOf t) $ 400 witness testOptions t 401