1/* 2 * Copyright 2001-2008 Artima, Inc. 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16package org.scalatest.testng 17 18import org.scalatest._ 19import org.scalatest.Suite 20import org.scalatest.TestRerunner 21import org.scalatest.events._ 22import Suite.getIndentedText 23import Suite.formatterForSuiteAborted 24import Suite.formatterForSuiteStarting 25import Suite.formatterForSuiteCompleted 26import events.MotionToSuppress 27 28import org.testng.TestNG 29import org.testng.TestListenerAdapter 30 31/** 32 * A suite of tests that can be run with either TestNG or ScalaTest. This trait allows you to mark any 33 * method as a test using TestNG's <code>@Test</code> annotation, and supports all other TestNG annotations. 34 * Here's an example: 35 * 36 * <p><b>BECAUSE OF A SCALADOC BUG IN SCALA 2.8, I HAD TO PUT A SPACE AFTER THE AT SIGN IN ANNOTATION EXAMPLES. IF YOU 37 * WANT TO COPY AND PASTE FROM THESE EXAMPLES, YOU'LL NEED TO REMOVE THE SPACE BY HAND, OR COPY FROM 38 * THE <a href="http://www.scalatest.org/scaladoc/doc-1.1/org/scalatest/testng/TestNGSuite.html">TESTNGSUITE SCALADOC FOR VERSION 1.1</a> INSTEAD, WHICH IS ALSO VALID FOR 1.3. - Bill Venners</b></p> 39 * 40 * <pre class="stHighlight"> 41 * import org.scalatest.testng.TestNGSuite 42 * import org.testng.annotations.Test 43 * import org.testng.annotations.Configuration 44 * import scala.collection.mutable.ListBuffer 45 * 46 * class MySuite extends TestNGSuite { 47 * 48 * var sb: StringBuilder = _ 49 * var lb: ListBuffer[String] = _ 50 * 51 * @ Configuration(beforeTestMethod = true) 52 * def setUpFixture() { 53 * sb = new StringBuilder("ScalaTest is ") 54 * lb = new ListBuffer[String] 55 * } 56 * 57 * @ Test(invocationCount = 3) 58 * def easyTest() { 59 * sb.append("easy!") 60 * assert(sb.toString === "ScalaTest is easy!") 61 * assert(lb.isEmpty) 62 * lb += "sweet" 63 * } 64 * 65 * @ Test(groups = Array("com.mycompany.groups.SlowTest")) 66 * def funTest() { 67 * sb.append("fun!") 68 * assert(sb.toString === "ScalaTest is fun!") 69 * assert(lb.isEmpty) 70 * } 71 * } 72 * </pre> 73 * 74 * <p> 75 * To execute <code>TestNGSuite</code>s with ScalaTest's <code>Runner</code>, you must include TestNG's jar file on the class path or runpath. 76 * This version of <code>TestNGSuite</code> was tested with TestNG version 5.7. 77 * </p> 78 * 79 * <p> 80 * See also: <a href="http://www.scalatest.org/getting_started_with_testng" target="_blank">Getting started with TestNG and ScalaTest.</a> 81 * </p> 82 * 83 * @author Josh Cough 84 * @author Bill Venners 85 */ 86trait TestNGSuite extends Suite { thisSuite => 87 88 /** 89 * Execute this <code>TestNGSuite</code>. 90 * 91 * @param testName an optional name of one test to execute. If <code>None</code>, this class will execute all relevant tests. 92 * I.e., <code>None</code> acts like a wildcard that means execute all relevant tests in this <code>TestNGSuite</code>. 93 * @param reporter The reporter to be notified of test events (success, failure, etc). 94 * @param groupsToInclude Contains the names of groups to run. Only tests in these groups will be executed. 95 * @param groupsToExclude Tests in groups in this Set will not be executed. 96 * 97 * @param stopper the <code>Stopper</code> may be used to request an early termination of a suite of tests. However, because TestNG does 98 * not support the notion of aborting a run early, this class ignores this parameter. 99 * @param properties a <code>Map</code> of properties that can be used by the executing <code>Suite</code> of tests. This class 100 * does not use this parameter. 101 * @param distributor an optional <code>Distributor</code>, into which nested <code>Suite</code>s could be put to be executed 102 * by another entity, such as concurrently by a pool of threads. If <code>None</code>, nested <code>Suite</code>s will be executed sequentially. 103 * Because TestNG handles its own concurrency, this class ignores this parameter. 104 * <br><br> 105 */ 106 override def run(testName: Option[String], reporter: Reporter, stopper: Stopper, 107 filter: Filter, properties: Map[String, Any], distributor: Option[Distributor], tracker: Tracker) { 108 109 runTestNG(testName, reporter, filter, tracker) 110 } 111 112 /** 113 * Runs TestNG with no test name, no groups. All tests in the class will be executed. 114 * @param reporter the reporter to be notified of test events (success, failure, etc) 115 */ 116 private[testng] def runTestNG(reporter: Reporter, tracker: Tracker) { 117 runTestNG(None, reporter, Filter(), tracker) 118 } 119 120 /** 121 * Runs TestNG, running only the test method with the given name. 122 * @param testName the name of the method to run 123 * @param reporter the reporter to be notified of test events (success, failure, etc) 124 */ 125 private[testng] def runTestNG(testName: String, reporter: Reporter, tracker: Tracker) { 126 runTestNG(Some(testName), reporter, Filter(), tracker) 127 } 128 129 /** 130 * Runs TestNG. The meat and potatoes. 131 * 132 * @param testName if present (Some), then only the method with the supplied name is executed and groups will be ignored 133 * @param reporter the reporter to be notified of test events (success, failure, etc) 134 * @param groupsToInclude contains the names of groups to run. only tests in these groups will be executed 135 * @param groupsToExclude tests in groups in this Set will not be executed 136 */ 137 private[testng] def runTestNG(testName: Option[String], reporter: Reporter, 138 filter: Filter, tracker: Tracker) { 139 140 val tagsToInclude = 141 filter.tagsToInclude match { 142 case None => Set[String]() 143 case Some(tti) => tti 144 } 145 val tagsToExclude = filter.tagsToExclude 146 147 val testng = new TestNG() 148 149 // only run the test methods in this class 150 testng.setTestClasses(Array(this.getClass)) 151 152 // if testName is supplied, ignore groups. 153 testName match { 154 case Some(tn) => setupTestNGToRunSingleMethod(tn, testng) 155 case None => handleGroups(tagsToInclude, tagsToExclude, testng) 156 } 157 158 this.run(testng, reporter, tracker) 159 } 160 161 /** 162 * Runs the TestNG object which calls back to the given Reporter. 163 */ 164 private[testng] def run(testng: TestNG, reporter: Reporter, tracker: Tracker) { 165 166 // setup the callback mechanism 167 val tla = new MyTestListenerAdapter(reporter, tracker) 168 testng.addListener(tla) 169 170 // finally, run TestNG 171 testng.run() 172 } 173 174 /** 175 * Tells TestNG which groups to include and exclude, which is directly a one-to-one mapping. 176 */ 177 private[testng] def handleGroups(groupsToInclude: Set[String], groupsToExclude: Set[String], testng: TestNG) { 178 testng.setGroups(groupsToInclude.mkString(",")) 179 testng.setExcludedGroups(groupsToExclude.mkString(",")) 180 } 181 182 /** 183 * This method ensures that TestNG will only run the test method whos name matches testName. 184 * 185 * The approach is a bit odd however because TestNG doesn't have an easy API for 186 * running a single method. To get around that we chose to use an AnnotationTransformer 187 * to add a secret group to the test method's annotation. We then set up TestNG to run only that group. 188 * 189 * NOTE: There was another option - we could TestNG's XmlSuites to specify which method to run. 190 * This approach was about as much work, offered no clear benefits, and no additional problems either. 191 * 192 * @param testName the name of the test method to be executed 193 */ 194 private def setupTestNGToRunSingleMethod(testName: String, testng: TestNG) = { 195 196 import org.testng.internal.annotations.IAnnotationTransformer 197 import org.testng.internal.annotations.ITest 198 import java.lang.reflect.Method 199 import java.lang.reflect.Constructor 200 201 class MyTransformer extends IAnnotationTransformer { 202 override def transform( annotation: ITest, testClass: java.lang.Class[_], testConstructor: Constructor[_], testMethod: Method){ 203 if (testName.equals(testMethod.getName)) { 204 annotation.setGroups(Array("org.scalatest.testng.singlemethodrun.methodname")) 205 } 206 } 207 } 208 testng.setGroups("org.scalatest.testng.singlemethodrun.methodname") 209 testng.setAnnotationTransformer(new MyTransformer()) 210 } 211 212 /** 213 * This class hooks TestNG's callback mechanism (TestListenerAdapter) to ScalaTest's 214 * reporting mechanism. TestNG has many different callback points which are a near one-to-one 215 * mapping with ScalaTest. At each callback point, this class simply creates ScalaTest 216 * reports and calls the appropriate method on the Reporter. 217 * 218 * TODO: 219 * (12:02:27 AM) bvenners: onTestFailedButWithinSuccessPercentage(ITestResult tr) 220 * (12:02:34 AM) bvenners: maybe a TestSucceeded with some extra info in the report 221 */ 222 private[testng] class MyTestListenerAdapter(reporter: Reporter, tracker: Tracker) extends TestListenerAdapter { 223 224 // TODO: Put the tracker in an atomic, because TestNG can go multithreaded? 225 226 val report = reporter 227 228 import org.testng.ITestContext 229 import org.testng.ITestResult 230 231 private val className = TestNGSuite.this.getClass.getName 232 233 /** 234 * This method is called when TestNG starts, and maps to ScalaTest's suiteStarting. 235 * @TODO TestNG doesn't seem to know how many tests are going to be executed. 236 * We are currently telling ScalaTest that 0 tests are about to be run. Investigate 237 * and/or chat with Cedric to determine if its possible to get this number from TestNG. 238 */ 239 override def onStart(itc: ITestContext) = { 240 val formatter = formatterForSuiteStarting(thisSuite) 241 report(SuiteStarting(tracker.nextOrdinal(), thisSuite.suiteName, Some(thisSuite.getClass.getName), formatter)) 242 } 243 244 /** 245 * TestNG's onFinish maps cleanly to suiteCompleted. 246 * TODO: TestNG does have some extra info here. One thing we could do is map the info 247 * in the ITestContext object into ScalaTest Reports and fire InfoProvided 248 */ 249 override def onFinish(itc: ITestContext) = { 250 val formatter = formatterForSuiteCompleted(thisSuite) 251 report(SuiteCompleted(tracker.nextOrdinal(), thisSuite.suiteName, Some(thisSuite.getClass.getName), None, formatter)) 252 } 253 254 /** 255 * TestNG's onTestStart maps cleanly to TestStarting. Simply build a report 256 * and pass it to the Reporter. 257 */ 258 override def onTestStart(result: ITestResult) = { 259 report(TestStarting(tracker.nextOrdinal(), thisSuite.suiteName, Some(thisSuite.getClass.getName), result.getName + params(result), 260 Some(MotionToSuppress), Some(new TestRerunner(className, result.getName)))) 261 } 262 263 /** 264 * TestNG's onTestSuccess maps cleanly to TestSucceeded. Again, simply build 265 * a report and pass it to the Reporter. 266 */ 267 override def onTestSuccess(result: ITestResult) = { 268 val testName = result.getName + params(result) 269 val formatter = getIndentedText(testName, 1, true) 270 report(TestSucceeded(tracker.nextOrdinal(), thisSuite.suiteName, Some(thisSuite.getClass.getName), testName, 271 None, Some(formatter), Some(new TestRerunner(className, result.getName)))) // Can I add a duration? 272 } 273 274 /** 275 * TestNG's onTestSkipped maps cleanly to TestIgnored. Again, simply build 276 * a report and pass it to the Reporter. 277 */ 278 override def onTestSkipped(result: ITestResult) = { 279 val testName = result.getName + params(result) 280 val formatter = getIndentedText(testName, 1, true) 281 report(TestIgnored(tracker.nextOrdinal(), thisSuite.suiteName, Some(thisSuite.getClass.getName), testName, Some(formatter))) 282 } 283 284 /** 285 * TestNG's onTestFailure maps cleanly to TestFailed. 286 */ 287 override def onTestFailure(result: ITestResult) = { 288 val throwableOrNull = result.getThrowable 289 val throwable = if (throwableOrNull != null) Some(throwableOrNull) else None 290 val message = if (throwableOrNull != null && throwableOrNull.getMessage != null) throwableOrNull.getMessage else Resources("testNGConfigFailed") 291 val testName = result.getName + params(result) 292 val formatter = getIndentedText(testName, 1, true) 293 report(TestFailed(tracker.nextOrdinal(), message, thisSuite.suiteName, Some(thisSuite.getClass.getName), testName, throwable, None, Some(formatter), Some(new TestRerunner(className, result.getName)))) // Can I add a duration? 294 } 295 296 /** 297 * A TestNG setup method resulted in an exception, and a test method will later fail to run. 298 * This TestNG callback method has the exception that caused the problem, as well 299 * as the name of the method that failed. Create a Report with the method name and the 300 * exception and call reporter(SuiteAborted). 301 */ 302 override def onConfigurationFailure(result: ITestResult) = { 303 val throwableOrNull = result.getThrowable 304 val throwable = if (throwableOrNull != null) Some(throwableOrNull) else None 305 val message = if (throwableOrNull != null && throwableOrNull.getMessage != null) throwableOrNull.getMessage else Resources("testNGConfigFailed") 306 val formatter = formatterForSuiteAborted(thisSuite, message) 307 report(SuiteAborted(tracker.nextOrdinal(), message, thisSuite.suiteName, Some(thisSuite.getClass.getName), throwable, None, formatter)) 308 } 309 310 /** 311 * TestNG's onConfigurationSuccess doesn't have a clean mapping in ScalaTest. 312 * Simply create a Report and fire InfoProvided. This works well 313 * because there may be a large number of setup methods and InfoProvided doesn't 314 * show up in your face on the UI, and so doesn't clutter the UI. 315 */ 316 override def onConfigurationSuccess(result: ITestResult) = { // TODO: Work on this report 317 // For now don't print anything. Succeed with silence. Is adding clutter. 318 // report(InfoProvided(tracker.nextOrdinal(), result.getName, Some(NameInfo(thisSuite.suiteName, Some(thisSuite.getClass.getName), None)))) 319 } 320 321 private def params(itr: ITestResult): String = { 322 itr.getParameters match { 323 case Array() => "" 324 case _ => "(" + itr.getParameters.mkString(",") + ")" 325 } 326 } 327 } 328 329 /** 330 TODO 331 (12:02:27 AM) bvenners: onTestFailedButWithinSuccessPercentage(ITestResult tr) 332 (12:02:34 AM) bvenners: maybe a TestSucceeded with some extra info in the report 333 **/ 334 335 /** 336 * Throws <code>UnsupportedOperationException</code>, because this method is unused by this 337 * class, given this class's <code>run</code> method delegates to JUnit to run 338 * its tests. 339 * 340 * <p> 341 * The main purpose of this method implementation is to render a compiler error an attempt 342 * to mix in a trait that overrides <code>withFixture</code>. Because this 343 * trait does not actually use <code>withFixture</code>, the attempt to mix 344 * in behavior would very likely not work. 345 * </p> 346 * 347 * 348 * @param test the no-arg test function to run with a fixture 349 */ 350 override final protected def withFixture(test: NoArgTest) { 351 throw new UnsupportedOperationException 352 } 353 354 /** 355 * Throws <code>UnsupportedOperationException</code>, because this method is unused by this 356 * trait, given this trait's <code>run</code> method delegates to TestNG to run 357 * its tests. 358 * 359 * <p> 360 * The main purpose of this method implementation is to render a compiler error an attempt 361 * to mix in a trait that overrides <code>runNestedSuites</code>. Because this 362 * trait does not actually use <code>runNestedSuites</code>, the attempt to mix 363 * in behavior would very likely not work. 364 * </p> 365 * 366 * @param reporter the <code>Reporter</code> to which results will be reported 367 * @param stopper the <code>Stopper</code> that will be consulted to determine whether to stop execution early. 368 * @param filter a <code>Filter</code> with which to filter tests based on their tags 369 * @param configMap a <code>Map</code> of key-value pairs that can be used by the executing <code>Suite</code> of tests. 370 * @param distributor an optional <code>Distributor</code>, into which to put nested <code>Suite</code>s to be run 371 * by another entity, such as concurrently by a pool of threads. If <code>None</code>, nested <code>Suite</code>s will be run sequentially. 372 * @param tracker a <code>Tracker</code> tracking <code>Ordinal</code>s being fired by the current thread. 373 * 374 * @throws UnsupportedOperationException always. 375 */ 376 override final protected def runNestedSuites(reporter: Reporter, stopper: Stopper, filter: Filter, 377 configMap: Map[String, Any], distributor: Option[Distributor], tracker: Tracker) { 378 379 throw new UnsupportedOperationException 380 } 381 382 /** 383 * Throws <code>UnsupportedOperationException</code>, because this method is unused by this 384 * trait, given this trait's <code>run</code> method delegates to TestNG to run 385 * its tests. 386 * 387 * <p> 388 * The main purpose of this method implementation is to render a compiler error an attempt 389 * to mix in a trait that overrides <code>runTests</code>. Because this 390 * trait does not actually use <code>runTests</code>, the attempt to mix 391 * in behavior would very likely not work. 392 * </p> 393 * 394 * @param testName an optional name of one test to run. If <code>None</code>, all relevant tests should be run. 395 * I.e., <code>None</code> acts like a wildcard that means run all relevant tests in this <code>Suite</code>. 396 * @param reporter the <code>Reporter</code> to which results will be reported 397 * @param stopper the <code>Stopper</code> that will be consulted to determine whether to stop execution early. 398 * @param filter a <code>Filter</code> with which to filter tests based on their tags 399 * @param configMap a <code>Map</code> of key-value pairs that can be used by the executing <code>Suite</code> of tests. 400 * @param distributor an optional <code>Distributor</code>, into which to put nested <code>Suite</code>s to be run 401 * by another entity, such as concurrently by a pool of threads. If <code>None</code>, nested <code>Suite</code>s will be run sequentially. 402 * @param tracker a <code>Tracker</code> tracking <code>Ordinal</code>s being fired by the current thread. 403 * @throws UnsupportedOperationException always. 404 */ 405 override protected final def runTests(testName: Option[String], reporter: Reporter, stopper: Stopper, filter: Filter, 406 configMap: Map[String, Any], distributor: Option[Distributor], tracker: Tracker) { 407 throw new UnsupportedOperationException 408 } 409 410 /** 411 * Throws <code>UnsupportedOperationException</code>, because this method is unused by this 412 * trait, given this trait's <code>run</code> method delegates to TestNG to run 413 * its tests. 414 * 415 * <p> 416 * The main purpose of this method implementation is to render a compiler error an attempt 417 * to mix in a trait that overrides <code>runTest</code>. Because this 418 * trait does not actually use <code>runTest</code>, the attempt to mix 419 * in behavior would very likely not work. 420 * </p> 421 * 422 * @param testName the name of one test to run. 423 * @param reporter the <code>Reporter</code> to which results will be reported 424 * @param stopper the <code>Stopper</code> that will be consulted to determine whether to stop execution early. 425 * @param configMap a <code>Map</code> of key-value pairs that can be used by the executing <code>Suite</code> of tests. 426 * @param tracker a <code>Tracker</code> tracking <code>Ordinal</code>s being fired by the current thread. 427 * @throws UnsupportedOperationException always. 428 */ 429 override protected final def runTest(testName: String, reporter: Reporter, stopper: Stopper, configMap: Map[String, Any], tracker: Tracker) { 430 431 throw new UnsupportedOperationException 432 } 433 434} 435