1package integration_test 2 3import ( 4 "fmt" 5 "io/ioutil" 6 "os" 7 "regexp" 8 "runtime" 9 "strings" 10 11 . "github.com/onsi/ginkgo" 12 "github.com/onsi/ginkgo/types" 13 . "github.com/onsi/gomega" 14 "github.com/onsi/gomega/gbytes" 15 "github.com/onsi/gomega/gexec" 16) 17 18var _ = Describe("Running Specs", func() { 19 var pathToTest string 20 21 isWindows := (runtime.GOOS == "windows") 22 denoter := "•" 23 24 if isWindows { 25 denoter = "+" 26 } 27 28 Context("when pointed at the current directory", func() { 29 BeforeEach(func() { 30 pathToTest = tmpPath("ginkgo") 31 copyIn(fixturePath("passing_ginkgo_tests"), pathToTest, false) 32 }) 33 34 It("should run the tests in the working directory", func() { 35 session := startGinkgo(pathToTest, "--noColor") 36 Eventually(session).Should(gexec.Exit(0)) 37 output := string(session.Out.Contents()) 38 39 Ω(output).Should(ContainSubstring("Running Suite: Passing_ginkgo_tests Suite")) 40 Ω(output).Should(ContainSubstring(strings.Repeat(denoter, 4))) 41 Ω(output).Should(ContainSubstring("SUCCESS! -- 4 Passed")) 42 Ω(output).Should(ContainSubstring("Test Suite Passed")) 43 }) 44 }) 45 46 Context("when passed an explicit package to run", func() { 47 BeforeEach(func() { 48 pathToTest = tmpPath("ginkgo") 49 copyIn(fixturePath("passing_ginkgo_tests"), pathToTest, false) 50 }) 51 52 It("should run the ginkgo style tests", func() { 53 session := startGinkgo(tmpDir, "--noColor", "ginkgo") 54 Eventually(session).Should(gexec.Exit(0)) 55 output := string(session.Out.Contents()) 56 57 Ω(output).Should(ContainSubstring("Running Suite: Passing_ginkgo_tests Suite")) 58 Ω(output).Should(ContainSubstring(strings.Repeat(denoter, 4))) 59 Ω(output).Should(ContainSubstring("SUCCESS! -- 4 Passed")) 60 Ω(output).Should(ContainSubstring("Test Suite Passed")) 61 }) 62 }) 63 64 Context("when passed a number of packages to run", func() { 65 BeforeEach(func() { 66 pathToTest = tmpPath("ginkgo") 67 otherPathToTest := tmpPath("other") 68 copyIn(fixturePath("passing_ginkgo_tests"), pathToTest, false) 69 copyIn(fixturePath("more_ginkgo_tests"), otherPathToTest, false) 70 }) 71 72 It("should run the ginkgo style tests", func() { 73 session := startGinkgo(tmpDir, "--noColor", "--succinct=false", "ginkgo", "./other") 74 Eventually(session).Should(gexec.Exit(0)) 75 output := string(session.Out.Contents()) 76 77 Ω(output).Should(ContainSubstring("Running Suite: Passing_ginkgo_tests Suite")) 78 Ω(output).Should(ContainSubstring("Running Suite: More_ginkgo_tests Suite")) 79 Ω(output).Should(ContainSubstring("Test Suite Passed")) 80 }) 81 }) 82 83 Context("when passed a number of packages to run, some of which have focused tests", func() { 84 BeforeEach(func() { 85 pathToTest = tmpPath("ginkgo") 86 otherPathToTest := tmpPath("other") 87 focusedPathToTest := tmpPath("focused") 88 copyIn(fixturePath("passing_ginkgo_tests"), pathToTest, false) 89 copyIn(fixturePath("more_ginkgo_tests"), otherPathToTest, false) 90 copyIn(fixturePath("focused_fixture"), focusedPathToTest, false) 91 }) 92 93 It("should exit with a status code of 2 and explain why", func() { 94 session := startGinkgo(tmpDir, "--noColor", "--succinct=false", "-r") 95 Eventually(session).Should(gexec.Exit(types.GINKGO_FOCUS_EXIT_CODE)) 96 output := string(session.Out.Contents()) 97 98 Ω(output).Should(ContainSubstring("Running Suite: Passing_ginkgo_tests Suite")) 99 Ω(output).Should(ContainSubstring("Running Suite: More_ginkgo_tests Suite")) 100 Ω(output).Should(ContainSubstring("Test Suite Passed")) 101 Ω(output).Should(ContainSubstring("Detected Programmatic Focus - setting exit status to %d", types.GINKGO_FOCUS_EXIT_CODE)) 102 }) 103 104 Context("when the GINKGO_EDITOR_INTEGRATION environment variable is set", func() { 105 BeforeEach(func() { 106 os.Setenv("GINKGO_EDITOR_INTEGRATION", "true") 107 }) 108 AfterEach(func() { 109 os.Setenv("GINKGO_EDITOR_INTEGRATION", "") 110 }) 111 It("should exit with a status code of 0 to allow a coverage file to be generated", func() { 112 session := startGinkgo(tmpDir, "--noColor", "--succinct=false", "-r") 113 Eventually(session).Should(gexec.Exit(0)) 114 output := string(session.Out.Contents()) 115 116 Ω(output).Should(ContainSubstring("Running Suite: Passing_ginkgo_tests Suite")) 117 Ω(output).Should(ContainSubstring("Running Suite: More_ginkgo_tests Suite")) 118 Ω(output).Should(ContainSubstring("Test Suite Passed")) 119 }) 120 }) 121 }) 122 123 Context("when told to skipPackages", func() { 124 BeforeEach(func() { 125 pathToTest = tmpPath("ginkgo") 126 otherPathToTest := tmpPath("other") 127 focusedPathToTest := tmpPath("focused") 128 copyIn(fixturePath("passing_ginkgo_tests"), pathToTest, false) 129 copyIn(fixturePath("more_ginkgo_tests"), otherPathToTest, false) 130 copyIn(fixturePath("focused_fixture"), focusedPathToTest, false) 131 }) 132 133 It("should skip packages that match the list", func() { 134 session := startGinkgo(tmpDir, "--noColor", "--skipPackage=other,focused", "-r") 135 Eventually(session).Should(gexec.Exit(0)) 136 output := string(session.Out.Contents()) 137 138 Ω(output).Should(ContainSubstring("Passing_ginkgo_tests Suite")) 139 Ω(output).ShouldNot(ContainSubstring("More_ginkgo_tests Suite")) 140 Ω(output).ShouldNot(ContainSubstring("Focused_fixture Suite")) 141 Ω(output).Should(ContainSubstring("Test Suite Passed")) 142 }) 143 144 Context("when all packages are skipped", func() { 145 It("should not run anything, but still exit 0", func() { 146 session := startGinkgo(tmpDir, "--noColor", "--skipPackage=other,focused,ginkgo", "-r") 147 Eventually(session).Should(gexec.Exit(0)) 148 output := string(session.Out.Contents()) 149 150 Ω(output).Should(ContainSubstring("All tests skipped!")) 151 Ω(output).ShouldNot(ContainSubstring("Passing_ginkgo_tests Suite")) 152 Ω(output).ShouldNot(ContainSubstring("More_ginkgo_tests Suite")) 153 Ω(output).ShouldNot(ContainSubstring("Focused_fixture Suite")) 154 Ω(output).ShouldNot(ContainSubstring("Test Suite Passed")) 155 }) 156 }) 157 }) 158 159 Context("when there are no tests to run", func() { 160 It("should exit 1", func() { 161 session := startGinkgo(tmpDir, "--noColor", "--skipPackage=other,focused", "-r") 162 Eventually(session).Should(gexec.Exit(1)) 163 output := string(session.Err.Contents()) 164 165 Ω(output).Should(ContainSubstring("Found no test suites")) 166 }) 167 }) 168 169 Context("when there are test files but `go test` reports there are no tests to run", func() { 170 BeforeEach(func() { 171 pathToTest = tmpPath("ginkgo") 172 copyIn(fixturePath("no_test_fn"), pathToTest, false) 173 }) 174 175 It("suggests running ginkgo bootstrap", func() { 176 session := startGinkgo(tmpDir, "--noColor", "--skipPackage=other,focused", "-r") 177 Eventually(session).Should(gexec.Exit(0)) 178 output := string(session.Err.Contents()) 179 180 Ω(output).Should(ContainSubstring(`Found no test suites, did you forget to run "ginkgo bootstrap"?`)) 181 }) 182 183 It("fails if told to requireSuite", func() { 184 session := startGinkgo(tmpDir, "--noColor", "--skipPackage=other,focused", "-r", "-requireSuite") 185 Eventually(session).Should(gexec.Exit(1)) 186 output := string(session.Err.Contents()) 187 188 Ω(output).Should(ContainSubstring(`Found no test suites, did you forget to run "ginkgo bootstrap"?`)) 189 }) 190 }) 191 192 Context("when told to randomizeSuites", func() { 193 BeforeEach(func() { 194 pathToTest = tmpPath("ginkgo") 195 otherPathToTest := tmpPath("other") 196 copyIn(fixturePath("passing_ginkgo_tests"), pathToTest, false) 197 copyIn(fixturePath("more_ginkgo_tests"), otherPathToTest, false) 198 }) 199 200 It("should skip packages that match the regexp", func() { 201 session := startGinkgo(tmpDir, "--noColor", "--randomizeSuites", "-r", "--seed=2") 202 Eventually(session).Should(gexec.Exit(0)) 203 204 Ω(session).Should(gbytes.Say("More_ginkgo_tests Suite")) 205 Ω(session).Should(gbytes.Say("Passing_ginkgo_tests Suite")) 206 207 session = startGinkgo(tmpDir, "--noColor", "--randomizeSuites", "-r", "--seed=3") 208 Eventually(session).Should(gexec.Exit(0)) 209 210 Ω(session).Should(gbytes.Say("Passing_ginkgo_tests Suite")) 211 Ω(session).Should(gbytes.Say("More_ginkgo_tests Suite")) 212 }) 213 }) 214 215 Context("when pointed at a package with xunit style tests", func() { 216 BeforeEach(func() { 217 pathToTest = tmpPath("xunit") 218 copyIn(fixturePath("xunit_tests"), pathToTest, false) 219 }) 220 221 It("should run the xunit style tests", func() { 222 session := startGinkgo(pathToTest) 223 Eventually(session).Should(gexec.Exit(0)) 224 output := string(session.Out.Contents()) 225 226 Ω(output).Should(ContainSubstring("--- PASS: TestAlwaysTrue")) 227 Ω(output).Should(ContainSubstring("Test Suite Passed")) 228 }) 229 }) 230 231 Context("when pointed at a package with no tests", func() { 232 BeforeEach(func() { 233 pathToTest = tmpPath("no_tests") 234 copyIn(fixturePath("no_tests"), pathToTest, false) 235 }) 236 237 It("should fail", func() { 238 session := startGinkgo(pathToTest, "--noColor") 239 Eventually(session).Should(gexec.Exit(1)) 240 241 Ω(session.Err.Contents()).Should(ContainSubstring("Found no test suites")) 242 }) 243 }) 244 245 Context("when pointed at a package that fails to compile", func() { 246 BeforeEach(func() { 247 pathToTest = tmpPath("does_not_compile") 248 copyIn(fixturePath("does_not_compile"), pathToTest, false) 249 }) 250 251 It("should fail", func() { 252 session := startGinkgo(pathToTest, "--noColor") 253 Eventually(session).Should(gexec.Exit(1)) 254 output := string(session.Out.Contents()) 255 256 Ω(output).Should(ContainSubstring("Failed to compile")) 257 }) 258 }) 259 260 Context("when running in parallel", func() { 261 BeforeEach(func() { 262 pathToTest = tmpPath("ginkgo") 263 copyIn(fixturePath("passing_ginkgo_tests"), pathToTest, false) 264 }) 265 266 Context("with a specific number of -nodes", func() { 267 It("should use the specified number of nodes", func() { 268 session := startGinkgo(pathToTest, "--noColor", "-succinct", "-nodes=2") 269 Eventually(session).Should(gexec.Exit(0)) 270 output := string(session.Out.Contents()) 271 272 Ω(output).Should(MatchRegexp(`\[\d+\] Passing_ginkgo_tests Suite - 4 specs - 2 nodes [%s]{4} SUCCESS! \d+(\.\d+)?[muµ]s`, regexp.QuoteMeta(denoter))) 273 Ω(output).Should(ContainSubstring("Test Suite Passed")) 274 }) 275 }) 276 277 Context("with -p", func() { 278 It("it should autocompute the number of nodes", func() { 279 session := startGinkgo(pathToTest, "--noColor", "-succinct", "-p") 280 Eventually(session).Should(gexec.Exit(0)) 281 output := string(session.Out.Contents()) 282 283 nodes := runtime.NumCPU() 284 if nodes == 1 { 285 Skip("Can't test parallel testings with 1 CPU") 286 } 287 if nodes > 4 { 288 nodes = nodes - 1 289 } 290 Ω(output).Should(MatchRegexp(`\[\d+\] Passing_ginkgo_tests Suite - 4 specs - %d nodes [%s]{4} SUCCESS! \d+(\.\d+)?[muµ]?s`, nodes, regexp.QuoteMeta(denoter))) 291 Ω(output).Should(ContainSubstring("Test Suite Passed")) 292 }) 293 }) 294 }) 295 296 Context("when running in parallel with -debug", func() { 297 BeforeEach(func() { 298 pathToTest = tmpPath("ginkgo") 299 copyIn(fixturePath("debug_parallel_fixture"), pathToTest, false) 300 }) 301 302 Context("without -v", func() { 303 It("should emit node output to files on disk", func() { 304 session := startGinkgo(pathToTest, "--nodes=2", "--debug") 305 Eventually(session).Should(gexec.Exit(0)) 306 307 f0, err := ioutil.ReadFile(pathToTest + "/ginkgo-node-1.log") 308 Ω(err).ShouldNot(HaveOccurred()) 309 f1, err := ioutil.ReadFile(pathToTest + "/ginkgo-node-2.log") 310 Ω(err).ShouldNot(HaveOccurred()) 311 content := string(append(f0, f1...)) 312 313 for i := 0; i < 10; i += 1 { 314 Ω(content).Should(ContainSubstring("StdOut %d\n", i)) 315 Ω(content).Should(ContainSubstring("GinkgoWriter %d\n", i)) 316 } 317 }) 318 }) 319 320 Context("without -v", func() { 321 It("should emit node output to files on disk, without duplicating the GinkgoWriter output", func() { 322 session := startGinkgo(pathToTest, "--nodes=2", "--debug", "-v") 323 Eventually(session).Should(gexec.Exit(0)) 324 325 f0, err := ioutil.ReadFile(pathToTest + "/ginkgo-node-1.log") 326 Ω(err).ShouldNot(HaveOccurred()) 327 f1, err := ioutil.ReadFile(pathToTest + "/ginkgo-node-2.log") 328 Ω(err).ShouldNot(HaveOccurred()) 329 content := string(append(f0, f1...)) 330 331 out := strings.Split(content, "GinkgoWriter 2") 332 Ω(out).Should(HaveLen(2)) 333 }) 334 }) 335 }) 336 337 Context("when streaming in parallel", func() { 338 BeforeEach(func() { 339 pathToTest = tmpPath("ginkgo") 340 copyIn(fixturePath("passing_ginkgo_tests"), pathToTest, false) 341 }) 342 343 It("should print output in realtime", func() { 344 session := startGinkgo(pathToTest, "--noColor", "-stream", "-nodes=2") 345 Eventually(session).Should(gexec.Exit(0)) 346 output := string(session.Out.Contents()) 347 348 Ω(output).Should(ContainSubstring(`[1] Parallel test node 1/2.`)) 349 Ω(output).Should(ContainSubstring(`[2] Parallel test node 2/2.`)) 350 Ω(output).Should(ContainSubstring(`[1] SUCCESS!`)) 351 Ω(output).Should(ContainSubstring(`[2] SUCCESS!`)) 352 Ω(output).Should(ContainSubstring("Test Suite Passed")) 353 }) 354 }) 355 356 Context("when running recursively", func() { 357 BeforeEach(func() { 358 passingTest := tmpPath("A") 359 otherPassingTest := tmpPath("E") 360 copyIn(fixturePath("passing_ginkgo_tests"), passingTest, false) 361 copyIn(fixturePath("more_ginkgo_tests"), otherPassingTest, false) 362 }) 363 364 Context("when all the tests pass", func() { 365 Context("with the -r flag", func() { 366 It("should run all the tests (in succinct mode) and succeed", func() { 367 session := startGinkgo(tmpDir, "--noColor", "-r", ".") 368 Eventually(session).Should(gexec.Exit(0)) 369 output := string(session.Out.Contents()) 370 371 outputLines := strings.Split(output, "\n") 372 Ω(outputLines[0]).Should(MatchRegexp(`\[\d+\] Passing_ginkgo_tests Suite - 4/4 specs [%s]{4} SUCCESS! \d+(\.\d+)?[muµ]s PASS`, regexp.QuoteMeta(denoter))) 373 Ω(outputLines[1]).Should(MatchRegexp(`\[\d+\] More_ginkgo_tests Suite - 2/2 specs [%s]{2} SUCCESS! \d+(\.\d+)?[muµ]s PASS`, regexp.QuoteMeta(denoter))) 374 Ω(output).Should(ContainSubstring("Test Suite Passed")) 375 }) 376 }) 377 Context("with a trailing /...", func() { 378 It("should run all the tests (in succinct mode) and succeed", func() { 379 session := startGinkgo(tmpDir, "--noColor", "./...") 380 Eventually(session).Should(gexec.Exit(0)) 381 output := string(session.Out.Contents()) 382 383 outputLines := strings.Split(output, "\n") 384 Ω(outputLines[0]).Should(MatchRegexp(`\[\d+\] Passing_ginkgo_tests Suite - 4/4 specs [%s]{4} SUCCESS! \d+(\.\d+)?[muµ]s PASS`, regexp.QuoteMeta(denoter))) 385 Ω(outputLines[1]).Should(MatchRegexp(`\[\d+\] More_ginkgo_tests Suite - 2/2 specs [%s]{2} SUCCESS! \d+(\.\d+)?[muµ]s PASS`, regexp.QuoteMeta(denoter))) 386 Ω(output).Should(ContainSubstring("Test Suite Passed")) 387 }) 388 }) 389 }) 390 391 Context("when one of the packages has a failing tests", func() { 392 BeforeEach(func() { 393 failingTest := tmpPath("C") 394 copyIn(fixturePath("failing_ginkgo_tests"), failingTest, false) 395 }) 396 397 It("should fail and stop running tests", func() { 398 session := startGinkgo(tmpDir, "--noColor", "-r") 399 Eventually(session).Should(gexec.Exit(1)) 400 output := string(session.Out.Contents()) 401 402 outputLines := strings.Split(output, "\n") 403 Ω(outputLines[0]).Should(MatchRegexp(`\[\d+\] Passing_ginkgo_tests Suite - 4/4 specs [%s]{4} SUCCESS! \d+(\.\d+)?[muµ]s PASS`, regexp.QuoteMeta(denoter))) 404 Ω(outputLines[1]).Should(MatchRegexp(`\[\d+\] Failing_ginkgo_tests Suite - 2/2 specs`)) 405 Ω(output).Should(ContainSubstring(fmt.Sprintf("%s Failure", denoter))) 406 Ω(output).ShouldNot(ContainSubstring("More_ginkgo_tests Suite")) 407 Ω(output).Should(ContainSubstring("Test Suite Failed")) 408 409 Ω(output).Should(ContainSubstring("Summarizing 1 Failure:")) 410 Ω(output).Should(ContainSubstring("[Fail] FailingGinkgoTests [It] should fail")) 411 }) 412 }) 413 414 Context("when one of the packages fails to compile", func() { 415 BeforeEach(func() { 416 doesNotCompileTest := tmpPath("C") 417 copyIn(fixturePath("does_not_compile"), doesNotCompileTest, false) 418 }) 419 420 It("should fail and stop running tests", func() { 421 session := startGinkgo(tmpDir, "--noColor", "-r") 422 Eventually(session).Should(gexec.Exit(1)) 423 output := string(session.Out.Contents()) 424 425 outputLines := strings.Split(output, "\n") 426 Ω(outputLines[0]).Should(MatchRegexp(`\[\d+\] Passing_ginkgo_tests Suite - 4/4 specs [%s]{4} SUCCESS! \d+(\.\d+)?[muµ]s PASS`, regexp.QuoteMeta(denoter))) 427 Ω(outputLines[1]).Should(ContainSubstring("Failed to compile C:")) 428 Ω(output).ShouldNot(ContainSubstring("More_ginkgo_tests Suite")) 429 Ω(output).Should(ContainSubstring("Test Suite Failed")) 430 }) 431 }) 432 433 Context("when either is the case, but the keepGoing flag is set", func() { 434 BeforeEach(func() { 435 doesNotCompileTest := tmpPath("B") 436 copyIn(fixturePath("does_not_compile"), doesNotCompileTest, false) 437 438 failingTest := tmpPath("C") 439 copyIn(fixturePath("failing_ginkgo_tests"), failingTest, false) 440 }) 441 442 It("should soldier on", func() { 443 session := startGinkgo(tmpDir, "--noColor", "-r", "-keepGoing") 444 Eventually(session).Should(gexec.Exit(1)) 445 output := string(session.Out.Contents()) 446 447 outputLines := strings.Split(output, "\n") 448 Ω(outputLines[0]).Should(MatchRegexp(`\[\d+\] Passing_ginkgo_tests Suite - 4/4 specs [%s]{4} SUCCESS! \d+(\.\d+)?[muµ]s PASS`, regexp.QuoteMeta(denoter))) 449 Ω(outputLines[1]).Should(ContainSubstring("Failed to compile B:")) 450 Ω(output).Should(MatchRegexp(`\[\d+\] Failing_ginkgo_tests Suite - 2/2 specs`)) 451 Ω(output).Should(ContainSubstring(fmt.Sprintf("%s Failure", denoter))) 452 Ω(output).Should(MatchRegexp(`\[\d+\] More_ginkgo_tests Suite - 2/2 specs [%s]{2} SUCCESS! \d+(\.\d+)?[muµ]s PASS`, regexp.QuoteMeta(denoter))) 453 Ω(output).Should(ContainSubstring("Test Suite Failed")) 454 }) 455 }) 456 }) 457 458 Context("when told to keep going --untilItFails", func() { 459 BeforeEach(func() { 460 copyIn(fixturePath("eventually_failing"), tmpDir, false) 461 }) 462 463 It("should keep rerunning the tests, until a failure occurs", func() { 464 session := startGinkgo(tmpDir, "--untilItFails", "--noColor") 465 Eventually(session).Should(gexec.Exit(1)) 466 Ω(session).Should(gbytes.Say("This was attempt #1")) 467 Ω(session).Should(gbytes.Say("This was attempt #2")) 468 Ω(session).Should(gbytes.Say("Tests failed on attempt #3")) 469 470 //it should change the random seed between each test 471 lines := strings.Split(string(session.Out.Contents()), "\n") 472 randomSeeds := []string{} 473 for _, line := range lines { 474 if strings.Contains(line, "Random Seed:") { 475 randomSeeds = append(randomSeeds, strings.Split(line, ": ")[1]) 476 } 477 } 478 Ω(randomSeeds[0]).ShouldNot(Equal(randomSeeds[1])) 479 Ω(randomSeeds[1]).ShouldNot(Equal(randomSeeds[2])) 480 Ω(randomSeeds[0]).ShouldNot(Equal(randomSeeds[2])) 481 }) 482 }) 483}) 484