1// Copyright 2011 Aaron Jacobs. All Rights Reserved. 2// Author: aaronjjacobs@gmail.com (Aaron Jacobs) 3// 4// Licensed under the Apache License, Version 2.0 (the "License"); 5// you may not use this file except in compliance with the License. 6// You may obtain a copy of the License at 7// 8// http://www.apache.org/licenses/LICENSE-2.0 9// 10// Unless required by applicable law or agreed to in writing, software 11// distributed under the License is distributed on an "AS IS" BASIS, 12// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13// See the License for the specific language governing permissions and 14// limitations under the License. 15 16package ogletest 17 18import ( 19 "bytes" 20 "flag" 21 "fmt" 22 "os" 23 "path" 24 "regexp" 25 "runtime" 26 "sync" 27 "sync/atomic" 28 "testing" 29 "time" 30 31 "github.com/smartystreets/assertions/internal/reqtrace" 32) 33 34var fTestFilter = flag.String( 35 "ogletest.run", 36 "", 37 "Regexp for matching tests to run.") 38 39var fStopEarly = flag.Bool( 40 "ogletest.stop_early", 41 false, 42 "If true, stop after the first failure.") 43 44// runTestsOnce protects RunTests from executing multiple times. 45var runTestsOnce sync.Once 46 47func isAbortError(x interface{}) bool { 48 _, ok := x.(abortError) 49 return ok 50} 51 52// Run a single test function, returning a slice of failure records. 53func runTestFunction(tf TestFunction) (failures []FailureRecord) { 54 // Set up a clean slate for this test. Make sure to reset it after everything 55 // below is finished, so we don't accidentally use it elsewhere. 56 currentlyRunningTest = newTestInfo() 57 defer func() { 58 currentlyRunningTest = nil 59 }() 60 61 ti := currentlyRunningTest 62 63 // Start a trace. 64 var reportOutcome reqtrace.ReportFunc 65 ti.Ctx, reportOutcome = reqtrace.Trace(ti.Ctx, tf.Name) 66 67 // Run the SetUp function, if any, paying attention to whether it panics. 68 setUpPanicked := false 69 if tf.SetUp != nil { 70 setUpPanicked = runWithProtection(func() { tf.SetUp(ti) }) 71 } 72 73 // Run the test function itself, but only if the SetUp function didn't panic. 74 // (This includes AssertThat errors.) 75 if !setUpPanicked { 76 runWithProtection(tf.Run) 77 } 78 79 // Run the TearDown function, if any. 80 if tf.TearDown != nil { 81 runWithProtection(tf.TearDown) 82 } 83 84 // Tell the mock controller for the tests to report any errors it's sitting 85 // on. 86 ti.MockController.Finish() 87 88 // Report the outcome to reqtrace. 89 if len(ti.failureRecords) == 0 { 90 reportOutcome(nil) 91 } else { 92 reportOutcome(fmt.Errorf("%v failure records", len(ti.failureRecords))) 93 } 94 95 return ti.failureRecords 96} 97 98// Run everything registered with Register (including via the wrapper 99// RegisterTestSuite). 100// 101// Failures are communicated to the supplied testing.T object. This is the 102// bridge between ogletest and the testing package (and `go test`); you should 103// ensure that it's called at least once by creating a test function compatible 104// with `go test` and calling it there. 105// 106// For example: 107// 108// import ( 109// "github.com/smartystreets/assertions/internal/ogletest" 110// "testing" 111// ) 112// 113// func TestOgletest(t *testing.T) { 114// ogletest.RunTests(t) 115// } 116// 117func RunTests(t *testing.T) { 118 runTestsOnce.Do(func() { runTestsInternal(t) }) 119} 120 121// Signalling between RunTests and StopRunningTests. 122var gStopRunning uint64 123 124// Request that RunTests stop what it's doing. After the currently running test 125// is finished, including tear-down, the program will exit with an error code. 126func StopRunningTests() { 127 atomic.StoreUint64(&gStopRunning, 1) 128} 129 130// runTestsInternal does the real work of RunTests, which simply wraps it in a 131// sync.Once. 132func runTestsInternal(t *testing.T) { 133 // Process each registered suite. 134 for _, suite := range registeredSuites { 135 // Stop now if we've already seen a failure and we've been told to stop 136 // early. 137 if t.Failed() && *fStopEarly { 138 break 139 } 140 141 // Print a banner. 142 fmt.Printf("[----------] Running tests from %s\n", suite.Name) 143 144 // Run the SetUp function, if any. 145 if suite.SetUp != nil { 146 suite.SetUp() 147 } 148 149 // Run each test function that the user has not told us to skip. 150 stoppedEarly := false 151 for _, tf := range filterTestFunctions(suite) { 152 // Did the user request that we stop running tests? If so, skip the rest 153 // of this suite (and exit after tearing it down). 154 if atomic.LoadUint64(&gStopRunning) != 0 { 155 stoppedEarly = true 156 break 157 } 158 159 // Print a banner for the start of this test function. 160 fmt.Printf("[ RUN ] %s.%s\n", suite.Name, tf.Name) 161 162 // Run the test function. 163 startTime := time.Now() 164 failures := runTestFunction(tf) 165 runDuration := time.Since(startTime) 166 167 // Print any failures, and mark the test as having failed if there are any. 168 for _, record := range failures { 169 t.Fail() 170 fmt.Printf( 171 "%s:%d:\n%s\n\n", 172 record.FileName, 173 record.LineNumber, 174 record.Error) 175 } 176 177 // Print a banner for the end of the test. 178 bannerMessage := "[ OK ]" 179 if len(failures) != 0 { 180 bannerMessage = "[ FAILED ]" 181 } 182 183 // Print a summary of the time taken, if long enough. 184 var timeMessage string 185 if runDuration >= 25*time.Millisecond { 186 timeMessage = fmt.Sprintf(" (%s)", runDuration.String()) 187 } 188 189 fmt.Printf( 190 "%s %s.%s%s\n", 191 bannerMessage, 192 suite.Name, 193 tf.Name, 194 timeMessage) 195 196 // Stop running tests from this suite if we've been told to stop early 197 // and this test failed. 198 if t.Failed() && *fStopEarly { 199 break 200 } 201 } 202 203 // Run the suite's TearDown function, if any. 204 if suite.TearDown != nil { 205 suite.TearDown() 206 } 207 208 // Were we told to exit early? 209 if stoppedEarly { 210 fmt.Println("Exiting early due to user request.") 211 os.Exit(1) 212 } 213 214 fmt.Printf("[----------] Finished with tests from %s\n", suite.Name) 215 } 216} 217 218// Return true iff the supplied program counter appears to lie within panic(). 219func isPanic(pc uintptr) bool { 220 f := runtime.FuncForPC(pc) 221 if f == nil { 222 return false 223 } 224 225 return f.Name() == "runtime.gopanic" || f.Name() == "runtime.sigpanic" 226} 227 228// Find the deepest stack frame containing something that appears to be a 229// panic. Return the 'skip' value that a caller to this function would need 230// to supply to runtime.Caller for that frame, or a negative number if not found. 231func findPanic() int { 232 localSkip := -1 233 for i := 0; ; i++ { 234 // Stop if we've passed the base of the stack. 235 pc, _, _, ok := runtime.Caller(i) 236 if !ok { 237 break 238 } 239 240 // Is this a panic? 241 if isPanic(pc) { 242 localSkip = i 243 } 244 } 245 246 return localSkip - 1 247} 248 249// Attempt to find the file base name and line number for the ultimate source 250// of a panic, on the panicking stack. Return a human-readable sentinel if 251// unsuccessful. 252func findPanicFileLine() (string, int) { 253 panicSkip := findPanic() 254 if panicSkip < 0 { 255 return "(unknown)", 0 256 } 257 258 // Find the trigger of the panic. 259 _, file, line, ok := runtime.Caller(panicSkip + 1) 260 if !ok { 261 return "(unknown)", 0 262 } 263 264 return path.Base(file), line 265} 266 267// Run the supplied function, catching panics (including AssertThat errors) and 268// reporting them to the currently-running test as appropriate. Return true iff 269// the function panicked. 270func runWithProtection(f func()) (panicked bool) { 271 defer func() { 272 // If the test didn't panic, we're done. 273 r := recover() 274 if r == nil { 275 return 276 } 277 278 panicked = true 279 280 // We modify the currently running test below. 281 currentlyRunningTest.mu.Lock() 282 defer currentlyRunningTest.mu.Unlock() 283 284 // If the function panicked (and the panic was not due to an AssertThat 285 // failure), add a failure for the panic. 286 if !isAbortError(r) { 287 var panicRecord FailureRecord 288 panicRecord.FileName, panicRecord.LineNumber = findPanicFileLine() 289 panicRecord.Error = fmt.Sprintf( 290 "panic: %v\n\n%s", r, formatPanicStack()) 291 292 currentlyRunningTest.failureRecords = append( 293 currentlyRunningTest.failureRecords, 294 panicRecord) 295 } 296 }() 297 298 f() 299 return 300} 301 302func formatPanicStack() string { 303 buf := new(bytes.Buffer) 304 305 // Find the panic. If successful, we'll skip to below it. Otherwise, we'll 306 // format everything. 307 var initialSkip int 308 if panicSkip := findPanic(); panicSkip >= 0 { 309 initialSkip = panicSkip + 1 310 } 311 312 for i := initialSkip; ; i++ { 313 pc, file, line, ok := runtime.Caller(i) 314 if !ok { 315 break 316 } 317 318 // Choose a function name to display. 319 funcName := "(unknown)" 320 if f := runtime.FuncForPC(pc); f != nil { 321 funcName = f.Name() 322 } 323 324 // Stop if we've gotten as far as the test runner code. 325 if funcName == "github.com/smartystreets/assertions/internal/ogletest.runTestMethod" || 326 funcName == "github.com/smartystreets/assertions/internal/ogletest.runWithProtection" { 327 break 328 } 329 330 // Add an entry for this frame. 331 fmt.Fprintf(buf, "%s\n\t%s:%d\n", funcName, file, line) 332 } 333 334 return buf.String() 335} 336 337// Filter test functions according to the user-supplied filter flag. 338func filterTestFunctions(suite TestSuite) (out []TestFunction) { 339 re, err := regexp.Compile(*fTestFilter) 340 if err != nil { 341 panic("Invalid value for --ogletest.run: " + err.Error()) 342 } 343 344 for _, tf := range suite.TestFunctions { 345 fullName := fmt.Sprintf("%s.%s", suite.Name, tf.Name) 346 if !re.MatchString(fullName) { 347 continue 348 } 349 350 out = append(out, tf) 351 } 352 353 return 354} 355