1// Copyright 2016 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5package testing
6
7import (
8	"bytes"
9	"fmt"
10	"regexp"
11	"runtime"
12	"strings"
13	"sync"
14	"sync/atomic"
15	"time"
16)
17
18func init() {
19	// Make benchmark tests run 10* faster.
20	*benchTime = 100 * time.Millisecond
21}
22
23func TestTestContext(t *T) {
24	const (
25		add1 = 0
26		done = 1
27	)
28	// After each of the calls are applied to the context, the
29	type call struct {
30		typ int // run or done
31		// result from applying the call
32		running int
33		waiting int
34		started bool
35	}
36	testCases := []struct {
37		max int
38		run []call
39	}{{
40		max: 1,
41		run: []call{
42			{typ: add1, running: 1, waiting: 0, started: true},
43			{typ: done, running: 0, waiting: 0, started: false},
44		},
45	}, {
46		max: 1,
47		run: []call{
48			{typ: add1, running: 1, waiting: 0, started: true},
49			{typ: add1, running: 1, waiting: 1, started: false},
50			{typ: done, running: 1, waiting: 0, started: true},
51			{typ: done, running: 0, waiting: 0, started: false},
52			{typ: add1, running: 1, waiting: 0, started: true},
53		},
54	}, {
55		max: 3,
56		run: []call{
57			{typ: add1, running: 1, waiting: 0, started: true},
58			{typ: add1, running: 2, waiting: 0, started: true},
59			{typ: add1, running: 3, waiting: 0, started: true},
60			{typ: add1, running: 3, waiting: 1, started: false},
61			{typ: add1, running: 3, waiting: 2, started: false},
62			{typ: add1, running: 3, waiting: 3, started: false},
63			{typ: done, running: 3, waiting: 2, started: true},
64			{typ: add1, running: 3, waiting: 3, started: false},
65			{typ: done, running: 3, waiting: 2, started: true},
66			{typ: done, running: 3, waiting: 1, started: true},
67			{typ: done, running: 3, waiting: 0, started: true},
68			{typ: done, running: 2, waiting: 0, started: false},
69			{typ: done, running: 1, waiting: 0, started: false},
70			{typ: done, running: 0, waiting: 0, started: false},
71		},
72	}}
73	for i, tc := range testCases {
74		ctx := &testContext{
75			startParallel: make(chan bool),
76			maxParallel:   tc.max,
77		}
78		for j, call := range tc.run {
79			doCall := func(f func()) chan bool {
80				done := make(chan bool)
81				go func() {
82					f()
83					done <- true
84				}()
85				return done
86			}
87			started := false
88			switch call.typ {
89			case add1:
90				signal := doCall(ctx.waitParallel)
91				select {
92				case <-signal:
93					started = true
94				case ctx.startParallel <- true:
95					<-signal
96				}
97			case done:
98				signal := doCall(ctx.release)
99				select {
100				case <-signal:
101				case <-ctx.startParallel:
102					started = true
103					<-signal
104				}
105			}
106			if started != call.started {
107				t.Errorf("%d:%d:started: got %v; want %v", i, j, started, call.started)
108			}
109			if ctx.running != call.running {
110				t.Errorf("%d:%d:running: got %v; want %v", i, j, ctx.running, call.running)
111			}
112			if ctx.numWaiting != call.waiting {
113				t.Errorf("%d:%d:waiting: got %v; want %v", i, j, ctx.numWaiting, call.waiting)
114			}
115		}
116	}
117}
118
119func TestTRun(t *T) {
120	realTest := t
121	testCases := []struct {
122		desc   string
123		ok     bool
124		maxPar int
125		chatty bool
126		output string
127		f      func(*T)
128	}{{
129		desc:   "failnow skips future sequential and parallel tests at same level",
130		ok:     false,
131		maxPar: 1,
132		output: `
133--- FAIL: failnow skips future sequential and parallel tests at same level (N.NNs)
134    --- FAIL: failnow skips future sequential and parallel tests at same level/#00 (N.NNs)
135    `,
136		f: func(t *T) {
137			ranSeq := false
138			ranPar := false
139			t.Run("", func(t *T) {
140				t.Run("par", func(t *T) {
141					t.Parallel()
142					ranPar = true
143				})
144				t.Run("seq", func(t *T) {
145					ranSeq = true
146				})
147				t.FailNow()
148				t.Run("seq", func(t *T) {
149					realTest.Error("test must be skipped")
150				})
151				t.Run("par", func(t *T) {
152					t.Parallel()
153					realTest.Error("test must be skipped.")
154				})
155			})
156			if !ranPar {
157				realTest.Error("parallel test was not run")
158			}
159			if !ranSeq {
160				realTest.Error("sequential test was not run")
161			}
162		},
163	}, {
164		desc:   "failure in parallel test propagates upwards",
165		ok:     false,
166		maxPar: 1,
167		output: `
168--- FAIL: failure in parallel test propagates upwards (N.NNs)
169    --- FAIL: failure in parallel test propagates upwards/#00 (N.NNs)
170        --- FAIL: failure in parallel test propagates upwards/#00/par (N.NNs)
171		`,
172		f: func(t *T) {
173			t.Run("", func(t *T) {
174				t.Parallel()
175				t.Run("par", func(t *T) {
176					t.Parallel()
177					t.Fail()
178				})
179			})
180		},
181	}, {
182		desc:   "skipping without message, chatty",
183		ok:     true,
184		chatty: true,
185		output: `
186=== RUN   skipping without message, chatty
187--- SKIP: skipping without message, chatty (N.NNs)`,
188		f: func(t *T) { t.SkipNow() },
189	}, {
190		desc:   "chatty with recursion",
191		ok:     true,
192		chatty: true,
193		output: `
194=== RUN   chatty with recursion
195=== RUN   chatty with recursion/#00
196=== RUN   chatty with recursion/#00/#00
197--- PASS: chatty with recursion (N.NNs)
198    --- PASS: chatty with recursion/#00 (N.NNs)
199        --- PASS: chatty with recursion/#00/#00 (N.NNs)`,
200		f: func(t *T) {
201			t.Run("", func(t *T) {
202				t.Run("", func(t *T) {})
203			})
204		},
205	}, {
206		desc: "skipping without message, not chatty",
207		ok:   true,
208		f:    func(t *T) { t.SkipNow() },
209	}, {
210		desc: "skipping after error",
211		output: `
212--- FAIL: skipping after error (N.NNs)
213	sub_test.go:NNN: an error
214	sub_test.go:NNN: skipped`,
215		f: func(t *T) {
216			t.Error("an error")
217			t.Skip("skipped")
218		},
219	}, {
220		desc:   "use Run to locally synchronize parallelism",
221		ok:     true,
222		maxPar: 1,
223		f: func(t *T) {
224			var count uint32
225			t.Run("waitGroup", func(t *T) {
226				for i := 0; i < 4; i++ {
227					t.Run("par", func(t *T) {
228						t.Parallel()
229						atomic.AddUint32(&count, 1)
230					})
231				}
232			})
233			if count != 4 {
234				t.Errorf("count was %d; want 4", count)
235			}
236		},
237	}, {
238		desc: "alternate sequential and parallel",
239		// Sequential tests should partake in the counting of running threads.
240		// Otherwise, if one runs parallel subtests in sequential tests that are
241		// itself subtests of parallel tests, the counts can get askew.
242		ok:     true,
243		maxPar: 1,
244		f: func(t *T) {
245			t.Run("a", func(t *T) {
246				t.Parallel()
247				t.Run("b", func(t *T) {
248					// Sequential: ensure running count is decremented.
249					t.Run("c", func(t *T) {
250						t.Parallel()
251					})
252
253				})
254			})
255		},
256	}, {
257		desc: "alternate sequential and parallel 2",
258		// Sequential tests should partake in the counting of running threads.
259		// Otherwise, if one runs parallel subtests in sequential tests that are
260		// itself subtests of parallel tests, the counts can get askew.
261		ok:     true,
262		maxPar: 2,
263		f: func(t *T) {
264			for i := 0; i < 2; i++ {
265				t.Run("a", func(t *T) {
266					t.Parallel()
267					time.Sleep(time.Nanosecond)
268					for i := 0; i < 2; i++ {
269						t.Run("b", func(t *T) {
270							time.Sleep(time.Nanosecond)
271							for i := 0; i < 2; i++ {
272								t.Run("c", func(t *T) {
273									t.Parallel()
274									time.Sleep(time.Nanosecond)
275								})
276							}
277
278						})
279					}
280				})
281			}
282		},
283	}, {
284		desc:   "stress test",
285		ok:     true,
286		maxPar: 4,
287		f: func(t *T) {
288			t.Parallel()
289			for i := 0; i < 12; i++ {
290				t.Run("a", func(t *T) {
291					t.Parallel()
292					time.Sleep(time.Nanosecond)
293					for i := 0; i < 12; i++ {
294						t.Run("b", func(t *T) {
295							time.Sleep(time.Nanosecond)
296							for i := 0; i < 12; i++ {
297								t.Run("c", func(t *T) {
298									t.Parallel()
299									time.Sleep(time.Nanosecond)
300									t.Run("d1", func(t *T) {})
301									t.Run("d2", func(t *T) {})
302									t.Run("d3", func(t *T) {})
303									t.Run("d4", func(t *T) {})
304								})
305							}
306						})
307					}
308				})
309			}
310		},
311	}, {
312		desc:   "skip output",
313		ok:     true,
314		maxPar: 4,
315		f: func(t *T) {
316			t.Skip()
317		},
318	}, {
319		desc:   "panic on goroutine fail after test exit",
320		ok:     false,
321		maxPar: 4,
322		f: func(t *T) {
323			ch := make(chan bool)
324			t.Run("", func(t *T) {
325				go func() {
326					<-ch
327					defer func() {
328						if r := recover(); r == nil {
329							realTest.Errorf("expected panic")
330						}
331						ch <- true
332					}()
333					t.Errorf("failed after success")
334				}()
335			})
336			ch <- true
337			<-ch
338		},
339	}}
340	for _, tc := range testCases {
341		ctx := newTestContext(tc.maxPar, newMatcher(regexp.MatchString, "", ""))
342		buf := &bytes.Buffer{}
343		root := &T{
344			common: common{
345				signal: make(chan bool),
346				name:   "Test",
347				w:      buf,
348				chatty: tc.chatty,
349			},
350			context: ctx,
351		}
352		ok := root.Run(tc.desc, tc.f)
353		ctx.release()
354
355		if ok != tc.ok {
356			t.Errorf("%s:ok: got %v; want %v", tc.desc, ok, tc.ok)
357		}
358		if ok != !root.Failed() {
359			t.Errorf("%s:root failed: got %v; want %v", tc.desc, !ok, root.Failed())
360		}
361		if ctx.running != 0 || ctx.numWaiting != 0 {
362			t.Errorf("%s:running and waiting non-zero: got %d and %d", tc.desc, ctx.running, ctx.numWaiting)
363		}
364		got := strings.TrimSpace(buf.String())
365		want := strings.TrimSpace(tc.output)
366		re := makeRegexp(want)
367		if ok, err := regexp.MatchString(re, got); !ok || err != nil {
368			t.Errorf("%s:output:\ngot:\n%s\nwant:\n%s", tc.desc, got, want)
369		}
370	}
371}
372
373func TestBRun(t *T) {
374	work := func(b *B) {
375		for i := 0; i < b.N; i++ {
376			time.Sleep(time.Nanosecond)
377		}
378	}
379	testCases := []struct {
380		desc   string
381		failed bool
382		chatty bool
383		output string
384		f      func(*B)
385	}{{
386		desc: "simulate sequential run of subbenchmarks.",
387		f: func(b *B) {
388			b.Run("", func(b *B) { work(b) })
389			time1 := b.result.NsPerOp()
390			b.Run("", func(b *B) { work(b) })
391			time2 := b.result.NsPerOp()
392			if time1 >= time2 {
393				t.Errorf("no time spent in benchmark t1 >= t2 (%d >= %d)", time1, time2)
394			}
395		},
396	}, {
397		desc: "bytes set by all benchmarks",
398		f: func(b *B) {
399			b.Run("", func(b *B) { b.SetBytes(10); work(b) })
400			b.Run("", func(b *B) { b.SetBytes(10); work(b) })
401			if b.result.Bytes != 20 {
402				t.Errorf("bytes: got: %d; want 20", b.result.Bytes)
403			}
404		},
405	}, {
406		desc: "bytes set by some benchmarks",
407		// In this case the bytes result is meaningless, so it must be 0.
408		f: func(b *B) {
409			b.Run("", func(b *B) { b.SetBytes(10); work(b) })
410			b.Run("", func(b *B) { work(b) })
411			b.Run("", func(b *B) { b.SetBytes(10); work(b) })
412			if b.result.Bytes != 0 {
413				t.Errorf("bytes: got: %d; want 0", b.result.Bytes)
414			}
415		},
416	}, {
417		desc:   "failure carried over to root",
418		failed: true,
419		output: "--- FAIL: root",
420		f:      func(b *B) { b.Fail() },
421	}, {
422		desc:   "skipping without message, chatty",
423		chatty: true,
424		output: "--- SKIP: root",
425		f:      func(b *B) { b.SkipNow() },
426	}, {
427		desc:   "skipping with message, chatty",
428		chatty: true,
429		output: `
430--- SKIP: root
431	sub_test.go:NNN: skipping`,
432		f: func(b *B) { b.Skip("skipping") },
433	}, {
434		desc:   "chatty with recursion",
435		chatty: true,
436		f: func(b *B) {
437			b.Run("", func(b *B) {
438				b.Run("", func(b *B) {})
439			})
440		},
441	}, {
442		desc: "skipping without message, not chatty",
443		f:    func(b *B) { b.SkipNow() },
444	}, {
445		desc:   "skipping after error",
446		failed: true,
447		output: `
448--- FAIL: root
449	sub_test.go:NNN: an error
450	sub_test.go:NNN: skipped`,
451		f: func(b *B) {
452			b.Error("an error")
453			b.Skip("skipped")
454		},
455	}, {
456		desc: "memory allocation",
457		f: func(b *B) {
458			const bufSize = 256
459			alloc := func(b *B) {
460				var buf [bufSize]byte
461				for i := 0; i < b.N; i++ {
462					_ = append([]byte(nil), buf[:]...)
463				}
464			}
465			b.Run("", func(b *B) {
466				alloc(b)
467				b.ReportAllocs()
468			})
469			b.Run("", func(b *B) {
470				alloc(b)
471				b.ReportAllocs()
472			})
473			// runtime.MemStats sometimes reports more allocations than the
474			// benchmark is responsible for. Luckily the point of this test is
475			// to ensure that the results are not underreported, so we can
476			// simply verify the lower bound.
477			if got := b.result.MemAllocs; got < 2 {
478				t.Errorf("MemAllocs was %v; want 2", got)
479			}
480			if got := b.result.MemBytes; got < 2*bufSize {
481				t.Errorf("MemBytes was %v; want %v", got, 2*bufSize)
482			}
483		},
484	}}
485	for _, tc := range testCases {
486		var ok bool
487		buf := &bytes.Buffer{}
488		// This is almost like the Benchmark function, except that we override
489		// the benchtime and catch the failure result of the subbenchmark.
490		root := &B{
491			common: common{
492				signal: make(chan bool),
493				name:   "root",
494				w:      buf,
495				chatty: tc.chatty,
496			},
497			benchFunc: func(b *B) { ok = b.Run("test", tc.f) }, // Use Run to catch failure.
498			benchTime: time.Microsecond,
499		}
500		root.runN(1)
501		if ok != !tc.failed {
502			t.Errorf("%s:ok: got %v; want %v", tc.desc, ok, !tc.failed)
503		}
504		if !ok != root.Failed() {
505			t.Errorf("%s:root failed: got %v; want %v", tc.desc, !ok, root.Failed())
506		}
507		// All tests are run as subtests
508		if root.result.N != 1 {
509			t.Errorf("%s: N for parent benchmark was %d; want 1", tc.desc, root.result.N)
510		}
511		got := strings.TrimSpace(buf.String())
512		want := strings.TrimSpace(tc.output)
513		re := makeRegexp(want)
514		if ok, err := regexp.MatchString(re, got); !ok || err != nil {
515			t.Errorf("%s:output:\ngot:\n%s\nwant:\n%s", tc.desc, got, want)
516		}
517	}
518}
519
520func makeRegexp(s string) string {
521	s = strings.Replace(s, ":NNN:", `:\d\d\d:`, -1)
522	s = strings.Replace(s, "(N.NNs)", `\(\d*\.\d*s\)`, -1)
523	return s
524}
525
526func TestBenchmarkOutput(t *T) {
527	// Ensure Benchmark initialized common.w by invoking it with an error and
528	// normal case.
529	Benchmark(func(b *B) { b.Error("do not print this output") })
530	Benchmark(func(b *B) {})
531}
532
533func TestBenchmarkStartsFrom1(t *T) {
534	var first = true
535	Benchmark(func(b *B) {
536		if first && b.N != 1 {
537			panic(fmt.Sprintf("Benchmark() first N=%v; want 1", b.N))
538		}
539		first = false
540	})
541}
542
543func TestBenchmarkReadMemStatsBeforeFirstRun(t *T) {
544	var first = true
545	Benchmark(func(b *B) {
546		if first && (b.startAllocs == 0 || b.startBytes == 0) {
547			panic(fmt.Sprintf("ReadMemStats not called before first run"))
548		}
549		first = false
550	})
551}
552
553func TestParallelSub(t *T) {
554	c := make(chan int)
555	block := make(chan int)
556	for i := 0; i < 10; i++ {
557		go func(i int) {
558			<-block
559			t.Run(fmt.Sprint(i), func(t *T) {})
560			c <- 1
561		}(i)
562	}
563	close(block)
564	for i := 0; i < 10; i++ {
565		<-c
566	}
567}
568
569type funcWriter func([]byte) (int, error)
570
571func (fw funcWriter) Write(b []byte) (int, error) { return fw(b) }
572
573func TestRacyOutput(t *T) {
574	var runs int32  // The number of running Writes
575	var races int32 // Incremented for each race detected
576	raceDetector := func(b []byte) (int, error) {
577		// Check if some other goroutine is concurrently calling Write.
578		if atomic.LoadInt32(&runs) > 0 {
579			atomic.AddInt32(&races, 1) // Race detected!
580		}
581		atomic.AddInt32(&runs, 1)
582		defer atomic.AddInt32(&runs, -1)
583		runtime.Gosched() // Increase probability of a race
584		return len(b), nil
585	}
586
587	var wg sync.WaitGroup
588	root := &T{
589		common:  common{w: funcWriter(raceDetector), chatty: true},
590		context: newTestContext(1, newMatcher(regexp.MatchString, "", "")),
591	}
592	root.Run("", func(t *T) {
593		for i := 0; i < 100; i++ {
594			wg.Add(1)
595			go func(i int) {
596				defer wg.Done()
597				t.Run(fmt.Sprint(i), func(t *T) {
598					t.Logf("testing run %d", i)
599				})
600			}(i)
601		}
602	})
603	wg.Wait()
604
605	if races > 0 {
606		t.Errorf("detected %d racy Writes", races)
607	}
608}
609
610func TestBenchmark(t *T) {
611	res := Benchmark(func(b *B) {
612		for i := 0; i < 5; i++ {
613			b.Run("", func(b *B) {
614				for i := 0; i < b.N; i++ {
615					time.Sleep(time.Millisecond)
616				}
617			})
618		}
619	})
620	if res.NsPerOp() < 4000000 {
621		t.Errorf("want >5ms; got %v", time.Duration(res.NsPerOp()))
622	}
623}
624