1// Copyright 2014 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5// Implementation of runtime/debug.WriteHeapDump. Writes all
6// objects in the heap plus additional info (roots, threads,
7// finalizers, etc.) to a file.
8
9// The format of the dumped file is described at
10// https://golang.org/s/go15heapdump.
11
12package runtime
13
14import (
15	"runtime/internal/sys"
16	"unsafe"
17)
18
19//go:linkname runtime_debug_WriteHeapDump runtime..z2fdebug.WriteHeapDump
20func runtime_debug_WriteHeapDump(fd uintptr) {
21	stopTheWorld("write heap dump")
22
23	systemstack(func() {
24		writeheapdump_m(fd)
25	})
26
27	startTheWorld()
28}
29
30const (
31	fieldKindEol       = 0
32	fieldKindPtr       = 1
33	fieldKindIface     = 2
34	fieldKindEface     = 3
35	tagEOF             = 0
36	tagObject          = 1
37	tagOtherRoot       = 2
38	tagType            = 3
39	tagGoroutine       = 4
40	tagStackFrame      = 5
41	tagParams          = 6
42	tagFinalizer       = 7
43	tagItab            = 8
44	tagOSThread        = 9
45	tagMemStats        = 10
46	tagQueuedFinalizer = 11
47	tagData            = 12
48	tagBSS             = 13
49	tagDefer           = 14
50	tagPanic           = 15
51	tagMemProf         = 16
52	tagAllocSample     = 17
53)
54
55var dumpfd uintptr // fd to write the dump to.
56var tmpbuf []byte
57
58// buffer of pending write data
59const (
60	bufSize = 4096
61)
62
63var buf [bufSize]byte
64var nbuf uintptr
65
66func dwrite(data unsafe.Pointer, len uintptr) {
67	if len == 0 {
68		return
69	}
70	if nbuf+len <= bufSize {
71		copy(buf[nbuf:], (*[bufSize]byte)(data)[:len])
72		nbuf += len
73		return
74	}
75
76	write(dumpfd, unsafe.Pointer(&buf), int32(nbuf))
77	if len >= bufSize {
78		write(dumpfd, data, int32(len))
79		nbuf = 0
80	} else {
81		copy(buf[:], (*[bufSize]byte)(data)[:len])
82		nbuf = len
83	}
84}
85
86func dwritebyte(b byte) {
87	dwrite(unsafe.Pointer(&b), 1)
88}
89
90func flush() {
91	write(dumpfd, unsafe.Pointer(&buf), int32(nbuf))
92	nbuf = 0
93}
94
95// Cache of types that have been serialized already.
96// We use a type's hash field to pick a bucket.
97// Inside a bucket, we keep a list of types that
98// have been serialized so far, most recently used first.
99// Note: when a bucket overflows we may end up
100// serializing a type more than once. That's ok.
101const (
102	typeCacheBuckets = 256
103	typeCacheAssoc   = 4
104)
105
106type typeCacheBucket struct {
107	t [typeCacheAssoc]*_type
108}
109
110var typecache [typeCacheBuckets]typeCacheBucket
111
112// dump a uint64 in a varint format parseable by encoding/binary
113func dumpint(v uint64) {
114	var buf [10]byte
115	var n int
116	for v >= 0x80 {
117		buf[n] = byte(v | 0x80)
118		n++
119		v >>= 7
120	}
121	buf[n] = byte(v)
122	n++
123	dwrite(unsafe.Pointer(&buf), uintptr(n))
124}
125
126func dumpbool(b bool) {
127	if b {
128		dumpint(1)
129	} else {
130		dumpint(0)
131	}
132}
133
134// dump varint uint64 length followed by memory contents
135func dumpmemrange(data unsafe.Pointer, len uintptr) {
136	dumpint(uint64(len))
137	dwrite(data, len)
138}
139
140func dumpslice(b []byte) {
141	dumpint(uint64(len(b)))
142	if len(b) > 0 {
143		dwrite(unsafe.Pointer(&b[0]), uintptr(len(b)))
144	}
145}
146
147func dumpstr(s string) {
148	sp := stringStructOf(&s)
149	dumpmemrange(sp.str, uintptr(sp.len))
150}
151
152// dump information for a type
153func dumptype(t *_type) {
154	if t == nil {
155		return
156	}
157
158	// If we've definitely serialized the type before,
159	// no need to do it again.
160	b := &typecache[t.hash&(typeCacheBuckets-1)]
161	if t == b.t[0] {
162		return
163	}
164	for i := 1; i < typeCacheAssoc; i++ {
165		if t == b.t[i] {
166			// Move-to-front
167			for j := i; j > 0; j-- {
168				b.t[j] = b.t[j-1]
169			}
170			b.t[0] = t
171			return
172		}
173	}
174
175	// Might not have been dumped yet. Dump it and
176	// remember we did so.
177	for j := typeCacheAssoc - 1; j > 0; j-- {
178		b.t[j] = b.t[j-1]
179	}
180	b.t[0] = t
181
182	// dump the type
183	dumpint(tagType)
184	dumpint(uint64(uintptr(unsafe.Pointer(t))))
185	dumpint(uint64(t.size))
186	if x := t.uncommontype; x == nil || t.pkgPath == nil || *t.pkgPath == "" {
187		dumpstr(t.string())
188	} else {
189		pkgpathstr := *t.pkgPath
190		pkgpath := stringStructOf(&pkgpathstr)
191		namestr := *t.name
192		name := stringStructOf(&namestr)
193		dumpint(uint64(uintptr(pkgpath.len) + 1 + uintptr(name.len)))
194		dwrite(pkgpath.str, uintptr(pkgpath.len))
195		dwritebyte('.')
196		dwrite(name.str, uintptr(name.len))
197	}
198	dumpbool(t.kind&kindDirectIface == 0 || t.kind&kindNoPointers == 0)
199}
200
201// dump an object
202func dumpobj(obj unsafe.Pointer, size uintptr, bv bitvector) {
203	dumpint(tagObject)
204	dumpint(uint64(uintptr(obj)))
205	dumpmemrange(obj, size)
206	dumpfields(bv)
207}
208
209func dumpotherroot(description string, to unsafe.Pointer) {
210	dumpint(tagOtherRoot)
211	dumpstr(description)
212	dumpint(uint64(uintptr(to)))
213}
214
215func dumpfinalizer(obj unsafe.Pointer, fn *funcval, ft *functype, ot *ptrtype) {
216	dumpint(tagFinalizer)
217	dumpint(uint64(uintptr(obj)))
218	dumpint(uint64(uintptr(unsafe.Pointer(fn))))
219	dumpint(uint64(uintptr(unsafe.Pointer(fn.fn))))
220	dumpint(uint64(uintptr(unsafe.Pointer(ft))))
221	dumpint(uint64(uintptr(unsafe.Pointer(ot))))
222}
223
224type childInfo struct {
225	// Information passed up from the callee frame about
226	// the layout of the outargs region.
227	argoff uintptr   // where the arguments start in the frame
228	arglen uintptr   // size of args region
229	args   bitvector // if args.n >= 0, pointer map of args region
230	sp     *uint8    // callee sp
231	depth  uintptr   // depth in call stack (0 == most recent)
232}
233
234// dump kinds & offsets of interesting fields in bv
235func dumpbv(cbv *bitvector, offset uintptr) {
236	for i := uintptr(0); i < uintptr(cbv.n); i++ {
237		if cbv.ptrbit(i) == 1 {
238			dumpint(fieldKindPtr)
239			dumpint(uint64(offset + i*sys.PtrSize))
240		}
241	}
242}
243
244func dumpgoroutine(gp *g) {
245	sp := gp.syscallsp
246
247	dumpint(tagGoroutine)
248	dumpint(uint64(uintptr(unsafe.Pointer(gp))))
249	dumpint(uint64(sp))
250	dumpint(uint64(gp.goid))
251	dumpint(uint64(gp.gopc))
252	dumpint(uint64(readgstatus(gp)))
253	dumpbool(isSystemGoroutine(gp, false))
254	dumpbool(false) // isbackground
255	dumpint(uint64(gp.waitsince))
256	dumpstr(gp.waitreason.String())
257	dumpint(0)
258	dumpint(uint64(uintptr(unsafe.Pointer(gp.m))))
259	dumpint(uint64(uintptr(unsafe.Pointer(gp._defer))))
260	dumpint(uint64(uintptr(unsafe.Pointer(gp._panic))))
261
262	// dump defer & panic records
263	for d := gp._defer; d != nil; d = d.link {
264		dumpint(tagDefer)
265		dumpint(uint64(uintptr(unsafe.Pointer(d))))
266		dumpint(uint64(uintptr(unsafe.Pointer(gp))))
267		dumpint(0)
268		dumpint(0)
269		dumpint(uint64(uintptr(unsafe.Pointer(d.pfn))))
270		dumpint(0)
271		dumpint(uint64(uintptr(unsafe.Pointer(d.link))))
272	}
273	for p := gp._panic; p != nil; p = p.link {
274		dumpint(tagPanic)
275		dumpint(uint64(uintptr(unsafe.Pointer(p))))
276		dumpint(uint64(uintptr(unsafe.Pointer(gp))))
277		eface := efaceOf(&p.arg)
278		dumpint(uint64(uintptr(unsafe.Pointer(eface._type))))
279		dumpint(uint64(uintptr(unsafe.Pointer(eface.data))))
280		dumpint(0) // was p->defer, no longer recorded
281		dumpint(uint64(uintptr(unsafe.Pointer(p.link))))
282	}
283}
284
285func dumpgs() {
286	// goroutines & stacks
287	for i := 0; uintptr(i) < allglen; i++ {
288		gp := allgs[i]
289		status := readgstatus(gp) // The world is stopped so gp will not be in a scan state.
290		switch status {
291		default:
292			print("runtime: unexpected G.status ", hex(status), "\n")
293			throw("dumpgs in STW - bad status")
294		case _Gdead:
295			// ok
296		case _Grunnable,
297			_Gsyscall,
298			_Gwaiting:
299			dumpgoroutine(gp)
300		}
301	}
302}
303
304func finq_callback(fn *funcval, obj unsafe.Pointer, ft *functype, ot *ptrtype) {
305	dumpint(tagQueuedFinalizer)
306	dumpint(uint64(uintptr(obj)))
307	dumpint(uint64(uintptr(unsafe.Pointer(fn))))
308	dumpint(uint64(uintptr(unsafe.Pointer(fn.fn))))
309	dumpint(uint64(uintptr(unsafe.Pointer(ft))))
310	dumpint(uint64(uintptr(unsafe.Pointer(ot))))
311}
312
313func dumproots() {
314	// MSpan.types
315	for _, s := range mheap_.allspans {
316		if s.state == mSpanInUse {
317			// Finalizers
318			for sp := s.specials; sp != nil; sp = sp.next {
319				if sp.kind != _KindSpecialFinalizer {
320					continue
321				}
322				spf := (*specialfinalizer)(unsafe.Pointer(sp))
323				p := unsafe.Pointer(s.base() + uintptr(spf.special.offset))
324				dumpfinalizer(p, spf.fn, spf.ft, spf.ot)
325			}
326		}
327	}
328
329	// Finalizer queue
330	iterate_finq(finq_callback)
331}
332
333// Bit vector of free marks.
334// Needs to be as big as the largest number of objects per span.
335var freemark [_PageSize / 8]bool
336
337func dumpobjs() {
338	for _, s := range mheap_.allspans {
339		if s.state != mSpanInUse {
340			continue
341		}
342		p := s.base()
343		size := s.elemsize
344		n := (s.npages << _PageShift) / size
345		if n > uintptr(len(freemark)) {
346			throw("freemark array doesn't have enough entries")
347		}
348
349		for freeIndex := uintptr(0); freeIndex < s.nelems; freeIndex++ {
350			if s.isFree(freeIndex) {
351				freemark[freeIndex] = true
352			}
353		}
354
355		for j := uintptr(0); j < n; j, p = j+1, p+size {
356			if freemark[j] {
357				freemark[j] = false
358				continue
359			}
360			dumpobj(unsafe.Pointer(p), size, makeheapobjbv(p, size))
361		}
362	}
363}
364
365func dumpparams() {
366	dumpint(tagParams)
367	x := uintptr(1)
368	if *(*byte)(unsafe.Pointer(&x)) == 1 {
369		dumpbool(false) // little-endian ptrs
370	} else {
371		dumpbool(true) // big-endian ptrs
372	}
373	dumpint(sys.PtrSize)
374	var arenaStart, arenaEnd uintptr
375	for i1 := range mheap_.arenas {
376		if mheap_.arenas[i1] == nil {
377			continue
378		}
379		for i, ha := range mheap_.arenas[i1] {
380			if ha == nil {
381				continue
382			}
383			base := arenaBase(arenaIdx(i1)<<arenaL1Shift | arenaIdx(i))
384			if arenaStart == 0 || base < arenaStart {
385				arenaStart = base
386			}
387			if base+heapArenaBytes > arenaEnd {
388				arenaEnd = base + heapArenaBytes
389			}
390		}
391	}
392	dumpint(uint64(arenaStart))
393	dumpint(uint64(arenaEnd))
394	dumpstr(sys.GOARCH)
395	dumpstr(sys.Goexperiment)
396	dumpint(uint64(ncpu))
397}
398
399func dumpms() {
400	for mp := allm; mp != nil; mp = mp.alllink {
401		dumpint(tagOSThread)
402		dumpint(uint64(uintptr(unsafe.Pointer(mp))))
403		dumpint(uint64(mp.id))
404		dumpint(mp.procid)
405	}
406}
407
408func dumpmemstats() {
409	dumpint(tagMemStats)
410	dumpint(memstats.alloc)
411	dumpint(memstats.total_alloc)
412	dumpint(memstats.sys)
413	dumpint(memstats.nlookup)
414	dumpint(memstats.nmalloc)
415	dumpint(memstats.nfree)
416	dumpint(memstats.heap_alloc)
417	dumpint(memstats.heap_sys)
418	dumpint(memstats.heap_idle)
419	dumpint(memstats.heap_inuse)
420	dumpint(memstats.heap_released)
421	dumpint(memstats.heap_objects)
422	dumpint(memstats.stacks_inuse)
423	dumpint(memstats.stacks_sys)
424	dumpint(memstats.mspan_inuse)
425	dumpint(memstats.mspan_sys)
426	dumpint(memstats.mcache_inuse)
427	dumpint(memstats.mcache_sys)
428	dumpint(memstats.buckhash_sys)
429	dumpint(memstats.gc_sys)
430	dumpint(memstats.other_sys)
431	dumpint(memstats.next_gc)
432	dumpint(memstats.last_gc_unix)
433	dumpint(memstats.pause_total_ns)
434	for i := 0; i < 256; i++ {
435		dumpint(memstats.pause_ns[i])
436	}
437	dumpint(uint64(memstats.numgc))
438}
439
440func dumpmemprof_callback(b *bucket, nstk uintptr, pstk *location, size, allocs, frees uintptr) {
441	stk := (*[100000]location)(unsafe.Pointer(pstk))
442	dumpint(tagMemProf)
443	dumpint(uint64(uintptr(unsafe.Pointer(b))))
444	dumpint(uint64(size))
445	dumpint(uint64(nstk))
446	for i := uintptr(0); i < nstk; i++ {
447		pc := stk[i].pc
448		fn := stk[i].function
449		file := stk[i].filename
450		line := stk[i].lineno
451		if fn == "" {
452			var buf [64]byte
453			n := len(buf)
454			n--
455			buf[n] = ')'
456			if pc == 0 {
457				n--
458				buf[n] = '0'
459			} else {
460				for pc > 0 {
461					n--
462					buf[n] = "0123456789abcdef"[pc&15]
463					pc >>= 4
464				}
465			}
466			n--
467			buf[n] = 'x'
468			n--
469			buf[n] = '0'
470			n--
471			buf[n] = '('
472			dumpslice(buf[n:])
473			dumpstr("?")
474			dumpint(0)
475		} else {
476			dumpstr(fn)
477			dumpstr(file)
478			dumpint(uint64(line))
479		}
480	}
481	dumpint(uint64(allocs))
482	dumpint(uint64(frees))
483}
484
485func dumpmemprof() {
486	iterate_memprof(dumpmemprof_callback)
487	for _, s := range mheap_.allspans {
488		if s.state != mSpanInUse {
489			continue
490		}
491		for sp := s.specials; sp != nil; sp = sp.next {
492			if sp.kind != _KindSpecialProfile {
493				continue
494			}
495			spp := (*specialprofile)(unsafe.Pointer(sp))
496			p := s.base() + uintptr(spp.special.offset)
497			dumpint(tagAllocSample)
498			dumpint(uint64(p))
499			dumpint(uint64(uintptr(unsafe.Pointer(spp.b))))
500		}
501	}
502}
503
504var dumphdr = []byte("go1.7 heap dump\n")
505
506func mdump() {
507	// make sure we're done sweeping
508	for _, s := range mheap_.allspans {
509		if s.state == mSpanInUse {
510			s.ensureSwept()
511		}
512	}
513	memclrNoHeapPointers(unsafe.Pointer(&typecache), unsafe.Sizeof(typecache))
514	dwrite(unsafe.Pointer(&dumphdr[0]), uintptr(len(dumphdr)))
515	dumpparams()
516	dumpobjs()
517	dumpgs()
518	dumpms()
519	dumproots()
520	dumpmemstats()
521	dumpmemprof()
522	dumpint(tagEOF)
523	flush()
524}
525
526func writeheapdump_m(fd uintptr) {
527	_g_ := getg()
528	casgstatus(_g_.m.curg, _Grunning, _Gwaiting)
529	_g_.waitreason = waitReasonDumpingHeap
530
531	// Update stats so we can dump them.
532	// As a side effect, flushes all the mcaches so the mspan.freelist
533	// lists contain all the free objects.
534	updatememstats()
535
536	// Set dump file.
537	dumpfd = fd
538
539	// Call dump routine.
540	mdump()
541
542	// Reset dump file.
543	dumpfd = 0
544	if tmpbuf != nil {
545		sysFree(unsafe.Pointer(&tmpbuf[0]), uintptr(len(tmpbuf)), &memstats.other_sys)
546		tmpbuf = nil
547	}
548
549	casgstatus(_g_.m.curg, _Gwaiting, _Grunning)
550}
551
552// dumpint() the kind & offset of each field in an object.
553func dumpfields(bv bitvector) {
554	dumpbv(&bv, 0)
555	dumpint(fieldKindEol)
556}
557
558func makeheapobjbv(p uintptr, size uintptr) bitvector {
559	// Extend the temp buffer if necessary.
560	nptr := size / sys.PtrSize
561	if uintptr(len(tmpbuf)) < nptr/8+1 {
562		if tmpbuf != nil {
563			sysFree(unsafe.Pointer(&tmpbuf[0]), uintptr(len(tmpbuf)), &memstats.other_sys)
564		}
565		n := nptr/8 + 1
566		p := sysAlloc(n, &memstats.other_sys)
567		if p == nil {
568			throw("heapdump: out of memory")
569		}
570		tmpbuf = (*[1 << 30]byte)(p)[:n]
571	}
572	// Convert heap bitmap to pointer bitmap.
573	for i := uintptr(0); i < nptr/8+1; i++ {
574		tmpbuf[i] = 0
575	}
576	i := uintptr(0)
577	hbits := heapBitsForAddr(p)
578	for ; i < nptr; i++ {
579		if i != 1 && !hbits.morePointers() {
580			break // end of object
581		}
582		if hbits.isPointer() {
583			tmpbuf[i/8] |= 1 << (i % 8)
584		}
585		hbits = hbits.next()
586	}
587	return bitvector{int32(i), &tmpbuf[0]}
588}
589
590type gobitvector struct {
591	n        uintptr
592	bytedata []uint8
593}
594
595func gobv(bv bitvector) gobitvector {
596	return gobitvector{
597		uintptr(bv.n),
598		(*[1 << 30]byte)(unsafe.Pointer(bv.bytedata))[:(bv.n+7)/8],
599	}
600}
601