1// Copyright 2019 The Go Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style 3// license that can be found in the LICENSE file. 4 5package debug 6 7import ( 8 "bytes" 9 "context" 10 "fmt" 11 "go/token" 12 "html/template" 13 "io" 14 stdlog "log" 15 "net" 16 "net/http" 17 "net/http/pprof" 18 _ "net/http/pprof" // pull in the standard pprof handlers 19 "os" 20 "path" 21 "path/filepath" 22 "reflect" 23 "runtime" 24 rpprof "runtime/pprof" 25 "strconv" 26 "strings" 27 "sync" 28 "time" 29 30 "golang.org/x/tools/internal/span" 31 "golang.org/x/tools/internal/telemetry/export" 32 "golang.org/x/tools/internal/telemetry/export/ocagent" 33 "golang.org/x/tools/internal/telemetry/export/prometheus" 34 "golang.org/x/tools/internal/telemetry/log" 35 "golang.org/x/tools/internal/telemetry/tag" 36) 37 38type Instance struct { 39 Logfile string 40 StartTime time.Time 41 ServerAddress string 42 DebugAddress string 43 Workdir string 44 OCAgentConfig string 45 46 LogWriter io.Writer 47 48 ocagent export.Exporter 49 prometheus *prometheus.Exporter 50 rpcs *rpcs 51 traces *traces 52} 53 54type Cache interface { 55 ID() string 56 FileSet() *token.FileSet 57 MemStats() map[reflect.Type]int 58} 59 60type Session interface { 61 ID() string 62 Cache() Cache 63 Files() []*File 64 File(hash string) *File 65} 66 67type View interface { 68 ID() string 69 Name() string 70 Folder() span.URI 71 Session() Session 72} 73 74type File struct { 75 Session Session 76 URI span.URI 77 Data string 78 Error error 79 Hash string 80} 81 82var ( 83 mu sync.Mutex 84 data = struct { 85 Caches []Cache 86 Sessions []Session 87 Views []View 88 }{} 89) 90 91// AddCache adds a cache to the set being served 92func AddCache(cache Cache) { 93 mu.Lock() 94 defer mu.Unlock() 95 data.Caches = append(data.Caches, cache) 96} 97 98// DropCache drops a cache from the set being served 99func DropCache(cache Cache) { 100 mu.Lock() 101 defer mu.Unlock() 102 //find and remove the cache 103 if i, _ := findCache(cache.ID()); i >= 0 { 104 copy(data.Caches[i:], data.Caches[i+1:]) 105 data.Caches[len(data.Caches)-1] = nil 106 data.Caches = data.Caches[:len(data.Caches)-1] 107 } 108} 109 110func findCache(id string) (int, Cache) { 111 for i, c := range data.Caches { 112 if c.ID() == id { 113 return i, c 114 } 115 } 116 return -1, nil 117} 118 119func getCache(r *http.Request) interface{} { 120 mu.Lock() 121 defer mu.Unlock() 122 id := path.Base(r.URL.Path) 123 result := struct { 124 Cache 125 Sessions []Session 126 }{} 127 _, result.Cache = findCache(id) 128 129 // now find all the views that belong to this session 130 for _, v := range data.Sessions { 131 if v.Cache().ID() == id { 132 result.Sessions = append(result.Sessions, v) 133 } 134 } 135 return result 136} 137 138func findSession(id string) (int, Session) { 139 for i, c := range data.Sessions { 140 if c.ID() == id { 141 return i, c 142 } 143 } 144 return -1, nil 145} 146 147func getSession(r *http.Request) interface{} { 148 mu.Lock() 149 defer mu.Unlock() 150 id := path.Base(r.URL.Path) 151 _, session := findSession(id) 152 result := struct { 153 Session 154 Views []View 155 }{ 156 Session: session, 157 } 158 // now find all the views that belong to this session 159 for _, v := range data.Views { 160 if v.Session().ID() == id { 161 result.Views = append(result.Views, v) 162 } 163 } 164 return result 165} 166 167func findView(id string) (int, View) { 168 for i, c := range data.Views { 169 if c.ID() == id { 170 return i, c 171 } 172 } 173 return -1, nil 174} 175 176func getView(r *http.Request) interface{} { 177 mu.Lock() 178 defer mu.Unlock() 179 id := path.Base(r.URL.Path) 180 _, v := findView(id) 181 return v 182} 183 184func getFile(r *http.Request) interface{} { 185 mu.Lock() 186 defer mu.Unlock() 187 hash := path.Base(r.URL.Path) 188 sid := path.Base(path.Dir(r.URL.Path)) 189 _, session := findSession(sid) 190 return session.File(hash) 191} 192 193func (i *Instance) getInfo() dataFunc { 194 return func(r *http.Request) interface{} { 195 buf := &bytes.Buffer{} 196 i.PrintServerInfo(buf) 197 return template.HTML(buf.String()) 198 } 199} 200 201func getMemory(r *http.Request) interface{} { 202 var m runtime.MemStats 203 runtime.ReadMemStats(&m) 204 return m 205} 206 207// AddSession adds a session to the set being served 208func AddSession(session Session) { 209 mu.Lock() 210 defer mu.Unlock() 211 data.Sessions = append(data.Sessions, session) 212} 213 214// DropSession drops a session from the set being served 215func DropSession(session Session) { 216 mu.Lock() 217 defer mu.Unlock() 218 219 if i, _ := findSession(session.ID()); i >= 0 { 220 copy(data.Sessions[i:], data.Sessions[i+1:]) 221 data.Sessions[len(data.Sessions)-1] = nil 222 data.Sessions = data.Sessions[:len(data.Sessions)-1] 223 } 224} 225 226// AddView adds a view to the set being served 227func AddView(view View) { 228 mu.Lock() 229 defer mu.Unlock() 230 data.Views = append(data.Views, view) 231} 232 233// DropView drops a view from the set being served 234func DropView(view View) { 235 mu.Lock() 236 defer mu.Unlock() 237 238 //find and remove the view 239 if i, _ := findView(view.ID()); i >= 0 { 240 copy(data.Views[i:], data.Views[i+1:]) 241 data.Views[len(data.Views)-1] = nil 242 data.Views = data.Views[:len(data.Views)-1] 243 } 244} 245 246// Prepare gets a debug instance ready for use using the supplied configuration. 247func (i *Instance) Prepare(ctx context.Context) { 248 mu.Lock() 249 defer mu.Unlock() 250 i.LogWriter = os.Stderr 251 ocConfig := ocagent.Discover() 252 //TODO: we should not need to adjust the discovered configuration 253 ocConfig.Address = i.OCAgentConfig 254 i.ocagent = ocagent.Connect(ocConfig) 255 i.prometheus = prometheus.New() 256 i.rpcs = &rpcs{} 257 i.traces = &traces{} 258 export.AddExporters(i.ocagent, i.prometheus, i.rpcs, i.traces) 259} 260 261func (i *Instance) SetLogFile(logfile string) (func(), error) { 262 // TODO: probably a better solution for deferring closure to the caller would 263 // be for the debug instance to itself be closed, but this fixes the 264 // immediate bug of logs not being captured. 265 closeLog := func() {} 266 if logfile != "" { 267 if logfile == "auto" { 268 logfile = filepath.Join(os.TempDir(), fmt.Sprintf("gopls-%d.log", os.Getpid())) 269 } 270 f, err := os.Create(logfile) 271 if err != nil { 272 return nil, fmt.Errorf("unable to create log file: %v", err) 273 } 274 closeLog = func() { 275 defer f.Close() 276 } 277 stdlog.SetOutput(io.MultiWriter(os.Stderr, f)) 278 i.LogWriter = f 279 } 280 i.Logfile = logfile 281 return closeLog, nil 282} 283 284// Serve starts and runs a debug server in the background. 285// It also logs the port the server starts on, to allow for :0 auto assigned 286// ports. 287func (i *Instance) Serve(ctx context.Context) error { 288 mu.Lock() 289 defer mu.Unlock() 290 if i.DebugAddress == "" { 291 return nil 292 } 293 listener, err := net.Listen("tcp", i.DebugAddress) 294 if err != nil { 295 return err 296 } 297 298 port := listener.Addr().(*net.TCPAddr).Port 299 if strings.HasSuffix(i.DebugAddress, ":0") { 300 stdlog.Printf("debug server listening on port %d", port) 301 } 302 log.Print(ctx, "Debug serving", tag.Of("Port", port)) 303 go func() { 304 mux := http.NewServeMux() 305 mux.HandleFunc("/", render(mainTmpl, func(*http.Request) interface{} { return data })) 306 mux.HandleFunc("/debug/", render(debugTmpl, nil)) 307 mux.HandleFunc("/debug/pprof/", pprof.Index) 308 mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) 309 mux.HandleFunc("/debug/pprof/profile", pprof.Profile) 310 mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) 311 mux.HandleFunc("/debug/pprof/trace", pprof.Trace) 312 if i.prometheus != nil { 313 mux.HandleFunc("/metrics/", i.prometheus.Serve) 314 } 315 if i.rpcs != nil { 316 mux.HandleFunc("/rpc/", render(rpcTmpl, i.rpcs.getData)) 317 } 318 if i.traces != nil { 319 mux.HandleFunc("/trace/", render(traceTmpl, i.traces.getData)) 320 } 321 mux.HandleFunc("/cache/", render(cacheTmpl, getCache)) 322 mux.HandleFunc("/session/", render(sessionTmpl, getSession)) 323 mux.HandleFunc("/view/", render(viewTmpl, getView)) 324 mux.HandleFunc("/file/", render(fileTmpl, getFile)) 325 mux.HandleFunc("/info", render(infoTmpl, i.getInfo())) 326 mux.HandleFunc("/memory", render(memoryTmpl, getMemory)) 327 if err := http.Serve(listener, mux); err != nil { 328 log.Error(ctx, "Debug server failed", err) 329 return 330 } 331 log.Print(ctx, "Debug server finished") 332 }() 333 return nil 334} 335 336func (i *Instance) MonitorMemory(ctx context.Context) { 337 tick := time.NewTicker(time.Second) 338 nextThresholdGiB := uint64(1) 339 go func() { 340 for { 341 <-tick.C 342 var mem runtime.MemStats 343 runtime.ReadMemStats(&mem) 344 if mem.HeapAlloc < nextThresholdGiB*1<<30 { 345 continue 346 } 347 i.writeMemoryDebug(nextThresholdGiB) 348 log.Print(ctx, fmt.Sprintf("Wrote memory usage debug info to %v", os.TempDir())) 349 nextThresholdGiB++ 350 } 351 }() 352} 353 354func (i *Instance) writeMemoryDebug(threshold uint64) error { 355 fname := func(t string) string { 356 return fmt.Sprintf("gopls.%d-%dGiB-%s", os.Getpid(), threshold, t) 357 } 358 359 f, err := os.Create(filepath.Join(os.TempDir(), fname("heap.pb.gz"))) 360 if err != nil { 361 return err 362 } 363 defer f.Close() 364 if err := rpprof.Lookup("heap").WriteTo(f, 0); err != nil { 365 return err 366 } 367 368 f, err = os.Create(filepath.Join(os.TempDir(), fname("goroutines.txt"))) 369 if err != nil { 370 return err 371 } 372 defer f.Close() 373 if err := rpprof.Lookup("goroutine").WriteTo(f, 1); err != nil { 374 return err 375 } 376 return nil 377} 378 379type dataFunc func(*http.Request) interface{} 380 381func render(tmpl *template.Template, fun dataFunc) func(http.ResponseWriter, *http.Request) { 382 return func(w http.ResponseWriter, r *http.Request) { 383 var data interface{} 384 if fun != nil { 385 data = fun(r) 386 } 387 if err := tmpl.Execute(w, data); err != nil { 388 log.Error(context.Background(), "", err) 389 } 390 } 391} 392 393func commas(s string) string { 394 for i := len(s); i > 3; { 395 i -= 3 396 s = s[:i] + "," + s[i:] 397 } 398 return s 399} 400 401func fuint64(v uint64) string { 402 return commas(strconv.FormatUint(v, 10)) 403} 404 405func fuint32(v uint32) string { 406 return commas(strconv.FormatUint(uint64(v), 10)) 407} 408 409var baseTemplate = template.Must(template.New("").Parse(` 410<html> 411<head> 412<title>{{template "title" .}}</title> 413<style> 414.profile-name{ 415 display:inline-block; 416 width:6rem; 417} 418td.value { 419 text-align: right; 420} 421ul.events { 422 list-style-type: none; 423} 424 425</style> 426{{block "head" .}}{{end}} 427</head> 428<body> 429<a href="/">Main</a> 430<a href="/info">Info</a> 431<a href="/memory">Memory</a> 432<a href="/metrics">Metrics</a> 433<a href="/rpc">RPC</a> 434<a href="/trace">Trace</a> 435<hr> 436<h1>{{template "title" .}}</h1> 437{{block "body" .}} 438Unknown page 439{{end}} 440</body> 441</html> 442 443{{define "cachelink"}}<a href="/cache/{{.}}">Cache {{.}}</a>{{end}} 444{{define "sessionlink"}}<a href="/session/{{.}}">Session {{.}}</a>{{end}} 445{{define "viewlink"}}<a href="/view/{{.}}">View {{.}}</a>{{end}} 446{{define "filelink"}}<a href="/file/{{.Session.ID}}/{{.Hash}}">{{.URI}}</a>{{end}} 447`)).Funcs(template.FuncMap{ 448 "fuint64": fuint64, 449 "fuint32": fuint32, 450}) 451 452var mainTmpl = template.Must(template.Must(baseTemplate.Clone()).Parse(` 453{{define "title"}}GoPls server information{{end}} 454{{define "body"}} 455<h2>Caches</h2> 456<ul>{{range .Caches}}<li>{{template "cachelink" .ID}}</li>{{end}}</ul> 457<h2>Sessions</h2> 458<ul>{{range .Sessions}}<li>{{template "sessionlink" .ID}} from {{template "cachelink" .Cache.ID}}</li>{{end}}</ul> 459<h2>Views</h2> 460<ul>{{range .Views}}<li>{{.Name}} is {{template "viewlink" .ID}} from {{template "sessionlink" .Session.ID}} in {{.Folder}}</li>{{end}}</ul> 461{{end}} 462`)) 463 464var infoTmpl = template.Must(template.Must(baseTemplate.Clone()).Parse(` 465{{define "title"}}GoPls version information{{end}} 466{{define "body"}} 467{{.}} 468{{end}} 469`)) 470 471var memoryTmpl = template.Must(template.Must(baseTemplate.Clone()).Parse(` 472{{define "title"}}GoPls memory usage{{end}} 473{{define "head"}}<meta http-equiv="refresh" content="5">{{end}} 474{{define "body"}} 475<h2>Stats</h2> 476<table> 477<tr><td class="label">Allocated bytes</td><td class="value">{{fuint64 .HeapAlloc}}</td></tr> 478<tr><td class="label">Total allocated bytes</td><td class="value">{{fuint64 .TotalAlloc}}</td></tr> 479<tr><td class="label">System bytes</td><td class="value">{{fuint64 .Sys}}</td></tr> 480<tr><td class="label">Heap system bytes</td><td class="value">{{fuint64 .HeapSys}}</td></tr> 481<tr><td class="label">Malloc calls</td><td class="value">{{fuint64 .Mallocs}}</td></tr> 482<tr><td class="label">Frees</td><td class="value">{{fuint64 .Frees}}</td></tr> 483<tr><td class="label">Idle heap bytes</td><td class="value">{{fuint64 .HeapIdle}}</td></tr> 484<tr><td class="label">In use bytes</td><td class="value">{{fuint64 .HeapInuse}}</td></tr> 485<tr><td class="label">Released to system bytes</td><td class="value">{{fuint64 .HeapReleased}}</td></tr> 486<tr><td class="label">Heap object count</td><td class="value">{{fuint64 .HeapObjects}}</td></tr> 487<tr><td class="label">Stack in use bytes</td><td class="value">{{fuint64 .StackInuse}}</td></tr> 488<tr><td class="label">Stack from system bytes</td><td class="value">{{fuint64 .StackSys}}</td></tr> 489<tr><td class="label">Bucket hash bytes</td><td class="value">{{fuint64 .BuckHashSys}}</td></tr> 490<tr><td class="label">GC metadata bytes</td><td class="value">{{fuint64 .GCSys}}</td></tr> 491<tr><td class="label">Off heap bytes</td><td class="value">{{fuint64 .OtherSys}}</td></tr> 492</table> 493<h2>By size</h2> 494<table> 495<tr><th>Size</th><th>Mallocs</th><th>Frees</th></tr> 496{{range .BySize}}<tr><td class="value">{{fuint32 .Size}}</td><td class="value">{{fuint64 .Mallocs}}</td><td class="value">{{fuint64 .Frees}}</td></tr>{{end}} 497</table> 498{{end}} 499`)) 500 501var debugTmpl = template.Must(template.Must(baseTemplate.Clone()).Parse(` 502{{define "title"}}GoPls Debug pages{{end}} 503{{define "body"}} 504<a href="/debug/pprof">Profiling</a> 505{{end}} 506`)) 507 508var cacheTmpl = template.Must(template.Must(baseTemplate.Clone()).Parse(` 509{{define "title"}}Cache {{.ID}}{{end}} 510{{define "body"}} 511<h2>Sessions</h2> 512<ul>{{range .Sessions}}<li>{{template "sessionlink" .ID}}</li>{{end}}</ul> 513<h2>memoize.Store entries</h2> 514<ul>{{range $k,$v := .MemStats}}<li>{{$k}} - {{$v}}</li>{{end}}</ul> 515{{end}} 516`)) 517 518var sessionTmpl = template.Must(template.Must(baseTemplate.Clone()).Parse(` 519{{define "title"}}Session {{.ID}}{{end}} 520{{define "body"}} 521From: <b>{{template "cachelink" .Cache.ID}}</b><br> 522<h2>Views</h2> 523<ul>{{range .Views}}<li>{{.Name}} is {{template "viewlink" .ID}} in {{.Folder}}</li>{{end}}</ul> 524<h2>Files</h2> 525<ul>{{range .Files}}<li>{{template "filelink" .}}</li>{{end}}</ul> 526{{end}} 527`)) 528 529var viewTmpl = template.Must(template.Must(baseTemplate.Clone()).Parse(` 530{{define "title"}}View {{.ID}}{{end}} 531{{define "body"}} 532Name: <b>{{.Name}}</b><br> 533Folder: <b>{{.Folder}}</b><br> 534From: <b>{{template "sessionlink" .Session.ID}}</b><br> 535<h2>Environment</h2> 536<ul>{{range .Env}}<li>{{.}}</li>{{end}}</ul> 537{{end}} 538`)) 539 540var fileTmpl = template.Must(template.Must(baseTemplate.Clone()).Parse(` 541{{define "title"}}File {{.Hash}}{{end}} 542{{define "body"}} 543From: <b>{{template "sessionlink" .Session.ID}}</b><br> 544URI: <b>{{.URI}}</b><br> 545Hash: <b>{{.Hash}}</b><br> 546Error: <b>{{.Error}}</b><br> 547<h3>Contents</h3> 548<pre>{{.Data}}</pre> 549{{end}} 550`)) 551