1package server 2 3import ( 4 "fmt" 5 "net/http" 6 "os" 7 "runtime" 8 9 grpc_recovery "github.com/grpc-ecosystem/go-grpc-middleware/recovery" 10 "github.com/prometheus/client_golang/prometheus" 11 "github.com/prometheus/client_golang/prometheus/promauto" 12 "github.com/weaveworks/common/httpgrpc" 13 "github.com/weaveworks/common/middleware" 14) 15 16const maxStacksize = 8 * 1024 17 18var ( 19 panicTotal = promauto.NewCounter(prometheus.CounterOpts{ 20 Namespace: "loki", 21 Name: "panic_total", 22 Help: "The total number of panic triggered", 23 }) 24 25 RecoveryHTTPMiddleware = middleware.Func(func(next http.Handler) http.Handler { 26 return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { 27 defer func() { 28 if p := recover(); p != nil { 29 WriteError(onPanic(p), w) 30 } 31 }() 32 next.ServeHTTP(w, req) 33 }) 34 }) 35 RecoveryGRPCStreamInterceptor = grpc_recovery.StreamServerInterceptor(grpc_recovery.WithRecoveryHandler(onPanic)) 36 RecoveryGRPCUnaryInterceptor = grpc_recovery.UnaryServerInterceptor(grpc_recovery.WithRecoveryHandler(onPanic)) 37) 38 39func onPanic(p interface{}) error { 40 stack := make([]byte, maxStacksize) 41 stack = stack[:runtime.Stack(stack, true)] 42 // keep a multiline stack 43 fmt.Fprintf(os.Stderr, "panic: %v\n%s", p, stack) 44 panicTotal.Inc() 45 return httpgrpc.Errorf(http.StatusInternalServerError, "error while processing request: %v", p) 46} 47