1package http 2 3import ( 4 "bytes" 5 "encoding/json" 6 "fmt" 7 "io/ioutil" 8 "net/http" 9 "sort" 10 "strings" 11 "time" 12 13 "github.com/ansel1/merry" 14 "github.com/go-graphite/carbonapi/carbonapipb" 15 "github.com/go-graphite/carbonapi/cmd/carbonapi/config" 16 "github.com/go-graphite/carbonapi/date" 17 "github.com/go-graphite/carbonapi/intervalset" 18 utilctx "github.com/go-graphite/carbonapi/util/ctx" 19 pbv2 "github.com/go-graphite/protocol/carbonapi_v2_pb" 20 pbv3 "github.com/go-graphite/protocol/carbonapi_v3_pb" 21 pickle "github.com/lomik/og-rek" 22 "github.com/lomik/zapwriter" 23 "github.com/maruel/natural" 24 uuid "github.com/satori/go.uuid" 25) 26 27// Find handler and it's helper functions 28type treejson struct { 29 AllowChildren int `json:"allowChildren"` 30 Expandable int `json:"expandable"` 31 Leaf int `json:"leaf"` 32 ID string `json:"id"` 33 Text string `json:"text"` 34 Context map[string]int `json:"context"` // unused 35} 36 37var treejsonContext = make(map[string]int) 38 39func findTreejson(multiGlobs *pbv3.MultiGlobResponse) ([]byte, error) { 40 var b bytes.Buffer 41 42 var tree = make([]treejson, 0) 43 44 seen := make(map[string]struct{}) 45 46 for _, globs := range multiGlobs.Metrics { 47 basepath := globs.Name 48 49 if i := strings.LastIndex(basepath, "."); i != -1 { 50 basepath = basepath[:i+1] 51 } else { 52 basepath = "" 53 } 54 55 for _, g := range globs.Matches { 56 if strings.HasPrefix(g.Path, "_tag") { 57 continue 58 } 59 60 name := g.Path 61 62 if i := strings.LastIndex(name, "."); i != -1 { 63 name = name[i+1:] 64 } 65 66 if _, ok := seen[name]; ok { 67 continue 68 } 69 seen[name] = struct{}{} 70 71 t := treejson{ 72 ID: basepath + name, 73 Context: treejsonContext, 74 Text: name, 75 } 76 77 if g.IsLeaf { 78 t.Leaf = 1 79 } else { 80 t.AllowChildren = 1 81 t.Expandable = 1 82 } 83 84 tree = append(tree, t) 85 } 86 } 87 88 sort.Slice(tree, func(i, j int) bool { 89 if tree[i].Leaf < tree[j].Leaf { 90 return true 91 } 92 if tree[i].Leaf > tree[j].Leaf { 93 return false 94 } 95 return natural.Less(tree[i].Text, tree[j].Text) 96 }) 97 98 err := json.NewEncoder(&b).Encode(tree) 99 return b.Bytes(), err 100} 101 102type completer struct { 103 Path string `json:"path"` 104 Name string `json:"name"` 105 IsLeaf string `json:"is_leaf"` 106} 107 108func findCompleter(multiGlobs *pbv3.MultiGlobResponse) ([]byte, error) { 109 var b bytes.Buffer 110 111 var complete = make([]completer, 0) 112 113 for _, globs := range multiGlobs.Metrics { 114 for _, g := range globs.Matches { 115 if strings.HasPrefix(g.Path, "_tag") { 116 continue 117 } 118 path := g.Path 119 if !g.IsLeaf && path[len(path)-1:] != "." { 120 path = g.Path + "." 121 } 122 c := completer{ 123 Path: path, 124 } 125 126 if g.IsLeaf { 127 c.IsLeaf = "1" 128 } else { 129 c.IsLeaf = "0" 130 } 131 132 i := strings.LastIndex(c.Path, ".") 133 134 if i != -1 { 135 c.Name = c.Path[i+1:] 136 } else { 137 c.Name = g.Path 138 } 139 140 complete = append(complete, c) 141 } 142 } 143 144 err := json.NewEncoder(&b).Encode(struct { 145 Metrics []completer `json:"metrics"` 146 }{ 147 Metrics: complete}, 148 ) 149 return b.Bytes(), err 150} 151 152func findList(multiGlobs *pbv3.MultiGlobResponse) ([]byte, error) { 153 var b bytes.Buffer 154 155 for _, globs := range multiGlobs.Metrics { 156 for _, g := range globs.Matches { 157 if strings.HasPrefix(g.Path, "_tag") { 158 continue 159 } 160 161 var dot string 162 // make sure non-leaves end in one dot 163 if !g.IsLeaf && !strings.HasSuffix(g.Path, ".") { 164 dot = "." 165 } 166 167 fmt.Fprintln(&b, g.Path+dot) 168 } 169 } 170 171 return b.Bytes(), nil 172} 173 174func findHandler(w http.ResponseWriter, r *http.Request) { 175 t0 := time.Now() 176 uid := uuid.NewV4() 177 // TODO: Migrate to context.WithTimeout 178 // ctx, _ := context.WithTimeout(context.TODO(), config.Config.ZipperTimeout) 179 ctx := utilctx.SetUUID(r.Context(), uid.String()) 180 username, _, _ := r.BasicAuth() 181 requestHeaders := utilctx.GetLogHeaders(ctx) 182 183 format, ok, formatRaw := getFormat(r, treejsonFormat) 184 jsonp := r.FormValue("jsonp") 185 186 qtz := r.FormValue("tz") 187 from := r.FormValue("from") 188 until := r.FormValue("until") 189 from64 := date.DateParamToEpoch(from, qtz, timeNow().Add(-time.Hour).Unix(), config.Config.DefaultTimeZone) 190 until64 := date.DateParamToEpoch(until, qtz, timeNow().Unix(), config.Config.DefaultTimeZone) 191 192 query := r.Form["query"] 193 srcIP, srcPort := splitRemoteAddr(r.RemoteAddr) 194 195 accessLogger := zapwriter.Logger("access") 196 var accessLogDetails = carbonapipb.AccessLogDetails{ 197 Handler: "find", 198 Username: username, 199 CarbonapiUUID: uid.String(), 200 URL: r.URL.RequestURI(), 201 PeerIP: srcIP, 202 PeerPort: srcPort, 203 Host: r.Host, 204 Referer: r.Referer(), 205 URI: r.RequestURI, 206 Format: formatRaw, 207 RequestHeaders: requestHeaders, 208 } 209 210 logAsError := false 211 defer func() { 212 deferredAccessLogging(accessLogger, &accessLogDetails, t0, logAsError) 213 }() 214 215 if !ok || !format.ValidFindFormat() { 216 http.Error(w, "unsupported format: "+formatRaw, http.StatusBadRequest) 217 accessLogDetails.HTTPCode = http.StatusBadRequest 218 accessLogDetails.Reason = "unsupported format: " + formatRaw 219 logAsError = true 220 return 221 } 222 223 if format == completerFormat { 224 var replacer = strings.NewReplacer("/", ".") 225 for i := range query { 226 query[i] = replacer.Replace(query[i]) 227 if query[i] == "" || query[i] == "/" || query[i] == "." { 228 query[i] = ".*" 229 } else { 230 query[i] += "*" 231 } 232 } 233 } 234 235 var pv3Request pbv3.MultiGlobRequest 236 237 if format == protoV3Format { 238 body, err := ioutil.ReadAll(r.Body) 239 if err != nil { 240 accessLogDetails.HTTPCode = http.StatusBadRequest 241 accessLogDetails.Reason = "failed to parse message body: " + err.Error() 242 http.Error(w, "bad request (failed to parse format): "+err.Error(), http.StatusBadRequest) 243 return 244 } 245 246 err = pv3Request.Unmarshal(body) 247 if err != nil { 248 accessLogDetails.HTTPCode = http.StatusBadRequest 249 accessLogDetails.Reason = "failed to parse message body: " + err.Error() 250 http.Error(w, "bad request (failed to parse format): "+err.Error(), http.StatusBadRequest) 251 return 252 } 253 } else { 254 pv3Request.Metrics = query 255 pv3Request.StartTime = from64 256 pv3Request.StopTime = until64 257 } 258 259 if len(pv3Request.Metrics) == 0 { 260 http.Error(w, "missing parameter `query`", http.StatusBadRequest) 261 accessLogDetails.HTTPCode = http.StatusBadRequest 262 accessLogDetails.Reason = "missing parameter `query`" 263 logAsError = true 264 return 265 } 266 267 multiGlobs, stats, err := config.Config.ZipperInstance.Find(ctx, pv3Request) 268 if stats != nil { 269 accessLogDetails.ZipperRequests = stats.ZipperRequests 270 accessLogDetails.TotalMetricsCount += stats.TotalMetricsCount 271 } 272 if err != nil { 273 returnCode := merry.HTTPCode(err) 274 if returnCode != http.StatusOK || multiGlobs == nil { 275 // Allow override status code for 404-not-found replies. 276 if returnCode == http.StatusNotFound { 277 returnCode = config.Config.NotFoundStatusCode 278 } 279 280 if returnCode < 300 { 281 multiGlobs = &pbv3.MultiGlobResponse{Metrics: []pbv3.GlobResponse{}} 282 } else { 283 http.Error(w, http.StatusText(returnCode), returnCode) 284 accessLogDetails.HTTPCode = int32(returnCode) 285 accessLogDetails.Reason = err.Error() 286 // We don't want to log this as an error if it's something normal 287 // Normal is everything that is >= 500. So if config.Config.NotFoundStatusCode is 500 - this will be 288 // logged as error 289 if returnCode >= 500 { 290 logAsError = true 291 } 292 return 293 } 294 } 295 } 296 var b []byte 297 var err2 error 298 switch format { 299 case treejsonFormat, jsonFormat: 300 b, err2 = findTreejson(multiGlobs) 301 err = merry.Wrap(err2) 302 format = jsonFormat 303 case completerFormat: 304 b, err2 = findCompleter(multiGlobs) 305 err = merry.Wrap(err2) 306 format = jsonFormat 307 case rawFormat: 308 b, err2 = findList(multiGlobs) 309 err = merry.Wrap(err2) 310 format = rawFormat 311 case protoV2Format: 312 r := pbv2.GlobResponse{ 313 Name: multiGlobs.Metrics[0].Name, 314 Matches: make([]pbv2.GlobMatch, 0, len(multiGlobs.Metrics)), 315 } 316 317 for i := range multiGlobs.Metrics { 318 for _, m := range multiGlobs.Metrics[i].Matches { 319 r.Matches = append(r.Matches, pbv2.GlobMatch{IsLeaf: m.IsLeaf, Path: m.Path}) 320 } 321 } 322 b, err2 = r.Marshal() 323 err = merry.Wrap(err2) 324 case protoV3Format: 325 b, err2 = multiGlobs.Marshal() 326 err = merry.Wrap(err2) 327 case pickleFormat: 328 var result []map[string]interface{} 329 now := int32(time.Now().Unix() + 60) 330 for _, globs := range multiGlobs.Metrics { 331 for _, metric := range globs.Matches { 332 if strings.HasPrefix(metric.Path, "_tag") { 333 continue 334 } 335 // Tell graphite-web that we have everything 336 var mm map[string]interface{} 337 if config.Config.GraphiteWeb09Compatibility { 338 // graphite-web 0.9.x 339 mm = map[string]interface{}{ 340 // graphite-web 0.9.x 341 "metric_path": metric.Path, 342 "isLeaf": metric.IsLeaf, 343 } 344 } else { 345 // graphite-web 1.0 346 interval := &intervalset.IntervalSet{Start: 0, End: now} 347 mm = map[string]interface{}{ 348 "is_leaf": metric.IsLeaf, 349 "path": metric.Path, 350 "intervals": interval, 351 } 352 } 353 result = append(result, mm) 354 } 355 } 356 357 p := bytes.NewBuffer(b) 358 pEnc := pickle.NewEncoder(p) 359 err = merry.Wrap(pEnc.Encode(result)) 360 b = p.Bytes() 361 } 362 363 if err != nil { 364 http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) 365 accessLogDetails.HTTPCode = http.StatusInternalServerError 366 accessLogDetails.Reason = err.Error() 367 logAsError = true 368 return 369 } 370 371 writeResponse(w, http.StatusOK, b, format, jsonp) 372} 373