1{ 2 "annotations": { 3 "list": [ ] 4 }, 5 "editable": true, 6 "gnetId": null, 7 "graphTooltip": 0, 8 "hideControls": false, 9 "links": [ ], 10 "refresh": "10s", 11 "rows": [ 12 { 13 "collapse": false, 14 "height": "250px", 15 "panels": [ 16 { 17 "aliasColors": { }, 18 "bars": false, 19 "dashLength": 10, 20 "dashes": false, 21 "datasource": "$datasource", 22 "description": "Shows rate of incoming requests.", 23 "fill": 10, 24 "id": 1, 25 "legend": { 26 "avg": false, 27 "current": false, 28 "max": false, 29 "min": false, 30 "show": true, 31 "total": false, 32 "values": false 33 }, 34 "lines": true, 35 "linewidth": 0, 36 "links": [ ], 37 "nullPointMode": "null as zero", 38 "percentage": false, 39 "pointradius": 5, 40 "points": false, 41 "renderer": "flot", 42 "seriesOverrides": [ 43 { 44 "alias": "/1../", 45 "color": "#EAB839" 46 }, 47 { 48 "alias": "/2../", 49 "color": "#37872D" 50 }, 51 { 52 "alias": "/3../", 53 "color": "#E0B400" 54 }, 55 { 56 "alias": "/4../", 57 "color": "#1F60C4" 58 }, 59 { 60 "alias": "/5../", 61 "color": "#C4162A" 62 } 63 ], 64 "spaceLength": 10, 65 "span": 4, 66 "stack": true, 67 "steppedLine": false, 68 "targets": [ 69 { 70 "expr": "sum by (job, handler, code) (rate(http_requests_total{job=\"$job\", handler=\"receive\"}[$interval]))", 71 "format": "time_series", 72 "intervalFactor": 2, 73 "legendFormat": "{{job}} {{handler}} {{code}}", 74 "step": 10 75 } 76 ], 77 "thresholds": [ ], 78 "timeFrom": null, 79 "timeShift": null, 80 "title": "Rate", 81 "tooltip": { 82 "shared": false, 83 "sort": 0, 84 "value_type": "individual" 85 }, 86 "type": "graph", 87 "xaxis": { 88 "buckets": null, 89 "mode": "time", 90 "name": null, 91 "show": true, 92 "values": [ ] 93 }, 94 "yaxes": [ 95 { 96 "format": "short", 97 "label": null, 98 "logBase": 1, 99 "max": null, 100 "min": 0, 101 "show": true 102 }, 103 { 104 "format": "short", 105 "label": null, 106 "logBase": 1, 107 "max": null, 108 "min": null, 109 "show": false 110 } 111 ] 112 }, 113 { 114 "aliasColors": { 115 "error": "#E24D42" 116 }, 117 "bars": false, 118 "dashLength": 10, 119 "dashes": false, 120 "datasource": "$datasource", 121 "description": "Shows ratio of errors compared to the total number of handled incoming requests.", 122 "fill": 10, 123 "id": 2, 124 "legend": { 125 "avg": false, 126 "current": false, 127 "max": false, 128 "min": false, 129 "show": true, 130 "total": false, 131 "values": false 132 }, 133 "lines": true, 134 "linewidth": 0, 135 "links": [ ], 136 "nullPointMode": "null as zero", 137 "percentage": false, 138 "pointradius": 5, 139 "points": false, 140 "renderer": "flot", 141 "seriesOverrides": [ ], 142 "spaceLength": 10, 143 "span": 4, 144 "stack": true, 145 "steppedLine": false, 146 "targets": [ 147 { 148 "expr": "sum by (job) (rate(http_requests_total{job=\"$job\", handler=\"receive\",code=~\"5..\"}[$interval])) / sum by (job) (rate(http_requests_total{job=\"$job\", handler=\"receive\"}[$interval]))", 149 "format": "time_series", 150 "intervalFactor": 2, 151 "legendFormat": "error", 152 "step": 10 153 } 154 ], 155 "thresholds": [ ], 156 "timeFrom": null, 157 "timeShift": null, 158 "title": "Errors", 159 "tooltip": { 160 "shared": false, 161 "sort": 0, 162 "value_type": "individual" 163 }, 164 "type": "graph", 165 "xaxis": { 166 "buckets": null, 167 "mode": "time", 168 "name": null, 169 "show": true, 170 "values": [ ] 171 }, 172 "yaxes": [ 173 { 174 "format": "percentunit", 175 "label": null, 176 "logBase": 1, 177 "max": null, 178 "min": 0, 179 "show": true 180 }, 181 { 182 "format": "short", 183 "label": null, 184 "logBase": 1, 185 "max": null, 186 "min": null, 187 "show": false 188 } 189 ] 190 }, 191 { 192 "aliasColors": { }, 193 "bars": false, 194 "dashLength": 10, 195 "dashes": false, 196 "datasource": "$datasource", 197 "description": "Shows how long has it taken to handle incoming requests in quantiles.", 198 "fill": 1, 199 "id": 3, 200 "legend": { 201 "avg": false, 202 "current": false, 203 "max": false, 204 "min": false, 205 "show": true, 206 "total": false, 207 "values": false 208 }, 209 "lines": true, 210 "linewidth": 1, 211 "links": [ ], 212 "nullPointMode": "null as zero", 213 "percentage": false, 214 "pointradius": 5, 215 "points": false, 216 "renderer": "flot", 217 "seriesOverrides": [ 218 { 219 "alias": "p99", 220 "color": "#FA6400", 221 "fill": 1, 222 "fillGradient": 1 223 }, 224 { 225 "alias": "p90", 226 "color": "#E0B400", 227 "fill": 1, 228 "fillGradient": 1 229 }, 230 { 231 "alias": "p50", 232 "color": "#37872D", 233 "fill": 10, 234 "fillGradient": 0 235 } 236 ], 237 "spaceLength": 10, 238 "span": 4, 239 "stack": false, 240 "steppedLine": false, 241 "targets": [ 242 { 243 "expr": "histogram_quantile(0.50, sum by (job, le) (rate(http_request_duration_seconds_bucket{job=\"$job\", handler=\"receive\"}[$interval]))) * 1", 244 "format": "time_series", 245 "intervalFactor": 2, 246 "legendFormat": "p50 {{job}}", 247 "logBase": 10, 248 "max": null, 249 "min": null, 250 "step": 10 251 }, 252 { 253 "expr": "histogram_quantile(0.90, sum by (job, le) (rate(http_request_duration_seconds_bucket{job=\"$job\", handler=\"receive\"}[$interval]))) * 1", 254 "format": "time_series", 255 "intervalFactor": 2, 256 "legendFormat": "p90 {{job}}", 257 "logBase": 10, 258 "max": null, 259 "min": null, 260 "step": 10 261 }, 262 { 263 "expr": "histogram_quantile(0.99, sum by (job, le) (rate(http_request_duration_seconds_bucket{job=\"$job\", handler=\"receive\"}[$interval]))) * 1", 264 "format": "time_series", 265 "intervalFactor": 2, 266 "legendFormat": "p99 {{job}}", 267 "logBase": 10, 268 "max": null, 269 "min": null, 270 "step": 10 271 } 272 ], 273 "thresholds": [ ], 274 "timeFrom": null, 275 "timeShift": null, 276 "title": "Duration", 277 "tooltip": { 278 "shared": false, 279 "sort": 0, 280 "value_type": "individual" 281 }, 282 "type": "graph", 283 "xaxis": { 284 "buckets": null, 285 "mode": "time", 286 "name": null, 287 "show": true, 288 "values": [ ] 289 }, 290 "yaxes": [ 291 { 292 "format": "s", 293 "label": null, 294 "logBase": 1, 295 "max": null, 296 "min": 0, 297 "show": true 298 }, 299 { 300 "format": "short", 301 "label": null, 302 "logBase": 1, 303 "max": null, 304 "min": null, 305 "show": false 306 } 307 ] 308 } 309 ], 310 "repeat": null, 311 "repeatIteration": null, 312 "repeatRowId": null, 313 "showTitle": true, 314 "title": "WRITE - Incoming Request", 315 "titleSize": "h6" 316 }, 317 { 318 "collapse": false, 319 "height": "250px", 320 "panels": [ 321 { 322 "aliasColors": { }, 323 "bars": false, 324 "dashLength": 10, 325 "dashes": false, 326 "datasource": "$datasource", 327 "description": "Shows rate of replications to other receive nodes.", 328 "fill": 1, 329 "id": 4, 330 "legend": { 331 "avg": false, 332 "current": false, 333 "max": false, 334 "min": false, 335 "show": true, 336 "total": false, 337 "values": false 338 }, 339 "lines": true, 340 "linewidth": 1, 341 "links": [ ], 342 "nullPointMode": "null as zero", 343 "percentage": false, 344 "pointradius": 5, 345 "points": false, 346 "renderer": "flot", 347 "seriesOverrides": [ ], 348 "spaceLength": 10, 349 "span": 6, 350 "stack": false, 351 "steppedLine": false, 352 "targets": [ 353 { 354 "expr": "sum by (job) (rate(thanos_receive_replications_total{job=\"$job\"}[$interval]))", 355 "format": "time_series", 356 "intervalFactor": 2, 357 "legendFormat": "all {{job}}", 358 "legendLink": null, 359 "step": 10 360 } 361 ], 362 "thresholds": [ ], 363 "timeFrom": null, 364 "timeShift": null, 365 "title": "Rate", 366 "tooltip": { 367 "shared": false, 368 "sort": 0, 369 "value_type": "individual" 370 }, 371 "type": "graph", 372 "xaxis": { 373 "buckets": null, 374 "mode": "time", 375 "name": null, 376 "show": true, 377 "values": [ ] 378 }, 379 "yaxes": [ 380 { 381 "format": "short", 382 "label": null, 383 "logBase": 1, 384 "max": null, 385 "min": 0, 386 "show": true 387 }, 388 { 389 "format": "short", 390 "label": null, 391 "logBase": 1, 392 "max": null, 393 "min": null, 394 "show": false 395 } 396 ] 397 }, 398 { 399 "aliasColors": { 400 "error": "#E24D42" 401 }, 402 "bars": false, 403 "dashLength": 10, 404 "dashes": false, 405 "datasource": "$datasource", 406 "description": "Shows ratio of errors compared to the total number of replications to other receive nodes.", 407 "fill": 10, 408 "id": 5, 409 "legend": { 410 "avg": false, 411 "current": false, 412 "max": false, 413 "min": false, 414 "show": true, 415 "total": false, 416 "values": false 417 }, 418 "lines": true, 419 "linewidth": 0, 420 "links": [ ], 421 "nullPointMode": "null as zero", 422 "percentage": false, 423 "pointradius": 5, 424 "points": false, 425 "renderer": "flot", 426 "seriesOverrides": [ ], 427 "spaceLength": 10, 428 "span": 6, 429 "stack": true, 430 "steppedLine": false, 431 "targets": [ 432 { 433 "expr": "sum by (job) (rate(thanos_receive_replications_total{job=\"$job\", result=\"error\"}[$interval])) / sum by (job) (rate(thanos_receive_replications_total{job=\"$job\"}[$interval]))", 434 "format": "time_series", 435 "intervalFactor": 2, 436 "legendFormat": "error", 437 "step": 10 438 } 439 ], 440 "thresholds": [ ], 441 "timeFrom": null, 442 "timeShift": null, 443 "title": "Errors", 444 "tooltip": { 445 "shared": false, 446 "sort": 0, 447 "value_type": "individual" 448 }, 449 "type": "graph", 450 "xaxis": { 451 "buckets": null, 452 "mode": "time", 453 "name": null, 454 "show": true, 455 "values": [ ] 456 }, 457 "yaxes": [ 458 { 459 "format": "percentunit", 460 "label": null, 461 "logBase": 1, 462 "max": null, 463 "min": 0, 464 "show": true 465 }, 466 { 467 "format": "short", 468 "label": null, 469 "logBase": 1, 470 "max": null, 471 "min": null, 472 "show": false 473 } 474 ] 475 } 476 ], 477 "repeat": null, 478 "repeatIteration": null, 479 "repeatRowId": null, 480 "showTitle": true, 481 "title": "WRITE - Replication", 482 "titleSize": "h6" 483 }, 484 { 485 "collapse": false, 486 "height": "250px", 487 "panels": [ 488 { 489 "aliasColors": { }, 490 "bars": false, 491 "dashLength": 10, 492 "dashes": false, 493 "datasource": "$datasource", 494 "description": "Shows rate of forwarded requests to other receive nodes.", 495 "fill": 1, 496 "id": 6, 497 "legend": { 498 "avg": false, 499 "current": false, 500 "max": false, 501 "min": false, 502 "show": true, 503 "total": false, 504 "values": false 505 }, 506 "lines": true, 507 "linewidth": 1, 508 "links": [ ], 509 "nullPointMode": "null as zero", 510 "percentage": false, 511 "pointradius": 5, 512 "points": false, 513 "renderer": "flot", 514 "seriesOverrides": [ ], 515 "spaceLength": 10, 516 "span": 6, 517 "stack": false, 518 "steppedLine": false, 519 "targets": [ 520 { 521 "expr": "sum by (job) (rate(thanos_receive_forward_requests_total{job=\"$job\"}[$interval]))", 522 "format": "time_series", 523 "intervalFactor": 2, 524 "legendFormat": "all {{job}}", 525 "legendLink": null, 526 "step": 10 527 } 528 ], 529 "thresholds": [ ], 530 "timeFrom": null, 531 "timeShift": null, 532 "title": "Rate", 533 "tooltip": { 534 "shared": false, 535 "sort": 0, 536 "value_type": "individual" 537 }, 538 "type": "graph", 539 "xaxis": { 540 "buckets": null, 541 "mode": "time", 542 "name": null, 543 "show": true, 544 "values": [ ] 545 }, 546 "yaxes": [ 547 { 548 "format": "short", 549 "label": null, 550 "logBase": 1, 551 "max": null, 552 "min": 0, 553 "show": true 554 }, 555 { 556 "format": "short", 557 "label": null, 558 "logBase": 1, 559 "max": null, 560 "min": null, 561 "show": false 562 } 563 ] 564 }, 565 { 566 "aliasColors": { 567 "error": "#E24D42" 568 }, 569 "bars": false, 570 "dashLength": 10, 571 "dashes": false, 572 "datasource": "$datasource", 573 "description": "Shows ratio of errors compared to the total number of forwareded requests to other receive nodes.", 574 "fill": 10, 575 "id": 7, 576 "legend": { 577 "avg": false, 578 "current": false, 579 "max": false, 580 "min": false, 581 "show": true, 582 "total": false, 583 "values": false 584 }, 585 "lines": true, 586 "linewidth": 0, 587 "links": [ ], 588 "nullPointMode": "null as zero", 589 "percentage": false, 590 "pointradius": 5, 591 "points": false, 592 "renderer": "flot", 593 "seriesOverrides": [ ], 594 "spaceLength": 10, 595 "span": 6, 596 "stack": true, 597 "steppedLine": false, 598 "targets": [ 599 { 600 "expr": "sum by (job) (rate(thanos_receive_forward_requests_total{job=\"$job\", result=\"error\"}[$interval])) / sum by (job) (rate(thanos_receive_forward_requests_total{job=\"$job\"}[$interval]))", 601 "format": "time_series", 602 "intervalFactor": 2, 603 "legendFormat": "error", 604 "step": 10 605 } 606 ], 607 "thresholds": [ ], 608 "timeFrom": null, 609 "timeShift": null, 610 "title": "Errors", 611 "tooltip": { 612 "shared": false, 613 "sort": 0, 614 "value_type": "individual" 615 }, 616 "type": "graph", 617 "xaxis": { 618 "buckets": null, 619 "mode": "time", 620 "name": null, 621 "show": true, 622 "values": [ ] 623 }, 624 "yaxes": [ 625 { 626 "format": "percentunit", 627 "label": null, 628 "logBase": 1, 629 "max": null, 630 "min": 0, 631 "show": true 632 }, 633 { 634 "format": "short", 635 "label": null, 636 "logBase": 1, 637 "max": null, 638 "min": null, 639 "show": false 640 } 641 ] 642 } 643 ], 644 "repeat": null, 645 "repeatIteration": null, 646 "repeatRowId": null, 647 "showTitle": true, 648 "title": "WRITE - Forward Request", 649 "titleSize": "h6" 650 }, 651 { 652 "collapse": false, 653 "height": "250px", 654 "panels": [ 655 { 656 "aliasColors": { }, 657 "bars": false, 658 "dashLength": 10, 659 "dashes": false, 660 "datasource": "$datasource", 661 "description": "Shows rate of handled Unary gRPC requests from queriers.", 662 "fill": 10, 663 "id": 8, 664 "legend": { 665 "avg": false, 666 "current": false, 667 "max": false, 668 "min": false, 669 "show": true, 670 "total": false, 671 "values": false 672 }, 673 "lines": true, 674 "linewidth": 0, 675 "links": [ ], 676 "nullPointMode": "null as zero", 677 "percentage": false, 678 "pointradius": 5, 679 "points": false, 680 "renderer": "flot", 681 "seriesOverrides": [ 682 { 683 "alias": "/Aborted/", 684 "color": "#EAB839" 685 }, 686 { 687 "alias": "/AlreadyExists/", 688 "color": "#37872D" 689 }, 690 { 691 "alias": "/FailedPrecondition/", 692 "color": "#E0B400" 693 }, 694 { 695 "alias": "/Unimplemented/", 696 "color": "#E0B400" 697 }, 698 { 699 "alias": "/InvalidArgument/", 700 "color": "#1F60C4" 701 }, 702 { 703 "alias": "/NotFound/", 704 "color": "#1F60C4" 705 }, 706 { 707 "alias": "/PermissionDenied/", 708 "color": "#1F60C4" 709 }, 710 { 711 "alias": "/Unauthenticated/", 712 "color": "#1F60C4" 713 }, 714 { 715 "alias": "/Canceled/", 716 "color": "#C4162A" 717 }, 718 { 719 "alias": "/DataLoss/", 720 "color": "#C4162A" 721 }, 722 { 723 "alias": "/DeadlineExceeded/", 724 "color": "#C4162A" 725 }, 726 { 727 "alias": "/Internal/", 728 "color": "#C4162A" 729 }, 730 { 731 "alias": "/OutOfRange/", 732 "color": "#C4162A" 733 }, 734 { 735 "alias": "/ResourceExhausted/", 736 "color": "#C4162A" 737 }, 738 { 739 "alias": "/Unavailable/", 740 "color": "#C4162A" 741 }, 742 { 743 "alias": "/Unknown/", 744 "color": "#C4162A" 745 }, 746 { 747 "alias": "/OK/", 748 "color": "#37872D" 749 }, 750 { 751 "alias": "error", 752 "color": "#C4162A" 753 } 754 ], 755 "spaceLength": 10, 756 "span": 4, 757 "stack": true, 758 "steppedLine": false, 759 "targets": [ 760 { 761 "expr": "sum by (job, grpc_method, grpc_code) (rate(grpc_server_handled_total{job=\"$job\", grpc_type=\"unary\", grpc_method=\"RemoteWrite\"}[$interval]))", 762 "format": "time_series", 763 "intervalFactor": 2, 764 "legendFormat": "{{job}} {{grpc_method}} {{grpc_code}}", 765 "step": 10 766 } 767 ], 768 "thresholds": [ ], 769 "timeFrom": null, 770 "timeShift": null, 771 "title": "Rate", 772 "tooltip": { 773 "shared": false, 774 "sort": 0, 775 "value_type": "individual" 776 }, 777 "type": "graph", 778 "xaxis": { 779 "buckets": null, 780 "mode": "time", 781 "name": null, 782 "show": true, 783 "values": [ ] 784 }, 785 "yaxes": [ 786 { 787 "format": "short", 788 "label": null, 789 "logBase": 1, 790 "max": null, 791 "min": 0, 792 "show": true 793 }, 794 { 795 "format": "short", 796 "label": null, 797 "logBase": 1, 798 "max": null, 799 "min": null, 800 "show": false 801 } 802 ] 803 }, 804 { 805 "aliasColors": { 806 "error": "#E24D42" 807 }, 808 "bars": false, 809 "dashLength": 10, 810 "dashes": false, 811 "datasource": "$datasource", 812 "description": "Shows ratio of errors compared to the total number of handled requests from queriers.", 813 "fill": 10, 814 "id": 9, 815 "legend": { 816 "avg": false, 817 "current": false, 818 "max": false, 819 "min": false, 820 "show": true, 821 "total": false, 822 "values": false 823 }, 824 "lines": true, 825 "linewidth": 0, 826 "links": [ ], 827 "nullPointMode": "null as zero", 828 "percentage": false, 829 "pointradius": 5, 830 "points": false, 831 "renderer": "flot", 832 "seriesOverrides": [ ], 833 "spaceLength": 10, 834 "span": 4, 835 "stack": true, 836 "steppedLine": false, 837 "targets": [ 838 { 839 "expr": "sum by (job) (rate(grpc_server_handled_total{grpc_code=~\"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss\",job=\"$job\", grpc_type=\"unary\", grpc_method=\"RemoteWrite\"}[$interval])) / sum by (job) (rate(grpc_server_handled_total{job=\"$job\", grpc_type=\"unary\", grpc_method=\"RemoteWrite\"}[$interval]))", 840 "format": "time_series", 841 "intervalFactor": 2, 842 "legendFormat": "error", 843 "step": 10 844 } 845 ], 846 "thresholds": [ ], 847 "timeFrom": null, 848 "timeShift": null, 849 "title": "Errors", 850 "tooltip": { 851 "shared": false, 852 "sort": 0, 853 "value_type": "individual" 854 }, 855 "type": "graph", 856 "xaxis": { 857 "buckets": null, 858 "mode": "time", 859 "name": null, 860 "show": true, 861 "values": [ ] 862 }, 863 "yaxes": [ 864 { 865 "format": "percentunit", 866 "label": null, 867 "logBase": 1, 868 "max": null, 869 "min": 0, 870 "show": true 871 }, 872 { 873 "format": "short", 874 "label": null, 875 "logBase": 1, 876 "max": null, 877 "min": null, 878 "show": false 879 } 880 ] 881 }, 882 { 883 "aliasColors": { }, 884 "bars": false, 885 "dashLength": 10, 886 "dashes": false, 887 "datasource": "$datasource", 888 "description": "Shows how long has it taken to handle requests from queriers, in quantiles.", 889 "fill": 1, 890 "id": 10, 891 "legend": { 892 "avg": false, 893 "current": false, 894 "max": false, 895 "min": false, 896 "show": true, 897 "total": false, 898 "values": false 899 }, 900 "lines": true, 901 "linewidth": 1, 902 "links": [ ], 903 "nullPointMode": "null as zero", 904 "percentage": false, 905 "pointradius": 5, 906 "points": false, 907 "renderer": "flot", 908 "seriesOverrides": [ 909 { 910 "alias": "p99", 911 "color": "#FA6400", 912 "fill": 1, 913 "fillGradient": 1 914 }, 915 { 916 "alias": "p90", 917 "color": "#E0B400", 918 "fill": 1, 919 "fillGradient": 1 920 }, 921 { 922 "alias": "p50", 923 "color": "#37872D", 924 "fill": 10, 925 "fillGradient": 0 926 } 927 ], 928 "spaceLength": 10, 929 "span": 4, 930 "stack": false, 931 "steppedLine": false, 932 "targets": [ 933 { 934 "expr": "histogram_quantile(0.50, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=\"$job\", grpc_type=\"unary\", grpc_method=\"RemoteWrite\"}[$interval]))) * 1", 935 "format": "time_series", 936 "intervalFactor": 2, 937 "legendFormat": "p50 {{job}}", 938 "logBase": 10, 939 "max": null, 940 "min": null, 941 "step": 10 942 }, 943 { 944 "expr": "histogram_quantile(0.90, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=\"$job\", grpc_type=\"unary\", grpc_method=\"RemoteWrite\"}[$interval]))) * 1", 945 "format": "time_series", 946 "intervalFactor": 2, 947 "legendFormat": "p90 {{job}}", 948 "logBase": 10, 949 "max": null, 950 "min": null, 951 "step": 10 952 }, 953 { 954 "expr": "histogram_quantile(0.99, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=\"$job\", grpc_type=\"unary\", grpc_method=\"RemoteWrite\"}[$interval]))) * 1", 955 "format": "time_series", 956 "intervalFactor": 2, 957 "legendFormat": "p99 {{job}}", 958 "logBase": 10, 959 "max": null, 960 "min": null, 961 "step": 10 962 } 963 ], 964 "thresholds": [ ], 965 "timeFrom": null, 966 "timeShift": null, 967 "title": "Duration", 968 "tooltip": { 969 "shared": false, 970 "sort": 0, 971 "value_type": "individual" 972 }, 973 "type": "graph", 974 "xaxis": { 975 "buckets": null, 976 "mode": "time", 977 "name": null, 978 "show": true, 979 "values": [ ] 980 }, 981 "yaxes": [ 982 { 983 "format": "s", 984 "label": null, 985 "logBase": 1, 986 "max": null, 987 "min": 0, 988 "show": true 989 }, 990 { 991 "format": "short", 992 "label": null, 993 "logBase": 1, 994 "max": null, 995 "min": null, 996 "show": false 997 } 998 ] 999 } 1000 ], 1001 "repeat": null, 1002 "repeatIteration": null, 1003 "repeatRowId": null, 1004 "showTitle": true, 1005 "title": "WRITE - gRPC (Unary)", 1006 "titleSize": "h6" 1007 }, 1008 { 1009 "collapse": false, 1010 "height": "250px", 1011 "panels": [ 1012 { 1013 "aliasColors": { }, 1014 "bars": false, 1015 "dashLength": 10, 1016 "dashes": false, 1017 "datasource": "$datasource", 1018 "description": "Shows rate of handled Unary gRPC requests from queriers.", 1019 "fill": 10, 1020 "id": 11, 1021 "legend": { 1022 "avg": false, 1023 "current": false, 1024 "max": false, 1025 "min": false, 1026 "show": true, 1027 "total": false, 1028 "values": false 1029 }, 1030 "lines": true, 1031 "linewidth": 0, 1032 "links": [ ], 1033 "nullPointMode": "null as zero", 1034 "percentage": false, 1035 "pointradius": 5, 1036 "points": false, 1037 "renderer": "flot", 1038 "seriesOverrides": [ 1039 { 1040 "alias": "/Aborted/", 1041 "color": "#EAB839" 1042 }, 1043 { 1044 "alias": "/AlreadyExists/", 1045 "color": "#37872D" 1046 }, 1047 { 1048 "alias": "/FailedPrecondition/", 1049 "color": "#E0B400" 1050 }, 1051 { 1052 "alias": "/Unimplemented/", 1053 "color": "#E0B400" 1054 }, 1055 { 1056 "alias": "/InvalidArgument/", 1057 "color": "#1F60C4" 1058 }, 1059 { 1060 "alias": "/NotFound/", 1061 "color": "#1F60C4" 1062 }, 1063 { 1064 "alias": "/PermissionDenied/", 1065 "color": "#1F60C4" 1066 }, 1067 { 1068 "alias": "/Unauthenticated/", 1069 "color": "#1F60C4" 1070 }, 1071 { 1072 "alias": "/Canceled/", 1073 "color": "#C4162A" 1074 }, 1075 { 1076 "alias": "/DataLoss/", 1077 "color": "#C4162A" 1078 }, 1079 { 1080 "alias": "/DeadlineExceeded/", 1081 "color": "#C4162A" 1082 }, 1083 { 1084 "alias": "/Internal/", 1085 "color": "#C4162A" 1086 }, 1087 { 1088 "alias": "/OutOfRange/", 1089 "color": "#C4162A" 1090 }, 1091 { 1092 "alias": "/ResourceExhausted/", 1093 "color": "#C4162A" 1094 }, 1095 { 1096 "alias": "/Unavailable/", 1097 "color": "#C4162A" 1098 }, 1099 { 1100 "alias": "/Unknown/", 1101 "color": "#C4162A" 1102 }, 1103 { 1104 "alias": "/OK/", 1105 "color": "#37872D" 1106 }, 1107 { 1108 "alias": "error", 1109 "color": "#C4162A" 1110 } 1111 ], 1112 "spaceLength": 10, 1113 "span": 4, 1114 "stack": true, 1115 "steppedLine": false, 1116 "targets": [ 1117 { 1118 "expr": "sum by (job, grpc_method, grpc_code) (rate(grpc_server_handled_total{job=\"$job\", grpc_type=\"unary\", grpc_method!=\"RemoteWrite\"}[$interval]))", 1119 "format": "time_series", 1120 "intervalFactor": 2, 1121 "legendFormat": "{{job}} {{grpc_method}} {{grpc_code}}", 1122 "step": 10 1123 } 1124 ], 1125 "thresholds": [ ], 1126 "timeFrom": null, 1127 "timeShift": null, 1128 "title": "Rate", 1129 "tooltip": { 1130 "shared": false, 1131 "sort": 0, 1132 "value_type": "individual" 1133 }, 1134 "type": "graph", 1135 "xaxis": { 1136 "buckets": null, 1137 "mode": "time", 1138 "name": null, 1139 "show": true, 1140 "values": [ ] 1141 }, 1142 "yaxes": [ 1143 { 1144 "format": "short", 1145 "label": null, 1146 "logBase": 1, 1147 "max": null, 1148 "min": 0, 1149 "show": true 1150 }, 1151 { 1152 "format": "short", 1153 "label": null, 1154 "logBase": 1, 1155 "max": null, 1156 "min": null, 1157 "show": false 1158 } 1159 ] 1160 }, 1161 { 1162 "aliasColors": { 1163 "error": "#E24D42" 1164 }, 1165 "bars": false, 1166 "dashLength": 10, 1167 "dashes": false, 1168 "datasource": "$datasource", 1169 "description": "Shows ratio of errors compared to the total number of handled requests from queriers.", 1170 "fill": 10, 1171 "id": 12, 1172 "legend": { 1173 "avg": false, 1174 "current": false, 1175 "max": false, 1176 "min": false, 1177 "show": true, 1178 "total": false, 1179 "values": false 1180 }, 1181 "lines": true, 1182 "linewidth": 0, 1183 "links": [ ], 1184 "nullPointMode": "null as zero", 1185 "percentage": false, 1186 "pointradius": 5, 1187 "points": false, 1188 "renderer": "flot", 1189 "seriesOverrides": [ ], 1190 "spaceLength": 10, 1191 "span": 4, 1192 "stack": true, 1193 "steppedLine": false, 1194 "targets": [ 1195 { 1196 "expr": "sum by (job) (rate(grpc_server_handled_total{grpc_code=~\"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss\",job=\"$job\", grpc_type=\"unary\", grpc_method!=\"RemoteWrite\"}[$interval])) / sum by (job) (rate(grpc_server_handled_total{job=\"$job\", grpc_type=\"unary\", grpc_method!=\"RemoteWrite\"}[$interval]))", 1197 "format": "time_series", 1198 "intervalFactor": 2, 1199 "legendFormat": "error", 1200 "step": 10 1201 } 1202 ], 1203 "thresholds": [ ], 1204 "timeFrom": null, 1205 "timeShift": null, 1206 "title": "Errors", 1207 "tooltip": { 1208 "shared": false, 1209 "sort": 0, 1210 "value_type": "individual" 1211 }, 1212 "type": "graph", 1213 "xaxis": { 1214 "buckets": null, 1215 "mode": "time", 1216 "name": null, 1217 "show": true, 1218 "values": [ ] 1219 }, 1220 "yaxes": [ 1221 { 1222 "format": "percentunit", 1223 "label": null, 1224 "logBase": 1, 1225 "max": null, 1226 "min": 0, 1227 "show": true 1228 }, 1229 { 1230 "format": "short", 1231 "label": null, 1232 "logBase": 1, 1233 "max": null, 1234 "min": null, 1235 "show": false 1236 } 1237 ] 1238 }, 1239 { 1240 "aliasColors": { }, 1241 "bars": false, 1242 "dashLength": 10, 1243 "dashes": false, 1244 "datasource": "$datasource", 1245 "description": "Shows how long has it taken to handle requests from queriers, in quantiles.", 1246 "fill": 1, 1247 "id": 13, 1248 "legend": { 1249 "avg": false, 1250 "current": false, 1251 "max": false, 1252 "min": false, 1253 "show": true, 1254 "total": false, 1255 "values": false 1256 }, 1257 "lines": true, 1258 "linewidth": 1, 1259 "links": [ ], 1260 "nullPointMode": "null as zero", 1261 "percentage": false, 1262 "pointradius": 5, 1263 "points": false, 1264 "renderer": "flot", 1265 "seriesOverrides": [ 1266 { 1267 "alias": "p99", 1268 "color": "#FA6400", 1269 "fill": 1, 1270 "fillGradient": 1 1271 }, 1272 { 1273 "alias": "p90", 1274 "color": "#E0B400", 1275 "fill": 1, 1276 "fillGradient": 1 1277 }, 1278 { 1279 "alias": "p50", 1280 "color": "#37872D", 1281 "fill": 10, 1282 "fillGradient": 0 1283 } 1284 ], 1285 "spaceLength": 10, 1286 "span": 4, 1287 "stack": false, 1288 "steppedLine": false, 1289 "targets": [ 1290 { 1291 "expr": "histogram_quantile(0.50, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=\"$job\", grpc_type=\"unary\", grpc_method!=\"RemoteWrite\"}[$interval]))) * 1", 1292 "format": "time_series", 1293 "intervalFactor": 2, 1294 "legendFormat": "p50 {{job}}", 1295 "logBase": 10, 1296 "max": null, 1297 "min": null, 1298 "step": 10 1299 }, 1300 { 1301 "expr": "histogram_quantile(0.90, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=\"$job\", grpc_type=\"unary\", grpc_method!=\"RemoteWrite\"}[$interval]))) * 1", 1302 "format": "time_series", 1303 "intervalFactor": 2, 1304 "legendFormat": "p90 {{job}}", 1305 "logBase": 10, 1306 "max": null, 1307 "min": null, 1308 "step": 10 1309 }, 1310 { 1311 "expr": "histogram_quantile(0.99, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=\"$job\", grpc_type=\"unary\", grpc_method!=\"RemoteWrite\"}[$interval]))) * 1", 1312 "format": "time_series", 1313 "intervalFactor": 2, 1314 "legendFormat": "p99 {{job}}", 1315 "logBase": 10, 1316 "max": null, 1317 "min": null, 1318 "step": 10 1319 } 1320 ], 1321 "thresholds": [ ], 1322 "timeFrom": null, 1323 "timeShift": null, 1324 "title": "Duration", 1325 "tooltip": { 1326 "shared": false, 1327 "sort": 0, 1328 "value_type": "individual" 1329 }, 1330 "type": "graph", 1331 "xaxis": { 1332 "buckets": null, 1333 "mode": "time", 1334 "name": null, 1335 "show": true, 1336 "values": [ ] 1337 }, 1338 "yaxes": [ 1339 { 1340 "format": "s", 1341 "label": null, 1342 "logBase": 1, 1343 "max": null, 1344 "min": 0, 1345 "show": true 1346 }, 1347 { 1348 "format": "short", 1349 "label": null, 1350 "logBase": 1, 1351 "max": null, 1352 "min": null, 1353 "show": false 1354 } 1355 ] 1356 } 1357 ], 1358 "repeat": null, 1359 "repeatIteration": null, 1360 "repeatRowId": null, 1361 "showTitle": true, 1362 "title": "READ - gRPC (Unary)", 1363 "titleSize": "h6" 1364 }, 1365 { 1366 "collapse": false, 1367 "height": "250px", 1368 "panels": [ 1369 { 1370 "aliasColors": { }, 1371 "bars": false, 1372 "dashLength": 10, 1373 "dashes": false, 1374 "datasource": "$datasource", 1375 "description": "Shows rate of handled Streamed gRPC requests from queriers.", 1376 "fill": 10, 1377 "id": 14, 1378 "legend": { 1379 "avg": false, 1380 "current": false, 1381 "max": false, 1382 "min": false, 1383 "show": true, 1384 "total": false, 1385 "values": false 1386 }, 1387 "lines": true, 1388 "linewidth": 0, 1389 "links": [ ], 1390 "nullPointMode": "null as zero", 1391 "percentage": false, 1392 "pointradius": 5, 1393 "points": false, 1394 "renderer": "flot", 1395 "seriesOverrides": [ 1396 { 1397 "alias": "/Aborted/", 1398 "color": "#EAB839" 1399 }, 1400 { 1401 "alias": "/AlreadyExists/", 1402 "color": "#37872D" 1403 }, 1404 { 1405 "alias": "/FailedPrecondition/", 1406 "color": "#E0B400" 1407 }, 1408 { 1409 "alias": "/Unimplemented/", 1410 "color": "#E0B400" 1411 }, 1412 { 1413 "alias": "/InvalidArgument/", 1414 "color": "#1F60C4" 1415 }, 1416 { 1417 "alias": "/NotFound/", 1418 "color": "#1F60C4" 1419 }, 1420 { 1421 "alias": "/PermissionDenied/", 1422 "color": "#1F60C4" 1423 }, 1424 { 1425 "alias": "/Unauthenticated/", 1426 "color": "#1F60C4" 1427 }, 1428 { 1429 "alias": "/Canceled/", 1430 "color": "#C4162A" 1431 }, 1432 { 1433 "alias": "/DataLoss/", 1434 "color": "#C4162A" 1435 }, 1436 { 1437 "alias": "/DeadlineExceeded/", 1438 "color": "#C4162A" 1439 }, 1440 { 1441 "alias": "/Internal/", 1442 "color": "#C4162A" 1443 }, 1444 { 1445 "alias": "/OutOfRange/", 1446 "color": "#C4162A" 1447 }, 1448 { 1449 "alias": "/ResourceExhausted/", 1450 "color": "#C4162A" 1451 }, 1452 { 1453 "alias": "/Unavailable/", 1454 "color": "#C4162A" 1455 }, 1456 { 1457 "alias": "/Unknown/", 1458 "color": "#C4162A" 1459 }, 1460 { 1461 "alias": "/OK/", 1462 "color": "#37872D" 1463 }, 1464 { 1465 "alias": "error", 1466 "color": "#C4162A" 1467 } 1468 ], 1469 "spaceLength": 10, 1470 "span": 4, 1471 "stack": true, 1472 "steppedLine": false, 1473 "targets": [ 1474 { 1475 "expr": "sum by (job, grpc_method, grpc_code) (rate(grpc_server_handled_total{job=\"$job\", grpc_type=\"server_stream\"}[$interval]))", 1476 "format": "time_series", 1477 "intervalFactor": 2, 1478 "legendFormat": "{{job}} {{grpc_method}} {{grpc_code}}", 1479 "step": 10 1480 } 1481 ], 1482 "thresholds": [ ], 1483 "timeFrom": null, 1484 "timeShift": null, 1485 "title": "Rate", 1486 "tooltip": { 1487 "shared": false, 1488 "sort": 0, 1489 "value_type": "individual" 1490 }, 1491 "type": "graph", 1492 "xaxis": { 1493 "buckets": null, 1494 "mode": "time", 1495 "name": null, 1496 "show": true, 1497 "values": [ ] 1498 }, 1499 "yaxes": [ 1500 { 1501 "format": "short", 1502 "label": null, 1503 "logBase": 1, 1504 "max": null, 1505 "min": 0, 1506 "show": true 1507 }, 1508 { 1509 "format": "short", 1510 "label": null, 1511 "logBase": 1, 1512 "max": null, 1513 "min": null, 1514 "show": false 1515 } 1516 ] 1517 }, 1518 { 1519 "aliasColors": { 1520 "error": "#E24D42" 1521 }, 1522 "bars": false, 1523 "dashLength": 10, 1524 "dashes": false, 1525 "datasource": "$datasource", 1526 "description": "Shows ratio of errors compared to the total number of handled requests from queriers.", 1527 "fill": 10, 1528 "id": 15, 1529 "legend": { 1530 "avg": false, 1531 "current": false, 1532 "max": false, 1533 "min": false, 1534 "show": true, 1535 "total": false, 1536 "values": false 1537 }, 1538 "lines": true, 1539 "linewidth": 0, 1540 "links": [ ], 1541 "nullPointMode": "null as zero", 1542 "percentage": false, 1543 "pointradius": 5, 1544 "points": false, 1545 "renderer": "flot", 1546 "seriesOverrides": [ ], 1547 "spaceLength": 10, 1548 "span": 4, 1549 "stack": true, 1550 "steppedLine": false, 1551 "targets": [ 1552 { 1553 "expr": "sum by (job) (rate(grpc_server_handled_total{grpc_code=~\"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss\",job=\"$job\", grpc_type=\"server_stream\"}[$interval])) / sum by (job) (rate(grpc_server_handled_total{job=\"$job\", grpc_type=\"server_stream\"}[$interval]))", 1554 "format": "time_series", 1555 "intervalFactor": 2, 1556 "legendFormat": "error", 1557 "step": 10 1558 } 1559 ], 1560 "thresholds": [ ], 1561 "timeFrom": null, 1562 "timeShift": null, 1563 "title": "Errors", 1564 "tooltip": { 1565 "shared": false, 1566 "sort": 0, 1567 "value_type": "individual" 1568 }, 1569 "type": "graph", 1570 "xaxis": { 1571 "buckets": null, 1572 "mode": "time", 1573 "name": null, 1574 "show": true, 1575 "values": [ ] 1576 }, 1577 "yaxes": [ 1578 { 1579 "format": "percentunit", 1580 "label": null, 1581 "logBase": 1, 1582 "max": null, 1583 "min": 0, 1584 "show": true 1585 }, 1586 { 1587 "format": "short", 1588 "label": null, 1589 "logBase": 1, 1590 "max": null, 1591 "min": null, 1592 "show": false 1593 } 1594 ] 1595 }, 1596 { 1597 "aliasColors": { }, 1598 "bars": false, 1599 "dashLength": 10, 1600 "dashes": false, 1601 "datasource": "$datasource", 1602 "description": "Shows how long has it taken to handle requests from queriers, in quantiles.", 1603 "fill": 1, 1604 "id": 16, 1605 "legend": { 1606 "avg": false, 1607 "current": false, 1608 "max": false, 1609 "min": false, 1610 "show": true, 1611 "total": false, 1612 "values": false 1613 }, 1614 "lines": true, 1615 "linewidth": 1, 1616 "links": [ ], 1617 "nullPointMode": "null as zero", 1618 "percentage": false, 1619 "pointradius": 5, 1620 "points": false, 1621 "renderer": "flot", 1622 "seriesOverrides": [ 1623 { 1624 "alias": "p99", 1625 "color": "#FA6400", 1626 "fill": 1, 1627 "fillGradient": 1 1628 }, 1629 { 1630 "alias": "p90", 1631 "color": "#E0B400", 1632 "fill": 1, 1633 "fillGradient": 1 1634 }, 1635 { 1636 "alias": "p50", 1637 "color": "#37872D", 1638 "fill": 10, 1639 "fillGradient": 0 1640 } 1641 ], 1642 "spaceLength": 10, 1643 "span": 4, 1644 "stack": false, 1645 "steppedLine": false, 1646 "targets": [ 1647 { 1648 "expr": "histogram_quantile(0.50, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=\"$job\", grpc_type=\"server_stream\"}[$interval]))) * 1", 1649 "format": "time_series", 1650 "intervalFactor": 2, 1651 "legendFormat": "p50 {{job}}", 1652 "logBase": 10, 1653 "max": null, 1654 "min": null, 1655 "step": 10 1656 }, 1657 { 1658 "expr": "histogram_quantile(0.90, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=\"$job\", grpc_type=\"server_stream\"}[$interval]))) * 1", 1659 "format": "time_series", 1660 "intervalFactor": 2, 1661 "legendFormat": "p90 {{job}}", 1662 "logBase": 10, 1663 "max": null, 1664 "min": null, 1665 "step": 10 1666 }, 1667 { 1668 "expr": "histogram_quantile(0.99, sum by (job, le) (rate(grpc_server_handling_seconds_bucket{job=\"$job\", grpc_type=\"server_stream\"}[$interval]))) * 1", 1669 "format": "time_series", 1670 "intervalFactor": 2, 1671 "legendFormat": "p99 {{job}}", 1672 "logBase": 10, 1673 "max": null, 1674 "min": null, 1675 "step": 10 1676 } 1677 ], 1678 "thresholds": [ ], 1679 "timeFrom": null, 1680 "timeShift": null, 1681 "title": "Duration", 1682 "tooltip": { 1683 "shared": false, 1684 "sort": 0, 1685 "value_type": "individual" 1686 }, 1687 "type": "graph", 1688 "xaxis": { 1689 "buckets": null, 1690 "mode": "time", 1691 "name": null, 1692 "show": true, 1693 "values": [ ] 1694 }, 1695 "yaxes": [ 1696 { 1697 "format": "s", 1698 "label": null, 1699 "logBase": 1, 1700 "max": null, 1701 "min": 0, 1702 "show": true 1703 }, 1704 { 1705 "format": "short", 1706 "label": null, 1707 "logBase": 1, 1708 "max": null, 1709 "min": null, 1710 "show": false 1711 } 1712 ] 1713 } 1714 ], 1715 "repeat": null, 1716 "repeatIteration": null, 1717 "repeatRowId": null, 1718 "showTitle": true, 1719 "title": "READ - gRPC (Stream)", 1720 "titleSize": "h6" 1721 }, 1722 { 1723 "collapse": false, 1724 "height": "250px", 1725 "panels": [ 1726 { 1727 "aliasColors": { }, 1728 "bars": false, 1729 "dashLength": 10, 1730 "dashes": false, 1731 "datasource": "$datasource", 1732 "description": "Shows the relative time of last successful upload to the object-store bucket.", 1733 "fill": 1, 1734 "id": 17, 1735 "legend": { 1736 "avg": false, 1737 "current": false, 1738 "max": false, 1739 "min": false, 1740 "show": true, 1741 "total": false, 1742 "values": false 1743 }, 1744 "lines": true, 1745 "linewidth": 1, 1746 "links": [ ], 1747 "nullPointMode": "null as zero", 1748 "percentage": false, 1749 "pointradius": 5, 1750 "points": false, 1751 "renderer": "flot", 1752 "seriesOverrides": [ ], 1753 "spaceLength": 10, 1754 "span": 12, 1755 "stack": false, 1756 "steppedLine": false, 1757 "styles": [ 1758 { 1759 "alias": "Time", 1760 "dateFormat": "YYYY-MM-DD HH:mm:ss", 1761 "pattern": "Time", 1762 "type": "hidden" 1763 }, 1764 { 1765 "alias": "Uploaded Ago", 1766 "colorMode": null, 1767 "colors": [ ], 1768 "dateFormat": "YYYY-MM-DD HH:mm:ss", 1769 "decimals": 2, 1770 "link": false, 1771 "linkTargetBlank": false, 1772 "linkTooltip": "Drill down", 1773 "linkUrl": "", 1774 "pattern": "Value", 1775 "thresholds": [ ], 1776 "type": "number", 1777 "unit": "s" 1778 }, 1779 { 1780 "alias": "", 1781 "colorMode": null, 1782 "colors": [ ], 1783 "dateFormat": "YYYY-MM-DD HH:mm:ss", 1784 "decimals": 2, 1785 "pattern": "/.*/", 1786 "thresholds": [ ], 1787 "type": "string", 1788 "unit": "short" 1789 } 1790 ], 1791 "targets": [ 1792 { 1793 "expr": "time() - max by (job, bucket) (thanos_objstore_bucket_last_successful_upload_time{job=\"$job\"})", 1794 "format": "table", 1795 "instant": true, 1796 "intervalFactor": 2, 1797 "legendFormat": "", 1798 "refId": "A", 1799 "step": 10 1800 } 1801 ], 1802 "thresholds": [ ], 1803 "timeFrom": null, 1804 "timeShift": null, 1805 "title": "Successful Upload", 1806 "tooltip": { 1807 "shared": false, 1808 "sort": 0, 1809 "value_type": "individual" 1810 }, 1811 "transform": "table", 1812 "type": "table", 1813 "xaxis": { 1814 "buckets": null, 1815 "mode": "time", 1816 "name": null, 1817 "show": true, 1818 "values": [ ] 1819 }, 1820 "yaxes": [ 1821 { 1822 "format": "short", 1823 "label": null, 1824 "logBase": 1, 1825 "max": null, 1826 "min": 0, 1827 "show": true 1828 }, 1829 { 1830 "format": "short", 1831 "label": null, 1832 "logBase": 1, 1833 "max": null, 1834 "min": null, 1835 "show": false 1836 } 1837 ] 1838 } 1839 ], 1840 "repeat": null, 1841 "repeatIteration": null, 1842 "repeatRowId": null, 1843 "showTitle": true, 1844 "title": "Last Updated", 1845 "titleSize": "h6" 1846 }, 1847 { 1848 "collapse": true, 1849 "height": "250px", 1850 "panels": [ 1851 { 1852 "aliasColors": { }, 1853 "bars": false, 1854 "dashLength": 10, 1855 "dashes": false, 1856 "datasource": "$datasource", 1857 "fill": 1, 1858 "id": 18, 1859 "legend": { 1860 "avg": false, 1861 "current": false, 1862 "max": false, 1863 "min": false, 1864 "show": true, 1865 "total": false, 1866 "values": false 1867 }, 1868 "lines": true, 1869 "linewidth": 1, 1870 "links": [ ], 1871 "nullPointMode": "null as zero", 1872 "percentage": false, 1873 "pointradius": 5, 1874 "points": false, 1875 "renderer": "flot", 1876 "seriesOverrides": [ ], 1877 "spaceLength": 10, 1878 "span": 4, 1879 "stack": false, 1880 "steppedLine": false, 1881 "targets": [ 1882 { 1883 "expr": "go_memstats_alloc_bytes{job=\"$job\"}", 1884 "format": "time_series", 1885 "intervalFactor": 2, 1886 "legendFormat": "alloc all {{instance}}", 1887 "legendLink": null, 1888 "step": 10 1889 }, 1890 { 1891 "expr": "go_memstats_heap_alloc_bytes{job=\"$job\"}", 1892 "format": "time_series", 1893 "intervalFactor": 2, 1894 "legendFormat": "alloc heap {{instance}}", 1895 "legendLink": null, 1896 "step": 10 1897 }, 1898 { 1899 "expr": "rate(go_memstats_alloc_bytes_total{job=\"$job\"}[30s])", 1900 "format": "time_series", 1901 "intervalFactor": 2, 1902 "legendFormat": "alloc rate all {{instance}}", 1903 "legendLink": null, 1904 "step": 10 1905 }, 1906 { 1907 "expr": "rate(go_memstats_heap_alloc_bytes{job=\"$job\"}[30s])", 1908 "format": "time_series", 1909 "intervalFactor": 2, 1910 "legendFormat": "alloc rate heap {{instance}}", 1911 "legendLink": null, 1912 "step": 10 1913 }, 1914 { 1915 "expr": "go_memstats_stack_inuse_bytes{job=\"$job\"}", 1916 "format": "time_series", 1917 "intervalFactor": 2, 1918 "legendFormat": "inuse heap {{instance}}", 1919 "legendLink": null, 1920 "step": 10 1921 }, 1922 { 1923 "expr": "go_memstats_heap_inuse_bytes{job=\"$job\"}", 1924 "format": "time_series", 1925 "intervalFactor": 2, 1926 "legendFormat": "inuse stack {{instance}}", 1927 "legendLink": null, 1928 "step": 10 1929 } 1930 ], 1931 "thresholds": [ ], 1932 "timeFrom": null, 1933 "timeShift": null, 1934 "title": "Memory Used", 1935 "tooltip": { 1936 "shared": false, 1937 "sort": 0, 1938 "value_type": "individual" 1939 }, 1940 "type": "graph", 1941 "xaxis": { 1942 "buckets": null, 1943 "mode": "time", 1944 "name": null, 1945 "show": true, 1946 "values": [ ] 1947 }, 1948 "yaxes": [ 1949 { 1950 "format": "bytes", 1951 "label": null, 1952 "logBase": 1, 1953 "max": null, 1954 "min": 0, 1955 "show": true 1956 }, 1957 { 1958 "format": "short", 1959 "label": null, 1960 "logBase": 1, 1961 "max": null, 1962 "min": null, 1963 "show": false 1964 } 1965 ] 1966 }, 1967 { 1968 "aliasColors": { }, 1969 "bars": false, 1970 "dashLength": 10, 1971 "dashes": false, 1972 "datasource": "$datasource", 1973 "fill": 1, 1974 "id": 19, 1975 "legend": { 1976 "avg": false, 1977 "current": false, 1978 "max": false, 1979 "min": false, 1980 "show": true, 1981 "total": false, 1982 "values": false 1983 }, 1984 "lines": true, 1985 "linewidth": 1, 1986 "links": [ ], 1987 "nullPointMode": "null as zero", 1988 "percentage": false, 1989 "pointradius": 5, 1990 "points": false, 1991 "renderer": "flot", 1992 "seriesOverrides": [ ], 1993 "spaceLength": 10, 1994 "span": 4, 1995 "stack": false, 1996 "steppedLine": false, 1997 "targets": [ 1998 { 1999 "expr": "go_goroutines{job=\"$job\"}", 2000 "format": "time_series", 2001 "intervalFactor": 2, 2002 "legendFormat": "{{instance}}", 2003 "legendLink": null, 2004 "step": 10 2005 } 2006 ], 2007 "thresholds": [ ], 2008 "timeFrom": null, 2009 "timeShift": null, 2010 "title": "Goroutines", 2011 "tooltip": { 2012 "shared": false, 2013 "sort": 0, 2014 "value_type": "individual" 2015 }, 2016 "type": "graph", 2017 "xaxis": { 2018 "buckets": null, 2019 "mode": "time", 2020 "name": null, 2021 "show": true, 2022 "values": [ ] 2023 }, 2024 "yaxes": [ 2025 { 2026 "format": "short", 2027 "label": null, 2028 "logBase": 1, 2029 "max": null, 2030 "min": 0, 2031 "show": true 2032 }, 2033 { 2034 "format": "short", 2035 "label": null, 2036 "logBase": 1, 2037 "max": null, 2038 "min": null, 2039 "show": false 2040 } 2041 ] 2042 }, 2043 { 2044 "aliasColors": { }, 2045 "bars": false, 2046 "dashLength": 10, 2047 "dashes": false, 2048 "datasource": "$datasource", 2049 "fill": 1, 2050 "id": 20, 2051 "legend": { 2052 "avg": false, 2053 "current": false, 2054 "max": false, 2055 "min": false, 2056 "show": true, 2057 "total": false, 2058 "values": false 2059 }, 2060 "lines": true, 2061 "linewidth": 1, 2062 "links": [ ], 2063 "nullPointMode": "null as zero", 2064 "percentage": false, 2065 "pointradius": 5, 2066 "points": false, 2067 "renderer": "flot", 2068 "seriesOverrides": [ ], 2069 "spaceLength": 10, 2070 "span": 4, 2071 "stack": false, 2072 "steppedLine": false, 2073 "targets": [ 2074 { 2075 "expr": "go_gc_duration_seconds{job=\"$job\"}", 2076 "format": "time_series", 2077 "intervalFactor": 2, 2078 "legendFormat": "{{quantile}} {{instance}}", 2079 "legendLink": null, 2080 "step": 10 2081 } 2082 ], 2083 "thresholds": [ ], 2084 "timeFrom": null, 2085 "timeShift": null, 2086 "title": "GC Time Quantiles", 2087 "tooltip": { 2088 "shared": false, 2089 "sort": 0, 2090 "value_type": "individual" 2091 }, 2092 "type": "graph", 2093 "xaxis": { 2094 "buckets": null, 2095 "mode": "time", 2096 "name": null, 2097 "show": true, 2098 "values": [ ] 2099 }, 2100 "yaxes": [ 2101 { 2102 "format": "short", 2103 "label": null, 2104 "logBase": 1, 2105 "max": null, 2106 "min": 0, 2107 "show": true 2108 }, 2109 { 2110 "format": "short", 2111 "label": null, 2112 "logBase": 1, 2113 "max": null, 2114 "min": null, 2115 "show": false 2116 } 2117 ] 2118 } 2119 ], 2120 "repeat": null, 2121 "repeatIteration": null, 2122 "repeatRowId": null, 2123 "showTitle": true, 2124 "title": "Resources", 2125 "titleSize": "h6" 2126 } 2127 ], 2128 "schemaVersion": 14, 2129 "style": "dark", 2130 "tags": [ 2131 "thanos-mixin" 2132 ], 2133 "templating": { 2134 "list": [ 2135 { 2136 "current": { 2137 "text": "default", 2138 "value": "default" 2139 }, 2140 "hide": 0, 2141 "label": null, 2142 "name": "datasource", 2143 "options": [ ], 2144 "query": "prometheus", 2145 "refresh": 1, 2146 "regex": "", 2147 "type": "datasource" 2148 }, 2149 { 2150 "auto": true, 2151 "auto_count": 300, 2152 "auto_min": "10s", 2153 "current": { 2154 "text": "5m", 2155 "value": "5m" 2156 }, 2157 "hide": 0, 2158 "label": "interval", 2159 "name": "interval", 2160 "query": "5m,10m,30m,1h,6h,12h", 2161 "refresh": 2, 2162 "type": "interval" 2163 }, 2164 { 2165 "allValue": null, 2166 "current": { 2167 "text": "all", 2168 "value": "$__all" 2169 }, 2170 "datasource": "$datasource", 2171 "hide": 0, 2172 "includeAll": true, 2173 "label": "job", 2174 "multi": false, 2175 "name": "job", 2176 "options": [ ], 2177 "query": "label_values(up{job=~\".*thanos-receive.*\"}, job)", 2178 "refresh": 1, 2179 "regex": "", 2180 "sort": 2, 2181 "tagValuesQuery": "", 2182 "tags": [ ], 2183 "tagsQuery": "", 2184 "type": "query", 2185 "useTags": false 2186 } 2187 ] 2188 }, 2189 "time": { 2190 "from": "now-1h", 2191 "to": "now" 2192 }, 2193 "timepicker": { 2194 "refresh_intervals": [ 2195 "5s", 2196 "10s", 2197 "30s", 2198 "1m", 2199 "5m", 2200 "15m", 2201 "30m", 2202 "1h", 2203 "2h", 2204 "1d" 2205 ], 2206 "time_options": [ 2207 "5m", 2208 "15m", 2209 "1h", 2210 "6h", 2211 "12h", 2212 "24h", 2213 "2d", 2214 "7d", 2215 "30d" 2216 ] 2217 }, 2218 "timezone": "UTC", 2219 "title": "Thanos / Receive", 2220 "uid": "916a852b00ccc5ed81056644718fa4fb", 2221 "version": 0 2222} 2223