1package testdata 2 3import "strings" 4 5// LogString returns a test log line. Returns the same line for the same index. 6func LogString(index int64) string { 7 if index > int64(len(Logs)-1) { 8 index = index % int64(len(Logs)) 9 } 10 return Logs[index] 11} 12 13var LogsBytes [][]byte 14 15func init() { 16 LogsBytes = make([][]byte, len(Logs)) 17 for i, l := range Logs { 18 LogsBytes[i] = []byte(l) 19 } 20} 21 22var Logs = strings.Split(`level=info ts=2019-12-12T15:00:08.325Z caller=compact.go:441 component=tsdb msg="compact blocks" count=3 mint=1576130400000 maxt=1576152000000 ulid=01DVX9ZHNM71GRCJS7M34Q0EV7 sources="[01DVWNC6NWY1A60AZV3Z6DGS65 01DVWW7XXX75GHA6ZDTD170CSZ 01DVX33N5W86CWJJVRPAVXJRWJ]" duration=2.897213221s 23level=info ts=2019-12-12T15:00:08.296Z caller=compact.go:441 component=tsdb msg="compact blocks" count=3 mint=1576130400000 maxt=1576152000000 ulid=01DVX9ZHQRVN42AF196NYJ9C4C sources="[01DVWNC6NSPJRCSBZ4QD3SXS66 01DVWW7XY69Y4YT09HR0RSR8KY 01DVX33N5SMVPB1TMD9J1M8GGK]" duration=2.800759388s 24level=info ts=2019-12-12T15:00:05.285Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=1037 last=1039 duration=3.030078405s 25level=info ts=2019-12-12T15:00:05.225Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=1037 last=1039 duration=3.019791992s 26level=info ts=2019-12-12T15:00:02.255Z caller=head.go:596 component=tsdb msg="head GC completed" duration=125.980176ms 27level=info ts=2019-12-12T15:00:02.206Z caller=head.go:596 component=tsdb msg="head GC completed" duration=127.111334ms 28level=info ts=2019-12-12T15:00:01.874Z caller=compact.go:496 component=tsdb msg="write block" mint=1576152000000 maxt=1576159200000 ulid=01DVX9ZCE8WZCTQJWSYDGHVQV8 duration=1.801853505s 29level=info ts=2019-12-12T15:00:01.854Z caller=compact.go:496 component=tsdb msg="write block" mint=1576152000000 maxt=1576159200000 ulid=01DVX9ZCDWEBXRYWA7585TN2RV duration=1.794588392s 30level=info ts=2019-12-12T13:00:05.461Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=1034 last=1036 duration=3.044019343s 31level=info ts=2019-12-12T13:00:05.332Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=1034 last=1036 duration=3.040243488s 32level=info ts=2019-12-12T13:00:02.417Z caller=head.go:596 component=tsdb msg="head GC completed" duration=128.883109ms 33level=info ts=2019-12-12T13:00:02.291Z caller=head.go:596 component=tsdb msg="head GC completed" duration=126.278558ms 34level=info ts=2019-12-12T13:00:02.048Z caller=compact.go:496 component=tsdb msg="write block" mint=1576144800000 maxt=1576152000000 ulid=01DVX33N5W86CWJJVRPAVXJRWJ duration=1.987867109s 35level=info ts=2019-12-12T13:00:01.914Z caller=compact.go:496 component=tsdb msg="write block" mint=1576144800000 maxt=1576152000000 ulid=01DVX33N5SMVPB1TMD9J1M8GGK duration=1.856432758s 36level=info ts=2019-12-12T12:58:04.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4 37level=info ts=2019-12-12T12:52:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3 38level=info ts=2019-12-12T11:00:05.320Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=1031 last=1033 duration=2.999621843s 39level=info ts=2019-12-12T11:00:05.315Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=1031 last=1033 duration=2.962560692s 40level=info ts=2019-12-12T11:00:02.352Z caller=head.go:596 component=tsdb msg="head GC completed" duration=131.600701ms 41level=info ts=2019-12-12T11:00:02.321Z caller=head.go:596 component=tsdb msg="head GC completed" duration=134.547131ms 42level=info ts=2019-12-12T11:00:01.975Z caller=compact.go:496 component=tsdb msg="write block" mint=1576137600000 maxt=1576144800000 ulid=01DVWW7XY69Y4YT09HR0RSR8KY duration=1.905948839s 43level=info ts=2019-12-12T11:00:01.889Z caller=compact.go:496 component=tsdb msg="write block" mint=1576137600000 maxt=1576144800000 ulid=01DVWW7XXX75GHA6ZDTD170CSZ duration=1.828298188s 44level=info ts=2019-12-12T10:55:24.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4 45level=info ts=2019-12-12T10:49:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3 46level=info ts=2019-12-12T10:33:24.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4 47level=info ts=2019-12-12T10:25:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3 48level=info ts=2019-12-12T10:21:24.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4 49level=info ts=2019-12-12T10:14:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3 50level=info ts=2019-12-12T09:00:16.465Z caller=compact.go:441 component=tsdb msg="compact blocks" count=3 mint=1576065600000 maxt=1576130400000 ulid=01DVWNCFJRNW4RP8C56D4QNRXH sources="[01DVVC60FYTRXZ9457XT10Y7AH 01DVW0S6A5HFTYBYD34SGAZJSR 01DVWNCC9SYJDQP0Y2RXK8XJC9]" duration=7.289011992s 51level=info ts=2019-12-12T09:00:15.812Z caller=compact.go:441 component=tsdb msg="compact blocks" count=3 mint=1576065600000 maxt=1576130400000 ulid=01DVWNCF9JNSMSKZHW8STXQARA sources="[01DVVC60DBGMXD5DXR6Y5XWNXF 01DVW0S67R7JFBFTFWMNVS8YR3 01DVWNCC599NDRZWRRSZF4XGHF]" duration=6.930550254s 52level=info ts=2019-12-12T09:00:08.717Z caller=compact.go:441 component=tsdb msg="compact blocks" count=3 mint=1576108800000 maxt=1576130400000 ulid=01DVWNCC9SYJDQP0Y2RXK8XJC9 sources="[01DVW0S0XW63CVRA3EPRSC8NWQ 01DVW7MR5W18322RVFY6WM9GR2 01DVWEGFDW0C09KSCRQ2F8DGN3]" duration=2.900180235s 53level=info ts=2019-12-12T09:00:08.440Z caller=compact.go:441 component=tsdb msg="compact blocks" count=3 mint=1576108800000 maxt=1576130400000 ulid=01DVWNCC599NDRZWRRSZF4XGHF sources="[01DVW0S0XS1SQQQK3CQYCHN9HV 01DVW7MR5ZN3K38ZHBJ243HDZJ 01DVWEGFE0DGKKDG4V9AGAPPBQ]" duration=2.767053211s 54level=info ts=2019-12-12T09:00:05.604Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=1028 last=1030 duration=2.998418095s 55level=info ts=2019-12-12T09:00:05.470Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=1028 last=1030 duration=3.008684806s 56level=info ts=2019-12-12T09:00:02.606Z caller=head.go:596 component=tsdb msg="head GC completed" duration=126.82085ms 57level=info ts=2019-12-12T09:00:02.461Z caller=head.go:596 component=tsdb msg="head GC completed" duration=127.770206ms 58level=info ts=2019-12-12T09:00:01.995Z caller=compact.go:496 component=tsdb msg="write block" mint=1576130400000 maxt=1576137600000 ulid=01DVWNC6NWY1A60AZV3Z6DGS65 duration=1.934602237s 59level=info ts=2019-12-12T09:00:01.960Z caller=compact.go:496 component=tsdb msg="write block" mint=1576130400000 maxt=1576137600000 ulid=01DVWNC6NSPJRCSBZ4QD3SXS66 duration=1.902822647s 60level=info ts=2019-12-12T08:59:54.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4 61level=info ts=2019-12-12T08:54:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3 62level=info ts=2019-12-12T08:12:34.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4 63level=info ts=2019-12-12T08:05:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3 64level=info ts=2019-12-12T07:00:05.421Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=1025 last=1027 duration=3.037037204s 65level=info ts=2019-12-12T07:00:05.263Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=1025 last=1027 duration=2.984857831s 66level=info ts=2019-12-12T07:00:02.383Z caller=head.go:596 component=tsdb msg="head GC completed" duration=126.79721ms 67level=info ts=2019-12-12T07:00:02.278Z caller=head.go:596 component=tsdb msg="head GC completed" duration=131.228064ms 68level=info ts=2019-12-12T07:00:02.052Z caller=compact.go:496 component=tsdb msg="write block" mint=1576123200000 maxt=1576130400000 ulid=01DVWEGFE0DGKKDG4V9AGAPPBQ duration=1.987940522s 69level=info ts=2019-12-12T07:00:01.927Z caller=compact.go:496 component=tsdb msg="write block" mint=1576123200000 maxt=1576130400000 ulid=01DVWEGFDW0C09KSCRQ2F8DGN3 duration=1.866990386s 70level=info ts=2019-12-12T05:00:05.355Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=1022 last=1024 duration=3.046145151s 71level=info ts=2019-12-12T05:00:05.309Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=1022 last=1024 duration=3.019897535s 72level=info ts=2019-12-12T05:00:02.309Z caller=head.go:596 component=tsdb msg="head GC completed" duration=128.294946ms 73level=info ts=2019-12-12T05:00:02.289Z caller=head.go:596 component=tsdb msg="head GC completed" duration=145.150847ms 74level=info ts=2019-12-12T05:00:01.939Z caller=compact.go:496 component=tsdb msg="write block" mint=1576116000000 maxt=1576123200000 ulid=01DVW7MR5ZN3K38ZHBJ243HDZJ duration=1.875204968s 75level=info ts=2019-12-12T05:00:01.813Z caller=compact.go:496 component=tsdb msg="write block" mint=1576116000000 maxt=1576123200000 ulid=01DVW7MR5W18322RVFY6WM9GR2 duration=1.753345795s 76level=info ts=2019-12-12T04:38:34.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4 77level=info ts=2019-12-12T04:33:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3 78level=info ts=2019-12-12T04:00:54.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4 79level=info ts=2019-12-12T03:56:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3 80level=info ts=2019-12-12T03:00:08.433Z caller=compact.go:441 component=tsdb msg="compact blocks" count=3 mint=1576087200000 maxt=1576108800000 ulid=01DVW0S6A5HFTYBYD34SGAZJSR sources="[01DVVC5V5WESMMH77FZVCJ80Q8 01DVVK1JDWNVFGWS4JPY2K4CAS 01DVVSX9NWR5V8SSJAPKQ2TCTH]" duration=2.860812672s 81level=info ts=2019-12-12T03:00:08.279Z caller=compact.go:441 component=tsdb msg="compact blocks" count=3 mint=1576087200000 maxt=1576108800000 ulid=01DVW0S67R7JFBFTFWMNVS8YR3 sources="[01DVVC5V6145SMRFE0WR0P3YTQ 01DVVK1JE1SSYY4EKS4HAT4SK3 01DVVSX9NRE3DWK67A2J17BE0T]" duration=2.782760638s 82level=info ts=2019-12-12T03:00:05.372Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=1019 last=1021 duration=2.990754756s 83level=info ts=2019-12-12T03:00:05.289Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=1019 last=1021 duration=3.007795347s 84level=info ts=2019-12-12T03:00:02.381Z caller=head.go:596 component=tsdb msg="head GC completed" duration=127.007667ms 85level=info ts=2019-12-12T03:00:02.282Z caller=head.go:596 component=tsdb msg="head GC completed" duration=133.138336ms 86level=info ts=2019-12-12T03:00:01.987Z caller=compact.go:496 component=tsdb msg="write block" mint=1576108800000 maxt=1576116000000 ulid=01DVW0S0XW63CVRA3EPRSC8NWQ duration=1.927367458s 87level=info ts=2019-12-12T03:00:01.906Z caller=compact.go:496 component=tsdb msg="write block" mint=1576108800000 maxt=1576116000000 ulid=01DVW0S0XS1SQQQK3CQYCHN9HV duration=1.84874308s 88level=info ts=2019-12-12T02:39:24.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4 89level=info ts=2019-12-12T02:33:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3 90level=info ts=2019-12-12T01:00:05.500Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=1016 last=1018 duration=3.027246961s 91level=info ts=2019-12-12T01:00:05.265Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=1016 last=1018 duration=2.989822576s 92level=info ts=2019-12-12T01:00:02.473Z caller=head.go:596 component=tsdb msg="head GC completed" duration=124.134851ms 93level=info ts=2019-12-12T01:00:02.275Z caller=head.go:596 component=tsdb msg="head GC completed" duration=126.268006ms 94level=info ts=2019-12-12T01:00:02.092Z caller=compact.go:496 component=tsdb msg="write block" mint=1576101600000 maxt=1576108800000 ulid=01DVVSX9NRE3DWK67A2J17BE0T duration=2.035218414s 95level=info ts=2019-12-12T01:00:01.907Z caller=compact.go:496 component=tsdb msg="write block" mint=1576101600000 maxt=1576108800000 ulid=01DVVSX9NWR5V8SSJAPKQ2TCTH duration=1.847566214s 96level=info ts=2019-12-11T23:00:05.552Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=1013 last=1015 duration=3.042911717s 97level=info ts=2019-12-11T23:00:05.255Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=1013 last=1015 duration=3.007686626s 98level=info ts=2019-12-11T23:00:02.509Z caller=head.go:596 component=tsdb msg="head GC completed" duration=135.735201ms 99level=info ts=2019-12-11T23:00:02.247Z caller=head.go:596 component=tsdb msg="head GC completed" duration=126.374582ms 100level=info ts=2019-12-11T23:00:02.154Z caller=compact.go:496 component=tsdb msg="write block" mint=1576094400000 maxt=1576101600000 ulid=01DVVK1JE1SSYY4EKS4HAT4SK3 duration=2.088724625s 101level=info ts=2019-12-11T23:00:01.873Z caller=compact.go:496 component=tsdb msg="write block" mint=1576094400000 maxt=1576101600000 ulid=01DVVK1JDWNVFGWS4JPY2K4CAS duration=1.813033164s 102level=info ts=2019-12-11T21:00:08.427Z caller=compact.go:441 component=tsdb msg="compact blocks" count=3 mint=1576065600000 maxt=1576087200000 ulid=01DVVC60FYTRXZ9457XT10Y7AH sources="[01DVTQJNDXSY7N5V60ZX7X1C3J 01DVTYECNW5T3AHHB2EXATYFMJ 01DVV5A3XWVRTNS7G7BBDQ9G2W]" duration=2.925663083s 103level=info ts=2019-12-11T21:00:08.281Z caller=compact.go:441 component=tsdb msg="compact blocks" count=3 mint=1576065600000 maxt=1576087200000 ulid=01DVVC60DBGMXD5DXR6Y5XWNXF sources="[01DVTQJNDRV9NDCK9H2BCH04R0 01DVTYECNS4AZH3ZMCER87DYWG 01DVV5A3XRVMTB2E7V3MZ6RGCA]" duration=2.862756811s 104level=info ts=2019-12-11T21:00:05.288Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=1010 last=1012 duration=2.998716456s 105level=info ts=2019-12-11T21:00:05.204Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=1010 last=1012 duration=3.013679702s 106level=info ts=2019-12-11T21:00:02.289Z caller=head.go:596 component=tsdb msg="head GC completed" duration=124.171081ms 107level=info ts=2019-12-11T21:00:02.190Z caller=head.go:596 component=tsdb msg="head GC completed" duration=114.925741ms 108level=info ts=2019-12-11T21:00:01.942Z caller=compact.go:496 component=tsdb msg="write block" mint=1576087200000 maxt=1576094400000 ulid=01DVVC5V5WESMMH77FZVCJ80Q8 duration=1.881893506s 109level=info ts=2019-12-11T21:00:01.837Z caller=compact.go:496 component=tsdb msg="write block" mint=1576087200000 maxt=1576094400000 ulid=01DVVC5V6145SMRFE0WR0P3YTQ duration=1.772164011s 110level=info ts=2019-12-11T19:00:05.276Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=1007 last=1009 duration=3.031727362s 111level=info ts=2019-12-11T19:00:05.222Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=1007 last=1009 duration=3.003072336s 112level=info ts=2019-12-11T19:00:02.244Z caller=head.go:596 component=tsdb msg="head GC completed" duration=125.675247ms 113level=info ts=2019-12-11T19:00:02.219Z caller=head.go:596 component=tsdb msg="head GC completed" duration=127.466308ms 114level=info ts=2019-12-11T19:00:01.888Z caller=compact.go:496 component=tsdb msg="write block" mint=1576080000000 maxt=1576087200000 ulid=01DVV5A3XRVMTB2E7V3MZ6RGCA duration=1.832443683s 115level=info ts=2019-12-11T19:00:01.845Z caller=compact.go:496 component=tsdb msg="write block" mint=1576080000000 maxt=1576087200000 ulid=01DVV5A3XWVRTNS7G7BBDQ9G2W duration=1.784935995s 116level=info ts=2019-12-11T18:31:34.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4 117level=info ts=2019-12-11T18:24:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3 118level=info ts=2019-12-11T17:00:05.233Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=1004 last=1006 duration=3.008189996s 119level=info ts=2019-12-11T17:00:05.223Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=1004 last=1006 duration=2.97892s 120level=info ts=2019-12-11T17:00:02.244Z caller=head.go:596 component=tsdb msg="head GC completed" duration=132.385042ms 121level=info ts=2019-12-11T17:00:02.225Z caller=head.go:596 component=tsdb msg="head GC completed" duration=125.500534ms 122level=info ts=2019-12-11T17:00:01.870Z caller=compact.go:496 component=tsdb msg="write block" mint=1576072800000 maxt=1576080000000 ulid=01DVTYECNW5T3AHHB2EXATYFMJ duration=1.810447322s 123level=info ts=2019-12-11T17:00:01.870Z caller=compact.go:496 component=tsdb msg="write block" mint=1576072800000 maxt=1576080000000 ulid=01DVTYECNS4AZH3ZMCER87DYWG duration=1.813347748s 124level=info ts=2019-12-11T15:00:16.297Z caller=compact.go:441 component=tsdb msg="compact blocks" count=3 mint=1576000800000 maxt=1576065600000 ulid=01DVTQJY4WBY96QVV4XQJTR2JC sources="[01DVSECF6Q4JXFDGMFQB3J1Z9E 01DVT2ZN0DMXXJJDHKS0M8JWMS 01DVTQJTX0GZ1S7J51CN1RJNQX]" duration=7.308935842s 125level=info ts=2019-12-11T15:00:15.941Z caller=compact.go:441 component=tsdb msg="compact blocks" count=3 mint=1576000800000 maxt=1576065600000 ulid=01DVTQJY58270MEPGVGGDZZRKJ sources="[01DVSECF6J4NZRNHABZ2MSG7V7 01DVT2ZN4RB65KG77XPHPNVSAM 01DVTQJTYGFS18MWME9Z2NFJSW]" duration=6.941637414s 126level=info ts=2019-12-11T15:00:08.544Z caller=compact.go:441 component=tsdb msg="compact blocks" count=3 mint=1576044000000 maxt=1576065600000 ulid=01DVTQJTX0GZ1S7J51CN1RJNQX sources="[01DVT2ZFNYB7DEH57ZX4HW2DAV 01DVT9V6XW9ENV15NHKR20T9B4 01DVTGPY5WTBSSEQ37JQ2VPCTQ]" duration=2.880290482s 127level=info ts=2019-12-11T15:00:08.541Z caller=compact.go:441 component=tsdb msg="compact blocks" count=3 mint=1576044000000 maxt=1576065600000 ulid=01DVTQJTYGFS18MWME9Z2NFJSW sources="[01DVT2ZFP3R7RB9H6BS3JVAMXJ 01DVT9V6Y21E8YXRKNGA9RPB7D 01DVTGPY5XGARMV8B8VBWQ23W3]" duration=2.829184147s 128level=info ts=2019-12-11T15:00:05.505Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=1001 last=1003 duration=3.006477625s 129level=info ts=2019-12-11T15:00:05.452Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=1001 last=1003 duration=2.990895181s 130level=info ts=2019-12-11T15:00:02.498Z caller=head.go:596 component=tsdb msg="head GC completed" duration=129.237566ms 131level=info ts=2019-12-11T15:00:02.461Z caller=head.go:596 component=tsdb msg="head GC completed" duration=129.961097ms 132level=info ts=2019-12-11T15:00:02.022Z caller=compact.go:496 component=tsdb msg="write block" mint=1576065600000 maxt=1576072800000 ulid=01DVTQJNDRV9NDCK9H2BCH04R0 duration=1.96598488s 133level=info ts=2019-12-11T15:00:01.933Z caller=compact.go:496 component=tsdb msg="write block" mint=1576065600000 maxt=1576072800000 ulid=01DVTQJNDXSY7N5V60ZX7X1C3J duration=1.871872199s 134level=info ts=2019-12-11T14:15:04.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4 135level=info ts=2019-12-11T14:07:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3 136level=info ts=2019-12-11T13:18:04.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4 137level=info ts=2019-12-11T13:12:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3 138level=info ts=2019-12-11T13:00:05.395Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=998 last=1000 duration=3.010358861s 139level=info ts=2019-12-11T13:00:05.249Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=998 last=1000 duration=3.032196282s 140level=info ts=2019-12-11T13:00:02.385Z caller=head.go:596 component=tsdb msg="head GC completed" duration=131.568186ms 141level=info ts=2019-12-11T13:00:02.217Z caller=head.go:596 component=tsdb msg="head GC completed" duration=136.017788ms 142level=info ts=2019-12-11T13:00:02.021Z caller=compact.go:496 component=tsdb msg="write block" mint=1576058400000 maxt=1576065600000 ulid=01DVTGPY5XGARMV8B8VBWQ23W3 duration=1.959903s 143level=info ts=2019-12-11T13:00:01.865Z caller=compact.go:496 component=tsdb msg="write block" mint=1576058400000 maxt=1576065600000 ulid=01DVTGPY5WTBSSEQ37JQ2VPCTQ duration=1.805149859s 144level=info ts=2019-12-11T11:46:34.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4 145level=info ts=2019-12-11T11:39:44.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3 146level=info ts=2019-12-11T11:35:04.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4 147level=info ts=2019-12-11T11:26:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3 148level=info ts=2019-12-11T11:15:34.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4 149level=info ts=2019-12-11T11:06:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3 150level=info ts=2019-12-11T11:01:24.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4 151level=info ts=2019-12-11T11:00:05.591Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=995 last=997 duration=3.063684941s 152level=info ts=2019-12-11T11:00:05.297Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=995 last=997 duration=3.051047495s 153level=info ts=2019-12-11T11:00:02.527Z caller=head.go:596 component=tsdb msg="head GC completed" duration=131.530749ms 154level=info ts=2019-12-11T11:00:02.246Z caller=head.go:596 component=tsdb msg="head GC completed" duration=123.08975ms 155level=info ts=2019-12-11T11:00:02.096Z caller=compact.go:496 component=tsdb msg="write block" mint=1576051200000 maxt=1576058400000 ulid=01DVT9V6Y21E8YXRKNGA9RPB7D duration=2.029825916s 156level=info ts=2019-12-11T11:00:01.819Z caller=compact.go:496 component=tsdb msg="write block" mint=1576051200000 maxt=1576058400000 ulid=01DVT9V6XW9ENV15NHKR20T9B4 duration=1.7583013s 157level=info ts=2019-12-11T10:54:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3 158level=info ts=2019-12-11T10:46:34.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4 159level=info ts=2019-12-11T10:39:44.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3 160level=info ts=2019-12-11T10:34:34.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4 161level=info ts=2019-12-11T10:26:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3 162level=info ts=2019-12-11T10:18:34.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4 163level=info ts=2019-12-11T10:12:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3 164level=info ts=2019-12-11T09:56:04.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4 165level=info ts=2019-12-11T09:48:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3 166level=info ts=2019-12-11T09:00:08.553Z caller=compact.go:441 component=tsdb msg="compact blocks" count=3 mint=1576022400000 maxt=1576044000000 ulid=01DVT2ZN4RB65KG77XPHPNVSAM sources="[01DVSEC9XXK1J3B0186KYQECZT 01DVSN815STR0D0B8245RWNF13 01DVSW3RDRKTJVEZWGYE07XBXE]" duration=2.896352595s 167level=info ts=2019-12-11T09:00:08.407Z caller=compact.go:441 component=tsdb msg="compact blocks" count=3 mint=1576022400000 maxt=1576044000000 ulid=01DVT2ZN0DMXXJJDHKS0M8JWMS sources="[01DVSEC9XW0W8V42SPRR0YMM0X 01DVSN815W8YTW3DPQTJVRNTS4 01DVSW3RDWE1WHSM8AEW0ARA3S]" duration=2.890101974s 168level=info ts=2019-12-11T09:00:05.444Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=992 last=994 duration=3.058184317s 169level=info ts=2019-12-11T09:00:05.306Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=992 last=994 duration=2.99204816s 170level=info ts=2019-12-11T09:00:02.385Z caller=head.go:596 component=tsdb msg="head GC completed" duration=128.295437ms 171level=info ts=2019-12-11T09:00:02.313Z caller=head.go:596 component=tsdb msg="head GC completed" duration=128.456638ms 172level=info ts=2019-12-11T09:00:02.023Z caller=compact.go:496 component=tsdb msg="write block" mint=1576044000000 maxt=1576051200000 ulid=01DVT2ZFP3R7RB9H6BS3JVAMXJ duration=1.955843851s 173level=info ts=2019-12-11T09:00:01.935Z caller=compact.go:496 component=tsdb msg="write block" mint=1576044000000 maxt=1576051200000 ulid=01DVT2ZFNYB7DEH57ZX4HW2DAV duration=1.873653026s 174level=info ts=2019-12-11T07:00:05.441Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=989 last=991 duration=3.013763908s 175level=info ts=2019-12-11T07:00:05.272Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=989 last=991 duration=2.979497994s 176level=info ts=2019-12-11T07:00:02.427Z caller=head.go:596 component=tsdb msg="head GC completed" duration=126.635643ms 177level=info ts=2019-12-11T07:00:02.293Z caller=head.go:596 component=tsdb msg="head GC completed" duration=121.051415ms 178level=info ts=2019-12-11T07:00:02.056Z caller=compact.go:496 component=tsdb msg="write block" mint=1576036800000 maxt=1576044000000 ulid=01DVSW3RDWE1WHSM8AEW0ARA3S duration=1.995603695s 179level=info ts=2019-12-11T07:00:01.941Z caller=compact.go:496 component=tsdb msg="write block" mint=1576036800000 maxt=1576044000000 ulid=01DVSW3RDRKTJVEZWGYE07XBXE duration=1.885680378s 180level=info ts=2019-12-11T06:20:04.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4 181level=info ts=2019-12-11T06:14:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3 182level=info ts=2019-12-11T05:02:04.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=5 to=4 183level=info ts=2019-12-11T05:01:04.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=5 184level=info ts=2019-12-11T05:00:05.488Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=986 last=988 duration=3.043360624s 185level=info ts=2019-12-11T05:00:05.288Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=986 last=988 duration=2.998209654s 186level=info ts=2019-12-11T05:00:02.445Z caller=head.go:596 component=tsdb msg="head GC completed" duration=130.642245ms 187level=info ts=2019-12-11T05:00:02.290Z caller=head.go:596 component=tsdb msg="head GC completed" duration=128.363621ms 188level=info ts=2019-12-11T05:00:02.066Z caller=compact.go:496 component=tsdb msg="write block" mint=1576029600000 maxt=1576036800000 ulid=01DVSN815STR0D0B8245RWNF13 duration=2.008689142s 189level=info ts=2019-12-11T05:00:01.938Z caller=compact.go:496 component=tsdb msg="write block" mint=1576029600000 maxt=1576036800000 ulid=01DVSN815W8YTW3DPQTJVRNTS4 duration=1.877943808s 190level=info ts=2019-12-11T04:55:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3 191level=info ts=2019-12-11T04:35:34.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4 192level=info ts=2019-12-11T04:28:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3 193level=info ts=2019-12-11T04:15:24.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4 194level=info ts=2019-12-11T04:07:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3 195level=info ts=2019-12-11T04:03:34.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4 196level=info ts=2019-12-11T03:57:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3 197level=info ts=2019-12-11T03:52:04.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4 198level=info ts=2019-12-11T03:43:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3 199level=info ts=2019-12-11T03:32:34.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4 200level=info ts=2019-12-11T03:24:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3 201level=info ts=2019-12-11T03:19:34.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4 202level=info ts=2019-12-11T03:12:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3 203level=info ts=2019-12-11T03:00:08.325Z caller=compact.go:441 component=tsdb msg="compact blocks" count=3 mint=1576000800000 maxt=1576022400000 ulid=01DVSECF6Q4JXFDGMFQB3J1Z9E sources="[01DVRSS45W7DXE05RGBYGH58PY 01DVS0MVDWGK47AZ3HY5GQEMK4 01DVS7GJNW7BF3R6KK7GW291R0]" duration=2.861556831s 204level=info ts=2019-12-11T03:00:08.255Z caller=compact.go:441 component=tsdb msg="compact blocks" count=3 mint=1576000800000 maxt=1576022400000 ulid=01DVSECF6J4NZRNHABZ2MSG7V7 sources="[01DVRSS4632MEB6SYC6SB7DTGE 01DVS0MVDR5Z67QJD6T94CXHRA 01DVS7GJNSYRFT48H9CDRP82YV]" duration=2.796902205s 205level=info ts=2019-12-11T03:00:05.253Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=983 last=985 duration=3.004398083s 206level=info ts=2019-12-11T03:00:05.245Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=983 last=985 duration=3.023743067s 207level=info ts=2019-12-11T03:00:02.248Z caller=head.go:596 component=tsdb msg="head GC completed" duration=127.893231ms 208level=info ts=2019-12-11T03:00:02.221Z caller=head.go:596 component=tsdb msg="head GC completed" duration=132.662929ms 209level=info ts=2019-12-11T03:00:01.903Z caller=compact.go:496 component=tsdb msg="write block" mint=1576022400000 maxt=1576029600000 ulid=01DVSEC9XW0W8V42SPRR0YMM0X duration=1.842688968s 210level=info ts=2019-12-11T03:00:01.847Z caller=compact.go:496 component=tsdb msg="write block" mint=1576022400000 maxt=1576029600000 ulid=01DVSEC9XXK1J3B0186KYQECZT duration=1.78558499s 211level=info ts=2019-12-11T02:18:44.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=5 to=4 212level=info ts=2019-12-11T02:18:04.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=5 213level=info ts=2019-12-11T02:11:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3 214level=info ts=2019-12-11T01:59:34.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4 215level=info ts=2019-12-11T01:52:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3 216level=info ts=2019-12-11T01:00:05.272Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=980 last=982 duration=3.025045534s 217level=info ts=2019-12-11T01:00:05.189Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=980 last=982 duration=2.992866718s 218level=info ts=2019-12-11T01:00:02.247Z caller=head.go:596 component=tsdb msg="head GC completed" duration=123.561834ms 219level=info ts=2019-12-11T01:00:02.196Z caller=head.go:596 component=tsdb msg="head GC completed" duration=108.589195ms 220level=info ts=2019-12-11T01:00:01.905Z caller=compact.go:496 component=tsdb msg="write block" mint=1576015200000 maxt=1576022400000 ulid=01DVS7GJNW7BF3R6KK7GW291R0 duration=1.844635186s 221level=info ts=2019-12-11T01:00:01.866Z caller=compact.go:496 component=tsdb msg="write block" mint=1576015200000 maxt=1576022400000 ulid=01DVS7GJNSYRFT48H9CDRP82YV duration=1.809175377s 222level=info ts=2019-12-11T00:31:25.063Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=7 to=5 223level=info ts=2019-12-11T00:30:25.063Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=10 to=7 224level=info ts=2019-12-11T00:29:55.063Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=14 to=10 225level=info ts=2019-12-11T00:29:25.064Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=85 to=14 226level=info ts=2019-12-11T00:29:15.063Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=5 to=85 227level=info ts=2019-12-10T23:00:05.385Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=977 last=979 duration=3.157877457s 228level=info ts=2019-12-10T23:00:05.136Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=977 last=979 duration=3.013728541s 229level=info ts=2019-12-10T23:00:02.227Z caller=head.go:596 component=tsdb msg="head GC completed" duration=133.349607ms 230level=info ts=2019-12-10T23:00:02.123Z caller=head.go:596 component=tsdb msg="head GC completed" duration=110.615384ms 231level=info ts=2019-12-10T23:00:01.832Z caller=compact.go:496 component=tsdb msg="write block" mint=1576008000000 maxt=1576015200000 ulid=01DVS0MVDWGK47AZ3HY5GQEMK4 duration=1.772580137s 232level=info ts=2019-12-10T23:00:01.780Z caller=compact.go:496 component=tsdb msg="write block" mint=1576008000000 maxt=1576015200000 ulid=01DVS0MVDR5Z67QJD6T94CXHRA duration=1.724738556s 233level=info ts=2019-12-10T21:00:18.426Z caller=compact.go:441 component=tsdb msg="compact blocks" count=3 mint=1575936000000 maxt=1576000800000 ulid=01DVRSSD7JM2Y5MQNCZ7QZRSWK sources="[01DVQGJZNRDFP0P161HP7GJX44 01DVR563YKZY789FPAM3DD8DKX 01DVRSS9Q04WN7F254ZCSQ4YP5]" duration=9.096172888s 234level=info ts=2019-12-10T21:00:16.394Z caller=compact.go:441 component=tsdb msg="compact blocks" count=3 mint=1575936000000 maxt=1576000800000 ulid=01DVRSSCTH9ED87EXNSR9J8PE6 sources="[01DVQGJY3RF9X7R93QY6V579W3 01DVR563QG1PZ1AY7RPKSCMKND 01DVRSS9HMN2EC6QQQ2XP7R90D]" duration=7.481260173s 235level=info ts=2019-12-10T21:00:08.859Z caller=compact.go:441 component=tsdb msg="compact blocks" count=3 mint=1575979200000 maxt=1576000800000 ulid=01DVRSS9Q04WN7F254ZCSQ4YP5 sources="[01DVR55YDW6Q96ZHGXD1T7HVF4 01DVRC1NNWE3ZMEQ6035ZJTF49 01DVRJXCXW4W35MBB4E9RXX1QD]" duration=3.130772971s 236level=info ts=2019-12-10T21:00:08.473Z caller=compact.go:441 component=tsdb msg="compact blocks" count=3 mint=1575979200000 maxt=1576000800000 ulid=01DVRSS9HMN2EC6QQQ2XP7R90D sources="[01DVR55YE3CA9B12S48FTTFSVD 01DVRC1NP4CCWPRMCC7667R1FZ 01DVRJXCY0DY2R6DVWWGNXNPRQ]" duration=2.917254733s 237level=info ts=2019-12-10T21:00:05.490Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=974 last=976 duration=3.159908932s 238level=info ts=2019-12-10T21:00:05.339Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=974 last=976 duration=3.090229598s 239level=info ts=2019-12-10T21:00:02.330Z caller=head.go:596 component=tsdb msg="head GC completed" duration=129.067188ms 240level=info ts=2019-12-10T21:00:02.249Z caller=head.go:596 component=tsdb msg="head GC completed" duration=108.272575ms 241level=info ts=2019-12-10T21:00:01.804Z caller=compact.go:496 component=tsdb msg="write block" mint=1576000800000 maxt=1576008000000 ulid=01DVRSS45W7DXE05RGBYGH58PY duration=1.743999568s 242level=info ts=2019-12-10T21:00:01.774Z caller=compact.go:496 component=tsdb msg="write block" mint=1576000800000 maxt=1576008000000 ulid=01DVRSS4632MEB6SYC6SB7DTGE duration=1.706876662s 243level=info ts=2019-12-10T20:35:34.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4 244level=info ts=2019-12-10T20:29:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3 245level=info ts=2019-12-10T19:00:06.012Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=971 last=973 duration=3.248527735s 246level=info ts=2019-12-10T19:00:05.641Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=971 last=973 duration=3.1047498s 247level=info ts=2019-12-10T19:00:02.763Z caller=head.go:596 component=tsdb msg="head GC completed" duration=210.397069ms 248level=info ts=2019-12-10T19:00:02.536Z caller=head.go:596 component=tsdb msg="head GC completed" duration=171.333573ms 249level=info ts=2019-12-10T19:00:02.259Z caller=compact.go:496 component=tsdb msg="write block" mint=1575993600000 maxt=1576000800000 ulid=01DVRJXCXW4W35MBB4E9RXX1QD duration=2.199162459s 250level=info ts=2019-12-10T19:00:02.080Z caller=compact.go:496 component=tsdb msg="write block" mint=1575993600000 maxt=1576000800000 ulid=01DVRJXCY0DY2R6DVWWGNXNPRQ duration=2.016321337s 251level=info ts=2019-12-10T17:00:05.549Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=968 last=970 duration=3.183706512s 252level=info ts=2019-12-10T17:00:05.319Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=968 last=970 duration=3.088304654s 253level=info ts=2019-12-10T17:00:02.365Z caller=head.go:596 component=tsdb msg="head GC completed" duration=133.008474ms 254level=info ts=2019-12-10T17:00:02.231Z caller=head.go:596 component=tsdb msg="head GC completed" duration=114.89207ms 255level=info ts=2019-12-10T17:00:01.942Z caller=compact.go:496 component=tsdb msg="write block" mint=1575986400000 maxt=1575993600000 ulid=01DVRC1NNWE3ZMEQ6035ZJTF49 duration=1.881731957s 256level=info ts=2019-12-10T17:00:01.864Z caller=compact.go:496 component=tsdb msg="write block" mint=1575986400000 maxt=1575993600000 ulid=01DVRC1NP4CCWPRMCC7667R1FZ duration=1.795832733s 257level=info ts=2019-12-10T15:00:09.507Z caller=compact.go:441 component=tsdb msg="compact blocks" count=3 mint=1575957600000 maxt=1575979200000 ulid=01DVR563YKZY789FPAM3DD8DKX sources="[01DVQGJRNW1DY7K8KW6B2RY4FF 01DVQQEFXWYP18MDFKY58VJCSG 01DVQYA75W7ERXK6FBEMYYEX6S]" duration=3.791514409s 258level=info ts=2019-12-10T15:00:08.520Z caller=compact.go:441 component=tsdb msg="compact blocks" count=3 mint=1575957600000 maxt=1575979200000 ulid=01DVR563QG1PZ1AY7RPKSCMKND sources="[01DVQGJRP3CC86KKVR5MZ1YYTK 01DVQQEFY0CEBFC4QE02GW9S4F 01DVQYA75TMRETNZRWPV46G5Y0]" duration=3.032106451s 259level=info ts=2019-12-10T15:00:05.484Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=965 last=967 duration=3.124836463s 260level=info ts=2019-12-10T15:00:05.277Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=965 last=967 duration=3.055778688s 261level=info ts=2019-12-10T15:00:02.359Z caller=head.go:596 component=tsdb msg="head GC completed" duration=130.554076ms 262level=info ts=2019-12-10T15:00:02.221Z caller=head.go:596 component=tsdb msg="head GC completed" duration=114.665423ms 263level=info ts=2019-12-10T15:00:01.972Z caller=compact.go:496 component=tsdb msg="write block" mint=1575979200000 maxt=1575986400000 ulid=01DVR55YDW6Q96ZHGXD1T7HVF4 duration=1.912209972s 264level=info ts=2019-12-10T15:00:01.878Z caller=compact.go:496 component=tsdb msg="write block" mint=1575979200000 maxt=1575986400000 ulid=01DVR55YE3CA9B12S48FTTFSVD duration=1.811316924s 2652019-12-10 13:50:13.598594155 +0000 UTC 266level=info ts=2019-12-10T13:50:13.596Z caller=main.go:771 msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml 267level=info ts=2019-12-10T13:50:13.553Z caller=kubernetes.go:192 component="discovery manager scrape" discovery=k8s msg="Using pod service account via in-cluster config" 268level=info ts=2019-12-10T13:50:13.552Z caller=kubernetes.go:192 component="discovery manager scrape" discovery=k8s msg="Using pod service account via in-cluster config" 269level=info ts=2019-12-10T13:50:13.551Z caller=kubernetes.go:192 component="discovery manager scrape" discovery=k8s msg="Using pod service account via in-cluster config" 270level=info ts=2019-12-10T13:50:13.554Z caller=kubernetes.go:192 component="discovery manager scrape" discovery=k8s msg="Using pod service account via in-cluster config" 271level=info ts=2019-12-10T13:50:13.546Z caller=main.go:743 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2721 curl -X POST --fail -o - -sS http://localhost:80/prometheus/-/reload 2732019/12/10 13:50:13 DEBUG: "/etc/prometheus/..2019_12_05_07_22_08.390693530": REMOVE at 2019-12-10 13:50:13.341280871 +0000 UTC 2742019/12/10 13:50:13 DEBUG: "/etc/prometheus/..data": REMOVE at 2019-12-10 13:50:13.341280871 +0000 UTC 2752019/12/10 13:50:13 DEBUG: "/etc/prometheus/..data/prometheus.yml": REMOVE at 2019-12-10 13:50:13.341280871 +0000 UTC 2762019/12/10 13:50:13 DEBUG: "/etc/prometheus/..data/recording.rules": REMOVE at 2019-12-10 13:50:13.341280871 +0000 UTC 2772019/12/10 13:50:13 DEBUG: "/etc/prometheus/..data/alerts.rules": REMOVE at 2019-12-10 13:50:13.341280871 +0000 UTC 2782019/12/10 13:50:13 DEBUG: Watching /etc/prometheus/..data 2792019/12/10 13:50:13 DEBUG: "/etc/prometheus/..data": CREATE at 2019-12-10 13:50:13.341280871 +0000 UTC 2802019/12/10 13:50:13 DEBUG: "/etc/prometheus/..data_tmp": RENAME at 2019-12-10 13:50:13.341280871 +0000 UTC 2812019/12/10 13:50:13 DEBUG: "/etc/prometheus/..2019_12_10_13_50_13.738754268": CHMOD at 2019-12-10 13:50:13.341280871 +0000 UTC 2822019/12/10 13:50:13 DEBUG: Watching /etc/prometheus/..2019_12_10_13_50_13.738754268 2832019/12/10 13:50:13 DEBUG: "/etc/prometheus/..2019_12_10_13_50_13.738754268": CREATE at 2019-12-10 13:50:13.341280871 +0000 UTC 2842019-12-10 13:49:53.296090488 +0000 UTC 285level=info ts=2019-12-10T13:49:53.294Z caller=main.go:771 msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml 286level=info ts=2019-12-10T13:49:53.254Z caller=kubernetes.go:192 component="discovery manager scrape" discovery=k8s msg="Using pod service account via in-cluster config" 287level=info ts=2019-12-10T13:49:53.253Z caller=kubernetes.go:192 component="discovery manager scrape" discovery=k8s msg="Using pod service account via in-cluster config" 288level=info ts=2019-12-10T13:49:53.252Z caller=kubernetes.go:192 component="discovery manager scrape" discovery=k8s msg="Using pod service account via in-cluster config" 289level=info ts=2019-12-10T13:49:53.251Z caller=kubernetes.go:192 component="discovery manager scrape" discovery=k8s msg="Using pod service account via in-cluster config" 290level=info ts=2019-12-10T13:49:53.248Z caller=main.go:743 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 291curl -X POST --fail -o - -sS http://localhost:80/prometheus/-/reload 2922019/12/10 13:49:53 DEBUG: "/etc/prometheus/..2019_12_05_07_22_36.096562123": REMOVE at 2019-12-10 13:49:53.044039978 +0000 UTC 2932019/12/10 13:49:53 DEBUG: "/etc/prometheus/..data": REMOVE at 2019-12-10 13:49:53.043039894 +0000 UTC 2942019/12/10 13:49:53 DEBUG: "/etc/prometheus/..data/prometheus.yml": REMOVE at 2019-12-10 13:49:53.043039894 +0000 UTC 2952019/12/10 13:49:53 DEBUG: "/etc/prometheus/..data/recording.rules": REMOVE at 2019-12-10 13:49:53.043039894 +0000 UTC 2962019/12/10 13:49:53 DEBUG: "/etc/prometheus/..data/alerts.rules": REMOVE at 2019-12-10 13:49:53.043039894 +0000 UTC 2972019/12/10 13:49:53 DEBUG: Watching /etc/prometheus/..data 2982019/12/10 13:49:53 DEBUG: "/etc/prometheus/..data": CREATE at 2019-12-10 13:49:53.043039894 +0000 UTC 2992019/12/10 13:49:53 DEBUG: "/etc/prometheus/..data_tmp": RENAME at 2019-12-10 13:49:53.044039978 +0000 UTC 3002019/12/10 13:49:53 DEBUG: "/etc/prometheus/..2019_12_10_13_49_53.355073198": CHMOD at 2019-12-10 13:49:53.043039894 +0000 UTC 3012019/12/10 13:49:53 DEBUG: Watching /etc/prometheus/..2019_12_10_13_49_53.355073198 3022019/12/10 13:49:53 DEBUG: "/etc/prometheus/..2019_12_10_13_49_53.355073198": CREATE at 2019-12-10 13:49:53.043039894 +0000 UTC 303level=info ts=2019-12-10T13:00:06.007Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=962 last=964 duration=3.145696569s 304level=info ts=2019-12-10T13:00:05.601Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=962 last=964 duration=3.062580976s 305level=info ts=2019-12-10T13:00:02.861Z caller=head.go:596 component=tsdb msg="head GC completed" duration=169.077152ms 306level=info ts=2019-12-10T13:00:02.539Z caller=head.go:596 component=tsdb msg="head GC completed" duration=152.173262ms 307level=info ts=2019-12-10T13:00:02.425Z caller=compact.go:496 component=tsdb msg="write block" mint=1575972000000 maxt=1575979200000 ulid=01DVQYA75W7ERXK6FBEMYYEX6S duration=2.364066751s 308level=info ts=2019-12-10T13:00:02.150Z caller=compact.go:496 component=tsdb msg="write block" mint=1575972000000 maxt=1575979200000 ulid=01DVQYA75TMRETNZRWPV46G5Y0 duration=2.092629264s 309level=info ts=2019-12-10T11:48:14.793Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=2 to=1 310level=info ts=2019-12-10T11:47:34.793Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=3 to=2 311level=info ts=2019-12-10T11:47:14.793Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=5 to=3 312level=info ts=2019-12-10T11:46:54.793Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=7 to=5 313level=info ts=2019-12-10T11:46:34.793Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=10 to=7 314level=info ts=2019-12-10T11:46:14.793Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=16 to=10 315level=info ts=2019-12-10T11:45:54.793Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=26 to=16 316level=info ts=2019-12-10T11:45:34.793Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=39 to=26 317level=info ts=2019-12-10T11:45:14.793Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=63 to=39 318level=info ts=2019-12-10T11:44:54.793Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=100 to=63 319level=info ts=2019-12-10T11:44:45.063Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=2 to=1 320level=info ts=2019-12-10T11:44:34.793Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=156 to=100 321level=info ts=2019-12-10T11:44:15.063Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=3 to=2 322level=info ts=2019-12-10T11:44:14.793Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=253 to=156 323level=info ts=2019-12-10T11:43:55.064Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=4 to=3 324level=info ts=2019-12-10T11:43:54.793Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=405 to=253 325level=info ts=2019-12-10T11:43:45.063Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=5 to=4 326level=info ts=2019-12-10T11:43:34.793Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=619 to=405 327level=info ts=2019-12-10T11:43:25.063Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=8 to=5 328level=info ts=2019-12-10T11:43:14.793Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=1000 to=619 329level=info ts=2019-12-10T11:43:05.063Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=11 to=8 330level=info ts=2019-12-10T11:42:45.063Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=22 to=11 331level=info ts=2019-12-10T11:42:25.063Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=16 to=22 332level=error ts=2019-12-10T11:42:11.074Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 400 Bad Request: out of order sample" 333level=error ts=2019-12-10T11:42:11.073Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 400 Bad Request: out of order sample" 334level=info ts=2019-12-10T11:42:05.063Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=11 to=16 335level=info ts=2019-12-10T11:41:55.063Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=8 to=11 336level=info ts=2019-12-10T11:39:35.063Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=6 to=8 337level=info ts=2019-12-10T11:38:55.063Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=4 to=6 338level=info ts=2019-12-10T11:38:35.064Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=2 to=4 339level=error ts=2019-12-10T11:38:12.281Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=84 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 84 samples" 340level=error ts=2019-12-10T11:38:12.281Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=69 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 69 samples" 341level=error ts=2019-12-10T11:38:12.235Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 342level=error ts=2019-12-10T11:38:12.236Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 343level=error ts=2019-12-10T11:38:12.204Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=97 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 97 samples" 344level=error ts=2019-12-10T11:38:12.183Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=89 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 89 samples" 345level=error ts=2019-12-10T11:38:12.129Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=88 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 88 samples" 346level=error ts=2019-12-10T11:38:12.127Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=91 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 91 samples" 347level=error ts=2019-12-10T11:38:12.128Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 348level=error ts=2019-12-10T11:38:12.129Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=86 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 86 samples" 349level=error ts=2019-12-10T11:38:12.139Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=90 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 90 samples" 350level=error ts=2019-12-10T11:38:12.140Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 351level=error ts=2019-12-10T11:38:12.141Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 352level=error ts=2019-12-10T11:38:12.142Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 353level=error ts=2019-12-10T11:38:12.143Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=56 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 56 samples" 354level=error ts=2019-12-10T11:38:12.125Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=52 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 52 samples" 355level=error ts=2019-12-10T11:38:12.124Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 356level=error ts=2019-12-10T11:38:12.123Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=75 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 75 samples" 357level=error ts=2019-12-10T11:38:12.122Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=69 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 69 samples" 358level=error ts=2019-12-10T11:38:12.120Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 359level=error ts=2019-12-10T11:38:12.1201Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=67 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 67 samples" 360level=error ts=2019-12-10T11:38:12.1202Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=78 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 78 samples" 361level=error ts=2019-12-10T11:38:12.1203Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 362level=error ts=2019-12-10T11:38:12.1191Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 363level=error ts=2019-12-10T11:38:12.1192Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 364level=error ts=2019-12-10T11:38:12.081Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=96 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 96 samples" 365level=error ts=2019-12-10T11:38:12.0371Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 366level=error ts=2019-12-10T11:38:12.0372Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 367level=error ts=2019-12-10T11:38:12.0373Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 368level=error ts=2019-12-10T11:38:12.024Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 369level=error ts=2019-12-10T11:38:12.025Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 370level=error ts=2019-12-10T11:38:12.026Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=89 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 89 samples" 371level=error ts=2019-12-10T11:38:11.920Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 372level=error ts=2019-12-10T11:38:11.917Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 373level=error ts=2019-12-10T11:38:11.916Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 374level=error ts=2019-12-10T11:38:11.913Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=95 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 95 samples" 375level=error ts=2019-12-10T11:38:11.913Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=82 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 82 samples" 376level=error ts=2019-12-10T11:38:11.722Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 377level=error ts=2019-12-10T11:38:11.721Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 378level=error ts=2019-12-10T11:38:11.176Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 379level=error ts=2019-12-10T11:38:11.1451Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 380level=error ts=2019-12-10T11:38:11.1452Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 381level=error ts=2019-12-10T11:38:11.1453Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=99 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 99 samples" 382level=error ts=2019-12-10T11:38:11.1454Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=85 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 85 samples" 383level=error ts=2019-12-10T11:38:11.1455Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 384level=error ts=2019-12-10T11:38:11.1441Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 385level=error ts=2019-12-10T11:38:11.1442Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=70 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 70 samples" 386level=error ts=2019-12-10T11:38:11.1443Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=72 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 72 samples" 387level=error ts=2019-12-10T11:38:11.1444Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=97 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 97 samples" 388level=error ts=2019-12-10T11:38:11.1445Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=46 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 46 samples" 389level=error ts=2019-12-10T11:38:11.1431Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 390level=error ts=2019-12-10T11:38:11.1432Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=83 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 83 samples" 391level=error ts=2019-12-10T11:38:11.1433Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 392level=error ts=2019-12-10T11:38:11.1434Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=92 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 92 samples" 393level=error ts=2019-12-10T11:38:11.0011Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 394level=error ts=2019-12-10T11:38:11.0012Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 395level=error ts=2019-12-10T11:38:11.0012Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=84 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 84 samples" 396level=error ts=2019-12-10T11:38:11.0013Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 397level=error ts=2019-12-10T11:38:10.9551Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 398level=error ts=2019-12-10T11:38:10.9532Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 399level=error ts=2019-12-10T11:38:10.9421Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=86 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 86 samples" 400level=error ts=2019-12-10T11:38:10.9422Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 401level=error ts=2019-12-10T11:38:10.9423Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 402level=error ts=2019-12-10T11:38:10.941Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=58 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 58 samples" 403level=error ts=2019-12-10T11:38:10.942Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 404level=error ts=2019-12-10T11:38:10.940Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 405level=error ts=2019-12-10T11:38:10.943Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 406level=error ts=2019-12-10T11:38:10.939Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 407level=error ts=2019-12-10T11:38:10.939Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=23 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 23 samples" 408level=error ts=2019-12-10T11:38:10.939Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=59 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 59 samples" 409level=error ts=2019-12-10T11:38:10.939Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=81 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 81 samples" 410level=error ts=2019-12-10T11:38:10.878Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 411level=error ts=2019-12-10T11:38:10.879Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 412level=error ts=2019-12-10T11:38:10.8781Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 413level=error ts=2019-12-10T11:38:10.8782Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 414level=error ts=2019-12-10T11:38:10.877Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=96 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 96 samples" 415level=error ts=2019-12-10T11:38:10.877Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=70 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 70 samples" 416level=error ts=2019-12-10T11:38:10.877Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=72 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 72 samples" 417level=error ts=2019-12-10T11:38:10.039Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 418level=error ts=2019-12-10T11:38:09.902Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=79 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 79 samples" 419level=error ts=2019-12-10T11:38:09.901Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 420level=error ts=2019-12-10T11:38:09.902Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 421level=error ts=2019-12-10T11:38:09.903Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=92 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 92 samples" 422level=error ts=2019-12-10T11:38:09.898Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 423level=error ts=2019-12-10T11:38:09.8971Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=95 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 95 samples" 424level=error ts=2019-12-10T11:38:09.8972Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=73 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 73 samples" 425level=error ts=2019-12-10T11:38:09.8973Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 426level=error ts=2019-12-10T11:38:09.8974Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=68 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 68 samples" 427level=error ts=2019-12-10T11:38:09.8975Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 428level=error ts=2019-12-10T11:38:09.8976Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=35 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 35 samples" 429level=error ts=2019-12-10T11:38:09.8977Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 430level=error ts=2019-12-10T11:38:09.8951Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 431level=error ts=2019-12-10T11:38:09.8952Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=98 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 98 samples" 432level=error ts=2019-12-10T11:38:09.8953Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=98 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 98 samples" 433level=error ts=2019-12-10T11:38:09.876Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 434level=error ts=2019-12-10T11:38:09.8401Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=88 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 88 samples" 435level=error ts=2019-12-10T11:38:09.8402Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=92 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 92 samples" 436level=error ts=2019-12-10T11:38:09.8403Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 437level=error ts=2019-12-10T11:38:09.8404Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=85 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 85 samples" 438level=error ts=2019-12-10T11:38:09.8406Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 439level=error ts=2019-12-10T11:38:09.8407Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 440level=error ts=2019-12-10T11:38:09.832Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 441level=error ts=2019-12-10T11:38:09.825Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 442level=error ts=2019-12-10T11:38:09.8251Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 443level=error ts=2019-12-10T11:38:09.8241Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 444level=error ts=2019-12-10T11:38:09.8242Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 445level=error ts=2019-12-10T11:38:09.8243Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=96 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 96 samples" 446level=error ts=2019-12-10T11:38:09.8244Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=17 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 17 samples" 447level=error ts=2019-12-10T11:38:09.8245Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 448level=error ts=2019-12-10T11:38:09.8246Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 449level=error ts=2019-12-10T11:38:09.815Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=94 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 94 samples" 450level=error ts=2019-12-10T11:38:09.806Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=81 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 81 samples" 451level=error ts=2019-12-10T11:38:09.736Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 452level=error ts=2019-12-10T11:38:09.736Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=48 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 48 samples" 453level=error ts=2019-12-10T11:38:09.735Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=97 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 97 samples" 454level=error ts=2019-12-10T11:38:09.735Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=67 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 67 samples" 455level=error ts=2019-12-10T11:38:09.735Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=43 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 43 samples" 456level=error ts=2019-12-10T11:38:09.735Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=90 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 90 samples" 457level=error ts=2019-12-10T11:38:09.735Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 458level=error ts=2019-12-10T11:38:09.735Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=19 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 19 samples" 459level=error ts=2019-12-10T11:38:09.733Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=79 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 79 samples" 460level=error ts=2019-12-10T11:38:09.700Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 461level=error ts=2019-12-10T11:38:09.700Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=50 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 50 samples" 462level=error ts=2019-12-10T11:38:09.700Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=77 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 77 samples" 463level=error ts=2019-12-10T11:38:08.797Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=86 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 86 samples" 464level=error ts=2019-12-10T11:38:08.797Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 101 samples" 465level=error ts=2019-12-10T11:38:08.797Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 102 samples" 466level=error ts=2019-12-10T11:38:08.797Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 103 samples" 467level=error ts=2019-12-10T11:38:08.797Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=83 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 83 samples" 468level=error ts=2019-12-10T11:38:08.797Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 113 samples" 469level=error ts=2019-12-10T11:38:08.796Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=90 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 90 samples" 470level=error ts=2019-12-10T11:38:08.796Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=37 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 37 samples" 471level=error ts=2019-12-10T11:38:08.796Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=67 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 67 samples" 472level=error ts=2019-12-10T11:38:08.796Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=77 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 77 samples" 473level=error ts=2019-12-10T11:38:08.795Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=74 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 74 samples" 474level=error ts=2019-12-10T11:38:08.710Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=94 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 94 samples" 475level=error ts=2019-12-10T11:38:08.710Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 476level=error ts=2019-12-10T11:38:08.710Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=89 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 89 samples" 477level=error ts=2019-12-10T11:38:08.710Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 101 samples" 478level=error ts=2019-12-10T11:38:08.694Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=77 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 77 samples" 479level=error ts=2019-12-10T11:38:08.694Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 102 samples" 480level=error ts=2019-12-10T11:38:08.694Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 103 samples" 481level=error ts=2019-12-10T11:38:08.693Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 104 samples" 482level=error ts=2019-12-10T11:38:08.692Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=58 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 58 samples" 483level=error ts=2019-12-10T11:38:08.692Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=91 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 91 samples" 484level=error ts=2019-12-10T11:38:08.692Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 485level=error ts=2019-12-10T11:38:08.692Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=59 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 59 samples" 486level=error ts=2019-12-10T11:38:08.692Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 101 samples" 487level=error ts=2019-12-10T11:38:08.692Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 102 samples" 488level=error ts=2019-12-10T11:38:08.692Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 103 samples" 489level=error ts=2019-12-10T11:38:08.692Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 104 samples" 490level=error ts=2019-12-10T11:38:08.691Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 105 samples" 491level=error ts=2019-12-10T11:38:08.678Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=75 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 75 samples" 492level=error ts=2019-12-10T11:38:08.677Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 106 samples" 493level=error ts=2019-12-10T11:38:08.648Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 107 samples" 494level=error ts=2019-12-10T11:38:08.647Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 108 samples" 495level=error ts=2019-12-10T11:38:08.644Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=90 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 90 samples" 496level=error ts=2019-12-10T11:38:08.603Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 109 samples" 497level=error ts=2019-12-10T11:38:08.603Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 110 samples" 498level=error ts=2019-12-10T11:38:08.603Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 111 samples" 499level=error ts=2019-12-10T11:38:08.603Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=71 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 71 samples" 500level=error ts=2019-12-10T11:38:08.603Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 112 samples" 501level=error ts=2019-12-10T11:38:08.602Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=83 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 83 samples" 502level=error ts=2019-12-10T11:38:08.596Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=84 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 84 samples" 503level=error ts=2019-12-10T11:38:08.595Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 113 samples" 504level=error ts=2019-12-10T11:38:08.595Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 114 samples" 505level=error ts=2019-12-10T11:38:08.553Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=95 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 95 samples" 506level=error ts=2019-12-10T11:38:08.552Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 115 samples" 507level=error ts=2019-12-10T11:38:08.552Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=73 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 73 samples" 508level=error ts=2019-12-10T11:38:08.552Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=87 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 87 samples" 509level=error ts=2019-12-10T11:38:08.541Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 116 samples" 510level=error ts=2019-12-10T11:38:08.501Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=96 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 96 samples" 511level=error ts=2019-12-10T11:38:08.501Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 117 samples" 512level=error ts=2019-12-10T11:38:08.497Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=91 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 91 samples" 513level=error ts=2019-12-10T11:38:08.497Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 118 samples" 514level=error ts=2019-12-10T11:38:08.497Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 119 samples" 515level=error ts=2019-12-10T11:38:08.497Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 120 samples" 516level=error ts=2019-12-10T11:38:08.447Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=72 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 72 samples" 517level=error ts=2019-12-10T11:38:08.447Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=97 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 97 samples" 518level=error ts=2019-12-10T11:38:08.447Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 130 samples" 519level=error ts=2019-12-10T11:38:08.447Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 131 samples" 520level=error ts=2019-12-10T11:38:08.447Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 132 samples" 521level=error ts=2019-12-10T11:38:08.447Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 133 samples" 522level=error ts=2019-12-10T11:38:08.446Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=70 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 70 samples" 523level=error ts=2019-12-10T11:38:08.445Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=65 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 65 samples" 524level=error ts=2019-12-10T11:38:08.441Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=88 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 88 samples" 525level=error ts=2019-12-10T11:38:08.433Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 526level=error ts=2019-12-10T11:38:08.431Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=89 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 89 samples" 527level=error ts=2019-12-10T11:38:08.407Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=80 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 80 samples" 528level=error ts=2019-12-10T11:38:08.393Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 529level=error ts=2019-12-10T11:38:08.3941Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 530level=error ts=2019-12-10T11:38:08.395Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 531level=error ts=2019-12-10T11:38:08.396Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=82 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 82 samples" 532level=error ts=2019-12-10T11:38:08.393Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=90 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 90 samples" 533level=error ts=2019-12-10T11:38:08.394Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 534level=error ts=2019-12-10T11:38:08.391Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 101 samples" 535level=error ts=2019-12-10T11:38:08.395Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 102 samples" 536level=error ts=2019-12-10T11:38:07.604Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 103 samples" 537level=error ts=2019-12-10T11:38:07.603Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 104 samples" 538level=error ts=2019-12-10T11:38:07.602Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=60 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 60 samples" 539level=error ts=2019-12-10T11:38:07.602Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 101 samples" 540level=error ts=2019-12-10T11:38:07.601Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 102 samples" 541level=error ts=2019-12-10T11:38:07.601Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 103 samples" 542level=error ts=2019-12-10T11:38:07.601Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 104 samples" 543level=error ts=2019-12-10T11:38:07.601Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 105 samples" 544level=error ts=2019-12-10T11:38:07.601Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 106 samples" 545level=error ts=2019-12-10T11:38:07.601Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=75 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 75 samples" 546level=error ts=2019-12-10T11:38:07.601Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=71 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 71 samples" 547level=error ts=2019-12-10T11:38:07.601Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=96 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 96 samples" 548level=error ts=2019-12-10T11:38:07.600Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 549level=error ts=2019-12-10T11:38:07.599Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 101 samples" 550level=error ts=2019-12-10T11:38:07.599Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=98 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 98 samples" 551level=error ts=2019-12-10T11:38:07.598Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=93 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 93 samples" 552level=error ts=2019-12-10T11:38:07.598Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=72 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 72 samples" 553level=error ts=2019-12-10T11:38:07.599Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 554level=error ts=2019-12-10T11:38:07.598Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 101 samples" 555level=error ts=2019-12-10T11:38:07.598Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 102 samples" 556level=error ts=2019-12-10T11:38:07.597Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 103 samples" 557level=error ts=2019-12-10T11:38:07.597Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 104 samples" 558level=error ts=2019-12-10T11:38:07.597Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 105 samples" 559level=error ts=2019-12-10T11:38:07.597Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 106 samples" 560level=error ts=2019-12-10T11:38:07.596Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=70 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 70 samples" 561level=error ts=2019-12-10T11:38:07.596Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 107 samples" 562level=error ts=2019-12-10T11:38:07.595Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 108 samples" 563level=error ts=2019-12-10T11:38:07.595Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=74 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 74 samples" 564level=error ts=2019-12-10T11:38:07.594Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=97 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 97 samples" 565level=error ts=2019-12-10T11:38:07.594Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=99 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 99 samples" 566level=error ts=2019-12-10T11:38:07.594Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 109 samples" 567level=error ts=2019-12-10T11:38:07.594Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 110 samples" 568level=error ts=2019-12-10T11:38:07.594Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=91 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 91 samples" 569level=error ts=2019-12-10T11:38:07.593Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=73 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 73 samples" 570level=error ts=2019-12-10T11:38:07.593Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 111 samples" 571level=error ts=2019-12-10T11:38:07.593Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=81 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 81 samples" 572level=error ts=2019-12-10T11:38:07.593Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=86 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 86 samples" 573level=error ts=2019-12-10T11:38:07.592Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 112 samples" 574level=error ts=2019-12-10T11:38:07.592Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 113 samples" 575level=error ts=2019-12-10T11:38:07.591Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=53 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 53 samples" 576level=error ts=2019-12-10T11:38:07.591Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 114 samples" 577level=error ts=2019-12-10T11:38:07.591Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 115 samples" 578level=error ts=2019-12-10T11:38:07.591Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 116 samples" 579level=error ts=2019-12-10T11:38:07.591Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 117 samples" 580level=error ts=2019-12-10T11:38:07.589Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 118 samples" 581level=error ts=2019-12-10T11:38:07.589Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=39 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 39 samples" 582level=error ts=2019-12-10T11:38:07.589Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 119 samples" 583level=error ts=2019-12-10T11:38:07.589Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 120 samples" 584level=error ts=2019-12-10T11:38:07.587Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 121 samples" 585level=error ts=2019-12-10T11:38:07.513Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 122 samples" 586level=error ts=2019-12-10T11:38:07.512Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=59 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 59 samples" 587level=error ts=2019-12-10T11:38:07.512Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 123 samples" 588level=error ts=2019-12-10T11:38:07.510Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=79 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 79 samples" 589level=error ts=2019-12-10T11:38:07.506Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 124 samples" 590level=error ts=2019-12-10T11:38:07.506Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 125 samples" 591level=error ts=2019-12-10T11:38:07.499Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 126 samples" 592level=error ts=2019-12-10T11:38:07.499Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=67 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 67 samples" 593level=error ts=2019-12-10T11:38:07.499Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=98 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 594level=error ts=2019-12-10T11:38:07.499Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=75 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 75 samples" 595level=error ts=2019-12-10T11:38:07.499Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=60 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 60 samples" 596level=error ts=2019-12-10T11:38:07.499Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=99 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 99 samples" 597level=error ts=2019-12-10T11:38:07.499Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=98 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 98 samples" 598level=error ts=2019-12-10T11:38:07.499Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 127 samples" 599level=error ts=2019-12-10T11:38:07.499Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 128 samples" 600level=error ts=2019-12-10T11:38:07.494Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 129 samples" 601level=error ts=2019-12-10T11:38:07.489Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 130 samples" 602level=error ts=2019-12-10T11:38:07.488Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 131 samples" 603level=error ts=2019-12-10T11:38:07.484Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 132 samples" 604level=error ts=2019-12-10T11:38:07.483Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 133 samples" 605level=error ts=2019-12-10T11:38:07.483Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 134 samples" 606level=error ts=2019-12-10T11:38:07.480Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 136 samples" 607level=error ts=2019-12-10T11:38:07.480Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=66 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 67 samples" 608level=error ts=2019-12-10T11:38:07.480Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=71 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 71 samples" 609level=error ts=2019-12-10T11:38:07.480Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=66 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 66 samples" 610level=error ts=2019-12-10T11:38:07.478Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=76 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 76 samples" 611level=error ts=2019-12-10T11:38:07.460Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=77 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 77 samples" 612level=error ts=2019-12-10T11:38:07.455Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 101 samples" 613level=error ts=2019-12-10T11:38:07.455Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 102 samples" 614level=error ts=2019-12-10T11:38:07.455Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=65 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 65 samples" 615level=error ts=2019-12-10T11:38:07.454Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 101 samples" 616level=error ts=2019-12-10T11:38:07.453Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 102 samples" 617level=error ts=2019-12-10T11:38:07.453Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=80 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 80 samples" 618level=error ts=2019-12-10T11:38:07.453Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 619level=error ts=2019-12-10T11:38:07.452Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=80 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 80 samples" 620level=error ts=2019-12-10T11:38:07.452Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 101 samples" 621level=error ts=2019-12-10T11:38:07.452Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 102 samples" 622level=error ts=2019-12-10T11:38:07.452Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=37 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 37 samples" 623level=error ts=2019-12-10T11:38:07.452Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=30 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 30 samples" 624level=error ts=2019-12-10T11:38:07.452Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=74 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 74 samples" 625level=error ts=2019-12-10T11:38:07.452Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 626level=error ts=2019-12-10T11:38:07.452Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=79 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 79 samples" 627level=error ts=2019-12-10T11:38:07.452Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=70 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 70 samples" 628level=error ts=2019-12-10T11:38:07.451Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 115 samples" 629level=error ts=2019-12-10T11:38:07.451Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 101 samples" 630level=error ts=2019-12-10T11:38:07.451Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=96 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 96 samples" 631level=error ts=2019-12-10T11:38:07.451Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=84 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 84 samples" 632level=error ts=2019-12-10T11:38:07.451Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=74 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 74 samples" 633level=error ts=2019-12-10T11:38:07.451Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=63 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 63 samples" 634level=error ts=2019-12-10T11:38:07.451Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 635level=error ts=2019-12-10T11:38:07.450Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 102 samples" 636level=error ts=2019-12-10T11:38:07.450Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 103 samples" 637level=error ts=2019-12-10T11:38:07.450Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=95 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 95 samples" 638level=error ts=2019-12-10T11:38:07.449Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 639level=error ts=2019-12-10T11:38:07.448Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=99 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 99 samples" 640level=error ts=2019-12-10T11:38:07.448Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 101 samples" 641level=error ts=2019-12-10T11:38:07.447Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 102 samples" 642level=error ts=2019-12-10T11:38:07.446Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 103 samples" 643level=error ts=2019-12-10T11:38:07.446Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 104 samples" 644level=error ts=2019-12-10T11:38:07.446Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 105 samples" 645level=error ts=2019-12-10T11:38:07.445Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=57 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 57 samples" 646level=error ts=2019-12-10T11:38:07.445Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 101 samples" 647level=error ts=2019-12-10T11:38:07.445Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 102 samples" 648level=error ts=2019-12-10T11:38:07.445Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 103 samples" 649level=error ts=2019-12-10T11:38:07.444Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 104 samples" 650level=error ts=2019-12-10T11:38:07.444Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 105 samples" 651level=error ts=2019-12-10T11:38:07.444Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=84 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 84 samples" 652level=error ts=2019-12-10T11:38:07.443Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 106 samples" 653level=error ts=2019-12-10T11:38:07.390Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 107 samples" 654level=error ts=2019-12-10T11:38:07.389Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 108 samples" 655level=error ts=2019-12-10T11:38:07.389Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 109 samples" 656level=error ts=2019-12-10T11:38:07.389Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 110 samples" 657level=error ts=2019-12-10T11:38:07.389Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 111 samples" 658level=error ts=2019-12-10T11:38:07.388Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=85 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 85 samples" 659level=error ts=2019-12-10T11:38:07.388Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 112 samples" 660level=error ts=2019-12-10T11:38:07.388Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=89 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 89 samples" 661level=error ts=2019-12-10T11:38:07.387Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 113 samples" 662level=error ts=2019-12-10T11:38:07.387Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=98 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 98 samples" 663level=error ts=2019-12-10T11:38:07.387Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=63 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 63 samples" 664level=error ts=2019-12-10T11:38:07.376Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 114 samples" 665level=error ts=2019-12-10T11:38:07.358Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=84 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 84 samples" 666level=error ts=2019-12-10T11:38:07.357Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 115 samples" 667level=error ts=2019-12-10T11:38:07.357Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 116 samples" 668level=error ts=2019-12-10T11:38:07.357Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=82 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 82 samples" 669level=error ts=2019-12-10T11:38:07.357Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 117 samples" 670level=error ts=2019-12-10T11:38:07.357Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=79 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 79 samples" 671level=error ts=2019-12-10T11:38:07.310Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 118 samples" 672level=error ts=2019-12-10T11:38:07.259Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 119 samples" 673level=error ts=2019-12-10T11:38:07.259Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=85 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 85 samples" 674level=error ts=2019-12-10T11:38:07.045Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 120 samples" 675level=error ts=2019-12-10T11:38:07.044Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=83 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 83 samples" 676level=error ts=2019-12-10T11:38:07.042Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 121 samples" 677level=error ts=2019-12-10T11:38:07.043Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 122 samples" 678level=error ts=2019-12-10T11:38:07.043Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=69 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 69 samples" 679level=error ts=2019-12-10T11:38:07.042Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 123 samples" 680level=error ts=2019-12-10T11:38:07.042Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 124 samples" 681level=error ts=2019-12-10T11:38:07.042Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 125 samples" 682level=error ts=2019-12-10T11:38:07.041Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=49 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 49 samples" 683level=error ts=2019-12-10T11:38:06.947Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=81 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 81 samples" 684level=error ts=2019-12-10T11:38:06.947Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 126 samples" 685level=error ts=2019-12-10T11:38:06.946Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 127 samples" 686level=error ts=2019-12-10T11:38:06.944Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 128 samples" 687level=error ts=2019-12-10T11:38:06.943Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=75 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 75 samples" 688level=error ts=2019-12-10T11:38:06.939Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 129 samples" 689level=error ts=2019-12-10T11:38:06.938Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=95 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 95 samples" 690level=error ts=2019-12-10T11:38:06.937Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 130 samples" 691level=error ts=2019-12-10T11:38:06.848Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=54 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 54 samples" 692level=error ts=2019-12-10T11:38:06.848Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 131 samples" 693level=error ts=2019-12-10T11:38:06.842Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 132 samples" 694level=error ts=2019-12-10T11:38:06.841Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 133 samples" 695level=error ts=2019-12-10T11:38:06.747Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 134 samples" 696level=error ts=2019-12-10T11:38:06.745Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=85 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 85 samples" 697level=error ts=2019-12-10T11:38:06.745Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 135 samples" 698level=error ts=2019-12-10T11:38:06.745Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 136 samples" 699level=error ts=2019-12-10T11:38:06.745Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 101 samples" 700level=error ts=2019-12-10T11:38:06.745Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=65 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 65 samples" 701level=error ts=2019-12-10T11:38:06.745Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 102 samples" 702level=error ts=2019-12-10T11:38:06.715Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 103 samples" 703level=error ts=2019-12-10T11:38:06.646Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=95 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 95 samples" 704level=error ts=2019-12-10T11:38:06.646Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=37 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 37 samples" 705level=error ts=2019-12-10T11:38:06.644Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 706level=error ts=2019-12-10T11:38:06.644Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=82 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 82 samples" 707level=error ts=2019-12-10T11:38:06.644Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=97 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 97 samples" 708level=error ts=2019-12-10T11:38:06.644Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 101 samples" 709level=error ts=2019-12-10T11:38:06.641Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 102 samples" 710level=error ts=2019-12-10T11:38:06.640Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 103 samples" 711level=error ts=2019-12-10T11:38:06.640Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 104 samples" 712level=error ts=2019-12-10T11:38:06.639Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 105 samples" 713level=error ts=2019-12-10T11:38:06.638Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 106 samples" 714level=error ts=2019-12-10T11:38:06.593Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=93 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 93 samples" 715level=error ts=2019-12-10T11:38:06.592Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 107 samples" 716level=error ts=2019-12-10T11:38:06.592Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 108 samples" 717level=error ts=2019-12-10T11:38:06.592Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=95 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 95 samples" 718level=error ts=2019-12-10T11:38:06.592Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 109 samples" 719level=error ts=2019-12-10T11:38:06.592Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 110 samples" 720level=error ts=2019-12-10T11:38:06.591Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 111 samples" 721level=error ts=2019-12-10T11:38:06.591Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=56 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 56 samples" 722level=error ts=2019-12-10T11:38:06.590Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 112 samples" 723level=error ts=2019-12-10T11:38:06.590Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=75 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 75 samples" 724level=error ts=2019-12-10T11:38:06.589Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 113 samples" 725level=error ts=2019-12-10T11:38:06.589Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 114 samples" 726level=error ts=2019-12-10T11:38:06.588Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=83 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 83 samples" 727level=error ts=2019-12-10T11:38:06.588Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 116 samples" 728level=error ts=2019-12-10T11:38:06.581Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 117 samples" 729level=error ts=2019-12-10T11:38:06.581Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 118 samples" 730level=error ts=2019-12-10T11:38:06.563Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=77 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 77 samples" 731level=error ts=2019-12-10T11:38:06.563Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 119 samples" 732level=error ts=2019-12-10T11:38:06.563Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 120 samples" 733level=error ts=2019-12-10T11:38:06.562Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 121 samples" 734level=error ts=2019-12-10T11:38:06.562Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 122 samples" 735level=error ts=2019-12-10T11:38:06.561Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 123 samples" 736level=error ts=2019-12-10T11:38:06.560Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 124 samples" 737level=error ts=2019-12-10T11:38:06.551Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 125 samples" 738level=error ts=2019-12-10T11:38:06.550Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 126 samples" 739level=error ts=2019-12-10T11:38:06.544Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 127 samples" 740level=error ts=2019-12-10T11:38:06.544Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 128 samples" 741level=error ts=2019-12-10T11:38:06.543Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 129 samples" 742level=error ts=2019-12-10T11:38:06.543Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 130 samples" 743level=error ts=2019-12-10T11:38:06.543Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 131 samples" 744level=error ts=2019-12-10T11:38:06.543Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=98 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 98 samples" 745level=error ts=2019-12-10T11:38:06.542Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 132 samples" 746level=error ts=2019-12-10T11:38:06.542Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 133 samples" 747level=error ts=2019-12-10T11:38:06.542Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 134 samples" 748level=error ts=2019-12-10T11:38:06.540Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 135 samples" 749level=error ts=2019-12-10T11:38:06.538Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 136 samples" 750level=error ts=2019-12-10T11:38:06.538Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 137 samples" 751level=error ts=2019-12-10T11:38:06.512Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 138 samples" 752level=error ts=2019-12-10T11:38:06.506Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 140 samples" 753level=error ts=2019-12-10T11:38:06.505Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 141 samples" 754level=error ts=2019-12-10T11:38:06.504Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=69 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 69 samples" 755level=error ts=2019-12-10T11:38:06.504Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 142 samples" 756level=error ts=2019-12-10T11:38:06.504Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 143 samples" 757level=error ts=2019-12-10T11:38:06.504Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=79 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 79 samples" 758level=error ts=2019-12-10T11:38:06.504Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=98 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 98 samples" 759level=error ts=2019-12-10T11:38:06.489Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 144 samples" 760level=error ts=2019-12-10T11:38:06.461Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 145 samples" 761level=error ts=2019-12-10T11:38:06.461Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 146 samples" 762level=error ts=2019-12-10T11:38:06.461Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 147 samples" 763level=error ts=2019-12-10T11:38:06.461Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=77 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 77 samples" 764level=error ts=2019-12-10T11:38:06.461Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=83 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 83 samples" 765level=error ts=2019-12-10T11:38:06.461Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=96 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 96 samples" 766level=error ts=2019-12-10T11:38:06.460Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 148 samples" 767level=error ts=2019-12-10T11:38:06.460Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 149 samples" 768level=error ts=2019-12-10T11:38:06.460Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=75 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 75 samples" 769level=error ts=2019-12-10T11:38:06.459Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 150 samples" 770level=error ts=2019-12-10T11:38:06.453Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 151 samples" 771level=error ts=2019-12-10T11:38:06.453Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 152 samples" 772level=error ts=2019-12-10T11:38:06.452Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 153 samples" 773level=error ts=2019-12-10T11:38:06.452Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 154 samples" 774level=error ts=2019-12-10T11:38:06.451Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=85 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 85 samples" 775level=error ts=2019-12-10T11:38:06.451Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 155 samples" 776level=error ts=2019-12-10T11:38:06.451Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 156 samples" 777level=error ts=2019-12-10T11:38:06.450Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 157 samples" 778level=error ts=2019-12-10T11:38:06.450Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=85 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 85 samples" 779level=error ts=2019-12-10T11:38:06.449Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 101 samples" 780level=error ts=2019-12-10T11:38:06.449Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 102 samples" 781level=error ts=2019-12-10T11:38:06.449Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 103 samples" 782level=error ts=2019-12-10T11:38:06.448Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 104 samples" 783level=error ts=2019-12-10T11:38:06.448Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 105 samples" 784level=error ts=2019-12-10T11:38:06.448Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=75 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 75 samples" 785level=error ts=2019-12-10T11:38:06.448Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 106 samples" 786level=error ts=2019-12-10T11:38:06.448Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=70 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 70 samples" 787level=error ts=2019-12-10T11:38:06.447Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=96 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 96 samples" 788level=error ts=2019-12-10T11:38:06.446Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=66 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 66 samples" 789level=error ts=2019-12-10T11:38:06.446Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 790level=error ts=2019-12-10T11:38:06.446Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=80 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 80 samples" 791level=error ts=2019-12-10T11:38:06.446Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=33 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 33 samples" 792level=error ts=2019-12-10T11:38:06.446Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=75 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 75 samples" 793level=error ts=2019-12-10T11:38:06.446Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=71 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 71 samples" 794level=error ts=2019-12-10T11:38:06.446Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=62 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 62 samples" 795level=error ts=2019-12-10T11:38:06.445Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=82 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 83 samples" 796level=error ts=2019-12-10T11:38:06.445Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=58 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 58 samples" 797level=error ts=2019-12-10T11:38:06.445Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=79 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 79 samples" 798level=error ts=2019-12-10T11:38:06.445Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=72 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 72 samples" 799level=error ts=2019-12-10T11:38:06.445Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 800level=error ts=2019-12-10T11:38:06.445Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=82 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 82 samples" 801level=error ts=2019-12-10T11:38:06.445Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=97 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 97 samples" 802level=error ts=2019-12-10T11:38:06.445Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=86 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 86 samples" 803level=error ts=2019-12-10T11:38:06.355Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 804level=error ts=2019-12-10T11:38:06.355Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 101 samples" 805level=error ts=2019-12-10T11:38:06.355Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 102 samples" 806level=error ts=2019-12-10T11:38:06.355Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 103 samples" 807level=error ts=2019-12-10T11:38:06.355Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 104 samples" 808level=error ts=2019-12-10T11:38:06.353Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 105 samples" 809level=error ts=2019-12-10T11:38:06.350Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 106 samples" 810level=error ts=2019-12-10T11:38:06.350Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 107 samples" 811level=error ts=2019-12-10T11:38:06.350Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 108 samples" 812level=error ts=2019-12-10T11:38:06.350Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 109 samples" 813level=error ts=2019-12-10T11:38:06.350Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 110 samples" 814level=error ts=2019-12-10T11:38:06.350Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 111 samples" 815level=error ts=2019-12-10T11:38:06.349Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 112 samples" 816level=error ts=2019-12-10T11:38:06.349Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 113 samples" 817level=error ts=2019-12-10T11:38:06.349Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 114 samples" 818level=error ts=2019-12-10T11:38:06.349Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 115 samples" 819level=error ts=2019-12-10T11:38:06.349Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 116 samples" 820level=error ts=2019-12-10T11:38:06.349Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=60 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 60 samples" 821level=error ts=2019-12-10T11:38:06.348Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=22 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 22 samples" 822level=error ts=2019-12-10T11:38:06.348Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 155 samples" 823level=error ts=2019-12-10T11:38:06.348Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=97 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 97 samples" 824level=error ts=2019-12-10T11:38:06.348Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 825level=error ts=2019-12-10T11:38:06.348Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 101 samples" 826level=error ts=2019-12-10T11:38:06.348Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 102 samples" 827level=error ts=2019-12-10T11:38:06.347Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 103 samples" 828level=error ts=2019-12-10T11:38:06.347Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 104 samples" 829level=error ts=2019-12-10T11:38:06.346Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=82 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 82 samples" 830level=error ts=2019-12-10T11:38:06.346Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 106 samples" 831level=error ts=2019-12-10T11:38:06.340Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 107 samples" 832level=error ts=2019-12-10T11:38:06.339Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 108 samples" 833level=error ts=2019-12-10T11:38:06.339Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 109 samples" 834level=error ts=2019-12-10T11:38:06.339Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=74 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 74 samples" 835level=error ts=2019-12-10T11:38:06.339Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 110 samples" 836level=error ts=2019-12-10T11:38:06.339Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 111 samples" 837level=error ts=2019-12-10T11:38:06.338Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 112 samples" 838level=error ts=2019-12-10T11:38:06.338Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 113 samples" 839level=error ts=2019-12-10T11:38:06.337Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 114 samples" 840level=error ts=2019-12-10T11:38:06.330Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 115 samples" 841level=error ts=2019-12-10T11:38:06.329Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 116 samples" 842level=error ts=2019-12-10T11:38:06.329Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 117 samples" 843level=error ts=2019-12-10T11:38:06.328Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=83 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 83 samples" 844level=error ts=2019-12-10T11:38:06.328Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 118 samples" 845level=error ts=2019-12-10T11:38:06.328Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 119 samples" 846level=error ts=2019-12-10T11:38:06.328Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=97 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 97 samples" 847level=error ts=2019-12-10T11:38:06.327Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=79 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 79 samples" 848level=error ts=2019-12-10T11:38:06.244Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 120 samples" 849level=error ts=2019-12-10T11:38:06.231Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 121 samples" 850level=error ts=2019-12-10T11:38:06.230Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 122 samples" 851level=error ts=2019-12-10T11:38:06.229Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 124 samples" 852level=error ts=2019-12-10T11:38:06.229Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 125 samples" 853level=error ts=2019-12-10T11:38:06.229Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=94 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 94 samples" 854level=error ts=2019-12-10T11:38:06.206Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 126 samples" 855level=error ts=2019-12-10T11:38:06.205Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 127 samples" 856level=error ts=2019-12-10T11:38:06.205Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=98 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 98 samples" 857level=error ts=2019-12-10T11:38:06.205Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 128 samples" 858level=error ts=2019-12-10T11:38:06.205Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 129 samples" 859level=error ts=2019-12-10T11:38:06.198Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 130 samples" 860level=error ts=2019-12-10T11:38:06.092Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 131 samples" 861level=error ts=2019-12-10T11:38:06.092Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 132 samples" 862level=error ts=2019-12-10T11:38:06.092Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 133 samples" 863level=error ts=2019-12-10T11:38:06.092Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 134 samples" 864level=error ts=2019-12-10T11:38:06.092Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 135 samples" 865level=error ts=2019-12-10T11:38:06.091Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=81 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 81 samples" 866level=error ts=2019-12-10T11:38:06.091Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 136 samples" 867level=error ts=2019-12-10T11:38:06.091Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 139 samples" 868level=error ts=2019-12-10T11:38:06.088Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 137 samples" 869level=error ts=2019-12-10T11:38:06.087Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 138 samples" 870level=error ts=2019-12-10T11:38:05.593Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 101 samples" 871level=error ts=2019-12-10T11:38:05.593Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 102 samples" 872level=error ts=2019-12-10T11:38:05.593Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=97 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 97 samples" 873level=error ts=2019-12-10T11:38:05.592Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 103 samples" 874level=info ts=2019-12-10T11:38:05.063Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=1 to=2 875level=error ts=2019-12-10T11:38:04.977Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 104 samples" 876level=error ts=2019-12-10T11:38:04.976Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 105 samples" 877level=error ts=2019-12-10T11:38:04.821Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 106 samples" 878level=error ts=2019-12-10T11:38:04.821Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=96 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 96 samples" 879level=error ts=2019-12-10T11:38:04.821Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 880level=error ts=2019-12-10T11:38:04.815Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 101 samples" 881level=error ts=2019-12-10T11:38:04.815Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 102 samples" 882level=error ts=2019-12-10T11:38:04.720Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=98 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 99 samples" 883level=error ts=2019-12-10T11:38:04.720Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 103 samples" 884level=error ts=2019-12-10T11:38:04.720Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 104 samples" 885level=error ts=2019-12-10T11:38:04.719Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=98 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 98 samples" 886level=error ts=2019-12-10T11:38:04.719Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 105 samples" 887level=error ts=2019-12-10T11:38:04.717Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 106 samples" 888level=error ts=2019-12-10T11:38:04.716Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=87 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 87 samples" 889level=error ts=2019-12-10T11:38:04.716Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 107 samples" 890level=error ts=2019-12-10T11:38:04.711Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 108 samples" 891level=error ts=2019-12-10T11:38:04.711Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=99 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 999 samples" 892level=error ts=2019-12-10T11:38:04.711Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 109 samples" 893level=error ts=2019-12-10T11:38:04.710Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 111 samples" 894level=error ts=2019-12-10T11:38:04.710Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 112 samples" 895level=error ts=2019-12-10T11:38:04.710Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 113 samples" 896level=error ts=2019-12-10T11:38:04.709Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=66 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 66 samples" 897level=error ts=2019-12-10T11:38:04.709Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=64 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 64 samples" 898level=error ts=2019-12-10T11:38:04.631Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 114 samples" 899level=error ts=2019-12-10T11:38:04.631Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=84 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 84 samples" 900level=error ts=2019-12-10T11:38:04.631Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 115 samples" 901level=error ts=2019-12-10T11:38:04.630Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 116 samples" 902level=error ts=2019-12-10T11:38:04.630Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 117 samples" 903level=error ts=2019-12-10T11:38:04.630Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 118 samples" 904level=error ts=2019-12-10T11:38:04.630Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 119 samples" 905level=error ts=2019-12-10T11:38:04.630Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 120 samples" 906level=error ts=2019-12-10T11:38:04.627Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 121 samples" 907level=error ts=2019-12-10T11:38:04.626Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 122 samples" 908level=error ts=2019-12-10T11:38:04.626Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 121 samples" 909level=error ts=2019-12-10T11:38:04.626Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 1244 samples" 910level=error ts=2019-12-10T11:38:04.621Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 123 samples" 911level=error ts=2019-12-10T11:38:04.620Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 124 samples" 912level=error ts=2019-12-10T11:38:04.620Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 125 samples" 913level=error ts=2019-12-10T11:38:04.620Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 126 samples" 914level=error ts=2019-12-10T11:38:04.619Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 127 samples" 915level=error ts=2019-12-10T11:38:04.136Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 131 samples" 916level=error ts=2019-12-10T11:38:04.136Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 132 samples" 917level=error ts=2019-12-10T11:38:04.135Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 133 samples" 918level=error ts=2019-12-10T11:38:04.018Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 134 samples" 919level=error ts=2019-12-10T11:38:04.018Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 135 samples" 920level=error ts=2019-12-10T11:38:04.017Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 141 samples" 921level=error ts=2019-12-10T11:38:04.016Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 142 samples" 922level=error ts=2019-12-10T11:38:04.015Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 153 samples" 923level=error ts=2019-12-10T11:38:04.015Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 1223 samples" 924level=error ts=2019-12-10T11:38:04.013Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 164 samples" 925level=error ts=2019-12-10T11:38:04.013Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 161 samples" 926level=error ts=2019-12-10T11:38:04.012Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 162 samples" 927level=error ts=2019-12-10T11:38:04.012Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 163 samples" 928level=error ts=2019-12-10T11:38:04.012Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 169 samples" 929level=error ts=2019-12-10T11:38:04.012Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 170 samples" 930level=error ts=2019-12-10T11:38:04.012Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=80 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 80 samples" 931level=error ts=2019-12-10T11:38:04.012Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples" 932level=error ts=2019-12-10T11:38:04.010Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 101 samples" 933level=error ts=2019-12-10T11:38:04.009Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 102 samples" 934level=error ts=2019-12-10T11:38:04.009Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 103 samples" 935level=error ts=2019-12-10T11:38:04.008Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 104 samples" 936level=error ts=2019-12-10T11:38:04.007Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=88 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 88 samples" 937level=error ts=2019-12-10T11:38:04.007Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 105 samples" 938level=error ts=2019-12-10T11:38:04.007Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 106 samples" 939level=error ts=2019-12-10T11:38:04.005Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 107 samples" 940level=error ts=2019-12-10T11:38:04.005Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 108 samples" 941level=error ts=2019-12-10T11:38:04.004Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=84 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 84 samples" 942level=error ts=2019-12-10T11:38:04.003Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 109 samples" 943level=error ts=2019-12-10T11:38:04.002Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 110 samples" 944level=error ts=2019-12-10T11:38:04.002Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 112 samples" 945level=error ts=2019-12-10T11:38:04.001Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 113 samples" 946level=error ts=2019-12-10T11:38:03.923Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 114 samples" 947level=error ts=2019-12-10T11:38:03.923Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=79 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 79 samples" 948level=error ts=2019-12-10T11:38:03.922Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 115 samples" 949level=error ts=2019-12-10T11:38:03.908Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 116 samples" 950level=error ts=2019-12-10T11:38:03.908Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 117 samples" 951level=error ts=2019-12-10T11:38:03.907Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=96 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 96 samples" 952level=error ts=2019-12-10T11:38:03.907Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 118 samples" 953level=error ts=2019-12-10T11:38:03.906Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 119 samples" 954level=error ts=2019-12-10T11:38:03.904Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 120 samples" 955level=error ts=2019-12-10T11:38:03.904Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=83 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 83 samples" 956level=error ts=2019-12-10T11:38:03.904Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 121 samples" 957level=error ts=2019-12-10T11:38:03.904Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 122 samples" 958level=error ts=2019-12-10T11:38:03.904Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=71 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 71 samples" 959level=error ts=2019-12-10T11:38:03.904Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 123 samples" 960level=error ts=2019-12-10T11:38:03.902Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 124 samples" 961level=error ts=2019-12-10T11:38:03.901Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 125 samples" 962level=error ts=2019-12-10T11:38:03.900Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 126 samples" 963level=error ts=2019-12-10T11:38:03.900Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 127 samples" 964level=error ts=2019-12-10T11:38:03.899Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=86 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 86 samples" 965level=error ts=2019-12-10T11:38:03.900Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=84 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 84 samples" 966level=error ts=2019-12-10T11:38:03.900Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 128 samples" 967level=error ts=2019-12-10T11:38:03.900Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 129 samples" 968level=error ts=2019-12-10T11:38:03.899Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 130 samples" 969level=error ts=2019-12-10T11:38:03.899Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 131 samples" 970level=error ts=2019-12-10T11:38:03.899Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 132 samples" 971level=error ts=2019-12-10T11:38:03.899Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 133 samples" 972level=error ts=2019-12-10T11:38:03.899Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 134 samples" 973level=error ts=2019-12-10T11:38:03.898Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 135 samples" 974level=error ts=2019-12-10T11:38:03.898Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=90 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 90 samples" 975level=error ts=2019-12-10T11:38:03.898Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 136 samples" 976level=error ts=2019-12-10T11:38:03.898Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 137 samples" 977level=error ts=2019-12-10T11:38:03.897Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 138 samples" 978level=error ts=2019-12-10T11:38:03.897Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 139 samples" 979level=error ts=2019-12-10T11:38:03.723Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=80 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 80 samples" 980level=error ts=2019-12-10T11:38:03.723Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 140 samples" 981level=error ts=2019-12-10T11:38:03.723Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 141 samples" 982level=error ts=2019-12-10T11:38:03.722Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 142 samples" 983level=error ts=2019-12-10T11:38:03.722Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 143 samples" 984level=error ts=2019-12-10T11:38:03.722Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 144 samples" 985level=error ts=2019-12-10T11:38:03.722Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 145 samples" 986level=error ts=2019-12-10T11:38:03.722Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 146 samples" 987level=error ts=2019-12-10T11:38:03.722Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=76 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 76 samples" 988level=error ts=2019-12-10T11:38:03.722Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 101 samples" 989level=error ts=2019-12-10T11:38:03.722Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 102 samples" 990level=error ts=2019-12-10T11:38:03.711Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 103 samples" 991level=error ts=2019-12-10T11:38:03.711Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 104 samples" 992level=error ts=2019-12-10T11:38:03.711Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 105 samples" 993level=error ts=2019-12-10T11:38:03.644Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 106 samples" 994level=error ts=2019-12-10T11:38:03.644Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=95 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 95 samples" 995level=error ts=2019-12-10T11:38:03.644Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 101 samples" 996level=error ts=2019-12-10T11:38:03.644Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 102 samples" 997level=error ts=2019-12-10T11:38:03.644Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 103 samples" 998level=error ts=2019-12-10T11:38:03.644Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 104 samples" 999level=error ts=2019-12-10T11:38:03.644Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 105 samples" 1000level=error ts=2019-12-10T11:38:03.643Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 106 samples" 1001level=error ts=2019-12-10T11:38:03.643Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 107 samples" 1002level=error ts=2019-12-10T11:38:03.643Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 108 samples" 1003level=error ts=2019-12-10T11:38:03.643Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 109 samples" 1004level=error ts=2019-12-10T11:38:03.642Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 110 samples" 1005level=error ts=2019-12-10T11:38:03.642Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 111 samples" 1006level=error ts=2019-12-10T11:38:03.642Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 112 samples" 1007level=error ts=2019-12-10T11:38:03.642Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 113 samples" 1008level=error ts=2019-12-10T11:38:03.642Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 114 samples" 1009level=error ts=2019-12-10T11:38:03.642Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 115 samples" 1010level=error ts=2019-12-10T11:38:03.641Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 116 samples" 1011level=error ts=2019-12-10T11:38:03.640Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 117 samples" 1012level=error ts=2019-12-10T11:38:03.641Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 118 samples" 1013level=error ts=2019-12-10T11:38:03.641Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 119 samples" 1014level=error ts=2019-12-10T11:38:03.640Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 120 samples" 1015level=error ts=2019-12-10T11:38:03.640Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 121 samples" 1016level=error ts=2019-12-10T11:38:03.640Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 122 samples" 1017level=error ts=2019-12-10T11:38:03.640Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=88 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 88 samples" 1018level=error ts=2019-12-10T11:38:03.640Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 123 samples" 1019level=error ts=2019-12-10T11:38:03.640Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 134 samples" 1020level=error ts=2019-12-10T11:38:03.640Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=78 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 78 samples" 1021level=error ts=2019-12-10T11:38:03.640Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 155 samples" 1022`, "\n") 1023