1package testdata
2
3import "strings"
4
5// LogString returns a test log line. Returns the same line for the same index.
6func LogString(index int64) string {
7	if index > int64(len(Logs)-1) {
8		index = index % int64(len(Logs))
9	}
10	return Logs[index]
11}
12
13var LogsBytes [][]byte
14
15func init() {
16	LogsBytes = make([][]byte, len(Logs))
17	for i, l := range Logs {
18		LogsBytes[i] = []byte(l)
19	}
20}
21
22var Logs = strings.Split(`level=info ts=2019-12-12T15:00:08.325Z caller=compact.go:441 component=tsdb msg="compact blocks" count=3 mint=1576130400000 maxt=1576152000000 ulid=01DVX9ZHNM71GRCJS7M34Q0EV7 sources="[01DVWNC6NWY1A60AZV3Z6DGS65 01DVWW7XXX75GHA6ZDTD170CSZ 01DVX33N5W86CWJJVRPAVXJRWJ]" duration=2.897213221s
23level=info ts=2019-12-12T15:00:08.296Z caller=compact.go:441 component=tsdb msg="compact blocks" count=3 mint=1576130400000 maxt=1576152000000 ulid=01DVX9ZHQRVN42AF196NYJ9C4C sources="[01DVWNC6NSPJRCSBZ4QD3SXS66 01DVWW7XY69Y4YT09HR0RSR8KY 01DVX33N5SMVPB1TMD9J1M8GGK]" duration=2.800759388s
24level=info ts=2019-12-12T15:00:05.285Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=1037 last=1039 duration=3.030078405s
25level=info ts=2019-12-12T15:00:05.225Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=1037 last=1039 duration=3.019791992s
26level=info ts=2019-12-12T15:00:02.255Z caller=head.go:596 component=tsdb msg="head GC completed" duration=125.980176ms
27level=info ts=2019-12-12T15:00:02.206Z caller=head.go:596 component=tsdb msg="head GC completed" duration=127.111334ms
28level=info ts=2019-12-12T15:00:01.874Z caller=compact.go:496 component=tsdb msg="write block" mint=1576152000000 maxt=1576159200000 ulid=01DVX9ZCE8WZCTQJWSYDGHVQV8 duration=1.801853505s
29level=info ts=2019-12-12T15:00:01.854Z caller=compact.go:496 component=tsdb msg="write block" mint=1576152000000 maxt=1576159200000 ulid=01DVX9ZCDWEBXRYWA7585TN2RV duration=1.794588392s
30level=info ts=2019-12-12T13:00:05.461Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=1034 last=1036 duration=3.044019343s
31level=info ts=2019-12-12T13:00:05.332Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=1034 last=1036 duration=3.040243488s
32level=info ts=2019-12-12T13:00:02.417Z caller=head.go:596 component=tsdb msg="head GC completed" duration=128.883109ms
33level=info ts=2019-12-12T13:00:02.291Z caller=head.go:596 component=tsdb msg="head GC completed" duration=126.278558ms
34level=info ts=2019-12-12T13:00:02.048Z caller=compact.go:496 component=tsdb msg="write block" mint=1576144800000 maxt=1576152000000 ulid=01DVX33N5W86CWJJVRPAVXJRWJ duration=1.987867109s
35level=info ts=2019-12-12T13:00:01.914Z caller=compact.go:496 component=tsdb msg="write block" mint=1576144800000 maxt=1576152000000 ulid=01DVX33N5SMVPB1TMD9J1M8GGK duration=1.856432758s
36level=info ts=2019-12-12T12:58:04.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4
37level=info ts=2019-12-12T12:52:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3
38level=info ts=2019-12-12T11:00:05.320Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=1031 last=1033 duration=2.999621843s
39level=info ts=2019-12-12T11:00:05.315Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=1031 last=1033 duration=2.962560692s
40level=info ts=2019-12-12T11:00:02.352Z caller=head.go:596 component=tsdb msg="head GC completed" duration=131.600701ms
41level=info ts=2019-12-12T11:00:02.321Z caller=head.go:596 component=tsdb msg="head GC completed" duration=134.547131ms
42level=info ts=2019-12-12T11:00:01.975Z caller=compact.go:496 component=tsdb msg="write block" mint=1576137600000 maxt=1576144800000 ulid=01DVWW7XY69Y4YT09HR0RSR8KY duration=1.905948839s
43level=info ts=2019-12-12T11:00:01.889Z caller=compact.go:496 component=tsdb msg="write block" mint=1576137600000 maxt=1576144800000 ulid=01DVWW7XXX75GHA6ZDTD170CSZ duration=1.828298188s
44level=info ts=2019-12-12T10:55:24.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4
45level=info ts=2019-12-12T10:49:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3
46level=info ts=2019-12-12T10:33:24.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4
47level=info ts=2019-12-12T10:25:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3
48level=info ts=2019-12-12T10:21:24.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4
49level=info ts=2019-12-12T10:14:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3
50level=info ts=2019-12-12T09:00:16.465Z caller=compact.go:441 component=tsdb msg="compact blocks" count=3 mint=1576065600000 maxt=1576130400000 ulid=01DVWNCFJRNW4RP8C56D4QNRXH sources="[01DVVC60FYTRXZ9457XT10Y7AH 01DVW0S6A5HFTYBYD34SGAZJSR 01DVWNCC9SYJDQP0Y2RXK8XJC9]" duration=7.289011992s
51level=info ts=2019-12-12T09:00:15.812Z caller=compact.go:441 component=tsdb msg="compact blocks" count=3 mint=1576065600000 maxt=1576130400000 ulid=01DVWNCF9JNSMSKZHW8STXQARA sources="[01DVVC60DBGMXD5DXR6Y5XWNXF 01DVW0S67R7JFBFTFWMNVS8YR3 01DVWNCC599NDRZWRRSZF4XGHF]" duration=6.930550254s
52level=info ts=2019-12-12T09:00:08.717Z caller=compact.go:441 component=tsdb msg="compact blocks" count=3 mint=1576108800000 maxt=1576130400000 ulid=01DVWNCC9SYJDQP0Y2RXK8XJC9 sources="[01DVW0S0XW63CVRA3EPRSC8NWQ 01DVW7MR5W18322RVFY6WM9GR2 01DVWEGFDW0C09KSCRQ2F8DGN3]" duration=2.900180235s
53level=info ts=2019-12-12T09:00:08.440Z caller=compact.go:441 component=tsdb msg="compact blocks" count=3 mint=1576108800000 maxt=1576130400000 ulid=01DVWNCC599NDRZWRRSZF4XGHF sources="[01DVW0S0XS1SQQQK3CQYCHN9HV 01DVW7MR5ZN3K38ZHBJ243HDZJ 01DVWEGFE0DGKKDG4V9AGAPPBQ]" duration=2.767053211s
54level=info ts=2019-12-12T09:00:05.604Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=1028 last=1030 duration=2.998418095s
55level=info ts=2019-12-12T09:00:05.470Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=1028 last=1030 duration=3.008684806s
56level=info ts=2019-12-12T09:00:02.606Z caller=head.go:596 component=tsdb msg="head GC completed" duration=126.82085ms
57level=info ts=2019-12-12T09:00:02.461Z caller=head.go:596 component=tsdb msg="head GC completed" duration=127.770206ms
58level=info ts=2019-12-12T09:00:01.995Z caller=compact.go:496 component=tsdb msg="write block" mint=1576130400000 maxt=1576137600000 ulid=01DVWNC6NWY1A60AZV3Z6DGS65 duration=1.934602237s
59level=info ts=2019-12-12T09:00:01.960Z caller=compact.go:496 component=tsdb msg="write block" mint=1576130400000 maxt=1576137600000 ulid=01DVWNC6NSPJRCSBZ4QD3SXS66 duration=1.902822647s
60level=info ts=2019-12-12T08:59:54.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4
61level=info ts=2019-12-12T08:54:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3
62level=info ts=2019-12-12T08:12:34.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4
63level=info ts=2019-12-12T08:05:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3
64level=info ts=2019-12-12T07:00:05.421Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=1025 last=1027 duration=3.037037204s
65level=info ts=2019-12-12T07:00:05.263Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=1025 last=1027 duration=2.984857831s
66level=info ts=2019-12-12T07:00:02.383Z caller=head.go:596 component=tsdb msg="head GC completed" duration=126.79721ms
67level=info ts=2019-12-12T07:00:02.278Z caller=head.go:596 component=tsdb msg="head GC completed" duration=131.228064ms
68level=info ts=2019-12-12T07:00:02.052Z caller=compact.go:496 component=tsdb msg="write block" mint=1576123200000 maxt=1576130400000 ulid=01DVWEGFE0DGKKDG4V9AGAPPBQ duration=1.987940522s
69level=info ts=2019-12-12T07:00:01.927Z caller=compact.go:496 component=tsdb msg="write block" mint=1576123200000 maxt=1576130400000 ulid=01DVWEGFDW0C09KSCRQ2F8DGN3 duration=1.866990386s
70level=info ts=2019-12-12T05:00:05.355Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=1022 last=1024 duration=3.046145151s
71level=info ts=2019-12-12T05:00:05.309Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=1022 last=1024 duration=3.019897535s
72level=info ts=2019-12-12T05:00:02.309Z caller=head.go:596 component=tsdb msg="head GC completed" duration=128.294946ms
73level=info ts=2019-12-12T05:00:02.289Z caller=head.go:596 component=tsdb msg="head GC completed" duration=145.150847ms
74level=info ts=2019-12-12T05:00:01.939Z caller=compact.go:496 component=tsdb msg="write block" mint=1576116000000 maxt=1576123200000 ulid=01DVW7MR5ZN3K38ZHBJ243HDZJ duration=1.875204968s
75level=info ts=2019-12-12T05:00:01.813Z caller=compact.go:496 component=tsdb msg="write block" mint=1576116000000 maxt=1576123200000 ulid=01DVW7MR5W18322RVFY6WM9GR2 duration=1.753345795s
76level=info ts=2019-12-12T04:38:34.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4
77level=info ts=2019-12-12T04:33:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3
78level=info ts=2019-12-12T04:00:54.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4
79level=info ts=2019-12-12T03:56:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3
80level=info ts=2019-12-12T03:00:08.433Z caller=compact.go:441 component=tsdb msg="compact blocks" count=3 mint=1576087200000 maxt=1576108800000 ulid=01DVW0S6A5HFTYBYD34SGAZJSR sources="[01DVVC5V5WESMMH77FZVCJ80Q8 01DVVK1JDWNVFGWS4JPY2K4CAS 01DVVSX9NWR5V8SSJAPKQ2TCTH]" duration=2.860812672s
81level=info ts=2019-12-12T03:00:08.279Z caller=compact.go:441 component=tsdb msg="compact blocks" count=3 mint=1576087200000 maxt=1576108800000 ulid=01DVW0S67R7JFBFTFWMNVS8YR3 sources="[01DVVC5V6145SMRFE0WR0P3YTQ 01DVVK1JE1SSYY4EKS4HAT4SK3 01DVVSX9NRE3DWK67A2J17BE0T]" duration=2.782760638s
82level=info ts=2019-12-12T03:00:05.372Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=1019 last=1021 duration=2.990754756s
83level=info ts=2019-12-12T03:00:05.289Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=1019 last=1021 duration=3.007795347s
84level=info ts=2019-12-12T03:00:02.381Z caller=head.go:596 component=tsdb msg="head GC completed" duration=127.007667ms
85level=info ts=2019-12-12T03:00:02.282Z caller=head.go:596 component=tsdb msg="head GC completed" duration=133.138336ms
86level=info ts=2019-12-12T03:00:01.987Z caller=compact.go:496 component=tsdb msg="write block" mint=1576108800000 maxt=1576116000000 ulid=01DVW0S0XW63CVRA3EPRSC8NWQ duration=1.927367458s
87level=info ts=2019-12-12T03:00:01.906Z caller=compact.go:496 component=tsdb msg="write block" mint=1576108800000 maxt=1576116000000 ulid=01DVW0S0XS1SQQQK3CQYCHN9HV duration=1.84874308s
88level=info ts=2019-12-12T02:39:24.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4
89level=info ts=2019-12-12T02:33:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3
90level=info ts=2019-12-12T01:00:05.500Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=1016 last=1018 duration=3.027246961s
91level=info ts=2019-12-12T01:00:05.265Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=1016 last=1018 duration=2.989822576s
92level=info ts=2019-12-12T01:00:02.473Z caller=head.go:596 component=tsdb msg="head GC completed" duration=124.134851ms
93level=info ts=2019-12-12T01:00:02.275Z caller=head.go:596 component=tsdb msg="head GC completed" duration=126.268006ms
94level=info ts=2019-12-12T01:00:02.092Z caller=compact.go:496 component=tsdb msg="write block" mint=1576101600000 maxt=1576108800000 ulid=01DVVSX9NRE3DWK67A2J17BE0T duration=2.035218414s
95level=info ts=2019-12-12T01:00:01.907Z caller=compact.go:496 component=tsdb msg="write block" mint=1576101600000 maxt=1576108800000 ulid=01DVVSX9NWR5V8SSJAPKQ2TCTH duration=1.847566214s
96level=info ts=2019-12-11T23:00:05.552Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=1013 last=1015 duration=3.042911717s
97level=info ts=2019-12-11T23:00:05.255Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=1013 last=1015 duration=3.007686626s
98level=info ts=2019-12-11T23:00:02.509Z caller=head.go:596 component=tsdb msg="head GC completed" duration=135.735201ms
99level=info ts=2019-12-11T23:00:02.247Z caller=head.go:596 component=tsdb msg="head GC completed" duration=126.374582ms
100level=info ts=2019-12-11T23:00:02.154Z caller=compact.go:496 component=tsdb msg="write block" mint=1576094400000 maxt=1576101600000 ulid=01DVVK1JE1SSYY4EKS4HAT4SK3 duration=2.088724625s
101level=info ts=2019-12-11T23:00:01.873Z caller=compact.go:496 component=tsdb msg="write block" mint=1576094400000 maxt=1576101600000 ulid=01DVVK1JDWNVFGWS4JPY2K4CAS duration=1.813033164s
102level=info ts=2019-12-11T21:00:08.427Z caller=compact.go:441 component=tsdb msg="compact blocks" count=3 mint=1576065600000 maxt=1576087200000 ulid=01DVVC60FYTRXZ9457XT10Y7AH sources="[01DVTQJNDXSY7N5V60ZX7X1C3J 01DVTYECNW5T3AHHB2EXATYFMJ 01DVV5A3XWVRTNS7G7BBDQ9G2W]" duration=2.925663083s
103level=info ts=2019-12-11T21:00:08.281Z caller=compact.go:441 component=tsdb msg="compact blocks" count=3 mint=1576065600000 maxt=1576087200000 ulid=01DVVC60DBGMXD5DXR6Y5XWNXF sources="[01DVTQJNDRV9NDCK9H2BCH04R0 01DVTYECNS4AZH3ZMCER87DYWG 01DVV5A3XRVMTB2E7V3MZ6RGCA]" duration=2.862756811s
104level=info ts=2019-12-11T21:00:05.288Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=1010 last=1012 duration=2.998716456s
105level=info ts=2019-12-11T21:00:05.204Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=1010 last=1012 duration=3.013679702s
106level=info ts=2019-12-11T21:00:02.289Z caller=head.go:596 component=tsdb msg="head GC completed" duration=124.171081ms
107level=info ts=2019-12-11T21:00:02.190Z caller=head.go:596 component=tsdb msg="head GC completed" duration=114.925741ms
108level=info ts=2019-12-11T21:00:01.942Z caller=compact.go:496 component=tsdb msg="write block" mint=1576087200000 maxt=1576094400000 ulid=01DVVC5V5WESMMH77FZVCJ80Q8 duration=1.881893506s
109level=info ts=2019-12-11T21:00:01.837Z caller=compact.go:496 component=tsdb msg="write block" mint=1576087200000 maxt=1576094400000 ulid=01DVVC5V6145SMRFE0WR0P3YTQ duration=1.772164011s
110level=info ts=2019-12-11T19:00:05.276Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=1007 last=1009 duration=3.031727362s
111level=info ts=2019-12-11T19:00:05.222Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=1007 last=1009 duration=3.003072336s
112level=info ts=2019-12-11T19:00:02.244Z caller=head.go:596 component=tsdb msg="head GC completed" duration=125.675247ms
113level=info ts=2019-12-11T19:00:02.219Z caller=head.go:596 component=tsdb msg="head GC completed" duration=127.466308ms
114level=info ts=2019-12-11T19:00:01.888Z caller=compact.go:496 component=tsdb msg="write block" mint=1576080000000 maxt=1576087200000 ulid=01DVV5A3XRVMTB2E7V3MZ6RGCA duration=1.832443683s
115level=info ts=2019-12-11T19:00:01.845Z caller=compact.go:496 component=tsdb msg="write block" mint=1576080000000 maxt=1576087200000 ulid=01DVV5A3XWVRTNS7G7BBDQ9G2W duration=1.784935995s
116level=info ts=2019-12-11T18:31:34.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4
117level=info ts=2019-12-11T18:24:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3
118level=info ts=2019-12-11T17:00:05.233Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=1004 last=1006 duration=3.008189996s
119level=info ts=2019-12-11T17:00:05.223Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=1004 last=1006 duration=2.97892s
120level=info ts=2019-12-11T17:00:02.244Z caller=head.go:596 component=tsdb msg="head GC completed" duration=132.385042ms
121level=info ts=2019-12-11T17:00:02.225Z caller=head.go:596 component=tsdb msg="head GC completed" duration=125.500534ms
122level=info ts=2019-12-11T17:00:01.870Z caller=compact.go:496 component=tsdb msg="write block" mint=1576072800000 maxt=1576080000000 ulid=01DVTYECNW5T3AHHB2EXATYFMJ duration=1.810447322s
123level=info ts=2019-12-11T17:00:01.870Z caller=compact.go:496 component=tsdb msg="write block" mint=1576072800000 maxt=1576080000000 ulid=01DVTYECNS4AZH3ZMCER87DYWG duration=1.813347748s
124level=info ts=2019-12-11T15:00:16.297Z caller=compact.go:441 component=tsdb msg="compact blocks" count=3 mint=1576000800000 maxt=1576065600000 ulid=01DVTQJY4WBY96QVV4XQJTR2JC sources="[01DVSECF6Q4JXFDGMFQB3J1Z9E 01DVT2ZN0DMXXJJDHKS0M8JWMS 01DVTQJTX0GZ1S7J51CN1RJNQX]" duration=7.308935842s
125level=info ts=2019-12-11T15:00:15.941Z caller=compact.go:441 component=tsdb msg="compact blocks" count=3 mint=1576000800000 maxt=1576065600000 ulid=01DVTQJY58270MEPGVGGDZZRKJ sources="[01DVSECF6J4NZRNHABZ2MSG7V7 01DVT2ZN4RB65KG77XPHPNVSAM 01DVTQJTYGFS18MWME9Z2NFJSW]" duration=6.941637414s
126level=info ts=2019-12-11T15:00:08.544Z caller=compact.go:441 component=tsdb msg="compact blocks" count=3 mint=1576044000000 maxt=1576065600000 ulid=01DVTQJTX0GZ1S7J51CN1RJNQX sources="[01DVT2ZFNYB7DEH57ZX4HW2DAV 01DVT9V6XW9ENV15NHKR20T9B4 01DVTGPY5WTBSSEQ37JQ2VPCTQ]" duration=2.880290482s
127level=info ts=2019-12-11T15:00:08.541Z caller=compact.go:441 component=tsdb msg="compact blocks" count=3 mint=1576044000000 maxt=1576065600000 ulid=01DVTQJTYGFS18MWME9Z2NFJSW sources="[01DVT2ZFP3R7RB9H6BS3JVAMXJ 01DVT9V6Y21E8YXRKNGA9RPB7D 01DVTGPY5XGARMV8B8VBWQ23W3]" duration=2.829184147s
128level=info ts=2019-12-11T15:00:05.505Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=1001 last=1003 duration=3.006477625s
129level=info ts=2019-12-11T15:00:05.452Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=1001 last=1003 duration=2.990895181s
130level=info ts=2019-12-11T15:00:02.498Z caller=head.go:596 component=tsdb msg="head GC completed" duration=129.237566ms
131level=info ts=2019-12-11T15:00:02.461Z caller=head.go:596 component=tsdb msg="head GC completed" duration=129.961097ms
132level=info ts=2019-12-11T15:00:02.022Z caller=compact.go:496 component=tsdb msg="write block" mint=1576065600000 maxt=1576072800000 ulid=01DVTQJNDRV9NDCK9H2BCH04R0 duration=1.96598488s
133level=info ts=2019-12-11T15:00:01.933Z caller=compact.go:496 component=tsdb msg="write block" mint=1576065600000 maxt=1576072800000 ulid=01DVTQJNDXSY7N5V60ZX7X1C3J duration=1.871872199s
134level=info ts=2019-12-11T14:15:04.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4
135level=info ts=2019-12-11T14:07:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3
136level=info ts=2019-12-11T13:18:04.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4
137level=info ts=2019-12-11T13:12:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3
138level=info ts=2019-12-11T13:00:05.395Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=998 last=1000 duration=3.010358861s
139level=info ts=2019-12-11T13:00:05.249Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=998 last=1000 duration=3.032196282s
140level=info ts=2019-12-11T13:00:02.385Z caller=head.go:596 component=tsdb msg="head GC completed" duration=131.568186ms
141level=info ts=2019-12-11T13:00:02.217Z caller=head.go:596 component=tsdb msg="head GC completed" duration=136.017788ms
142level=info ts=2019-12-11T13:00:02.021Z caller=compact.go:496 component=tsdb msg="write block" mint=1576058400000 maxt=1576065600000 ulid=01DVTGPY5XGARMV8B8VBWQ23W3 duration=1.959903s
143level=info ts=2019-12-11T13:00:01.865Z caller=compact.go:496 component=tsdb msg="write block" mint=1576058400000 maxt=1576065600000 ulid=01DVTGPY5WTBSSEQ37JQ2VPCTQ duration=1.805149859s
144level=info ts=2019-12-11T11:46:34.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4
145level=info ts=2019-12-11T11:39:44.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3
146level=info ts=2019-12-11T11:35:04.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4
147level=info ts=2019-12-11T11:26:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3
148level=info ts=2019-12-11T11:15:34.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4
149level=info ts=2019-12-11T11:06:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3
150level=info ts=2019-12-11T11:01:24.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4
151level=info ts=2019-12-11T11:00:05.591Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=995 last=997 duration=3.063684941s
152level=info ts=2019-12-11T11:00:05.297Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=995 last=997 duration=3.051047495s
153level=info ts=2019-12-11T11:00:02.527Z caller=head.go:596 component=tsdb msg="head GC completed" duration=131.530749ms
154level=info ts=2019-12-11T11:00:02.246Z caller=head.go:596 component=tsdb msg="head GC completed" duration=123.08975ms
155level=info ts=2019-12-11T11:00:02.096Z caller=compact.go:496 component=tsdb msg="write block" mint=1576051200000 maxt=1576058400000 ulid=01DVT9V6Y21E8YXRKNGA9RPB7D duration=2.029825916s
156level=info ts=2019-12-11T11:00:01.819Z caller=compact.go:496 component=tsdb msg="write block" mint=1576051200000 maxt=1576058400000 ulid=01DVT9V6XW9ENV15NHKR20T9B4 duration=1.7583013s
157level=info ts=2019-12-11T10:54:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3
158level=info ts=2019-12-11T10:46:34.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4
159level=info ts=2019-12-11T10:39:44.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3
160level=info ts=2019-12-11T10:34:34.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4
161level=info ts=2019-12-11T10:26:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3
162level=info ts=2019-12-11T10:18:34.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4
163level=info ts=2019-12-11T10:12:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3
164level=info ts=2019-12-11T09:56:04.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4
165level=info ts=2019-12-11T09:48:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3
166level=info ts=2019-12-11T09:00:08.553Z caller=compact.go:441 component=tsdb msg="compact blocks" count=3 mint=1576022400000 maxt=1576044000000 ulid=01DVT2ZN4RB65KG77XPHPNVSAM sources="[01DVSEC9XXK1J3B0186KYQECZT 01DVSN815STR0D0B8245RWNF13 01DVSW3RDRKTJVEZWGYE07XBXE]" duration=2.896352595s
167level=info ts=2019-12-11T09:00:08.407Z caller=compact.go:441 component=tsdb msg="compact blocks" count=3 mint=1576022400000 maxt=1576044000000 ulid=01DVT2ZN0DMXXJJDHKS0M8JWMS sources="[01DVSEC9XW0W8V42SPRR0YMM0X 01DVSN815W8YTW3DPQTJVRNTS4 01DVSW3RDWE1WHSM8AEW0ARA3S]" duration=2.890101974s
168level=info ts=2019-12-11T09:00:05.444Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=992 last=994 duration=3.058184317s
169level=info ts=2019-12-11T09:00:05.306Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=992 last=994 duration=2.99204816s
170level=info ts=2019-12-11T09:00:02.385Z caller=head.go:596 component=tsdb msg="head GC completed" duration=128.295437ms
171level=info ts=2019-12-11T09:00:02.313Z caller=head.go:596 component=tsdb msg="head GC completed" duration=128.456638ms
172level=info ts=2019-12-11T09:00:02.023Z caller=compact.go:496 component=tsdb msg="write block" mint=1576044000000 maxt=1576051200000 ulid=01DVT2ZFP3R7RB9H6BS3JVAMXJ duration=1.955843851s
173level=info ts=2019-12-11T09:00:01.935Z caller=compact.go:496 component=tsdb msg="write block" mint=1576044000000 maxt=1576051200000 ulid=01DVT2ZFNYB7DEH57ZX4HW2DAV duration=1.873653026s
174level=info ts=2019-12-11T07:00:05.441Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=989 last=991 duration=3.013763908s
175level=info ts=2019-12-11T07:00:05.272Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=989 last=991 duration=2.979497994s
176level=info ts=2019-12-11T07:00:02.427Z caller=head.go:596 component=tsdb msg="head GC completed" duration=126.635643ms
177level=info ts=2019-12-11T07:00:02.293Z caller=head.go:596 component=tsdb msg="head GC completed" duration=121.051415ms
178level=info ts=2019-12-11T07:00:02.056Z caller=compact.go:496 component=tsdb msg="write block" mint=1576036800000 maxt=1576044000000 ulid=01DVSW3RDWE1WHSM8AEW0ARA3S duration=1.995603695s
179level=info ts=2019-12-11T07:00:01.941Z caller=compact.go:496 component=tsdb msg="write block" mint=1576036800000 maxt=1576044000000 ulid=01DVSW3RDRKTJVEZWGYE07XBXE duration=1.885680378s
180level=info ts=2019-12-11T06:20:04.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4
181level=info ts=2019-12-11T06:14:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3
182level=info ts=2019-12-11T05:02:04.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=5 to=4
183level=info ts=2019-12-11T05:01:04.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=5
184level=info ts=2019-12-11T05:00:05.488Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=986 last=988 duration=3.043360624s
185level=info ts=2019-12-11T05:00:05.288Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=986 last=988 duration=2.998209654s
186level=info ts=2019-12-11T05:00:02.445Z caller=head.go:596 component=tsdb msg="head GC completed" duration=130.642245ms
187level=info ts=2019-12-11T05:00:02.290Z caller=head.go:596 component=tsdb msg="head GC completed" duration=128.363621ms
188level=info ts=2019-12-11T05:00:02.066Z caller=compact.go:496 component=tsdb msg="write block" mint=1576029600000 maxt=1576036800000 ulid=01DVSN815STR0D0B8245RWNF13 duration=2.008689142s
189level=info ts=2019-12-11T05:00:01.938Z caller=compact.go:496 component=tsdb msg="write block" mint=1576029600000 maxt=1576036800000 ulid=01DVSN815W8YTW3DPQTJVRNTS4 duration=1.877943808s
190level=info ts=2019-12-11T04:55:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3
191level=info ts=2019-12-11T04:35:34.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4
192level=info ts=2019-12-11T04:28:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3
193level=info ts=2019-12-11T04:15:24.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4
194level=info ts=2019-12-11T04:07:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3
195level=info ts=2019-12-11T04:03:34.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4
196level=info ts=2019-12-11T03:57:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3
197level=info ts=2019-12-11T03:52:04.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4
198level=info ts=2019-12-11T03:43:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3
199level=info ts=2019-12-11T03:32:34.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4
200level=info ts=2019-12-11T03:24:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3
201level=info ts=2019-12-11T03:19:34.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4
202level=info ts=2019-12-11T03:12:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3
203level=info ts=2019-12-11T03:00:08.325Z caller=compact.go:441 component=tsdb msg="compact blocks" count=3 mint=1576000800000 maxt=1576022400000 ulid=01DVSECF6Q4JXFDGMFQB3J1Z9E sources="[01DVRSS45W7DXE05RGBYGH58PY 01DVS0MVDWGK47AZ3HY5GQEMK4 01DVS7GJNW7BF3R6KK7GW291R0]" duration=2.861556831s
204level=info ts=2019-12-11T03:00:08.255Z caller=compact.go:441 component=tsdb msg="compact blocks" count=3 mint=1576000800000 maxt=1576022400000 ulid=01DVSECF6J4NZRNHABZ2MSG7V7 sources="[01DVRSS4632MEB6SYC6SB7DTGE 01DVS0MVDR5Z67QJD6T94CXHRA 01DVS7GJNSYRFT48H9CDRP82YV]" duration=2.796902205s
205level=info ts=2019-12-11T03:00:05.253Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=983 last=985 duration=3.004398083s
206level=info ts=2019-12-11T03:00:05.245Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=983 last=985 duration=3.023743067s
207level=info ts=2019-12-11T03:00:02.248Z caller=head.go:596 component=tsdb msg="head GC completed" duration=127.893231ms
208level=info ts=2019-12-11T03:00:02.221Z caller=head.go:596 component=tsdb msg="head GC completed" duration=132.662929ms
209level=info ts=2019-12-11T03:00:01.903Z caller=compact.go:496 component=tsdb msg="write block" mint=1576022400000 maxt=1576029600000 ulid=01DVSEC9XW0W8V42SPRR0YMM0X duration=1.842688968s
210level=info ts=2019-12-11T03:00:01.847Z caller=compact.go:496 component=tsdb msg="write block" mint=1576022400000 maxt=1576029600000 ulid=01DVSEC9XXK1J3B0186KYQECZT duration=1.78558499s
211level=info ts=2019-12-11T02:18:44.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=5 to=4
212level=info ts=2019-12-11T02:18:04.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=5
213level=info ts=2019-12-11T02:11:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3
214level=info ts=2019-12-11T01:59:34.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4
215level=info ts=2019-12-11T01:52:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3
216level=info ts=2019-12-11T01:00:05.272Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=980 last=982 duration=3.025045534s
217level=info ts=2019-12-11T01:00:05.189Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=980 last=982 duration=2.992866718s
218level=info ts=2019-12-11T01:00:02.247Z caller=head.go:596 component=tsdb msg="head GC completed" duration=123.561834ms
219level=info ts=2019-12-11T01:00:02.196Z caller=head.go:596 component=tsdb msg="head GC completed" duration=108.589195ms
220level=info ts=2019-12-11T01:00:01.905Z caller=compact.go:496 component=tsdb msg="write block" mint=1576015200000 maxt=1576022400000 ulid=01DVS7GJNW7BF3R6KK7GW291R0 duration=1.844635186s
221level=info ts=2019-12-11T01:00:01.866Z caller=compact.go:496 component=tsdb msg="write block" mint=1576015200000 maxt=1576022400000 ulid=01DVS7GJNSYRFT48H9CDRP82YV duration=1.809175377s
222level=info ts=2019-12-11T00:31:25.063Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=7 to=5
223level=info ts=2019-12-11T00:30:25.063Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=10 to=7
224level=info ts=2019-12-11T00:29:55.063Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=14 to=10
225level=info ts=2019-12-11T00:29:25.064Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=85 to=14
226level=info ts=2019-12-11T00:29:15.063Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=5 to=85
227level=info ts=2019-12-10T23:00:05.385Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=977 last=979 duration=3.157877457s
228level=info ts=2019-12-10T23:00:05.136Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=977 last=979 duration=3.013728541s
229level=info ts=2019-12-10T23:00:02.227Z caller=head.go:596 component=tsdb msg="head GC completed" duration=133.349607ms
230level=info ts=2019-12-10T23:00:02.123Z caller=head.go:596 component=tsdb msg="head GC completed" duration=110.615384ms
231level=info ts=2019-12-10T23:00:01.832Z caller=compact.go:496 component=tsdb msg="write block" mint=1576008000000 maxt=1576015200000 ulid=01DVS0MVDWGK47AZ3HY5GQEMK4 duration=1.772580137s
232level=info ts=2019-12-10T23:00:01.780Z caller=compact.go:496 component=tsdb msg="write block" mint=1576008000000 maxt=1576015200000 ulid=01DVS0MVDR5Z67QJD6T94CXHRA duration=1.724738556s
233level=info ts=2019-12-10T21:00:18.426Z caller=compact.go:441 component=tsdb msg="compact blocks" count=3 mint=1575936000000 maxt=1576000800000 ulid=01DVRSSD7JM2Y5MQNCZ7QZRSWK sources="[01DVQGJZNRDFP0P161HP7GJX44 01DVR563YKZY789FPAM3DD8DKX 01DVRSS9Q04WN7F254ZCSQ4YP5]" duration=9.096172888s
234level=info ts=2019-12-10T21:00:16.394Z caller=compact.go:441 component=tsdb msg="compact blocks" count=3 mint=1575936000000 maxt=1576000800000 ulid=01DVRSSCTH9ED87EXNSR9J8PE6 sources="[01DVQGJY3RF9X7R93QY6V579W3 01DVR563QG1PZ1AY7RPKSCMKND 01DVRSS9HMN2EC6QQQ2XP7R90D]" duration=7.481260173s
235level=info ts=2019-12-10T21:00:08.859Z caller=compact.go:441 component=tsdb msg="compact blocks" count=3 mint=1575979200000 maxt=1576000800000 ulid=01DVRSS9Q04WN7F254ZCSQ4YP5 sources="[01DVR55YDW6Q96ZHGXD1T7HVF4 01DVRC1NNWE3ZMEQ6035ZJTF49 01DVRJXCXW4W35MBB4E9RXX1QD]" duration=3.130772971s
236level=info ts=2019-12-10T21:00:08.473Z caller=compact.go:441 component=tsdb msg="compact blocks" count=3 mint=1575979200000 maxt=1576000800000 ulid=01DVRSS9HMN2EC6QQQ2XP7R90D sources="[01DVR55YE3CA9B12S48FTTFSVD 01DVRC1NP4CCWPRMCC7667R1FZ 01DVRJXCY0DY2R6DVWWGNXNPRQ]" duration=2.917254733s
237level=info ts=2019-12-10T21:00:05.490Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=974 last=976 duration=3.159908932s
238level=info ts=2019-12-10T21:00:05.339Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=974 last=976 duration=3.090229598s
239level=info ts=2019-12-10T21:00:02.330Z caller=head.go:596 component=tsdb msg="head GC completed" duration=129.067188ms
240level=info ts=2019-12-10T21:00:02.249Z caller=head.go:596 component=tsdb msg="head GC completed" duration=108.272575ms
241level=info ts=2019-12-10T21:00:01.804Z caller=compact.go:496 component=tsdb msg="write block" mint=1576000800000 maxt=1576008000000 ulid=01DVRSS45W7DXE05RGBYGH58PY duration=1.743999568s
242level=info ts=2019-12-10T21:00:01.774Z caller=compact.go:496 component=tsdb msg="write block" mint=1576000800000 maxt=1576008000000 ulid=01DVRSS4632MEB6SYC6SB7DTGE duration=1.706876662s
243level=info ts=2019-12-10T20:35:34.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=3 to=4
244level=info ts=2019-12-10T20:29:14.793Z caller=queue_manager.go:559 component=remote queue=1:https://ops-us-east4.grafana.net/api/prom/push msg="Remote storage resharding" from=4 to=3
245level=info ts=2019-12-10T19:00:06.012Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=971 last=973 duration=3.248527735s
246level=info ts=2019-12-10T19:00:05.641Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=971 last=973 duration=3.1047498s
247level=info ts=2019-12-10T19:00:02.763Z caller=head.go:596 component=tsdb msg="head GC completed" duration=210.397069ms
248level=info ts=2019-12-10T19:00:02.536Z caller=head.go:596 component=tsdb msg="head GC completed" duration=171.333573ms
249level=info ts=2019-12-10T19:00:02.259Z caller=compact.go:496 component=tsdb msg="write block" mint=1575993600000 maxt=1576000800000 ulid=01DVRJXCXW4W35MBB4E9RXX1QD duration=2.199162459s
250level=info ts=2019-12-10T19:00:02.080Z caller=compact.go:496 component=tsdb msg="write block" mint=1575993600000 maxt=1576000800000 ulid=01DVRJXCY0DY2R6DVWWGNXNPRQ duration=2.016321337s
251level=info ts=2019-12-10T17:00:05.549Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=968 last=970 duration=3.183706512s
252level=info ts=2019-12-10T17:00:05.319Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=968 last=970 duration=3.088304654s
253level=info ts=2019-12-10T17:00:02.365Z caller=head.go:596 component=tsdb msg="head GC completed" duration=133.008474ms
254level=info ts=2019-12-10T17:00:02.231Z caller=head.go:596 component=tsdb msg="head GC completed" duration=114.89207ms
255level=info ts=2019-12-10T17:00:01.942Z caller=compact.go:496 component=tsdb msg="write block" mint=1575986400000 maxt=1575993600000 ulid=01DVRC1NNWE3ZMEQ6035ZJTF49 duration=1.881731957s
256level=info ts=2019-12-10T17:00:01.864Z caller=compact.go:496 component=tsdb msg="write block" mint=1575986400000 maxt=1575993600000 ulid=01DVRC1NP4CCWPRMCC7667R1FZ duration=1.795832733s
257level=info ts=2019-12-10T15:00:09.507Z caller=compact.go:441 component=tsdb msg="compact blocks" count=3 mint=1575957600000 maxt=1575979200000 ulid=01DVR563YKZY789FPAM3DD8DKX sources="[01DVQGJRNW1DY7K8KW6B2RY4FF 01DVQQEFXWYP18MDFKY58VJCSG 01DVQYA75W7ERXK6FBEMYYEX6S]" duration=3.791514409s
258level=info ts=2019-12-10T15:00:08.520Z caller=compact.go:441 component=tsdb msg="compact blocks" count=3 mint=1575957600000 maxt=1575979200000 ulid=01DVR563QG1PZ1AY7RPKSCMKND sources="[01DVQGJRP3CC86KKVR5MZ1YYTK 01DVQQEFY0CEBFC4QE02GW9S4F 01DVQYA75TMRETNZRWPV46G5Y0]" duration=3.032106451s
259level=info ts=2019-12-10T15:00:05.484Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=965 last=967 duration=3.124836463s
260level=info ts=2019-12-10T15:00:05.277Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=965 last=967 duration=3.055778688s
261level=info ts=2019-12-10T15:00:02.359Z caller=head.go:596 component=tsdb msg="head GC completed" duration=130.554076ms
262level=info ts=2019-12-10T15:00:02.221Z caller=head.go:596 component=tsdb msg="head GC completed" duration=114.665423ms
263level=info ts=2019-12-10T15:00:01.972Z caller=compact.go:496 component=tsdb msg="write block" mint=1575979200000 maxt=1575986400000 ulid=01DVR55YDW6Q96ZHGXD1T7HVF4 duration=1.912209972s
264level=info ts=2019-12-10T15:00:01.878Z caller=compact.go:496 component=tsdb msg="write block" mint=1575979200000 maxt=1575986400000 ulid=01DVR55YE3CA9B12S48FTTFSVD duration=1.811316924s
2652019-12-10 13:50:13.598594155 +0000 UTC
266level=info ts=2019-12-10T13:50:13.596Z caller=main.go:771 msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml
267level=info ts=2019-12-10T13:50:13.553Z caller=kubernetes.go:192 component="discovery manager scrape" discovery=k8s msg="Using pod service account via in-cluster config"
268level=info ts=2019-12-10T13:50:13.552Z caller=kubernetes.go:192 component="discovery manager scrape" discovery=k8s msg="Using pod service account via in-cluster config"
269level=info ts=2019-12-10T13:50:13.551Z caller=kubernetes.go:192 component="discovery manager scrape" discovery=k8s msg="Using pod service account via in-cluster config"
270level=info ts=2019-12-10T13:50:13.554Z caller=kubernetes.go:192 component="discovery manager scrape" discovery=k8s msg="Using pod service account via in-cluster config"
271level=info ts=2019-12-10T13:50:13.546Z caller=main.go:743 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
2721 curl -X POST --fail -o - -sS http://localhost:80/prometheus/-/reload
2732019/12/10 13:50:13 DEBUG: "/etc/prometheus/..2019_12_05_07_22_08.390693530": REMOVE at 2019-12-10 13:50:13.341280871 +0000 UTC
2742019/12/10 13:50:13 DEBUG: "/etc/prometheus/..data": REMOVE at 2019-12-10 13:50:13.341280871 +0000 UTC
2752019/12/10 13:50:13 DEBUG: "/etc/prometheus/..data/prometheus.yml": REMOVE at 2019-12-10 13:50:13.341280871 +0000 UTC
2762019/12/10 13:50:13 DEBUG: "/etc/prometheus/..data/recording.rules": REMOVE at 2019-12-10 13:50:13.341280871 +0000 UTC
2772019/12/10 13:50:13 DEBUG: "/etc/prometheus/..data/alerts.rules": REMOVE at 2019-12-10 13:50:13.341280871 +0000 UTC
2782019/12/10 13:50:13 DEBUG: Watching /etc/prometheus/..data
2792019/12/10 13:50:13 DEBUG: "/etc/prometheus/..data": CREATE at 2019-12-10 13:50:13.341280871 +0000 UTC
2802019/12/10 13:50:13 DEBUG: "/etc/prometheus/..data_tmp": RENAME at 2019-12-10 13:50:13.341280871 +0000 UTC
2812019/12/10 13:50:13 DEBUG: "/etc/prometheus/..2019_12_10_13_50_13.738754268": CHMOD at 2019-12-10 13:50:13.341280871 +0000 UTC
2822019/12/10 13:50:13 DEBUG: Watching /etc/prometheus/..2019_12_10_13_50_13.738754268
2832019/12/10 13:50:13 DEBUG: "/etc/prometheus/..2019_12_10_13_50_13.738754268": CREATE at 2019-12-10 13:50:13.341280871 +0000 UTC
2842019-12-10 13:49:53.296090488 +0000 UTC
285level=info ts=2019-12-10T13:49:53.294Z caller=main.go:771 msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml
286level=info ts=2019-12-10T13:49:53.254Z caller=kubernetes.go:192 component="discovery manager scrape" discovery=k8s msg="Using pod service account via in-cluster config"
287level=info ts=2019-12-10T13:49:53.253Z caller=kubernetes.go:192 component="discovery manager scrape" discovery=k8s msg="Using pod service account via in-cluster config"
288level=info ts=2019-12-10T13:49:53.252Z caller=kubernetes.go:192 component="discovery manager scrape" discovery=k8s msg="Using pod service account via in-cluster config"
289level=info ts=2019-12-10T13:49:53.251Z caller=kubernetes.go:192 component="discovery manager scrape" discovery=k8s msg="Using pod service account via in-cluster config"
290level=info ts=2019-12-10T13:49:53.248Z caller=main.go:743 msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml
291curl -X POST --fail -o - -sS http://localhost:80/prometheus/-/reload
2922019/12/10 13:49:53 DEBUG: "/etc/prometheus/..2019_12_05_07_22_36.096562123": REMOVE at 2019-12-10 13:49:53.044039978 +0000 UTC
2932019/12/10 13:49:53 DEBUG: "/etc/prometheus/..data": REMOVE at 2019-12-10 13:49:53.043039894 +0000 UTC
2942019/12/10 13:49:53 DEBUG: "/etc/prometheus/..data/prometheus.yml": REMOVE at 2019-12-10 13:49:53.043039894 +0000 UTC
2952019/12/10 13:49:53 DEBUG: "/etc/prometheus/..data/recording.rules": REMOVE at 2019-12-10 13:49:53.043039894 +0000 UTC
2962019/12/10 13:49:53 DEBUG: "/etc/prometheus/..data/alerts.rules": REMOVE at 2019-12-10 13:49:53.043039894 +0000 UTC
2972019/12/10 13:49:53 DEBUG: Watching /etc/prometheus/..data
2982019/12/10 13:49:53 DEBUG: "/etc/prometheus/..data": CREATE at 2019-12-10 13:49:53.043039894 +0000 UTC
2992019/12/10 13:49:53 DEBUG: "/etc/prometheus/..data_tmp": RENAME at 2019-12-10 13:49:53.044039978 +0000 UTC
3002019/12/10 13:49:53 DEBUG: "/etc/prometheus/..2019_12_10_13_49_53.355073198": CHMOD at 2019-12-10 13:49:53.043039894 +0000 UTC
3012019/12/10 13:49:53 DEBUG: Watching /etc/prometheus/..2019_12_10_13_49_53.355073198
3022019/12/10 13:49:53 DEBUG: "/etc/prometheus/..2019_12_10_13_49_53.355073198": CREATE at 2019-12-10 13:49:53.043039894 +0000 UTC
303level=info ts=2019-12-10T13:00:06.007Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=962 last=964 duration=3.145696569s
304level=info ts=2019-12-10T13:00:05.601Z caller=head.go:666 component=tsdb msg="WAL checkpoint complete" first=962 last=964 duration=3.062580976s
305level=info ts=2019-12-10T13:00:02.861Z caller=head.go:596 component=tsdb msg="head GC completed" duration=169.077152ms
306level=info ts=2019-12-10T13:00:02.539Z caller=head.go:596 component=tsdb msg="head GC completed" duration=152.173262ms
307level=info ts=2019-12-10T13:00:02.425Z caller=compact.go:496 component=tsdb msg="write block" mint=1575972000000 maxt=1575979200000 ulid=01DVQYA75W7ERXK6FBEMYYEX6S duration=2.364066751s
308level=info ts=2019-12-10T13:00:02.150Z caller=compact.go:496 component=tsdb msg="write block" mint=1575972000000 maxt=1575979200000 ulid=01DVQYA75TMRETNZRWPV46G5Y0 duration=2.092629264s
309level=info ts=2019-12-10T11:48:14.793Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=2 to=1
310level=info ts=2019-12-10T11:47:34.793Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=3 to=2
311level=info ts=2019-12-10T11:47:14.793Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=5 to=3
312level=info ts=2019-12-10T11:46:54.793Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=7 to=5
313level=info ts=2019-12-10T11:46:34.793Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=10 to=7
314level=info ts=2019-12-10T11:46:14.793Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=16 to=10
315level=info ts=2019-12-10T11:45:54.793Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=26 to=16
316level=info ts=2019-12-10T11:45:34.793Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=39 to=26
317level=info ts=2019-12-10T11:45:14.793Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=63 to=39
318level=info ts=2019-12-10T11:44:54.793Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=100 to=63
319level=info ts=2019-12-10T11:44:45.063Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=2 to=1
320level=info ts=2019-12-10T11:44:34.793Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=156 to=100
321level=info ts=2019-12-10T11:44:15.063Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=3 to=2
322level=info ts=2019-12-10T11:44:14.793Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=253 to=156
323level=info ts=2019-12-10T11:43:55.064Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=4 to=3
324level=info ts=2019-12-10T11:43:54.793Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=405 to=253
325level=info ts=2019-12-10T11:43:45.063Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=5 to=4
326level=info ts=2019-12-10T11:43:34.793Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=619 to=405
327level=info ts=2019-12-10T11:43:25.063Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=8 to=5
328level=info ts=2019-12-10T11:43:14.793Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=1000 to=619
329level=info ts=2019-12-10T11:43:05.063Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=11 to=8
330level=info ts=2019-12-10T11:42:45.063Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=22 to=11
331level=info ts=2019-12-10T11:42:25.063Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=16 to=22
332level=error ts=2019-12-10T11:42:11.074Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 400 Bad Request: out of order sample"
333level=error ts=2019-12-10T11:42:11.073Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 400 Bad Request: out of order sample"
334level=info ts=2019-12-10T11:42:05.063Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=11 to=16
335level=info ts=2019-12-10T11:41:55.063Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=8 to=11
336level=info ts=2019-12-10T11:39:35.063Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=6 to=8
337level=info ts=2019-12-10T11:38:55.063Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=4 to=6
338level=info ts=2019-12-10T11:38:35.064Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=2 to=4
339level=error ts=2019-12-10T11:38:12.281Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=84 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 84 samples"
340level=error ts=2019-12-10T11:38:12.281Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=69 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 69 samples"
341level=error ts=2019-12-10T11:38:12.235Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
342level=error ts=2019-12-10T11:38:12.236Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
343level=error ts=2019-12-10T11:38:12.204Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=97 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 97 samples"
344level=error ts=2019-12-10T11:38:12.183Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=89 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 89 samples"
345level=error ts=2019-12-10T11:38:12.129Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=88 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 88 samples"
346level=error ts=2019-12-10T11:38:12.127Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=91 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 91 samples"
347level=error ts=2019-12-10T11:38:12.128Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
348level=error ts=2019-12-10T11:38:12.129Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=86 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 86 samples"
349level=error ts=2019-12-10T11:38:12.139Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=90 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 90 samples"
350level=error ts=2019-12-10T11:38:12.140Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
351level=error ts=2019-12-10T11:38:12.141Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
352level=error ts=2019-12-10T11:38:12.142Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
353level=error ts=2019-12-10T11:38:12.143Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=56 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 56 samples"
354level=error ts=2019-12-10T11:38:12.125Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=52 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 52 samples"
355level=error ts=2019-12-10T11:38:12.124Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
356level=error ts=2019-12-10T11:38:12.123Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=75 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 75 samples"
357level=error ts=2019-12-10T11:38:12.122Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=69 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 69 samples"
358level=error ts=2019-12-10T11:38:12.120Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
359level=error ts=2019-12-10T11:38:12.1201Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=67 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 67 samples"
360level=error ts=2019-12-10T11:38:12.1202Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=78 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 78 samples"
361level=error ts=2019-12-10T11:38:12.1203Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
362level=error ts=2019-12-10T11:38:12.1191Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
363level=error ts=2019-12-10T11:38:12.1192Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
364level=error ts=2019-12-10T11:38:12.081Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=96 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 96 samples"
365level=error ts=2019-12-10T11:38:12.0371Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
366level=error ts=2019-12-10T11:38:12.0372Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
367level=error ts=2019-12-10T11:38:12.0373Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
368level=error ts=2019-12-10T11:38:12.024Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
369level=error ts=2019-12-10T11:38:12.025Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
370level=error ts=2019-12-10T11:38:12.026Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=89 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 89 samples"
371level=error ts=2019-12-10T11:38:11.920Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
372level=error ts=2019-12-10T11:38:11.917Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
373level=error ts=2019-12-10T11:38:11.916Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
374level=error ts=2019-12-10T11:38:11.913Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=95 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 95 samples"
375level=error ts=2019-12-10T11:38:11.913Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=82 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 82 samples"
376level=error ts=2019-12-10T11:38:11.722Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
377level=error ts=2019-12-10T11:38:11.721Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
378level=error ts=2019-12-10T11:38:11.176Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
379level=error ts=2019-12-10T11:38:11.1451Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
380level=error ts=2019-12-10T11:38:11.1452Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
381level=error ts=2019-12-10T11:38:11.1453Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=99 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 99 samples"
382level=error ts=2019-12-10T11:38:11.1454Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=85 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 85 samples"
383level=error ts=2019-12-10T11:38:11.1455Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
384level=error ts=2019-12-10T11:38:11.1441Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
385level=error ts=2019-12-10T11:38:11.1442Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=70 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 70 samples"
386level=error ts=2019-12-10T11:38:11.1443Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=72 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 72 samples"
387level=error ts=2019-12-10T11:38:11.1444Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=97 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 97 samples"
388level=error ts=2019-12-10T11:38:11.1445Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=46 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 46 samples"
389level=error ts=2019-12-10T11:38:11.1431Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
390level=error ts=2019-12-10T11:38:11.1432Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=83 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 83 samples"
391level=error ts=2019-12-10T11:38:11.1433Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
392level=error ts=2019-12-10T11:38:11.1434Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=92 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 92 samples"
393level=error ts=2019-12-10T11:38:11.0011Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
394level=error ts=2019-12-10T11:38:11.0012Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
395level=error ts=2019-12-10T11:38:11.0012Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=84 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 84 samples"
396level=error ts=2019-12-10T11:38:11.0013Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
397level=error ts=2019-12-10T11:38:10.9551Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
398level=error ts=2019-12-10T11:38:10.9532Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
399level=error ts=2019-12-10T11:38:10.9421Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=86 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 86 samples"
400level=error ts=2019-12-10T11:38:10.9422Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
401level=error ts=2019-12-10T11:38:10.9423Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
402level=error ts=2019-12-10T11:38:10.941Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=58 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 58 samples"
403level=error ts=2019-12-10T11:38:10.942Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
404level=error ts=2019-12-10T11:38:10.940Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
405level=error ts=2019-12-10T11:38:10.943Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
406level=error ts=2019-12-10T11:38:10.939Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
407level=error ts=2019-12-10T11:38:10.939Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=23 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 23 samples"
408level=error ts=2019-12-10T11:38:10.939Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=59 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 59 samples"
409level=error ts=2019-12-10T11:38:10.939Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=81 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 81 samples"
410level=error ts=2019-12-10T11:38:10.878Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
411level=error ts=2019-12-10T11:38:10.879Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
412level=error ts=2019-12-10T11:38:10.8781Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
413level=error ts=2019-12-10T11:38:10.8782Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
414level=error ts=2019-12-10T11:38:10.877Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=96 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 96 samples"
415level=error ts=2019-12-10T11:38:10.877Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=70 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 70 samples"
416level=error ts=2019-12-10T11:38:10.877Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=72 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 72 samples"
417level=error ts=2019-12-10T11:38:10.039Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
418level=error ts=2019-12-10T11:38:09.902Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=79 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 79 samples"
419level=error ts=2019-12-10T11:38:09.901Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
420level=error ts=2019-12-10T11:38:09.902Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
421level=error ts=2019-12-10T11:38:09.903Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=92 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 92 samples"
422level=error ts=2019-12-10T11:38:09.898Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
423level=error ts=2019-12-10T11:38:09.8971Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=95 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 95 samples"
424level=error ts=2019-12-10T11:38:09.8972Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=73 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 73 samples"
425level=error ts=2019-12-10T11:38:09.8973Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
426level=error ts=2019-12-10T11:38:09.8974Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=68 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 68 samples"
427level=error ts=2019-12-10T11:38:09.8975Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
428level=error ts=2019-12-10T11:38:09.8976Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=35 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 35 samples"
429level=error ts=2019-12-10T11:38:09.8977Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
430level=error ts=2019-12-10T11:38:09.8951Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
431level=error ts=2019-12-10T11:38:09.8952Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=98 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 98 samples"
432level=error ts=2019-12-10T11:38:09.8953Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=98 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 98 samples"
433level=error ts=2019-12-10T11:38:09.876Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
434level=error ts=2019-12-10T11:38:09.8401Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=88 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 88 samples"
435level=error ts=2019-12-10T11:38:09.8402Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=92 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 92 samples"
436level=error ts=2019-12-10T11:38:09.8403Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
437level=error ts=2019-12-10T11:38:09.8404Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=85 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 85 samples"
438level=error ts=2019-12-10T11:38:09.8406Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
439level=error ts=2019-12-10T11:38:09.8407Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
440level=error ts=2019-12-10T11:38:09.832Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
441level=error ts=2019-12-10T11:38:09.825Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
442level=error ts=2019-12-10T11:38:09.8251Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
443level=error ts=2019-12-10T11:38:09.8241Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
444level=error ts=2019-12-10T11:38:09.8242Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
445level=error ts=2019-12-10T11:38:09.8243Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=96 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 96 samples"
446level=error ts=2019-12-10T11:38:09.8244Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=17 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 17 samples"
447level=error ts=2019-12-10T11:38:09.8245Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
448level=error ts=2019-12-10T11:38:09.8246Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
449level=error ts=2019-12-10T11:38:09.815Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=94 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 94 samples"
450level=error ts=2019-12-10T11:38:09.806Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=81 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 81 samples"
451level=error ts=2019-12-10T11:38:09.736Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
452level=error ts=2019-12-10T11:38:09.736Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=48 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 48 samples"
453level=error ts=2019-12-10T11:38:09.735Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=97 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 97 samples"
454level=error ts=2019-12-10T11:38:09.735Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=67 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 67 samples"
455level=error ts=2019-12-10T11:38:09.735Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=43 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 43 samples"
456level=error ts=2019-12-10T11:38:09.735Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=90 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 90 samples"
457level=error ts=2019-12-10T11:38:09.735Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
458level=error ts=2019-12-10T11:38:09.735Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=19 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 19 samples"
459level=error ts=2019-12-10T11:38:09.733Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=79 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 79 samples"
460level=error ts=2019-12-10T11:38:09.700Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
461level=error ts=2019-12-10T11:38:09.700Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=50 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 50 samples"
462level=error ts=2019-12-10T11:38:09.700Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=77 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 77 samples"
463level=error ts=2019-12-10T11:38:08.797Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=86 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 86 samples"
464level=error ts=2019-12-10T11:38:08.797Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 101 samples"
465level=error ts=2019-12-10T11:38:08.797Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 102 samples"
466level=error ts=2019-12-10T11:38:08.797Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 103 samples"
467level=error ts=2019-12-10T11:38:08.797Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=83 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 83 samples"
468level=error ts=2019-12-10T11:38:08.797Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 113 samples"
469level=error ts=2019-12-10T11:38:08.796Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=90 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 90 samples"
470level=error ts=2019-12-10T11:38:08.796Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=37 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 37 samples"
471level=error ts=2019-12-10T11:38:08.796Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=67 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 67 samples"
472level=error ts=2019-12-10T11:38:08.796Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=77 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 77 samples"
473level=error ts=2019-12-10T11:38:08.795Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=74 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 74 samples"
474level=error ts=2019-12-10T11:38:08.710Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=94 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 94 samples"
475level=error ts=2019-12-10T11:38:08.710Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
476level=error ts=2019-12-10T11:38:08.710Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=89 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 89 samples"
477level=error ts=2019-12-10T11:38:08.710Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 101 samples"
478level=error ts=2019-12-10T11:38:08.694Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=77 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 77 samples"
479level=error ts=2019-12-10T11:38:08.694Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 102 samples"
480level=error ts=2019-12-10T11:38:08.694Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 103 samples"
481level=error ts=2019-12-10T11:38:08.693Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 104 samples"
482level=error ts=2019-12-10T11:38:08.692Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=58 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 58 samples"
483level=error ts=2019-12-10T11:38:08.692Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=91 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 91 samples"
484level=error ts=2019-12-10T11:38:08.692Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
485level=error ts=2019-12-10T11:38:08.692Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=59 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 59 samples"
486level=error ts=2019-12-10T11:38:08.692Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 101 samples"
487level=error ts=2019-12-10T11:38:08.692Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 102 samples"
488level=error ts=2019-12-10T11:38:08.692Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 103 samples"
489level=error ts=2019-12-10T11:38:08.692Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 104 samples"
490level=error ts=2019-12-10T11:38:08.691Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 105 samples"
491level=error ts=2019-12-10T11:38:08.678Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=75 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 75 samples"
492level=error ts=2019-12-10T11:38:08.677Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 106 samples"
493level=error ts=2019-12-10T11:38:08.648Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 107 samples"
494level=error ts=2019-12-10T11:38:08.647Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 108 samples"
495level=error ts=2019-12-10T11:38:08.644Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=90 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 90 samples"
496level=error ts=2019-12-10T11:38:08.603Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 109 samples"
497level=error ts=2019-12-10T11:38:08.603Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 110 samples"
498level=error ts=2019-12-10T11:38:08.603Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 111 samples"
499level=error ts=2019-12-10T11:38:08.603Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=71 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 71 samples"
500level=error ts=2019-12-10T11:38:08.603Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 112 samples"
501level=error ts=2019-12-10T11:38:08.602Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=83 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 83 samples"
502level=error ts=2019-12-10T11:38:08.596Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=84 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 84 samples"
503level=error ts=2019-12-10T11:38:08.595Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 113 samples"
504level=error ts=2019-12-10T11:38:08.595Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 114 samples"
505level=error ts=2019-12-10T11:38:08.553Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=95 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 95 samples"
506level=error ts=2019-12-10T11:38:08.552Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 115 samples"
507level=error ts=2019-12-10T11:38:08.552Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=73 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 73 samples"
508level=error ts=2019-12-10T11:38:08.552Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=87 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 87 samples"
509level=error ts=2019-12-10T11:38:08.541Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 116 samples"
510level=error ts=2019-12-10T11:38:08.501Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=96 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 96 samples"
511level=error ts=2019-12-10T11:38:08.501Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 117 samples"
512level=error ts=2019-12-10T11:38:08.497Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=91 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 91 samples"
513level=error ts=2019-12-10T11:38:08.497Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 118 samples"
514level=error ts=2019-12-10T11:38:08.497Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 119 samples"
515level=error ts=2019-12-10T11:38:08.497Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 120 samples"
516level=error ts=2019-12-10T11:38:08.447Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=72 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 72 samples"
517level=error ts=2019-12-10T11:38:08.447Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=97 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 97 samples"
518level=error ts=2019-12-10T11:38:08.447Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 130 samples"
519level=error ts=2019-12-10T11:38:08.447Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 131 samples"
520level=error ts=2019-12-10T11:38:08.447Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 132 samples"
521level=error ts=2019-12-10T11:38:08.447Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 133 samples"
522level=error ts=2019-12-10T11:38:08.446Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=70 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 70 samples"
523level=error ts=2019-12-10T11:38:08.445Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=65 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 65 samples"
524level=error ts=2019-12-10T11:38:08.441Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=88 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 88 samples"
525level=error ts=2019-12-10T11:38:08.433Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
526level=error ts=2019-12-10T11:38:08.431Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=89 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 89 samples"
527level=error ts=2019-12-10T11:38:08.407Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=80 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 80 samples"
528level=error ts=2019-12-10T11:38:08.393Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
529level=error ts=2019-12-10T11:38:08.3941Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
530level=error ts=2019-12-10T11:38:08.395Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
531level=error ts=2019-12-10T11:38:08.396Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=82 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 82 samples"
532level=error ts=2019-12-10T11:38:08.393Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=90 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 90 samples"
533level=error ts=2019-12-10T11:38:08.394Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
534level=error ts=2019-12-10T11:38:08.391Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 101 samples"
535level=error ts=2019-12-10T11:38:08.395Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 102 samples"
536level=error ts=2019-12-10T11:38:07.604Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 103 samples"
537level=error ts=2019-12-10T11:38:07.603Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 104 samples"
538level=error ts=2019-12-10T11:38:07.602Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=60 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 60 samples"
539level=error ts=2019-12-10T11:38:07.602Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 101 samples"
540level=error ts=2019-12-10T11:38:07.601Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 102 samples"
541level=error ts=2019-12-10T11:38:07.601Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 103 samples"
542level=error ts=2019-12-10T11:38:07.601Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 104 samples"
543level=error ts=2019-12-10T11:38:07.601Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 105 samples"
544level=error ts=2019-12-10T11:38:07.601Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 106 samples"
545level=error ts=2019-12-10T11:38:07.601Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=75 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 75 samples"
546level=error ts=2019-12-10T11:38:07.601Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=71 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 71 samples"
547level=error ts=2019-12-10T11:38:07.601Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=96 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 96 samples"
548level=error ts=2019-12-10T11:38:07.600Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
549level=error ts=2019-12-10T11:38:07.599Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 101 samples"
550level=error ts=2019-12-10T11:38:07.599Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=98 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 98 samples"
551level=error ts=2019-12-10T11:38:07.598Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=93 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 93 samples"
552level=error ts=2019-12-10T11:38:07.598Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=72 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 72 samples"
553level=error ts=2019-12-10T11:38:07.599Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
554level=error ts=2019-12-10T11:38:07.598Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 101 samples"
555level=error ts=2019-12-10T11:38:07.598Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 102 samples"
556level=error ts=2019-12-10T11:38:07.597Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 103 samples"
557level=error ts=2019-12-10T11:38:07.597Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 104 samples"
558level=error ts=2019-12-10T11:38:07.597Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 105 samples"
559level=error ts=2019-12-10T11:38:07.597Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 106 samples"
560level=error ts=2019-12-10T11:38:07.596Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=70 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 70 samples"
561level=error ts=2019-12-10T11:38:07.596Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 107 samples"
562level=error ts=2019-12-10T11:38:07.595Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 108 samples"
563level=error ts=2019-12-10T11:38:07.595Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=74 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 74 samples"
564level=error ts=2019-12-10T11:38:07.594Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=97 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 97 samples"
565level=error ts=2019-12-10T11:38:07.594Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=99 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 99 samples"
566level=error ts=2019-12-10T11:38:07.594Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 109 samples"
567level=error ts=2019-12-10T11:38:07.594Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 110 samples"
568level=error ts=2019-12-10T11:38:07.594Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=91 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 91 samples"
569level=error ts=2019-12-10T11:38:07.593Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=73 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 73 samples"
570level=error ts=2019-12-10T11:38:07.593Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 111 samples"
571level=error ts=2019-12-10T11:38:07.593Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=81 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 81 samples"
572level=error ts=2019-12-10T11:38:07.593Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=86 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 86 samples"
573level=error ts=2019-12-10T11:38:07.592Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 112 samples"
574level=error ts=2019-12-10T11:38:07.592Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 113 samples"
575level=error ts=2019-12-10T11:38:07.591Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=53 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 53 samples"
576level=error ts=2019-12-10T11:38:07.591Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 114 samples"
577level=error ts=2019-12-10T11:38:07.591Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 115 samples"
578level=error ts=2019-12-10T11:38:07.591Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 116 samples"
579level=error ts=2019-12-10T11:38:07.591Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 117 samples"
580level=error ts=2019-12-10T11:38:07.589Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 118 samples"
581level=error ts=2019-12-10T11:38:07.589Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=39 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 39 samples"
582level=error ts=2019-12-10T11:38:07.589Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 119 samples"
583level=error ts=2019-12-10T11:38:07.589Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 120 samples"
584level=error ts=2019-12-10T11:38:07.587Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 121 samples"
585level=error ts=2019-12-10T11:38:07.513Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 122 samples"
586level=error ts=2019-12-10T11:38:07.512Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=59 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 59 samples"
587level=error ts=2019-12-10T11:38:07.512Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 123 samples"
588level=error ts=2019-12-10T11:38:07.510Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=79 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 79 samples"
589level=error ts=2019-12-10T11:38:07.506Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 124 samples"
590level=error ts=2019-12-10T11:38:07.506Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 125 samples"
591level=error ts=2019-12-10T11:38:07.499Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 126 samples"
592level=error ts=2019-12-10T11:38:07.499Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=67 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 67 samples"
593level=error ts=2019-12-10T11:38:07.499Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=98 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
594level=error ts=2019-12-10T11:38:07.499Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=75 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 75 samples"
595level=error ts=2019-12-10T11:38:07.499Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=60 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 60 samples"
596level=error ts=2019-12-10T11:38:07.499Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=99 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 99 samples"
597level=error ts=2019-12-10T11:38:07.499Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=98 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 98 samples"
598level=error ts=2019-12-10T11:38:07.499Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 127 samples"
599level=error ts=2019-12-10T11:38:07.499Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 128 samples"
600level=error ts=2019-12-10T11:38:07.494Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 129 samples"
601level=error ts=2019-12-10T11:38:07.489Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 130 samples"
602level=error ts=2019-12-10T11:38:07.488Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 131 samples"
603level=error ts=2019-12-10T11:38:07.484Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 132 samples"
604level=error ts=2019-12-10T11:38:07.483Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 133 samples"
605level=error ts=2019-12-10T11:38:07.483Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 134 samples"
606level=error ts=2019-12-10T11:38:07.480Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 136 samples"
607level=error ts=2019-12-10T11:38:07.480Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=66 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 67 samples"
608level=error ts=2019-12-10T11:38:07.480Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=71 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 71 samples"
609level=error ts=2019-12-10T11:38:07.480Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=66 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 66 samples"
610level=error ts=2019-12-10T11:38:07.478Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=76 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 76 samples"
611level=error ts=2019-12-10T11:38:07.460Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=77 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 77 samples"
612level=error ts=2019-12-10T11:38:07.455Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 101 samples"
613level=error ts=2019-12-10T11:38:07.455Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 102 samples"
614level=error ts=2019-12-10T11:38:07.455Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=65 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 65 samples"
615level=error ts=2019-12-10T11:38:07.454Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 101 samples"
616level=error ts=2019-12-10T11:38:07.453Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 102 samples"
617level=error ts=2019-12-10T11:38:07.453Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=80 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 80 samples"
618level=error ts=2019-12-10T11:38:07.453Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
619level=error ts=2019-12-10T11:38:07.452Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=80 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 80 samples"
620level=error ts=2019-12-10T11:38:07.452Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 101 samples"
621level=error ts=2019-12-10T11:38:07.452Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 102 samples"
622level=error ts=2019-12-10T11:38:07.452Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=37 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 37 samples"
623level=error ts=2019-12-10T11:38:07.452Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=30 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 30 samples"
624level=error ts=2019-12-10T11:38:07.452Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=74 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 74 samples"
625level=error ts=2019-12-10T11:38:07.452Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
626level=error ts=2019-12-10T11:38:07.452Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=79 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 79 samples"
627level=error ts=2019-12-10T11:38:07.452Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=70 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 70 samples"
628level=error ts=2019-12-10T11:38:07.451Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 115 samples"
629level=error ts=2019-12-10T11:38:07.451Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 101 samples"
630level=error ts=2019-12-10T11:38:07.451Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=96 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 96 samples"
631level=error ts=2019-12-10T11:38:07.451Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=84 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 84 samples"
632level=error ts=2019-12-10T11:38:07.451Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=74 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 74 samples"
633level=error ts=2019-12-10T11:38:07.451Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=63 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 63 samples"
634level=error ts=2019-12-10T11:38:07.451Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
635level=error ts=2019-12-10T11:38:07.450Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 102 samples"
636level=error ts=2019-12-10T11:38:07.450Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 103 samples"
637level=error ts=2019-12-10T11:38:07.450Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=95 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 95 samples"
638level=error ts=2019-12-10T11:38:07.449Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
639level=error ts=2019-12-10T11:38:07.448Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=99 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 99 samples"
640level=error ts=2019-12-10T11:38:07.448Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 101 samples"
641level=error ts=2019-12-10T11:38:07.447Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 102 samples"
642level=error ts=2019-12-10T11:38:07.446Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 103 samples"
643level=error ts=2019-12-10T11:38:07.446Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 104 samples"
644level=error ts=2019-12-10T11:38:07.446Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 105 samples"
645level=error ts=2019-12-10T11:38:07.445Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=57 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 57 samples"
646level=error ts=2019-12-10T11:38:07.445Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 101 samples"
647level=error ts=2019-12-10T11:38:07.445Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 102 samples"
648level=error ts=2019-12-10T11:38:07.445Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 103 samples"
649level=error ts=2019-12-10T11:38:07.444Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 104 samples"
650level=error ts=2019-12-10T11:38:07.444Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 105 samples"
651level=error ts=2019-12-10T11:38:07.444Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=84 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 84 samples"
652level=error ts=2019-12-10T11:38:07.443Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 106 samples"
653level=error ts=2019-12-10T11:38:07.390Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 107 samples"
654level=error ts=2019-12-10T11:38:07.389Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 108 samples"
655level=error ts=2019-12-10T11:38:07.389Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 109 samples"
656level=error ts=2019-12-10T11:38:07.389Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 110 samples"
657level=error ts=2019-12-10T11:38:07.389Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 111 samples"
658level=error ts=2019-12-10T11:38:07.388Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=85 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 85 samples"
659level=error ts=2019-12-10T11:38:07.388Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 112 samples"
660level=error ts=2019-12-10T11:38:07.388Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=89 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 89 samples"
661level=error ts=2019-12-10T11:38:07.387Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 113 samples"
662level=error ts=2019-12-10T11:38:07.387Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=98 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 98 samples"
663level=error ts=2019-12-10T11:38:07.387Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=63 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 63 samples"
664level=error ts=2019-12-10T11:38:07.376Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 114 samples"
665level=error ts=2019-12-10T11:38:07.358Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=84 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 84 samples"
666level=error ts=2019-12-10T11:38:07.357Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 115 samples"
667level=error ts=2019-12-10T11:38:07.357Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 116 samples"
668level=error ts=2019-12-10T11:38:07.357Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=82 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 82 samples"
669level=error ts=2019-12-10T11:38:07.357Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 117 samples"
670level=error ts=2019-12-10T11:38:07.357Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=79 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 79 samples"
671level=error ts=2019-12-10T11:38:07.310Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 118 samples"
672level=error ts=2019-12-10T11:38:07.259Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 119 samples"
673level=error ts=2019-12-10T11:38:07.259Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=85 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 85 samples"
674level=error ts=2019-12-10T11:38:07.045Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 120 samples"
675level=error ts=2019-12-10T11:38:07.044Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=83 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 83 samples"
676level=error ts=2019-12-10T11:38:07.042Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 121 samples"
677level=error ts=2019-12-10T11:38:07.043Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 122 samples"
678level=error ts=2019-12-10T11:38:07.043Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=69 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 69 samples"
679level=error ts=2019-12-10T11:38:07.042Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 123 samples"
680level=error ts=2019-12-10T11:38:07.042Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 124 samples"
681level=error ts=2019-12-10T11:38:07.042Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 125 samples"
682level=error ts=2019-12-10T11:38:07.041Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=49 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 49 samples"
683level=error ts=2019-12-10T11:38:06.947Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=81 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 81 samples"
684level=error ts=2019-12-10T11:38:06.947Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 126 samples"
685level=error ts=2019-12-10T11:38:06.946Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 127 samples"
686level=error ts=2019-12-10T11:38:06.944Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 128 samples"
687level=error ts=2019-12-10T11:38:06.943Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=75 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 75 samples"
688level=error ts=2019-12-10T11:38:06.939Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 129 samples"
689level=error ts=2019-12-10T11:38:06.938Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=95 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 95 samples"
690level=error ts=2019-12-10T11:38:06.937Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 130 samples"
691level=error ts=2019-12-10T11:38:06.848Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=54 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 54 samples"
692level=error ts=2019-12-10T11:38:06.848Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 131 samples"
693level=error ts=2019-12-10T11:38:06.842Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 132 samples"
694level=error ts=2019-12-10T11:38:06.841Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 133 samples"
695level=error ts=2019-12-10T11:38:06.747Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 134 samples"
696level=error ts=2019-12-10T11:38:06.745Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=85 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 85 samples"
697level=error ts=2019-12-10T11:38:06.745Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 135 samples"
698level=error ts=2019-12-10T11:38:06.745Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 136 samples"
699level=error ts=2019-12-10T11:38:06.745Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 101 samples"
700level=error ts=2019-12-10T11:38:06.745Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=65 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 65 samples"
701level=error ts=2019-12-10T11:38:06.745Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 102 samples"
702level=error ts=2019-12-10T11:38:06.715Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 103 samples"
703level=error ts=2019-12-10T11:38:06.646Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=95 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 95 samples"
704level=error ts=2019-12-10T11:38:06.646Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=37 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 37 samples"
705level=error ts=2019-12-10T11:38:06.644Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
706level=error ts=2019-12-10T11:38:06.644Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=82 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 82 samples"
707level=error ts=2019-12-10T11:38:06.644Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=97 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 97 samples"
708level=error ts=2019-12-10T11:38:06.644Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 101 samples"
709level=error ts=2019-12-10T11:38:06.641Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 102 samples"
710level=error ts=2019-12-10T11:38:06.640Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 103 samples"
711level=error ts=2019-12-10T11:38:06.640Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 104 samples"
712level=error ts=2019-12-10T11:38:06.639Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 105 samples"
713level=error ts=2019-12-10T11:38:06.638Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 106 samples"
714level=error ts=2019-12-10T11:38:06.593Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=93 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 93 samples"
715level=error ts=2019-12-10T11:38:06.592Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 107 samples"
716level=error ts=2019-12-10T11:38:06.592Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 108 samples"
717level=error ts=2019-12-10T11:38:06.592Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=95 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 95 samples"
718level=error ts=2019-12-10T11:38:06.592Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 109 samples"
719level=error ts=2019-12-10T11:38:06.592Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 110 samples"
720level=error ts=2019-12-10T11:38:06.591Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 111 samples"
721level=error ts=2019-12-10T11:38:06.591Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=56 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 56 samples"
722level=error ts=2019-12-10T11:38:06.590Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 112 samples"
723level=error ts=2019-12-10T11:38:06.590Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=75 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 75 samples"
724level=error ts=2019-12-10T11:38:06.589Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 113 samples"
725level=error ts=2019-12-10T11:38:06.589Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 114 samples"
726level=error ts=2019-12-10T11:38:06.588Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=83 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 83 samples"
727level=error ts=2019-12-10T11:38:06.588Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 116 samples"
728level=error ts=2019-12-10T11:38:06.581Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 117 samples"
729level=error ts=2019-12-10T11:38:06.581Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 118 samples"
730level=error ts=2019-12-10T11:38:06.563Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=77 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 77 samples"
731level=error ts=2019-12-10T11:38:06.563Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 119 samples"
732level=error ts=2019-12-10T11:38:06.563Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 120 samples"
733level=error ts=2019-12-10T11:38:06.562Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 121 samples"
734level=error ts=2019-12-10T11:38:06.562Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 122 samples"
735level=error ts=2019-12-10T11:38:06.561Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 123 samples"
736level=error ts=2019-12-10T11:38:06.560Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 124 samples"
737level=error ts=2019-12-10T11:38:06.551Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 125 samples"
738level=error ts=2019-12-10T11:38:06.550Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 126 samples"
739level=error ts=2019-12-10T11:38:06.544Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 127 samples"
740level=error ts=2019-12-10T11:38:06.544Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 128 samples"
741level=error ts=2019-12-10T11:38:06.543Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 129 samples"
742level=error ts=2019-12-10T11:38:06.543Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 130 samples"
743level=error ts=2019-12-10T11:38:06.543Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 131 samples"
744level=error ts=2019-12-10T11:38:06.543Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=98 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 98 samples"
745level=error ts=2019-12-10T11:38:06.542Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 132 samples"
746level=error ts=2019-12-10T11:38:06.542Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 133 samples"
747level=error ts=2019-12-10T11:38:06.542Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 134 samples"
748level=error ts=2019-12-10T11:38:06.540Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 135 samples"
749level=error ts=2019-12-10T11:38:06.538Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 136 samples"
750level=error ts=2019-12-10T11:38:06.538Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 137 samples"
751level=error ts=2019-12-10T11:38:06.512Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 138 samples"
752level=error ts=2019-12-10T11:38:06.506Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 140 samples"
753level=error ts=2019-12-10T11:38:06.505Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 141 samples"
754level=error ts=2019-12-10T11:38:06.504Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=69 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 69 samples"
755level=error ts=2019-12-10T11:38:06.504Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 142 samples"
756level=error ts=2019-12-10T11:38:06.504Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 143 samples"
757level=error ts=2019-12-10T11:38:06.504Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=79 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 79 samples"
758level=error ts=2019-12-10T11:38:06.504Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=98 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 98 samples"
759level=error ts=2019-12-10T11:38:06.489Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 144 samples"
760level=error ts=2019-12-10T11:38:06.461Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 145 samples"
761level=error ts=2019-12-10T11:38:06.461Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 146 samples"
762level=error ts=2019-12-10T11:38:06.461Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 147 samples"
763level=error ts=2019-12-10T11:38:06.461Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=77 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 77 samples"
764level=error ts=2019-12-10T11:38:06.461Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=83 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 83 samples"
765level=error ts=2019-12-10T11:38:06.461Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=96 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 96 samples"
766level=error ts=2019-12-10T11:38:06.460Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 148 samples"
767level=error ts=2019-12-10T11:38:06.460Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 149 samples"
768level=error ts=2019-12-10T11:38:06.460Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=75 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 75 samples"
769level=error ts=2019-12-10T11:38:06.459Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 150 samples"
770level=error ts=2019-12-10T11:38:06.453Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 151 samples"
771level=error ts=2019-12-10T11:38:06.453Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 152 samples"
772level=error ts=2019-12-10T11:38:06.452Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 153 samples"
773level=error ts=2019-12-10T11:38:06.452Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 154 samples"
774level=error ts=2019-12-10T11:38:06.451Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=85 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 85 samples"
775level=error ts=2019-12-10T11:38:06.451Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 155 samples"
776level=error ts=2019-12-10T11:38:06.451Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 156 samples"
777level=error ts=2019-12-10T11:38:06.450Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 157 samples"
778level=error ts=2019-12-10T11:38:06.450Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=85 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 85 samples"
779level=error ts=2019-12-10T11:38:06.449Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 101 samples"
780level=error ts=2019-12-10T11:38:06.449Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 102 samples"
781level=error ts=2019-12-10T11:38:06.449Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 103 samples"
782level=error ts=2019-12-10T11:38:06.448Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 104 samples"
783level=error ts=2019-12-10T11:38:06.448Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 105 samples"
784level=error ts=2019-12-10T11:38:06.448Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=75 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 75 samples"
785level=error ts=2019-12-10T11:38:06.448Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 106 samples"
786level=error ts=2019-12-10T11:38:06.448Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=70 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 70 samples"
787level=error ts=2019-12-10T11:38:06.447Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=96 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 96 samples"
788level=error ts=2019-12-10T11:38:06.446Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=66 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 66 samples"
789level=error ts=2019-12-10T11:38:06.446Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
790level=error ts=2019-12-10T11:38:06.446Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=80 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 80 samples"
791level=error ts=2019-12-10T11:38:06.446Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=33 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 33 samples"
792level=error ts=2019-12-10T11:38:06.446Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=75 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 75 samples"
793level=error ts=2019-12-10T11:38:06.446Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=71 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 71 samples"
794level=error ts=2019-12-10T11:38:06.446Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=62 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 62 samples"
795level=error ts=2019-12-10T11:38:06.445Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=82 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 83 samples"
796level=error ts=2019-12-10T11:38:06.445Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=58 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 58 samples"
797level=error ts=2019-12-10T11:38:06.445Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=79 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 79 samples"
798level=error ts=2019-12-10T11:38:06.445Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=72 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 72 samples"
799level=error ts=2019-12-10T11:38:06.445Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
800level=error ts=2019-12-10T11:38:06.445Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=82 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 82 samples"
801level=error ts=2019-12-10T11:38:06.445Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=97 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 97 samples"
802level=error ts=2019-12-10T11:38:06.445Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=86 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 86 samples"
803level=error ts=2019-12-10T11:38:06.355Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
804level=error ts=2019-12-10T11:38:06.355Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 101 samples"
805level=error ts=2019-12-10T11:38:06.355Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 102 samples"
806level=error ts=2019-12-10T11:38:06.355Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 103 samples"
807level=error ts=2019-12-10T11:38:06.355Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 104 samples"
808level=error ts=2019-12-10T11:38:06.353Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 105 samples"
809level=error ts=2019-12-10T11:38:06.350Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 106 samples"
810level=error ts=2019-12-10T11:38:06.350Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 107 samples"
811level=error ts=2019-12-10T11:38:06.350Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 108 samples"
812level=error ts=2019-12-10T11:38:06.350Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 109 samples"
813level=error ts=2019-12-10T11:38:06.350Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 110 samples"
814level=error ts=2019-12-10T11:38:06.350Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 111 samples"
815level=error ts=2019-12-10T11:38:06.349Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 112 samples"
816level=error ts=2019-12-10T11:38:06.349Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 113 samples"
817level=error ts=2019-12-10T11:38:06.349Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 114 samples"
818level=error ts=2019-12-10T11:38:06.349Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 115 samples"
819level=error ts=2019-12-10T11:38:06.349Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 116 samples"
820level=error ts=2019-12-10T11:38:06.349Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=60 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 60 samples"
821level=error ts=2019-12-10T11:38:06.348Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=22 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 22 samples"
822level=error ts=2019-12-10T11:38:06.348Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 155 samples"
823level=error ts=2019-12-10T11:38:06.348Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=97 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 97 samples"
824level=error ts=2019-12-10T11:38:06.348Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
825level=error ts=2019-12-10T11:38:06.348Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 101 samples"
826level=error ts=2019-12-10T11:38:06.348Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 102 samples"
827level=error ts=2019-12-10T11:38:06.347Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 103 samples"
828level=error ts=2019-12-10T11:38:06.347Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 104 samples"
829level=error ts=2019-12-10T11:38:06.346Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=82 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 82 samples"
830level=error ts=2019-12-10T11:38:06.346Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 106 samples"
831level=error ts=2019-12-10T11:38:06.340Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 107 samples"
832level=error ts=2019-12-10T11:38:06.339Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 108 samples"
833level=error ts=2019-12-10T11:38:06.339Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 109 samples"
834level=error ts=2019-12-10T11:38:06.339Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=74 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 74 samples"
835level=error ts=2019-12-10T11:38:06.339Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 110 samples"
836level=error ts=2019-12-10T11:38:06.339Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 111 samples"
837level=error ts=2019-12-10T11:38:06.338Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 112 samples"
838level=error ts=2019-12-10T11:38:06.338Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 113 samples"
839level=error ts=2019-12-10T11:38:06.337Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 114 samples"
840level=error ts=2019-12-10T11:38:06.330Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 115 samples"
841level=error ts=2019-12-10T11:38:06.329Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 116 samples"
842level=error ts=2019-12-10T11:38:06.329Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 117 samples"
843level=error ts=2019-12-10T11:38:06.328Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=83 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 83 samples"
844level=error ts=2019-12-10T11:38:06.328Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 118 samples"
845level=error ts=2019-12-10T11:38:06.328Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 119 samples"
846level=error ts=2019-12-10T11:38:06.328Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=97 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 97 samples"
847level=error ts=2019-12-10T11:38:06.327Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=79 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 79 samples"
848level=error ts=2019-12-10T11:38:06.244Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 120 samples"
849level=error ts=2019-12-10T11:38:06.231Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 121 samples"
850level=error ts=2019-12-10T11:38:06.230Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 122 samples"
851level=error ts=2019-12-10T11:38:06.229Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 124 samples"
852level=error ts=2019-12-10T11:38:06.229Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 125 samples"
853level=error ts=2019-12-10T11:38:06.229Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=94 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 94 samples"
854level=error ts=2019-12-10T11:38:06.206Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 126 samples"
855level=error ts=2019-12-10T11:38:06.205Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 127 samples"
856level=error ts=2019-12-10T11:38:06.205Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=98 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 98 samples"
857level=error ts=2019-12-10T11:38:06.205Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 128 samples"
858level=error ts=2019-12-10T11:38:06.205Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 129 samples"
859level=error ts=2019-12-10T11:38:06.198Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 130 samples"
860level=error ts=2019-12-10T11:38:06.092Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 131 samples"
861level=error ts=2019-12-10T11:38:06.092Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 132 samples"
862level=error ts=2019-12-10T11:38:06.092Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 133 samples"
863level=error ts=2019-12-10T11:38:06.092Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 134 samples"
864level=error ts=2019-12-10T11:38:06.092Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 135 samples"
865level=error ts=2019-12-10T11:38:06.091Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=81 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 81 samples"
866level=error ts=2019-12-10T11:38:06.091Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 136 samples"
867level=error ts=2019-12-10T11:38:06.091Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 139 samples"
868level=error ts=2019-12-10T11:38:06.088Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 137 samples"
869level=error ts=2019-12-10T11:38:06.087Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 138 samples"
870level=error ts=2019-12-10T11:38:05.593Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 101 samples"
871level=error ts=2019-12-10T11:38:05.593Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 102 samples"
872level=error ts=2019-12-10T11:38:05.593Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=97 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 97 samples"
873level=error ts=2019-12-10T11:38:05.592Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 103 samples"
874level=info ts=2019-12-10T11:38:05.063Z caller=queue_manager.go:559 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="Remote storage resharding" from=1 to=2
875level=error ts=2019-12-10T11:38:04.977Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 104 samples"
876level=error ts=2019-12-10T11:38:04.976Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 105 samples"
877level=error ts=2019-12-10T11:38:04.821Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 106 samples"
878level=error ts=2019-12-10T11:38:04.821Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=96 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 96 samples"
879level=error ts=2019-12-10T11:38:04.821Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
880level=error ts=2019-12-10T11:38:04.815Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 101 samples"
881level=error ts=2019-12-10T11:38:04.815Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 102 samples"
882level=error ts=2019-12-10T11:38:04.720Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=98 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 99 samples"
883level=error ts=2019-12-10T11:38:04.720Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 103 samples"
884level=error ts=2019-12-10T11:38:04.720Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 104 samples"
885level=error ts=2019-12-10T11:38:04.719Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=98 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 98 samples"
886level=error ts=2019-12-10T11:38:04.719Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 105 samples"
887level=error ts=2019-12-10T11:38:04.717Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 106 samples"
888level=error ts=2019-12-10T11:38:04.716Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=87 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 87 samples"
889level=error ts=2019-12-10T11:38:04.716Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 107 samples"
890level=error ts=2019-12-10T11:38:04.711Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 108 samples"
891level=error ts=2019-12-10T11:38:04.711Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=99 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 999 samples"
892level=error ts=2019-12-10T11:38:04.711Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 109 samples"
893level=error ts=2019-12-10T11:38:04.710Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 111 samples"
894level=error ts=2019-12-10T11:38:04.710Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 112 samples"
895level=error ts=2019-12-10T11:38:04.710Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 113 samples"
896level=error ts=2019-12-10T11:38:04.709Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=66 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 66 samples"
897level=error ts=2019-12-10T11:38:04.709Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=64 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 64 samples"
898level=error ts=2019-12-10T11:38:04.631Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 114 samples"
899level=error ts=2019-12-10T11:38:04.631Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=84 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 84 samples"
900level=error ts=2019-12-10T11:38:04.631Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 115 samples"
901level=error ts=2019-12-10T11:38:04.630Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 116 samples"
902level=error ts=2019-12-10T11:38:04.630Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 117 samples"
903level=error ts=2019-12-10T11:38:04.630Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 118 samples"
904level=error ts=2019-12-10T11:38:04.630Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 119 samples"
905level=error ts=2019-12-10T11:38:04.630Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 120 samples"
906level=error ts=2019-12-10T11:38:04.627Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 121 samples"
907level=error ts=2019-12-10T11:38:04.626Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 122 samples"
908level=error ts=2019-12-10T11:38:04.626Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 121 samples"
909level=error ts=2019-12-10T11:38:04.626Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 1244 samples"
910level=error ts=2019-12-10T11:38:04.621Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 123 samples"
911level=error ts=2019-12-10T11:38:04.620Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 124 samples"
912level=error ts=2019-12-10T11:38:04.620Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 125 samples"
913level=error ts=2019-12-10T11:38:04.620Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 126 samples"
914level=error ts=2019-12-10T11:38:04.619Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 127 samples"
915level=error ts=2019-12-10T11:38:04.136Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 131 samples"
916level=error ts=2019-12-10T11:38:04.136Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 132 samples"
917level=error ts=2019-12-10T11:38:04.135Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 133 samples"
918level=error ts=2019-12-10T11:38:04.018Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 134 samples"
919level=error ts=2019-12-10T11:38:04.018Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 135 samples"
920level=error ts=2019-12-10T11:38:04.017Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 141 samples"
921level=error ts=2019-12-10T11:38:04.016Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 142 samples"
922level=error ts=2019-12-10T11:38:04.015Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 153 samples"
923level=error ts=2019-12-10T11:38:04.015Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 1223 samples"
924level=error ts=2019-12-10T11:38:04.013Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 164 samples"
925level=error ts=2019-12-10T11:38:04.013Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 161 samples"
926level=error ts=2019-12-10T11:38:04.012Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 162 samples"
927level=error ts=2019-12-10T11:38:04.012Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 163 samples"
928level=error ts=2019-12-10T11:38:04.012Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 169 samples"
929level=error ts=2019-12-10T11:38:04.012Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 170 samples"
930level=error ts=2019-12-10T11:38:04.012Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=80 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 80 samples"
931level=error ts=2019-12-10T11:38:04.012Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 100 samples"
932level=error ts=2019-12-10T11:38:04.010Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 101 samples"
933level=error ts=2019-12-10T11:38:04.009Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 102 samples"
934level=error ts=2019-12-10T11:38:04.009Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 103 samples"
935level=error ts=2019-12-10T11:38:04.008Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 104 samples"
936level=error ts=2019-12-10T11:38:04.007Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=88 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 88 samples"
937level=error ts=2019-12-10T11:38:04.007Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 105 samples"
938level=error ts=2019-12-10T11:38:04.007Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 106 samples"
939level=error ts=2019-12-10T11:38:04.005Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 107 samples"
940level=error ts=2019-12-10T11:38:04.005Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 108 samples"
941level=error ts=2019-12-10T11:38:04.004Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=84 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 84 samples"
942level=error ts=2019-12-10T11:38:04.003Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 109 samples"
943level=error ts=2019-12-10T11:38:04.002Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 110 samples"
944level=error ts=2019-12-10T11:38:04.002Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 112 samples"
945level=error ts=2019-12-10T11:38:04.001Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 113 samples"
946level=error ts=2019-12-10T11:38:03.923Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 114 samples"
947level=error ts=2019-12-10T11:38:03.923Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=79 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 79 samples"
948level=error ts=2019-12-10T11:38:03.922Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 115 samples"
949level=error ts=2019-12-10T11:38:03.908Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 116 samples"
950level=error ts=2019-12-10T11:38:03.908Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 117 samples"
951level=error ts=2019-12-10T11:38:03.907Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=96 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 96 samples"
952level=error ts=2019-12-10T11:38:03.907Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 118 samples"
953level=error ts=2019-12-10T11:38:03.906Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 119 samples"
954level=error ts=2019-12-10T11:38:03.904Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 120 samples"
955level=error ts=2019-12-10T11:38:03.904Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=83 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 83 samples"
956level=error ts=2019-12-10T11:38:03.904Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 121 samples"
957level=error ts=2019-12-10T11:38:03.904Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 122 samples"
958level=error ts=2019-12-10T11:38:03.904Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=71 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 71 samples"
959level=error ts=2019-12-10T11:38:03.904Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 123 samples"
960level=error ts=2019-12-10T11:38:03.902Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 124 samples"
961level=error ts=2019-12-10T11:38:03.901Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 125 samples"
962level=error ts=2019-12-10T11:38:03.900Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 126 samples"
963level=error ts=2019-12-10T11:38:03.900Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 127 samples"
964level=error ts=2019-12-10T11:38:03.899Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=86 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 86 samples"
965level=error ts=2019-12-10T11:38:03.900Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=84 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 84 samples"
966level=error ts=2019-12-10T11:38:03.900Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 128 samples"
967level=error ts=2019-12-10T11:38:03.900Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 129 samples"
968level=error ts=2019-12-10T11:38:03.899Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 130 samples"
969level=error ts=2019-12-10T11:38:03.899Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 131 samples"
970level=error ts=2019-12-10T11:38:03.899Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 132 samples"
971level=error ts=2019-12-10T11:38:03.899Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 133 samples"
972level=error ts=2019-12-10T11:38:03.899Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 134 samples"
973level=error ts=2019-12-10T11:38:03.898Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 135 samples"
974level=error ts=2019-12-10T11:38:03.898Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=90 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 90 samples"
975level=error ts=2019-12-10T11:38:03.898Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 136 samples"
976level=error ts=2019-12-10T11:38:03.898Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 137 samples"
977level=error ts=2019-12-10T11:38:03.897Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 138 samples"
978level=error ts=2019-12-10T11:38:03.897Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 139 samples"
979level=error ts=2019-12-10T11:38:03.723Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=80 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 80 samples"
980level=error ts=2019-12-10T11:38:03.723Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 140 samples"
981level=error ts=2019-12-10T11:38:03.723Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 141 samples"
982level=error ts=2019-12-10T11:38:03.722Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 142 samples"
983level=error ts=2019-12-10T11:38:03.722Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 143 samples"
984level=error ts=2019-12-10T11:38:03.722Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 144 samples"
985level=error ts=2019-12-10T11:38:03.722Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 145 samples"
986level=error ts=2019-12-10T11:38:03.722Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 146 samples"
987level=error ts=2019-12-10T11:38:03.722Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=76 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 76 samples"
988level=error ts=2019-12-10T11:38:03.722Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 101 samples"
989level=error ts=2019-12-10T11:38:03.722Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 102 samples"
990level=error ts=2019-12-10T11:38:03.711Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 103 samples"
991level=error ts=2019-12-10T11:38:03.711Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 104 samples"
992level=error ts=2019-12-10T11:38:03.711Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 105 samples"
993level=error ts=2019-12-10T11:38:03.644Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 106 samples"
994level=error ts=2019-12-10T11:38:03.644Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=95 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 95 samples"
995level=error ts=2019-12-10T11:38:03.644Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 101 samples"
996level=error ts=2019-12-10T11:38:03.644Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 102 samples"
997level=error ts=2019-12-10T11:38:03.644Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 103 samples"
998level=error ts=2019-12-10T11:38:03.644Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 104 samples"
999level=error ts=2019-12-10T11:38:03.644Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 105 samples"
1000level=error ts=2019-12-10T11:38:03.643Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 106 samples"
1001level=error ts=2019-12-10T11:38:03.643Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 107 samples"
1002level=error ts=2019-12-10T11:38:03.643Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 108 samples"
1003level=error ts=2019-12-10T11:38:03.643Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 109 samples"
1004level=error ts=2019-12-10T11:38:03.642Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 110 samples"
1005level=error ts=2019-12-10T11:38:03.642Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 111 samples"
1006level=error ts=2019-12-10T11:38:03.642Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 112 samples"
1007level=error ts=2019-12-10T11:38:03.642Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 113 samples"
1008level=error ts=2019-12-10T11:38:03.642Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 114 samples"
1009level=error ts=2019-12-10T11:38:03.642Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 115 samples"
1010level=error ts=2019-12-10T11:38:03.641Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 116 samples"
1011level=error ts=2019-12-10T11:38:03.640Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 117 samples"
1012level=error ts=2019-12-10T11:38:03.641Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 118 samples"
1013level=error ts=2019-12-10T11:38:03.641Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 119 samples"
1014level=error ts=2019-12-10T11:38:03.640Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 120 samples"
1015level=error ts=2019-12-10T11:38:03.640Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 121 samples"
1016level=error ts=2019-12-10T11:38:03.640Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 122 samples"
1017level=error ts=2019-12-10T11:38:03.640Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=88 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 88 samples"
1018level=error ts=2019-12-10T11:38:03.640Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 123 samples"
1019level=error ts=2019-12-10T11:38:03.640Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 134 samples"
1020level=error ts=2019-12-10T11:38:03.640Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=78 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 78 samples"
1021level=error ts=2019-12-10T11:38:03.640Z caller=queue_manager.go:770 component=remote queue=0:http://cortex-gw.cortex-tsdb-dev.svc.cluster.local/api/prom/push msg="non-recoverable error" count=100 err="server returned HTTP status 429 Too Many Requests: ingestion rate limit (200000) exceeded while adding 155 samples"
1022`, "\n")
1023