1// Copyright 2018 Istio Authors
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15package perftests
16
17import (
18	"testing"
19	"time"
20
21	"istio.io/istio/mixer/pkg/adapter"
22	"istio.io/istio/mixer/pkg/perf"
23	spyadapter "istio.io/istio/mixer/test/spyAdapter"
24	"istio.io/istio/mixer/test/spyAdapter/template"
25)
26
27// Tests single report call into Mixer that dispatches report instances to multiple noop inproc adapters.
28func Benchmark_Report_1Client_1Call(b *testing.B) {
29	settings, spyAdapter := settingsWithAdapterAndTmpls()
30	settings.RunMode = perf.InProcess
31
32	setup := perf.Setup{
33		Config: perf.Config{
34			Global:  mixerGlobalCfg,
35			Service: logentryToNoop + metricsToSpyAdapter + attrGenToSpyAdapter,
36		},
37
38		Loads: []perf.Load{{
39			Multiplier: 1,
40			Requests: []perf.Request{
41				perf.BuildBasicReport(baseAttr),
42			},
43		}},
44	}
45
46	perf.Run(b, &setup, settings)
47
48	validateReportBehavior(spyAdapter, b)
49}
50
51// Tests 5 synchronous identical report call into Mixer that dispatches report instances to multiple noop inproc adapters.
52func Benchmark_Report_1Client_5SameCalls(b *testing.B) {
53	settings, spyAdapter := settingsWithAdapterAndTmpls()
54	settings.RunMode = perf.InProcess
55	setup := perf.Setup{
56		Config: perf.Config{
57			Global:  mixerGlobalCfg,
58			Service: logentryToNoop + metricsToSpyAdapter + attrGenToSpyAdapter,
59		},
60
61		Loads: []perf.Load{{
62			Multiplier: 5,
63			Requests: []perf.Request{
64				perf.BuildBasicReport(baseAttr),
65			},
66		}},
67	}
68
69	perf.Run(b, &setup, settings)
70
71	validateReportBehavior(spyAdapter, b)
72}
73
74// Tests 5 synchronous different report call into Mixer that dispatches report instances to multiple noop inproc adapters.
75func Benchmark_Report_1Client_5DifferentCalls(b *testing.B) {
76	settings, spyAdapter := settingsWithAdapterAndTmpls()
77	settings.RunMode = perf.InProcess
78	setup := perf.Setup{
79		Config: perf.Config{
80			Global:  mixerGlobalCfg,
81			Service: logentryToNoop + metricsToSpyAdapter + attrGenToSpyAdapter,
82		},
83
84		Loads: []perf.Load{
85			{
86				Multiplier: 1,
87				Requests: []perf.Request{
88					perf.BuildBasicReport(attr1),
89					perf.BuildBasicReport(attr2),
90					perf.BuildBasicReport(attr3),
91					perf.BuildBasicReport(attr4),
92					perf.BuildBasicReport(attr5),
93				},
94			},
95		},
96	}
97
98	perf.Run(b, &setup, settings)
99
100	validateReportBehavior(spyAdapter, b)
101}
102
103// Tests 4 async client, each sending 5 synchronous identical report call into Mixer that dispatches report instances to
104// multiple noop inproc adapters.
105func Benchmark_Report_4Clients_5SameCallsEach(b *testing.B) {
106	settings, spyAdapter := settingsWithAdapterAndTmpls()
107	settings.RunMode = perf.InProcess
108	setup := perf.Setup{
109		Config: perf.Config{
110			Global:  mixerGlobalCfg,
111			Service: logentryToNoop + metricsToSpyAdapter + attrGenToSpyAdapter,
112		},
113
114		Loads: []perf.Load{
115			{
116				Multiplier: 5,
117				Requests: []perf.Request{
118					perf.BuildBasicReport(baseAttr),
119				},
120			},
121			{
122				Multiplier: 5,
123				Requests: []perf.Request{
124					perf.BuildBasicReport(baseAttr),
125				},
126			},
127			{
128				Multiplier: 5,
129				Requests: []perf.Request{
130					perf.BuildBasicReport(baseAttr),
131				},
132			},
133			{
134				Multiplier: 5,
135				Requests: []perf.Request{
136					perf.BuildBasicReport(baseAttr),
137				},
138			},
139		},
140	}
141
142	perf.Run(b, &setup, settings)
143
144	validateReportBehavior(spyAdapter, b)
145}
146
147// Tests 4 async client, each sending 5 different report call into Mixer that dispatches report instances to
148// multiple noop inproc adapters.
149func Benchmark_Report_4Clients_5DifferentCallsEach(b *testing.B) {
150	settings, spyAdapter := settingsWithAdapterAndTmpls()
151	settings.RunMode = perf.InProcess
152	setup := perf.Setup{
153		Config: perf.Config{
154			Global:  mixerGlobalCfg,
155			Service: logentryToNoop + metricsToSpyAdapter + attrGenToSpyAdapter,
156		},
157
158		Loads: []perf.Load{
159			{
160				Multiplier: 1,
161				Requests: []perf.Request{
162					perf.BuildBasicReport(attr1),
163					perf.BuildBasicReport(attr2),
164					perf.BuildBasicReport(attr3),
165					perf.BuildBasicReport(attr4),
166					perf.BuildBasicReport(attr5),
167				},
168			},
169			{
170				Multiplier: 1,
171				Requests: []perf.Request{
172					perf.BuildBasicReport(attr1),
173					perf.BuildBasicReport(attr2),
174					perf.BuildBasicReport(attr3),
175					perf.BuildBasicReport(attr4),
176					perf.BuildBasicReport(attr5),
177				},
178			},
179			{
180				Multiplier: 1,
181				Requests: []perf.Request{
182					perf.BuildBasicReport(attr1),
183					perf.BuildBasicReport(attr2),
184					perf.BuildBasicReport(attr3),
185					perf.BuildBasicReport(attr4),
186					perf.BuildBasicReport(attr5),
187				},
188			},
189			{
190				Multiplier: 1,
191				Requests: []perf.Request{
192					perf.BuildBasicReport(attr1),
193					perf.BuildBasicReport(attr2),
194					perf.BuildBasicReport(attr3),
195					perf.BuildBasicReport(attr4),
196					perf.BuildBasicReport(attr5),
197				},
198			},
199		},
200	}
201
202	perf.Run(b, &setup, settings)
203
204	validateReportBehavior(spyAdapter, b)
205}
206
207// Tests 4 async client, each sending 5 identical report call into Mixer that dispatches report instances to
208// multiple noop inproc adapters. The APA in this case is a slow by 1ms.
209func Benchmark_Report_4Clients_5SameCallsEach_1MilliSecSlowApa(b *testing.B) {
210	settings, spyAdapter := settingsWith1milliSecApaAdapterAndTmpls()
211	settings.RunMode = perf.InProcess
212	setup := perf.Setup{
213		Config: perf.Config{
214			Global:  mixerGlobalCfg,
215			Service: logentryToNoop + metricsToSpyAdapter + attrGenToSpyAdapter,
216		},
217
218		Loads: []perf.Load{
219			{
220				Multiplier: 5,
221				Requests: []perf.Request{
222					perf.BuildBasicReport(baseAttr),
223				},
224			},
225			{
226				Multiplier: 5,
227				Requests: []perf.Request{
228					perf.BuildBasicReport(baseAttr),
229				},
230			},
231			{
232				Multiplier: 5,
233				Requests: []perf.Request{
234					perf.BuildBasicReport(baseAttr),
235				},
236			},
237			{
238				Multiplier: 5,
239				Requests: []perf.Request{
240					perf.BuildBasicReport(baseAttr),
241				},
242			},
243		},
244	}
245
246	perf.Run(b, &setup, settings)
247
248	validateReportBehavior(spyAdapter, b)
249}
250
251const (
252	// contains 2 rules that pass logentry instances to a noop handler.
253	logentryToNoop = `
254apiVersion: "config.istio.io/v1alpha2"
255kind: noop
256metadata:
257  name: handler
258  namespace: istio-system
259spec:
260---
261apiVersion: "config.istio.io/v1alpha2"
262kind: logentry
263metadata:
264  name: accesslog
265  namespace: istio-system
266spec:
267  severity: '"Default"'
268  # timestamp: request.time
269  variables:
270    sourceIp: source.ip | ip("0.0.0.0")
271    destinationIp: destination.ip | ip("0.0.0.0")
272    sourceUser: source.user | ""
273    method: request.method | ""
274    url: request.path | ""
275    protocol: request.scheme | "http"
276    responseCode: response.code | 0
277    responseSize: response.size | 0
278    requestSize: request.size | 0
279    latency: response.duration | "0ms"
280    connectionMtls: connection.mtls | false
281---
282apiVersion: "config.istio.io/v1alpha2"
283kind: rule
284metadata:
285  name: stdio
286  namespace: istio-system
287spec:
288  match: "true" # If omitted match is true.
289  actions:
290  - handler: handler.noop
291    instances:
292    - accesslog.logentry
293---
294
295# Configuration for logentry instances
296apiVersion: "config.istio.io/v1alpha2"
297kind: logentry
298metadata:
299  name: newlog
300  namespace: istio-system
301spec:
302  severity: '"warning"'
303  #timestamp: request.time
304  variables:
305    source: source.labels["app"] | source.service | "unknown"
306    user: source.user | "unknown"
307    destination: destination.labels["app"] | destination.service | "unknown"
308    responseCode: response.code | 0
309    responseSize: response.size | 0
310    latency: response.duration | "0ms"
311---
312# Configuration for a stdio handler
313apiVersion: "config.istio.io/v1alpha2"
314kind: noop
315metadata:
316  name: newhandler
317  namespace: istio-system
318spec:
319---
320# Rule to send logentry instances to a stdio handler
321apiVersion: "config.istio.io/v1alpha2"
322kind: rule
323metadata:
324  name: newlogstdio
325  namespace: istio-system
326spec:
327  match: "true" # match for all requests
328  actions:
329   - handler: newhandler.noop
330     instances:
331     - newlog.logentry
332---
333`
334
335	// contains 2 rules that pass instances to the same handler.
336	metricsToSpyAdapter = `
337apiVersion: "config.istio.io/v1alpha2"
338kind: samplereport
339metadata:
340  name: requestcount
341  namespace: istio-system
342spec:
343  value: "1"
344  dimensions:
345    source_service: source.service | "unknown"
346    source_version: source.labels["version"] | "unknown"
347    destination_service: destination.service | "unknown"
348    destination_version: destination.labels["version"] | "unknown"
349    response_code: response.code | 200
350    connection_mtls: connection.mtls | false
351---
352apiVersion: "config.istio.io/v1alpha2"
353kind: samplereport
354metadata:
355  name: requestduration
356  namespace: istio-system
357spec:
358  value: response.duration | "0ms"
359  dimensions:
360    source_service: source.service | "unknown"
361    source_version: source.labels["version"] | "unknown"
362    destination_service: destination.service | "unknown"
363    destination_version: destination.labels["version"] | "unknown"
364    response_code: response.code | 200
365    connection_mtls: connection.mtls | false
366---
367apiVersion: "config.istio.io/v1alpha2"
368kind: samplereport
369metadata:
370  name: requestsize
371  namespace: istio-system
372spec:
373  value: request.size | 0
374  dimensions:
375    source_service: source.service | "unknown"
376    source_version: source.labels["version"] | "unknown"
377    destination_service: destination.service | "unknown"
378    destination_version: destination.labels["version"] | "unknown"
379    response_code: response.code | 200
380    connection_mtls: connection.mtls | false
381---
382apiVersion: "config.istio.io/v1alpha2"
383kind: samplereport
384metadata:
385  name: responsesize
386  namespace: istio-system
387spec:
388  value: response.size | 0
389  dimensions:
390    source_service: source.service | "unknown"
391    source_version: source.labels["version"] | "unknown"
392    destination_service: destination.service | "unknown"
393    destination_version: destination.labels["version"] | "unknown"
394    response_code: response.code | 200
395    connection_mtls: connection.mtls | false
396---
397apiVersion: "config.istio.io/v1alpha2"
398kind: samplereport
399metadata:
400  name: tcpbytesent
401  namespace: istio-system
402spec:
403  value: connection.sent.bytes | 0
404  dimensions:
405    source_service: source.service | "unknown"
406    source_version: source.labels["version"] | "unknown"
407    destination_service: destination.service | "unknown"
408    destination_version: destination.labels["version"] | "unknown"
409    connection_mtls: connection.mtls | false
410---
411apiVersion: "config.istio.io/v1alpha2"
412kind: samplereport
413metadata:
414  name: tcpbytereceived
415  namespace: istio-system
416spec:
417  value: connection.received.bytes | 0
418  dimensions:
419    source_service: source.service | "unknown"
420    source_version: source.labels["version"] | "unknown"
421    destination_service: destination.service | "unknown"
422    destination_version: destination.labels["version"] | "unknown"
423    connection_mtls: connection.mtls | false
424---
425apiVersion: "config.istio.io/v1alpha2"
426kind: spyadapter
427metadata:
428  name: handler
429  namespace: istio-system
430spec:
431---
432apiVersion: "config.istio.io/v1alpha2"
433kind: rule
434metadata:
435  name: promhttp
436  namespace: istio-system
437spec:
438  match: context.protocol == "http"
439  actions:
440  - handler: handler.spyadapter
441    instances:
442    - requestcount.samplereport
443    - requestduration.samplereport
444    - requestsize.samplereport
445    - responsesize.samplereport
446---
447apiVersion: "config.istio.io/v1alpha2"
448kind: rule
449metadata:
450  name: promtcp
451  namespace: istio-system
452spec:
453  match: context.protocol == "tcp"
454  actions:
455  - handler: handler.spyadapter
456    instances:
457    - tcpbytesent.samplereport
458    - tcpbytereceived.samplereport
459---
460
461# Configuration for metric instances
462apiVersion: "config.istio.io/v1alpha2"
463kind: samplereport
464metadata:
465  name: doublerequestcount
466  namespace: istio-system
467spec:
468  value: "2" # count each request twice
469  dimensions:
470    source: source.service | "unknown"
471    destination: destination.service | "unknown"
472    message: '"twice the fun!"'
473---
474# Configuration for a Prometheus handler
475apiVersion: "config.istio.io/v1alpha2"
476kind: spyadapter
477metadata:
478  name: doublehandler
479  namespace: istio-system
480spec:
481---
482# Rule to send metric instances to a Prometheus handler
483apiVersion: "config.istio.io/v1alpha2"
484kind: rule
485metadata:
486  name: doubleprom
487  namespace: istio-system
488spec:
489  actions:
490  - handler: doublehandler.spyadapter
491    instances:
492    - doublerequestcount.samplereport
493---
494`
495
496	// contains 1 rules that pass instances to a apa adapter
497	attrGenToSpyAdapter = `
498apiVersion: "config.istio.io/v1alpha2"
499kind: spyadapter
500metadata:
501  name: handler
502  namespace: istio-system
503spec:
504
505---
506apiVersion: "config.istio.io/v1alpha2"
507kind: rule
508metadata:
509  name: kubeattrgenrulerule
510  namespace: istio-system
511spec:
512  actions:
513  - handler: handler.spyadapter
514    instances:
515    - attributes.sampleapa
516---
517apiVersion: "config.istio.io/v1alpha2"
518kind: sampleapa
519metadata:
520  name: attributes
521  namespace: istio-system
522spec:
523  # Pass the required attribute data to the adapter
524  boolPrimitive: connection.mtls | true
525  stringPrimitive: source.service | "unknown"
526  attribute_bindings:
527    connection.mtls: $out.boolPrimitive | true
528    origin.uid: $out.stringPrimitive | "unknown"
529---
530`
531
532	mixerGlobalCfg = `
533apiVersion: "config.istio.io/v1alpha2"
534kind: attributemanifest
535metadata:
536  name: istioproxy
537  namespace: istio-system
538spec:
539  attributes:
540    origin.ip:
541      valueType: IP_ADDRESS
542    origin.uid:
543      valueType: STRING
544    origin.user:
545      valueType: STRING
546    request.headers:
547      valueType: STRING_MAP
548    request.id:
549      valueType: STRING
550    request.host:
551      valueType: STRING
552    request.method:
553      valueType: STRING
554    request.path:
555      valueType: STRING
556    request.reason:
557      valueType: STRING
558    request.referer:
559      valueType: STRING
560    request.scheme:
561      valueType: STRING
562    request.size:
563      valueType: INT64
564    request.time:
565      valueType: TIMESTAMP
566    request.useragent:
567      valueType: STRING
568    response.code:
569      valueType: INT64
570    response.duration:
571      valueType: DURATION
572    response.headers:
573      valueType: STRING_MAP
574    response.size:
575      valueType: INT64
576    response.time:
577      valueType: TIMESTAMP
578    source.uid:
579      valueType: STRING
580    source.user:
581      valueType: STRING
582    destination.uid:
583      valueType: STRING
584    connection.id:
585      valueType: STRING
586    connection.received.bytes:
587      valueType: INT64
588    connection.received.bytes_total:
589      valueType: INT64
590    connection.sent.bytes:
591      valueType: INT64
592    connection.sent.bytes_total:
593      valueType: INT64
594    connection.duration:
595      valueType: DURATION
596    connection.mtls:
597      valueType: BOOL
598    context.protocol:
599      valueType: STRING
600    context.timestamp:
601      valueType: TIMESTAMP
602    context.time:
603      valueType: TIMESTAMP
604    api.service:
605      valueType: STRING
606    api.version:
607      valueType: STRING
608    api.operation:
609      valueType: STRING
610    api.protocol:
611      valueType: STRING
612    request.auth.principal:
613      valueType: STRING
614    request.auth.audiences:
615      valueType: STRING
616    request.auth.presenter:
617      valueType: STRING
618    request.api_key:
619      valueType: STRING
620
621---
622apiVersion: "config.istio.io/v1alpha2"
623kind: attributemanifest
624metadata:
625  name: kubernetes
626  namespace: istio-system
627spec:
628  attributes:
629    source.ip:
630      valueType: IP_ADDRESS
631    source.labels:
632      valueType: STRING_MAP
633    source.name:
634      valueType: STRING
635    source.namespace:
636      valueType: STRING
637    source.service:
638      valueType: STRING
639    source.serviceAccount:
640      valueType: STRING
641    destination.ip:
642      valueType: IP_ADDRESS
643    destination.labels:
644      valueType: STRING_MAP
645    destination.name:
646      valueType: STRING
647    destination.namespace:
648      valueType: STRING
649    destination.service:
650      valueType: STRING
651    destination.serviceAccount:
652      valueType: STRING
653---
654`
655)
656
657func settingsWithAdapterAndTmpls() (perf.Settings, *spyadapter.Adapter) {
658	setting := baseSettings
659	a := spyadapter.NewSpyAdapter(spyadapter.AdapterBehavior{Name: "spyadapter", Handler: spyadapter.HandlerBehavior{
660		HandleSampleCheckResult: adapter.CheckResult{ValidUseCount: 10000, ValidDuration: 5 * time.Minute}}})
661	setting.Adapters = append(setting.Adapters, a.GetAdptInfoFn())
662	for k, v := range template.SupportedTmplInfo {
663		setting.Templates[k] = v
664	}
665	return setting, a
666}
667
668func settingsWith1milliSecApaAdapterAndTmpls() (perf.Settings, *spyadapter.Adapter) {
669	setting := baseSettings
670
671	a := spyadapter.NewSpyAdapter(spyadapter.AdapterBehavior{Name: "spyadapter",
672		Handler: spyadapter.HandlerBehavior{GenerateSampleApaSleep: time.Millisecond,
673			HandleSampleCheckResult: adapter.CheckResult{ValidUseCount: 10000, ValidDuration: 5 * time.Minute}}})
674	setting.Adapters = append(setting.Adapters, a.GetAdptInfoFn())
675	for k, v := range template.SupportedTmplInfo {
676		setting.Templates[k] = v
677	}
678	return setting, a
679}
680
681func validateReportBehavior(spyAdapter *spyadapter.Adapter, b *testing.B) {
682	// validate all went as expected. Note: logentry goes to the noop adapter so we cannot inspect that. However,
683	// anything that is going to spy adapter, based on the config below, can be inspected.
684	//
685	// based on the config, there must be, for each Report call from client,
686	// * single attribute generation call
687	// * two metric handle call
688	//   * with 4 instances.
689	//   * with 1 instance.
690
691	foundAttrGenCall := false
692	foundReport1InstCall := false
693	foundReport4InstCall := false
694
695	for _, cc := range spyAdapter.HandlerData.CapturedCalls {
696		if cc.Name == "HandleSampleApaAttributes" && len(cc.Instances) == 1 {
697			foundAttrGenCall = true
698		}
699		if cc.Name == "HandleSampleReport" && len(cc.Instances) == 1 {
700			foundReport1InstCall = true
701		}
702		if cc.Name == "HandleSampleReport" && len(cc.Instances) == 4 {
703			foundReport4InstCall = true
704		}
705	}
706
707	if !foundAttrGenCall || !foundReport1InstCall || !foundReport4InstCall {
708		b.Errorf("got spy adapter calls %v; want calls  with HandleSampleApaAttributes:1 & HandleSampleReport:1"+
709			"& HandleSampleReport:4",
710			spyAdapter.HandlerData.CapturedCalls)
711	}
712}
713