1// Copyright 2018 Istio Authors
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14package v2_test
15
16import (
17	"encoding/json"
18	"fmt"
19	"io/ioutil"
20	"net"
21	"net/http"
22	"net/http/httputil"
23	"net/url"
24	"sync"
25	"testing"
26	"time"
27
28	testenv "istio.io/istio/mixer/test/client/env"
29	"istio.io/istio/pilot/pkg/bootstrap"
30	"istio.io/istio/pilot/pkg/model"
31	"istio.io/istio/pkg/config/constants"
32	"istio.io/istio/pkg/config/host"
33	"istio.io/istio/pkg/config/labels"
34	"istio.io/istio/pkg/config/mesh"
35	"istio.io/istio/pkg/config/protocol"
36	"istio.io/istio/pkg/test/env"
37	"istio.io/istio/tests/util"
38)
39
40// This file contains common helpers and initialization for the local/unit tests
41// for XDS. Tests start a Pilot, configured with an in-memory endpoint registry, and
42// using a file-based config, sourced from tests/testdata.
43// A single instance of pilot is used by all tests - similar with the e2e environment.
44// initLocalPilotTestEnv() must be called at the start of each test to ensure the
45// environment is configured. Tests making modifications to services should use
46// unique names for the service.
47// The tests can also use a local envoy process - see TestEnvoy as example, to verify
48// envoy accepts the config. Most tests are changing and checking the state of pilot.
49//
50// The pilot is accessible as pilotServer, which is an instance of bootstrap.Server.
51// The server has a field EnvoyXdsServer which is the configured instance of the XDS service.
52//
53// DiscoveryServer.MemRegistry has a memory registry that can be used by tests,
54// implemented in debug.go file.
55
56var (
57	// mixer-style test environment, includes mixer and envoy configs.
58	testEnv        *testenv.TestSetup
59	initMutex      sync.Mutex
60	initEnvoyMutex sync.Mutex
61
62	envoyStarted = false
63	// service1 and service2 are used by mixer tests. Use 'service3' and 'app3' for pilot
64	// local tests.
65
66	localIP = "10.3.0.3"
67)
68
69const (
70	// 10.10.0.0/24 is service CIDR range
71
72	// 10.0.0.0/9 is instance CIDR range
73	app3Ip    = "10.2.0.1"
74	gatewayIP = "10.3.0.1"
75	ingressIP = "10.3.0.2"
76)
77
78// Common code for the xds testing.
79// The tests in this package use an in-process pilot using mock service registry and
80// envoy, mixer setup using mixer local testing framework.
81
82// Additional servers may be added here.
83
84// One set of pilot/mixer/envoy is used for all tests, similar with the larger integration
85// tests in real docker/k8s environments
86
87// Common test environment, including Mixer and Watcher. This is a singleton, the env will be
88// used for multiple tests, for local integration testing.
89func startEnvoy(t *testing.T) {
90	initEnvoyMutex.Lock()
91	defer initEnvoyMutex.Unlock()
92
93	if envoyStarted {
94		return
95	}
96
97	tmplB, err := ioutil.ReadFile(env.IstioSrc + "/tests/testdata/bootstrap_tmpl.json")
98	if err != nil {
99		t.Fatal("Can't read bootstrap template", err)
100	}
101	testEnv.EnvoyTemplate = string(tmplB)
102	testEnv.Dir = env.IstioSrc
103	nodeID := sidecarID(app3Ip, "app3")
104	testEnv.EnvoyParams = []string{"--service-cluster", "serviceCluster", "--service-node", nodeID}
105	testEnv.EnvoyConfigOpt = map[string]interface{}{
106		"NodeID":  nodeID,
107		"BaseDir": env.IstioSrc + "/tests/testdata/local",
108		// Same value used in the real template
109		"meta_json_str": fmt.Sprintf(`"BASE": "%s", ISTIO_VERSION: 1.5.0`, env.IstioSrc+"/tests/testdata/local"),
110	}
111
112	// Mixer will push stats every 1 sec
113	testenv.SetStatsUpdateInterval(testEnv.MfConfig(), 1)
114	if err := testEnv.SetUp(); err != nil {
115		t.Fatalf("Failed to setup test: %v", err)
116	}
117	envoyStarted = true
118}
119
120func sidecarID(ip, deployment string) string {
121	return fmt.Sprintf("sidecar~%s~%s-644fc65469-96dza.testns~testns.svc.cluster.local", ip, deployment)
122}
123
124func gatewayID(ip string) string { //nolint: unparam
125	return fmt.Sprintf("router~%s~istio-gateway-644fc65469-96dzt.istio-system~istio-system.svc.cluster.local", ip)
126}
127
128// localPilotTestEnv builds a pilot testing environment and it initializes with registry with the passed in init function.
129func localPilotTestEnv(t *testing.T, initFunc func(*bootstrap.Server), additionalArgs ...func(*bootstrap.PilotArgs)) (*bootstrap.Server, util.TearDownFunc) {
130	initMutex.Lock()
131	defer initMutex.Unlock()
132
133	additionalArgs = append(additionalArgs, func(args *bootstrap.PilotArgs) {
134		args.Plugins = bootstrap.DefaultPlugins
135	})
136	server, tearDown := util.EnsureTestServer(additionalArgs...)
137	testEnv = testenv.NewTestSetup(testenv.XDSTest, t)
138	testEnv.Ports().PilotGrpcPort = uint16(util.MockPilotGrpcPort)
139	testEnv.Ports().PilotHTTPPort = uint16(util.MockPilotHTTPPort)
140	testEnv.IstioSrc = env.IstioSrc
141	testEnv.IstioOut = env.IstioOut
142
143	localIP = getLocalIP()
144
145	// Run the initialization function.
146	initFunc(server)
147
148	// Trigger a push, to initiate push context with contents of registry.
149	server.EnvoyXdsServer.Push(&model.PushRequest{Full: true})
150
151	// Wait till a push is propagated.
152	time.Sleep(200 * time.Millisecond)
153
154	// Add a dummy client connection to validate that push is triggered.
155	dummyClient := adsConnectAndWait(t, 0x0a0a0a0a)
156	defer dummyClient.Close()
157
158	return server, tearDown
159}
160
161// initLocalPilotTestEnv creates a local, in process Pilot with XDSv2 support and a set
162// of common test configs. This is a singleton server, reused for all tests in this package.
163//
164// The server will have a set of pre-defined instances and services, and read CRDs from the
165// common tests/testdata directory.
166func initLocalPilotTestEnv(t *testing.T) (*bootstrap.Server, util.TearDownFunc) {
167	return localPilotTestEnv(t, func(server *bootstrap.Server) {
168		// Service and endpoints for hello.default - used in v1 pilot tests
169		hostname := host.Name("hello.default.svc.cluster.local")
170		server.EnvoyXdsServer.MemRegistry.AddService(hostname, &model.Service{
171			Hostname: hostname,
172			Address:  "10.10.0.3",
173			Ports:    testPorts(0),
174			Attributes: model.ServiceAttributes{
175				Name:      "local",
176				Namespace: "default",
177			},
178		})
179
180		server.EnvoyXdsServer.MemRegistry.SetEndpoints(string(hostname), "default", []*model.IstioEndpoint{
181			{
182				Address:         "127.0.0.1",
183				EndpointPort:    uint32(testEnv.Ports().BackendPort),
184				ServicePortName: "http",
185				Locality:        model.Locality{Label: "az"},
186				ServiceAccount:  "hello-sa",
187			},
188		})
189
190		// "local" service points to the current host and the in-process mixer http test endpoint
191		hostname = "local.default.svc.cluster.local"
192		server.EnvoyXdsServer.MemRegistry.AddService(hostname, &model.Service{
193			Hostname: hostname,
194			Address:  "10.10.0.4",
195			Ports: []*model.Port{
196				{
197					Name:     "http",
198					Port:     80,
199					Protocol: protocol.HTTP,
200				}},
201			Attributes: model.ServiceAttributes{
202				Name:      "local",
203				Namespace: "default",
204			},
205		})
206
207		server.EnvoyXdsServer.MemRegistry.SetEndpoints(string(hostname), "default", []*model.IstioEndpoint{
208			{
209				Address:         localIP,
210				EndpointPort:    uint32(testEnv.Ports().BackendPort),
211				ServicePortName: "http",
212				Locality:        model.Locality{Label: "az"},
213			},
214		})
215
216		// Explicit test service, in the v2 memory registry. Similar with mock.MakeService,
217		// but easier to read.
218		hostname = "service3.default.svc.cluster.local"
219		server.EnvoyXdsServer.MemRegistry.AddService(hostname, &model.Service{
220			Hostname: hostname,
221			Address:  "10.10.0.1",
222			Ports:    testPorts(0),
223			Attributes: model.ServiceAttributes{
224				Name:      "service3",
225				Namespace: "default",
226			},
227		})
228
229		svc3Endpoints := make([]*model.IstioEndpoint, len(testPorts(0)))
230		for i, p := range testPorts(0) {
231			svc3Endpoints[i] = &model.IstioEndpoint{
232				Address:         app3Ip,
233				EndpointPort:    uint32(p.Port),
234				ServicePortName: p.Name,
235				Locality:        model.Locality{Label: "az"},
236			}
237		}
238
239		server.EnvoyXdsServer.MemRegistry.SetEndpoints(string(hostname), "default", svc3Endpoints)
240
241		// Mock ingress service
242		server.EnvoyXdsServer.MemRegistry.AddService("istio-ingress.istio-system.svc.cluster.local", &model.Service{
243			Hostname: "istio-ingress.istio-system.svc.cluster.local",
244			Address:  "10.10.0.2",
245			Ports: []*model.Port{
246				{
247					Name:     "http",
248					Port:     80,
249					Protocol: protocol.HTTP,
250				},
251				{
252					Name:     "https",
253					Port:     443,
254					Protocol: protocol.HTTPS,
255				},
256			},
257			// TODO: set attribute for this service. It may affect TestLDSIsolated as we now having service defined in istio-system namespaces
258		})
259		server.EnvoyXdsServer.MemRegistry.AddInstance("istio-ingress.istio-system.svc.cluster.local", &model.ServiceInstance{
260			Endpoint: &model.IstioEndpoint{
261				Address:         ingressIP,
262				EndpointPort:    80,
263				ServicePortName: "http",
264				Locality:        model.Locality{Label: "az"},
265				Labels:          labels.Instance{constants.IstioLabel: constants.IstioIngressLabelValue},
266			},
267			ServicePort: &model.Port{
268				Name:     "http",
269				Port:     80,
270				Protocol: protocol.HTTP,
271			},
272		})
273		server.EnvoyXdsServer.MemRegistry.AddInstance("istio-ingress.istio-system.svc.cluster.local", &model.ServiceInstance{
274			Endpoint: &model.IstioEndpoint{
275				Address:         ingressIP,
276				EndpointPort:    443,
277				ServicePortName: "https",
278				Locality:        model.Locality{Label: "az"},
279				Labels:          labels.Instance{constants.IstioLabel: constants.IstioIngressLabelValue},
280			},
281			ServicePort: &model.Port{
282				Name:     "https",
283				Port:     443,
284				Protocol: protocol.HTTPS,
285			},
286		})
287
288		// RouteConf Service4 is using port 80, to test that we generate multiple clusters (regression)
289		// service4 has no endpoints
290		server.EnvoyXdsServer.MemRegistry.AddHTTPService("service4.default.svc.cluster.local", "10.1.0.4", 80)
291	})
292}
293
294// nolint: unparam
295func testPorts(base int) []*model.Port {
296	return []*model.Port{
297		{
298			Name:     "http",
299			Port:     base + 80,
300			Protocol: protocol.HTTP,
301		}, {
302			Name:     "http-status",
303			Port:     base + 81,
304			Protocol: protocol.HTTP,
305		}, {
306			Name:     "custom",
307			Port:     base + 90,
308			Protocol: protocol.TCP,
309		}, {
310			Name:     "mongo",
311			Port:     base + 100,
312			Protocol: protocol.Mongo,
313		}, {
314			Name:     "redis",
315			Port:     base + 110,
316			Protocol: protocol.Redis,
317		}, {
318			Name:     "mysql",
319			Port:     base + 120,
320			Protocol: protocol.MySQL,
321		}, {
322			Name:     "h2port",
323			Port:     base + 66,
324			Protocol: protocol.GRPC,
325		}}
326}
327
328// Test XDS with real envoy and with mixer.
329func TestEnvoy(t *testing.T) {
330	mesh.TestMode = true
331	_, tearDown := initLocalPilotTestEnv(t)
332	defer func() {
333		if testEnv != nil {
334			testEnv.TearDown()
335		}
336		tearDown()
337	}()
338	startEnvoy(t)
339	// Make sure tcp port is ready before starting the test.
340	testenv.WaitForPort(testEnv.Ports().TCPProxyPort)
341
342	t.Run("envoyInit", envoyInit)
343	t.Run("service", testService)
344}
345
346// envoyInit verifies envoy has accepted the config from pilot by checking the stats.
347func envoyInit(t *testing.T) {
348	statsURL := fmt.Sprintf("http://localhost:%d/stats?format=json", testEnv.Ports().AdminPort)
349	res, err := http.Get(statsURL)
350	if err != nil {
351		t.Fatal("Failed to get stats, envoy not started")
352	}
353	statsBytes, err := ioutil.ReadAll(res.Body)
354	if err != nil {
355		t.Fatal("Failed to get stats, envoy not started")
356	}
357
358	statsMap := stats2map(statsBytes)
359
360	if statsMap["cluster_manager.cds.update_success"] < 1 {
361		t.Error("Failed cds update")
362	}
363	// Other interesting values for CDS: cluster_added: 19, active_clusters
364	// cds.update_attempt: 2, cds.update_rejected, cds.version
365	for _, port := range testPorts(0) {
366		stat := fmt.Sprintf("cluster.outbound|%d||service3.default.svc.cluster.local.update_success", port.Port)
367		if statsMap[stat] < 1 {
368			t.Error("Failed sds updates")
369		}
370	}
371
372	if statsMap["cluster.xds-grpc.update_failure"] > 0 {
373		t.Error("GRPC update failure")
374	}
375
376	if statsMap["listener_manager.lds.update_rejected"] > 0 {
377		t.Error("LDS update failure")
378	}
379	if statsMap["listener_manager.lds.update_success"] < 1 {
380		t.Error("LDS update failure")
381	}
382}
383
384// Example of using a local test connecting to the in-process test service, using Envoy http proxy
385// mode. This is also a test for http proxy (finally).
386func testService(t *testing.T) {
387	proxyURL, _ := url.Parse("http://localhost:17002")
388
389	client := &http.Client{Transport: &http.Transport{Proxy: http.ProxyURL(proxyURL)}}
390
391	res, err := client.Get("http://local.default.svc.cluster.local")
392	if err != nil {
393		t.Error("Failed to access proxy", err)
394		return
395	}
396	resdmp, _ := httputil.DumpResponse(res, true)
397	t.Log(string(resdmp))
398	if res.Status != "200 OK" {
399		t.Error("Proxy failed ", res.Status)
400	}
401}
402
403// EnvoyStat is used to parse envoy stats
404type EnvoyStat struct {
405	Name  string `json:"name"`
406	Value int    `json:"value"`
407}
408
409// stats2map parses envoy stats.
410func stats2map(stats []byte) map[string]int {
411	s := struct {
412		Stats []EnvoyStat `json:"stats"`
413	}{}
414	_ = json.Unmarshal(stats, &s)
415	m := map[string]int{}
416	for _, stat := range s.Stats {
417		m[stat.Name] = stat.Value
418	}
419	return m
420}
421
422func getLocalIP() string {
423	addrs, err := net.InterfaceAddrs()
424	if err != nil {
425		return ""
426	}
427	for _, address := range addrs {
428		// check the address type and if it is not a loopback the display it
429		if ipnet, ok := address.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
430			if ipnet.IP.To4() != nil {
431				return ipnet.IP.String()
432			}
433		}
434	}
435	return ""
436}
437
438// newEndpointWithAccount is a helper for IstioEndpoint creation. Creates endpoints with
439// port name "http", with the given IP, service account and a 'version' label.
440// nolint: unparam
441func newEndpointWithAccount(ip, account, version string) []*model.IstioEndpoint {
442	return []*model.IstioEndpoint{
443		{
444			Address:         ip,
445			ServicePortName: "http-main",
446			EndpointPort:    80,
447			Labels:          map[string]string{"version": version},
448			UID:             "uid1",
449			ServiceAccount:  account,
450		},
451	}
452}
453