1// Copyright 2018 Istio Authors 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14package v2_test 15 16import ( 17 "encoding/json" 18 "fmt" 19 "io/ioutil" 20 "net" 21 "net/http" 22 "net/http/httputil" 23 "net/url" 24 "sync" 25 "testing" 26 "time" 27 28 testenv "istio.io/istio/mixer/test/client/env" 29 "istio.io/istio/pilot/pkg/bootstrap" 30 "istio.io/istio/pilot/pkg/model" 31 "istio.io/istio/pkg/config/constants" 32 "istio.io/istio/pkg/config/host" 33 "istio.io/istio/pkg/config/labels" 34 "istio.io/istio/pkg/config/mesh" 35 "istio.io/istio/pkg/config/protocol" 36 "istio.io/istio/pkg/test/env" 37 "istio.io/istio/tests/util" 38) 39 40// This file contains common helpers and initialization for the local/unit tests 41// for XDS. Tests start a Pilot, configured with an in-memory endpoint registry, and 42// using a file-based config, sourced from tests/testdata. 43// A single instance of pilot is used by all tests - similar with the e2e environment. 44// initLocalPilotTestEnv() must be called at the start of each test to ensure the 45// environment is configured. Tests making modifications to services should use 46// unique names for the service. 47// The tests can also use a local envoy process - see TestEnvoy as example, to verify 48// envoy accepts the config. Most tests are changing and checking the state of pilot. 49// 50// The pilot is accessible as pilotServer, which is an instance of bootstrap.Server. 51// The server has a field EnvoyXdsServer which is the configured instance of the XDS service. 52// 53// DiscoveryServer.MemRegistry has a memory registry that can be used by tests, 54// implemented in debug.go file. 55 56var ( 57 // mixer-style test environment, includes mixer and envoy configs. 58 testEnv *testenv.TestSetup 59 initMutex sync.Mutex 60 initEnvoyMutex sync.Mutex 61 62 envoyStarted = false 63 // service1 and service2 are used by mixer tests. Use 'service3' and 'app3' for pilot 64 // local tests. 65 66 localIP = "10.3.0.3" 67) 68 69const ( 70 // 10.10.0.0/24 is service CIDR range 71 72 // 10.0.0.0/9 is instance CIDR range 73 app3Ip = "10.2.0.1" 74 gatewayIP = "10.3.0.1" 75 ingressIP = "10.3.0.2" 76) 77 78// Common code for the xds testing. 79// The tests in this package use an in-process pilot using mock service registry and 80// envoy, mixer setup using mixer local testing framework. 81 82// Additional servers may be added here. 83 84// One set of pilot/mixer/envoy is used for all tests, similar with the larger integration 85// tests in real docker/k8s environments 86 87// Common test environment, including Mixer and Watcher. This is a singleton, the env will be 88// used for multiple tests, for local integration testing. 89func startEnvoy(t *testing.T) { 90 initEnvoyMutex.Lock() 91 defer initEnvoyMutex.Unlock() 92 93 if envoyStarted { 94 return 95 } 96 97 tmplB, err := ioutil.ReadFile(env.IstioSrc + "/tests/testdata/bootstrap_tmpl.json") 98 if err != nil { 99 t.Fatal("Can't read bootstrap template", err) 100 } 101 testEnv.EnvoyTemplate = string(tmplB) 102 testEnv.Dir = env.IstioSrc 103 nodeID := sidecarID(app3Ip, "app3") 104 testEnv.EnvoyParams = []string{"--service-cluster", "serviceCluster", "--service-node", nodeID} 105 testEnv.EnvoyConfigOpt = map[string]interface{}{ 106 "NodeID": nodeID, 107 "BaseDir": env.IstioSrc + "/tests/testdata/local", 108 // Same value used in the real template 109 "meta_json_str": fmt.Sprintf(`"BASE": "%s", ISTIO_VERSION: 1.5.0`, env.IstioSrc+"/tests/testdata/local"), 110 } 111 112 // Mixer will push stats every 1 sec 113 testenv.SetStatsUpdateInterval(testEnv.MfConfig(), 1) 114 if err := testEnv.SetUp(); err != nil { 115 t.Fatalf("Failed to setup test: %v", err) 116 } 117 envoyStarted = true 118} 119 120func sidecarID(ip, deployment string) string { 121 return fmt.Sprintf("sidecar~%s~%s-644fc65469-96dza.testns~testns.svc.cluster.local", ip, deployment) 122} 123 124func gatewayID(ip string) string { //nolint: unparam 125 return fmt.Sprintf("router~%s~istio-gateway-644fc65469-96dzt.istio-system~istio-system.svc.cluster.local", ip) 126} 127 128// localPilotTestEnv builds a pilot testing environment and it initializes with registry with the passed in init function. 129func localPilotTestEnv(t *testing.T, initFunc func(*bootstrap.Server), additionalArgs ...func(*bootstrap.PilotArgs)) (*bootstrap.Server, util.TearDownFunc) { 130 initMutex.Lock() 131 defer initMutex.Unlock() 132 133 additionalArgs = append(additionalArgs, func(args *bootstrap.PilotArgs) { 134 args.Plugins = bootstrap.DefaultPlugins 135 }) 136 server, tearDown := util.EnsureTestServer(additionalArgs...) 137 testEnv = testenv.NewTestSetup(testenv.XDSTest, t) 138 testEnv.Ports().PilotGrpcPort = uint16(util.MockPilotGrpcPort) 139 testEnv.Ports().PilotHTTPPort = uint16(util.MockPilotHTTPPort) 140 testEnv.IstioSrc = env.IstioSrc 141 testEnv.IstioOut = env.IstioOut 142 143 localIP = getLocalIP() 144 145 // Run the initialization function. 146 initFunc(server) 147 148 // Trigger a push, to initiate push context with contents of registry. 149 server.EnvoyXdsServer.Push(&model.PushRequest{Full: true}) 150 151 // Wait till a push is propagated. 152 time.Sleep(200 * time.Millisecond) 153 154 // Add a dummy client connection to validate that push is triggered. 155 dummyClient := adsConnectAndWait(t, 0x0a0a0a0a) 156 defer dummyClient.Close() 157 158 return server, tearDown 159} 160 161// initLocalPilotTestEnv creates a local, in process Pilot with XDSv2 support and a set 162// of common test configs. This is a singleton server, reused for all tests in this package. 163// 164// The server will have a set of pre-defined instances and services, and read CRDs from the 165// common tests/testdata directory. 166func initLocalPilotTestEnv(t *testing.T) (*bootstrap.Server, util.TearDownFunc) { 167 return localPilotTestEnv(t, func(server *bootstrap.Server) { 168 // Service and endpoints for hello.default - used in v1 pilot tests 169 hostname := host.Name("hello.default.svc.cluster.local") 170 server.EnvoyXdsServer.MemRegistry.AddService(hostname, &model.Service{ 171 Hostname: hostname, 172 Address: "10.10.0.3", 173 Ports: testPorts(0), 174 Attributes: model.ServiceAttributes{ 175 Name: "local", 176 Namespace: "default", 177 }, 178 }) 179 180 server.EnvoyXdsServer.MemRegistry.SetEndpoints(string(hostname), "default", []*model.IstioEndpoint{ 181 { 182 Address: "127.0.0.1", 183 EndpointPort: uint32(testEnv.Ports().BackendPort), 184 ServicePortName: "http", 185 Locality: model.Locality{Label: "az"}, 186 ServiceAccount: "hello-sa", 187 }, 188 }) 189 190 // "local" service points to the current host and the in-process mixer http test endpoint 191 hostname = "local.default.svc.cluster.local" 192 server.EnvoyXdsServer.MemRegistry.AddService(hostname, &model.Service{ 193 Hostname: hostname, 194 Address: "10.10.0.4", 195 Ports: []*model.Port{ 196 { 197 Name: "http", 198 Port: 80, 199 Protocol: protocol.HTTP, 200 }}, 201 Attributes: model.ServiceAttributes{ 202 Name: "local", 203 Namespace: "default", 204 }, 205 }) 206 207 server.EnvoyXdsServer.MemRegistry.SetEndpoints(string(hostname), "default", []*model.IstioEndpoint{ 208 { 209 Address: localIP, 210 EndpointPort: uint32(testEnv.Ports().BackendPort), 211 ServicePortName: "http", 212 Locality: model.Locality{Label: "az"}, 213 }, 214 }) 215 216 // Explicit test service, in the v2 memory registry. Similar with mock.MakeService, 217 // but easier to read. 218 hostname = "service3.default.svc.cluster.local" 219 server.EnvoyXdsServer.MemRegistry.AddService(hostname, &model.Service{ 220 Hostname: hostname, 221 Address: "10.10.0.1", 222 Ports: testPorts(0), 223 Attributes: model.ServiceAttributes{ 224 Name: "service3", 225 Namespace: "default", 226 }, 227 }) 228 229 svc3Endpoints := make([]*model.IstioEndpoint, len(testPorts(0))) 230 for i, p := range testPorts(0) { 231 svc3Endpoints[i] = &model.IstioEndpoint{ 232 Address: app3Ip, 233 EndpointPort: uint32(p.Port), 234 ServicePortName: p.Name, 235 Locality: model.Locality{Label: "az"}, 236 } 237 } 238 239 server.EnvoyXdsServer.MemRegistry.SetEndpoints(string(hostname), "default", svc3Endpoints) 240 241 // Mock ingress service 242 server.EnvoyXdsServer.MemRegistry.AddService("istio-ingress.istio-system.svc.cluster.local", &model.Service{ 243 Hostname: "istio-ingress.istio-system.svc.cluster.local", 244 Address: "10.10.0.2", 245 Ports: []*model.Port{ 246 { 247 Name: "http", 248 Port: 80, 249 Protocol: protocol.HTTP, 250 }, 251 { 252 Name: "https", 253 Port: 443, 254 Protocol: protocol.HTTPS, 255 }, 256 }, 257 // TODO: set attribute for this service. It may affect TestLDSIsolated as we now having service defined in istio-system namespaces 258 }) 259 server.EnvoyXdsServer.MemRegistry.AddInstance("istio-ingress.istio-system.svc.cluster.local", &model.ServiceInstance{ 260 Endpoint: &model.IstioEndpoint{ 261 Address: ingressIP, 262 EndpointPort: 80, 263 ServicePortName: "http", 264 Locality: model.Locality{Label: "az"}, 265 Labels: labels.Instance{constants.IstioLabel: constants.IstioIngressLabelValue}, 266 }, 267 ServicePort: &model.Port{ 268 Name: "http", 269 Port: 80, 270 Protocol: protocol.HTTP, 271 }, 272 }) 273 server.EnvoyXdsServer.MemRegistry.AddInstance("istio-ingress.istio-system.svc.cluster.local", &model.ServiceInstance{ 274 Endpoint: &model.IstioEndpoint{ 275 Address: ingressIP, 276 EndpointPort: 443, 277 ServicePortName: "https", 278 Locality: model.Locality{Label: "az"}, 279 Labels: labels.Instance{constants.IstioLabel: constants.IstioIngressLabelValue}, 280 }, 281 ServicePort: &model.Port{ 282 Name: "https", 283 Port: 443, 284 Protocol: protocol.HTTPS, 285 }, 286 }) 287 288 // RouteConf Service4 is using port 80, to test that we generate multiple clusters (regression) 289 // service4 has no endpoints 290 server.EnvoyXdsServer.MemRegistry.AddHTTPService("service4.default.svc.cluster.local", "10.1.0.4", 80) 291 }) 292} 293 294// nolint: unparam 295func testPorts(base int) []*model.Port { 296 return []*model.Port{ 297 { 298 Name: "http", 299 Port: base + 80, 300 Protocol: protocol.HTTP, 301 }, { 302 Name: "http-status", 303 Port: base + 81, 304 Protocol: protocol.HTTP, 305 }, { 306 Name: "custom", 307 Port: base + 90, 308 Protocol: protocol.TCP, 309 }, { 310 Name: "mongo", 311 Port: base + 100, 312 Protocol: protocol.Mongo, 313 }, { 314 Name: "redis", 315 Port: base + 110, 316 Protocol: protocol.Redis, 317 }, { 318 Name: "mysql", 319 Port: base + 120, 320 Protocol: protocol.MySQL, 321 }, { 322 Name: "h2port", 323 Port: base + 66, 324 Protocol: protocol.GRPC, 325 }} 326} 327 328// Test XDS with real envoy and with mixer. 329func TestEnvoy(t *testing.T) { 330 mesh.TestMode = true 331 _, tearDown := initLocalPilotTestEnv(t) 332 defer func() { 333 if testEnv != nil { 334 testEnv.TearDown() 335 } 336 tearDown() 337 }() 338 startEnvoy(t) 339 // Make sure tcp port is ready before starting the test. 340 testenv.WaitForPort(testEnv.Ports().TCPProxyPort) 341 342 t.Run("envoyInit", envoyInit) 343 t.Run("service", testService) 344} 345 346// envoyInit verifies envoy has accepted the config from pilot by checking the stats. 347func envoyInit(t *testing.T) { 348 statsURL := fmt.Sprintf("http://localhost:%d/stats?format=json", testEnv.Ports().AdminPort) 349 res, err := http.Get(statsURL) 350 if err != nil { 351 t.Fatal("Failed to get stats, envoy not started") 352 } 353 statsBytes, err := ioutil.ReadAll(res.Body) 354 if err != nil { 355 t.Fatal("Failed to get stats, envoy not started") 356 } 357 358 statsMap := stats2map(statsBytes) 359 360 if statsMap["cluster_manager.cds.update_success"] < 1 { 361 t.Error("Failed cds update") 362 } 363 // Other interesting values for CDS: cluster_added: 19, active_clusters 364 // cds.update_attempt: 2, cds.update_rejected, cds.version 365 for _, port := range testPorts(0) { 366 stat := fmt.Sprintf("cluster.outbound|%d||service3.default.svc.cluster.local.update_success", port.Port) 367 if statsMap[stat] < 1 { 368 t.Error("Failed sds updates") 369 } 370 } 371 372 if statsMap["cluster.xds-grpc.update_failure"] > 0 { 373 t.Error("GRPC update failure") 374 } 375 376 if statsMap["listener_manager.lds.update_rejected"] > 0 { 377 t.Error("LDS update failure") 378 } 379 if statsMap["listener_manager.lds.update_success"] < 1 { 380 t.Error("LDS update failure") 381 } 382} 383 384// Example of using a local test connecting to the in-process test service, using Envoy http proxy 385// mode. This is also a test for http proxy (finally). 386func testService(t *testing.T) { 387 proxyURL, _ := url.Parse("http://localhost:17002") 388 389 client := &http.Client{Transport: &http.Transport{Proxy: http.ProxyURL(proxyURL)}} 390 391 res, err := client.Get("http://local.default.svc.cluster.local") 392 if err != nil { 393 t.Error("Failed to access proxy", err) 394 return 395 } 396 resdmp, _ := httputil.DumpResponse(res, true) 397 t.Log(string(resdmp)) 398 if res.Status != "200 OK" { 399 t.Error("Proxy failed ", res.Status) 400 } 401} 402 403// EnvoyStat is used to parse envoy stats 404type EnvoyStat struct { 405 Name string `json:"name"` 406 Value int `json:"value"` 407} 408 409// stats2map parses envoy stats. 410func stats2map(stats []byte) map[string]int { 411 s := struct { 412 Stats []EnvoyStat `json:"stats"` 413 }{} 414 _ = json.Unmarshal(stats, &s) 415 m := map[string]int{} 416 for _, stat := range s.Stats { 417 m[stat.Name] = stat.Value 418 } 419 return m 420} 421 422func getLocalIP() string { 423 addrs, err := net.InterfaceAddrs() 424 if err != nil { 425 return "" 426 } 427 for _, address := range addrs { 428 // check the address type and if it is not a loopback the display it 429 if ipnet, ok := address.(*net.IPNet); ok && !ipnet.IP.IsLoopback() { 430 if ipnet.IP.To4() != nil { 431 return ipnet.IP.String() 432 } 433 } 434 } 435 return "" 436} 437 438// newEndpointWithAccount is a helper for IstioEndpoint creation. Creates endpoints with 439// port name "http", with the given IP, service account and a 'version' label. 440// nolint: unparam 441func newEndpointWithAccount(ip, account, version string) []*model.IstioEndpoint { 442 return []*model.IstioEndpoint{ 443 { 444 Address: ip, 445 ServicePortName: "http-main", 446 EndpointPort: 80, 447 Labels: map[string]string{"version": version}, 448 UID: "uid1", 449 ServiceAccount: account, 450 }, 451 } 452} 453