1// Copyright 2019 Istio Authors. All Rights Reserved. 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15package outboundtrafficpolicy 16 17import ( 18 "bytes" 19 "fmt" 20 "html/template" 21 "io/ioutil" 22 "path" 23 "reflect" 24 "testing" 25 "time" 26 27 "istio.io/istio/pkg/test/echo/common" 28 "istio.io/istio/pkg/test/env" 29 util "istio.io/istio/tests/integration/mixer" 30 31 envoyAdmin "github.com/envoyproxy/go-control-plane/envoy/admin/v3" 32 33 "istio.io/istio/pkg/config/protocol" 34 "istio.io/istio/pkg/test/framework" 35 "istio.io/istio/pkg/test/framework/components/echo" 36 "istio.io/istio/pkg/test/framework/components/echo/echoboot" 37 "istio.io/istio/pkg/test/framework/components/environment/kube" 38 "istio.io/istio/pkg/test/framework/components/galley" 39 "istio.io/istio/pkg/test/framework/components/namespace" 40 "istio.io/istio/pkg/test/framework/components/pilot" 41 "istio.io/istio/pkg/test/framework/components/prometheus" 42 "istio.io/istio/pkg/test/framework/resource" 43 "istio.io/istio/pkg/test/util/retry" 44 "istio.io/istio/pkg/test/util/structpath" 45) 46 47const ( 48 // This service entry exists to create conflicts on various ports 49 // As defined below, the tcp-conflict and https-conflict ports are 9443 and 9091 50 ServiceEntry = ` 51apiVersion: networking.istio.io/v1alpha3 52kind: ServiceEntry 53metadata: 54 name: http 55spec: 56 hosts: 57 - istio.io 58 location: MESH_EXTERNAL 59 ports: 60 - name: http-for-https 61 number: 9443 62 protocol: HTTP 63 - name: http-for-tcp 64 number: 9091 65 protocol: HTTP 66 resolution: DNS 67` 68 SidecarScope = ` 69apiVersion: networking.istio.io/v1alpha3 70kind: Sidecar 71metadata: 72 name: restrict-to-service-entry-namespace 73spec: 74 egress: 75 - hosts: 76 - "{{.ImportNamespace}}/*" 77 - "istio-system/*" 78 outboundTrafficPolicy: 79 mode: "{{.TrafficPolicyMode}}" 80` 81 82 Gateway = ` 83apiVersion: networking.istio.io/v1alpha3 84kind: Gateway 85metadata: 86 name: istio-egressgateway 87spec: 88 selector: 89 istio: egressgateway 90 servers: 91 - port: 92 number: 80 93 name: http 94 protocol: HTTP 95 hosts: 96 - "some-external-site.com" 97--- 98apiVersion: networking.istio.io/v1alpha3 99kind: VirtualService 100metadata: 101 name: route-via-egressgateway 102spec: 103 hosts: 104 - "some-external-site.com" 105 gateways: 106 - istio-egressgateway 107 - mesh 108 http: 109 - match: 110 - gateways: 111 - mesh # from sidecars, route to egress gateway service 112 port: 80 113 route: 114 - destination: 115 host: istio-egressgateway.istio-system.svc.cluster.local 116 port: 117 number: 80 118 weight: 100 119 - match: 120 - gateways: 121 - istio-egressgateway 122 port: 80 123 route: 124 - destination: 125 host: destination.{{.AppNamespace}}.svc.cluster.local 126 port: 127 number: 80 128 weight: 100 129 headers: 130 request: 131 add: 132 handled-by-egress-gateway: "true" 133` 134) 135 136// TestCase represents what is being tested 137type TestCase struct { 138 Name string 139 PortName string 140 Host string 141 Gateway bool 142 Expected Expected 143} 144 145// Expected contains the metric and query to run against 146// prometheus to validate that expected telemetry information was gathered; 147// as well as the http response code 148type Expected struct { 149 Metric string 150 PromQueryFormat string 151 ResponseCode []string 152} 153 154// TrafficPolicy is the mode of the outbound traffic policy to use 155// when configuring the sidecar for the client 156type TrafficPolicy string 157 158const ( 159 AllowAny TrafficPolicy = "ALLOW_ANY" 160 RegistryOnly TrafficPolicy = "REGISTRY_ONLY" 161) 162 163// String implements fmt.Stringer 164func (t TrafficPolicy) String() string { 165 return string(t) 166} 167 168// We want to test "external" traffic. To do this without actually hitting an external endpoint, 169// we can import only the service namespace, so the apps are not known 170func createSidecarScope(t *testing.T, tPolicy TrafficPolicy, appsNamespace namespace.Instance, serviceNamespace namespace.Instance, g galley.Instance) { 171 tmpl, err := template.New("SidecarScope").Parse(SidecarScope) 172 if err != nil { 173 t.Errorf("failed to create template: %v", err) 174 } 175 176 var buf bytes.Buffer 177 if err := tmpl.Execute(&buf, map[string]string{"ImportNamespace": serviceNamespace.Name(), "TrafficPolicyMode": tPolicy.String()}); err != nil { 178 t.Errorf("failed to create template: %v", err) 179 } 180 if err := g.ApplyConfig(appsNamespace, buf.String()); err != nil { 181 t.Errorf("failed to apply service entries: %v", err) 182 } 183} 184 185func mustReadCert(t *testing.T, f string) string { 186 b, err := ioutil.ReadFile(path.Join(env.IstioSrc, "tests/testdata/certs", f)) 187 if err != nil { 188 t.Fatalf("failed to read %v: %v", f, err) 189 } 190 return string(b) 191} 192 193// We want to test "external" traffic. To do this without actually hitting an external endpoint, 194// we can import only the service namespace, so the apps are not known 195func createGateway(t *testing.T, appsNamespace namespace.Instance, serviceNamespace namespace.Instance, g galley.Instance) { 196 tmpl, err := template.New("Gateway").Parse(Gateway) 197 if err != nil { 198 t.Fatalf("failed to create template: %v", err) 199 } 200 201 var buf bytes.Buffer 202 if err := tmpl.Execute(&buf, map[string]string{"AppNamespace": appsNamespace.Name()}); err != nil { 203 t.Fatalf("failed to create template: %v", err) 204 } 205 if err := g.ApplyConfig(serviceNamespace, buf.String()); err != nil { 206 t.Fatalf("failed to apply gateway: %v. template: %v", err, buf.String()) 207 } 208} 209 210// TODO support native environment for registry only/gateway. Blocked by #13177 because the listeners for native use static 211// routes and this test relies on the dynamic routes sent through pilot to allow external traffic. 212 213func RunExternalRequest(cases []*TestCase, prometheus prometheus.Instance, mode TrafficPolicy, t *testing.T) { 214 215 // Testing of Blackhole and Passthrough clusters: 216 // Setup of environment: 217 // 1. client and destination are deployed to app-1-XXXX namespace 218 // 2. client is restricted to talk to destination via Sidecar scope where outbound policy is set (ALLOW_ANY, REGISTRY_ONLY) 219 // and clients' egress can only be to service-2-XXXX/* and istio-system/* 220 // 3. a namespace service-2-YYYY is created 221 // 4. A gateway is put in service-2-YYYY where its host is set for some-external-site.com on port 80 and 443 222 // 3. a VirtualService is also created in service-2-XXXX to: 223 // a) route requests for some-external-site.com to the istio-egressgateway 224 // * if the request on port 80, then it will add an http header `handled-by-egress-gateway` 225 // b) from the egressgateway it will forward the request to the destination pod deployed in the app-1-XXX 226 // namespace 227 228 // Test cases: 229 // 1. http case: 230 // client -------> Hits listener 0.0.0.0_80 cluster 231 // Metric is istio_requests_total i.e. HTTP 232 // 233 // 2. https case: 234 // client ----> Hits no listener -> 0.0.0.0_150001 -> ALLOW_ANY/REGISTRY_ONLY 235 // Metric is istio_tcp_connections_closed_total i.e. TCP 236 // 237 // 3. https conflict case: 238 // client ----> Hits listener 0.0.0.0_9443 239 // Metric is istio_tcp_connections_closed_total i.e. TCP 240 // 241 // 4. http_egress 242 // client ) ---HTTP request (Host: some-external-site.com----> Hits listener 0.0.0.0_80 -> 243 // VS Routing (add Egress Header) --> Egress Gateway --> destination 244 // Metric is istio_requests_total i.e. HTTP with destination as destination 245 // 246 // 5. TCP 247 // client ---TCP request at port 9090----> Matches no listener -> 0.0.0.0_150001 -> ALLOW_ANY/REGISTRY_ONLY 248 // Metric is istio_tcp_connections_closed_total i.e. TCP 249 // 250 // 5. TCP conflict 251 // client ---TCP request at port 9091 ----> Hits listener 0.0.0.0_9091 -> ALLOW_ANY/REGISTRY_ONLY 252 // Metric is istio_tcp_connections_closed_total i.e. TCP 253 // 254 framework. 255 NewTest(t). 256 Run(func(ctx framework.TestContext) { 257 client, dest := setupEcho(t, ctx, mode) 258 259 for _, tc := range cases { 260 t.Run(tc.Name, func(t *testing.T) { 261 if _, kube := ctx.Environment().(*kube.Environment); !kube && tc.Gateway { 262 t.Skip("Cannot run gateway in native environment.") 263 } 264 retry.UntilSuccessOrFail(t, func() error { 265 resp, err := client.Call(echo.CallOptions{ 266 Target: dest, 267 PortName: tc.PortName, 268 Headers: map[string][]string{ 269 "Host": {tc.Host}, 270 }, 271 }) 272 273 // the expected response from a blackhole test case will have err 274 // set; use the length of the expected code to ignore this condition 275 if err != nil && len(tc.Expected.ResponseCode) != 0 { 276 return fmt.Errorf("request failed: %v", err) 277 } 278 279 codes := make([]string, 0, len(resp)) 280 for _, r := range resp { 281 codes = append(codes, r.Code) 282 } 283 if !reflect.DeepEqual(codes, tc.Expected.ResponseCode) { 284 return fmt.Errorf("got codes %q, expected %q", codes, tc.Expected.ResponseCode) 285 } 286 287 for _, r := range resp { 288 if _, f := r.RawResponse["Handled-By-Egress-Gateway"]; tc.Gateway && !f { 289 return fmt.Errorf("expected to be handled by gateway. response: %+v", r.RawResponse) 290 } 291 } 292 return nil 293 }, retry.Delay(time.Second), retry.Timeout(20*time.Second)) 294 295 if tc.Expected.Metric != "" { 296 util.ValidateMetric(t, prometheus, tc.Expected.PromQueryFormat, tc.Expected.Metric, 1) 297 } 298 }) 299 } 300 }) 301} 302 303func setupEcho(t *testing.T, ctx resource.Context, mode TrafficPolicy) (echo.Instance, echo.Instance) { 304 g := galley.NewOrFail(t, ctx, galley.Config{}) 305 p := pilot.NewOrFail(t, ctx, pilot.Config{Galley: g}) 306 307 appsNamespace := namespace.NewOrFail(t, ctx, namespace.Config{ 308 Prefix: "app", 309 Inject: true, 310 }) 311 serviceNamespace := namespace.NewOrFail(t, ctx, namespace.Config{ 312 Prefix: "service", 313 Inject: true, 314 }) 315 316 var client, dest echo.Instance 317 echoboot.NewBuilderOrFail(t, ctx). 318 With(&client, echo.Config{ 319 Service: "client", 320 Namespace: appsNamespace, 321 Subsets: []echo.SubsetConfig{{}}, 322 Pilot: p, 323 Galley: g, 324 }). 325 With(&dest, echo.Config{ 326 Service: "destination", 327 Namespace: appsNamespace, 328 Subsets: []echo.SubsetConfig{{}}, 329 Pilot: p, 330 Galley: g, 331 Ports: []echo.Port{ 332 { 333 // Plain HTTP port, will match no listeners and fall through 334 Name: "http", 335 Protocol: protocol.HTTP, 336 ServicePort: 80, 337 InstancePort: 8080, 338 }, 339 { 340 // HTTPS port, will match no listeners and fall through 341 Name: "https", 342 Protocol: protocol.HTTPS, 343 ServicePort: 443, 344 InstancePort: 8443, 345 TLS: true, 346 }, 347 { 348 // HTTPS port, there will be an HTTP service defined on this port that will match 349 Name: "https-conflict", 350 Protocol: protocol.HTTPS, 351 ServicePort: 9443, 352 TLS: true, 353 }, 354 { 355 // TCP port, will match no listeners and fall through 356 Name: "tcp", 357 Protocol: protocol.TCP, 358 ServicePort: 9090, 359 }, 360 { 361 // TCP port, there will be an HTTP service defined on this port that will match 362 Name: "tcp-conflict", 363 Protocol: protocol.TCP, 364 ServicePort: 9091, 365 }, 366 }, 367 TLSSettings: &common.TLSSettings{ 368 // Echo has these test certs baked into the docker image 369 RootCert: mustReadCert(t, "cacert.pem"), 370 ClientCert: mustReadCert(t, "cert.crt"), 371 Key: mustReadCert(t, "cert.key"), 372 }, 373 }).BuildOrFail(t) 374 375 // External traffic should work even if we have service entries on the same ports 376 createSidecarScope(t, mode, appsNamespace, serviceNamespace, g) 377 if err := g.ApplyConfig(serviceNamespace, ServiceEntry); err != nil { 378 t.Errorf("failed to apply service entries: %v", err) 379 } 380 381 if _, kube := ctx.Environment().(*kube.Environment); kube { 382 createGateway(t, appsNamespace, serviceNamespace, g) 383 } 384 if err := WaitUntilNotCallable(client, dest); err != nil { 385 t.Fatalf("failed to apply sidecar, %v", err) 386 } 387 return client, dest 388} 389 390func clusterName(target echo.Instance, port echo.Port) string { 391 cfg := target.Config() 392 return fmt.Sprintf("outbound|%d||%s.%s.svc.%s", port.ServicePort, cfg.Service, cfg.Namespace.Name(), cfg.Domain) 393} 394 395// Wait for the destination to NOT be callable by the client. This allows us to simulate external traffic. 396// This essentially just waits for the Sidecar to be applied, without sleeping. 397func WaitUntilNotCallable(c echo.Instance, dest echo.Instance) error { 398 accept := func(cfg *envoyAdmin.ConfigDump) (bool, error) { 399 validator := structpath.ForProto(cfg) 400 for _, port := range dest.Config().Ports { 401 clusterName := clusterName(dest, port) 402 // Ensure that we have an outbound configuration for the target port. 403 err := validator.NotExists("{.configs[*].dynamicActiveClusters[?(@.cluster.Name == '%s')]}", clusterName).Check() 404 if err != nil { 405 return false, err 406 } 407 } 408 409 return true, nil 410 } 411 412 workloads, _ := c.Workloads() 413 // Wait for the outbound config to be received by each workload from Pilot. 414 for _, w := range workloads { 415 if w.Sidecar() != nil { 416 if err := w.Sidecar().WaitForConfig(accept, retry.Timeout(time.Second*10)); err != nil { 417 return err 418 } 419 } 420 } 421 422 return nil 423} 424