1package xds 2 3import ( 4 "bytes" 5 "path/filepath" 6 "sort" 7 "testing" 8 "text/template" 9 "time" 10 11 envoy_cluster_v3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" 12 13 "github.com/golang/protobuf/ptypes/wrappers" 14 testinf "github.com/mitchellh/go-testing-interface" 15 "github.com/stretchr/testify/require" 16 17 "github.com/hashicorp/consul/agent/connect" 18 "github.com/hashicorp/consul/agent/consul/discoverychain" 19 "github.com/hashicorp/consul/agent/proxycfg" 20 "github.com/hashicorp/consul/agent/structs" 21 "github.com/hashicorp/consul/agent/xds/proxysupport" 22 "github.com/hashicorp/consul/lib/stringslice" 23 "github.com/hashicorp/consul/sdk/testutil" 24) 25 26func TestClustersFromSnapshot(t *testing.T) { 27 if testing.Short() { 28 t.Skip("too slow for testing.Short") 29 } 30 31 tests := []struct { 32 name string 33 create func(t testinf.T) *proxycfg.ConfigSnapshot 34 // Setup is called before the test starts. It is passed the snapshot from 35 // create func and is allowed to modify it in any way to setup the 36 // test input. 37 setup func(snap *proxycfg.ConfigSnapshot) 38 overrideGoldenName string 39 }{ 40 { 41 name: "defaults", 42 create: proxycfg.TestConfigSnapshot, 43 setup: nil, // Default snapshot 44 }, 45 { 46 name: "custom-local-app", 47 create: proxycfg.TestConfigSnapshot, 48 setup: func(snap *proxycfg.ConfigSnapshot) { 49 snap.Proxy.Config["envoy_local_cluster_json"] = 50 customAppClusterJSON(t, customClusterJSONOptions{ 51 Name: "mylocal", 52 }) 53 }, 54 }, 55 { 56 name: "custom-upstream", 57 create: proxycfg.TestConfigSnapshot, 58 setup: func(snap *proxycfg.ConfigSnapshot) { 59 snap.Proxy.Upstreams[0].Config["envoy_cluster_json"] = 60 customAppClusterJSON(t, customClusterJSONOptions{ 61 Name: "myservice", 62 }) 63 }, 64 }, 65 { 66 name: "custom-upstream-default-chain", 67 create: proxycfg.TestConfigSnapshotDiscoveryChainDefault, 68 setup: func(snap *proxycfg.ConfigSnapshot) { 69 snap.Proxy.Upstreams[0].Config["envoy_cluster_json"] = 70 customAppClusterJSON(t, customClusterJSONOptions{ 71 Name: "myservice", 72 }) 73 snap.ConnectProxy.UpstreamConfig = map[string]*structs.Upstream{ 74 "db": { 75 Config: map[string]interface{}{ 76 "envoy_cluster_json": customAppClusterJSON(t, customClusterJSONOptions{ 77 Name: "myservice", 78 }), 79 }, 80 }, 81 } 82 }, 83 }, 84 { 85 name: "custom-upstream-ignores-tls", 86 create: proxycfg.TestConfigSnapshot, 87 overrideGoldenName: "custom-upstream", // should be the same 88 setup: func(snap *proxycfg.ConfigSnapshot) { 89 snap.Proxy.Upstreams[0].Config["envoy_cluster_json"] = 90 customAppClusterJSON(t, customClusterJSONOptions{ 91 Name: "myservice", 92 // Attempt to override the TLS context should be ignored 93 TLSContext: `"allowRenegotiation": false`, 94 }) 95 }, 96 }, 97 { 98 name: "custom-timeouts", 99 create: proxycfg.TestConfigSnapshot, 100 setup: func(snap *proxycfg.ConfigSnapshot) { 101 snap.Proxy.Config["local_connect_timeout_ms"] = 1234 102 snap.Proxy.Upstreams[0].Config["connect_timeout_ms"] = 2345 103 }, 104 }, 105 { 106 name: "custom-limits-max-connections-only", 107 create: proxycfg.TestConfigSnapshot, 108 setup: func(snap *proxycfg.ConfigSnapshot) { 109 for i := range snap.Proxy.Upstreams { 110 // We check if Config is nil because the prepared_query upstream is 111 // initialized without a Config map. Use Upstreams[i] syntax to 112 // modify the actual ConfigSnapshot instead of copying the Upstream 113 // in the range. 114 if snap.Proxy.Upstreams[i].Config == nil { 115 snap.Proxy.Upstreams[i].Config = map[string]interface{}{} 116 } 117 118 snap.Proxy.Upstreams[i].Config["limits"] = map[string]interface{}{ 119 "max_connections": 500, 120 } 121 } 122 }, 123 }, 124 { 125 name: "custom-limits-set-to-zero", 126 create: proxycfg.TestConfigSnapshot, 127 setup: func(snap *proxycfg.ConfigSnapshot) { 128 for i := range snap.Proxy.Upstreams { 129 if snap.Proxy.Upstreams[i].Config == nil { 130 snap.Proxy.Upstreams[i].Config = map[string]interface{}{} 131 } 132 133 snap.Proxy.Upstreams[i].Config["limits"] = map[string]interface{}{ 134 "max_connections": 0, 135 "max_pending_requests": 0, 136 "max_concurrent_requests": 0, 137 } 138 } 139 }, 140 }, 141 { 142 name: "custom-limits", 143 create: proxycfg.TestConfigSnapshot, 144 setup: func(snap *proxycfg.ConfigSnapshot) { 145 for i := range snap.Proxy.Upstreams { 146 if snap.Proxy.Upstreams[i].Config == nil { 147 snap.Proxy.Upstreams[i].Config = map[string]interface{}{} 148 } 149 150 snap.Proxy.Upstreams[i].Config["limits"] = map[string]interface{}{ 151 "max_connections": 500, 152 "max_pending_requests": 600, 153 "max_concurrent_requests": 700, 154 } 155 } 156 }, 157 }, 158 { 159 name: "connect-proxy-with-chain", 160 create: proxycfg.TestConfigSnapshotDiscoveryChain, 161 setup: nil, 162 }, 163 { 164 name: "connect-proxy-with-chain-external-sni", 165 create: proxycfg.TestConfigSnapshotDiscoveryChainExternalSNI, 166 setup: nil, 167 }, 168 { 169 name: "connect-proxy-with-chain-and-overrides", 170 create: proxycfg.TestConfigSnapshotDiscoveryChainWithOverrides, 171 setup: nil, 172 }, 173 { 174 name: "connect-proxy-with-chain-and-failover", 175 create: proxycfg.TestConfigSnapshotDiscoveryChainWithFailover, 176 setup: nil, 177 }, 178 { 179 name: "connect-proxy-with-tcp-chain-failover-through-remote-gateway", 180 create: proxycfg.TestConfigSnapshotDiscoveryChainWithFailoverThroughRemoteGateway, 181 setup: nil, 182 }, 183 { 184 name: "connect-proxy-with-tcp-chain-failover-through-remote-gateway-triggered", 185 create: proxycfg.TestConfigSnapshotDiscoveryChainWithFailoverThroughRemoteGatewayTriggered, 186 setup: nil, 187 }, 188 { 189 name: "connect-proxy-with-tcp-chain-double-failover-through-remote-gateway", 190 create: proxycfg.TestConfigSnapshotDiscoveryChainWithDoubleFailoverThroughRemoteGateway, 191 setup: nil, 192 }, 193 { 194 name: "connect-proxy-with-tcp-chain-double-failover-through-remote-gateway-triggered", 195 create: proxycfg.TestConfigSnapshotDiscoveryChainWithDoubleFailoverThroughRemoteGatewayTriggered, 196 setup: nil, 197 }, 198 { 199 name: "connect-proxy-with-tcp-chain-failover-through-local-gateway", 200 create: proxycfg.TestConfigSnapshotDiscoveryChainWithFailoverThroughLocalGateway, 201 setup: nil, 202 }, 203 { 204 name: "connect-proxy-with-tcp-chain-failover-through-local-gateway-triggered", 205 create: proxycfg.TestConfigSnapshotDiscoveryChainWithFailoverThroughLocalGatewayTriggered, 206 setup: nil, 207 }, 208 { 209 name: "connect-proxy-with-tcp-chain-double-failover-through-local-gateway", 210 create: proxycfg.TestConfigSnapshotDiscoveryChainWithDoubleFailoverThroughLocalGateway, 211 setup: nil, 212 }, 213 { 214 name: "connect-proxy-with-tcp-chain-double-failover-through-local-gateway-triggered", 215 create: proxycfg.TestConfigSnapshotDiscoveryChainWithDoubleFailoverThroughLocalGatewayTriggered, 216 setup: nil, 217 }, 218 { 219 name: "splitter-with-resolver-redirect", 220 create: proxycfg.TestConfigSnapshotDiscoveryChain_SplitterWithResolverRedirectMultiDC, 221 setup: nil, 222 }, 223 { 224 name: "connect-proxy-lb-in-resolver", 225 create: proxycfg.TestConfigSnapshotDiscoveryChainWithLB, 226 setup: nil, 227 }, 228 { 229 name: "expose-paths-local-app-paths", 230 create: proxycfg.TestConfigSnapshotExposeConfig, 231 }, 232 { 233 name: "downstream-service-with-unix-sockets", 234 create: proxycfg.TestConfigSnapshot, 235 setup: func(snap *proxycfg.ConfigSnapshot) { 236 snap.Address = "" 237 snap.Port = 0 238 snap.Proxy.LocalServiceAddress = "" 239 snap.Proxy.LocalServicePort = 0 240 snap.Proxy.LocalServiceSocketPath = "/tmp/downstream_proxy.sock" 241 }, 242 }, 243 { 244 name: "expose-paths-new-cluster-http2", 245 create: proxycfg.TestConfigSnapshotExposeConfig, 246 setup: func(snap *proxycfg.ConfigSnapshot) { 247 snap.Proxy.Expose.Paths[1] = structs.ExposePath{ 248 LocalPathPort: 9090, 249 Path: "/grpc.health.v1.Health/Check", 250 ListenerPort: 21501, 251 Protocol: "http2", 252 } 253 }, 254 }, 255 { 256 name: "expose-paths-grpc-new-cluster-http1", 257 create: proxycfg.TestConfigSnapshotGRPCExposeHTTP1, 258 }, 259 { 260 name: "mesh-gateway", 261 create: proxycfg.TestConfigSnapshotMeshGateway, 262 setup: nil, 263 }, 264 { 265 name: "mesh-gateway-using-federation-states", 266 create: proxycfg.TestConfigSnapshotMeshGatewayUsingFederationStates, 267 setup: nil, 268 }, 269 { 270 name: "mesh-gateway-no-services", 271 create: proxycfg.TestConfigSnapshotMeshGatewayNoServices, 272 setup: nil, 273 }, 274 { 275 name: "mesh-gateway-service-subsets", 276 create: proxycfg.TestConfigSnapshotMeshGateway, 277 setup: func(snap *proxycfg.ConfigSnapshot) { 278 snap.MeshGateway.ServiceResolvers = map[structs.ServiceName]*structs.ServiceResolverConfigEntry{ 279 structs.NewServiceName("bar", nil): { 280 Kind: structs.ServiceResolver, 281 Name: "bar", 282 Subsets: map[string]structs.ServiceResolverSubset{ 283 "v1": { 284 Filter: "Service.Meta.Version == 1", 285 }, 286 "v2": { 287 Filter: "Service.Meta.Version == 2", 288 OnlyPassing: true, 289 }, 290 }, 291 }, 292 } 293 }, 294 }, 295 { 296 name: "mesh-gateway-ignore-extra-resolvers", 297 create: proxycfg.TestConfigSnapshotMeshGateway, 298 setup: func(snap *proxycfg.ConfigSnapshot) { 299 snap.MeshGateway.ServiceResolvers = map[structs.ServiceName]*structs.ServiceResolverConfigEntry{ 300 structs.NewServiceName("bar", nil): { 301 Kind: structs.ServiceResolver, 302 Name: "bar", 303 DefaultSubset: "v2", 304 Subsets: map[string]structs.ServiceResolverSubset{ 305 "v1": { 306 Filter: "Service.Meta.Version == 1", 307 }, 308 "v2": { 309 Filter: "Service.Meta.Version == 2", 310 OnlyPassing: true, 311 }, 312 }, 313 }, 314 structs.NewServiceName("notfound", nil): { 315 Kind: structs.ServiceResolver, 316 Name: "notfound", 317 DefaultSubset: "v2", 318 Subsets: map[string]structs.ServiceResolverSubset{ 319 "v1": { 320 Filter: "Service.Meta.Version == 1", 321 }, 322 "v2": { 323 Filter: "Service.Meta.Version == 2", 324 OnlyPassing: true, 325 }, 326 }, 327 }, 328 } 329 }, 330 }, 331 { 332 name: "mesh-gateway-service-timeouts", 333 create: proxycfg.TestConfigSnapshotMeshGateway, 334 setup: func(snap *proxycfg.ConfigSnapshot) { 335 snap.MeshGateway.ServiceResolvers = map[structs.ServiceName]*structs.ServiceResolverConfigEntry{ 336 structs.NewServiceName("bar", nil): { 337 Kind: structs.ServiceResolver, 338 Name: "bar", 339 ConnectTimeout: 10 * time.Second, 340 Subsets: map[string]structs.ServiceResolverSubset{ 341 "v1": { 342 Filter: "Service.Meta.Version == 1", 343 }, 344 "v2": { 345 Filter: "Service.Meta.Version == 2", 346 OnlyPassing: true, 347 }, 348 }, 349 }, 350 } 351 }, 352 }, 353 { 354 name: "mesh-gateway-non-hash-lb-injected", 355 create: proxycfg.TestConfigSnapshotMeshGateway, 356 setup: func(snap *proxycfg.ConfigSnapshot) { 357 snap.MeshGateway.ServiceResolvers = map[structs.ServiceName]*structs.ServiceResolverConfigEntry{ 358 structs.NewServiceName("bar", nil): { 359 Kind: structs.ServiceResolver, 360 Name: "bar", 361 Subsets: map[string]structs.ServiceResolverSubset{ 362 "v1": { 363 Filter: "Service.Meta.Version == 1", 364 }, 365 "v2": { 366 Filter: "Service.Meta.Version == 2", 367 OnlyPassing: true, 368 }, 369 }, 370 LoadBalancer: &structs.LoadBalancer{ 371 Policy: "least_request", 372 LeastRequestConfig: &structs.LeastRequestConfig{ 373 ChoiceCount: 5, 374 }, 375 }, 376 }, 377 } 378 }, 379 }, 380 { 381 name: "mesh-gateway-hash-lb-ignored", 382 create: proxycfg.TestConfigSnapshotMeshGateway, 383 setup: func(snap *proxycfg.ConfigSnapshot) { 384 snap.MeshGateway.ServiceResolvers = map[structs.ServiceName]*structs.ServiceResolverConfigEntry{ 385 structs.NewServiceName("bar", nil): { 386 Kind: structs.ServiceResolver, 387 Name: "bar", 388 Subsets: map[string]structs.ServiceResolverSubset{ 389 "v1": { 390 Filter: "Service.Meta.Version == 1", 391 }, 392 "v2": { 393 Filter: "Service.Meta.Version == 2", 394 OnlyPassing: true, 395 }, 396 }, 397 LoadBalancer: &structs.LoadBalancer{ 398 Policy: "ring_hash", 399 RingHashConfig: &structs.RingHashConfig{ 400 MinimumRingSize: 20, 401 MaximumRingSize: 50, 402 }, 403 }, 404 }, 405 } 406 }, 407 }, 408 { 409 name: "ingress-gateway", 410 create: proxycfg.TestConfigSnapshotIngressGateway, 411 setup: nil, 412 }, 413 { 414 name: "ingress-gateway-no-services", 415 create: proxycfg.TestConfigSnapshotIngressGatewayNoServices, 416 setup: nil, 417 }, 418 { 419 name: "ingress-with-chain", 420 create: proxycfg.TestConfigSnapshotIngress, 421 setup: nil, 422 }, 423 { 424 name: "ingress-with-chain-external-sni", 425 create: proxycfg.TestConfigSnapshotIngressExternalSNI, 426 setup: nil, 427 }, 428 { 429 name: "ingress-with-chain-and-overrides", 430 create: proxycfg.TestConfigSnapshotIngressWithOverrides, 431 setup: nil, 432 }, 433 { 434 name: "ingress-with-chain-and-failover", 435 create: proxycfg.TestConfigSnapshotIngressWithFailover, 436 setup: nil, 437 }, 438 { 439 name: "ingress-with-tcp-chain-failover-through-remote-gateway", 440 create: proxycfg.TestConfigSnapshotIngressWithFailoverThroughRemoteGateway, 441 setup: nil, 442 }, 443 { 444 name: "ingress-with-tcp-chain-failover-through-remote-gateway-triggered", 445 create: proxycfg.TestConfigSnapshotIngressWithFailoverThroughRemoteGatewayTriggered, 446 setup: nil, 447 }, 448 { 449 name: "ingress-with-tcp-chain-double-failover-through-remote-gateway", 450 create: proxycfg.TestConfigSnapshotIngressWithDoubleFailoverThroughRemoteGateway, 451 setup: nil, 452 }, 453 { 454 name: "ingress-with-tcp-chain-double-failover-through-remote-gateway-triggered", 455 create: proxycfg.TestConfigSnapshotIngressWithDoubleFailoverThroughRemoteGatewayTriggered, 456 setup: nil, 457 }, 458 { 459 name: "ingress-with-tcp-chain-failover-through-local-gateway", 460 create: proxycfg.TestConfigSnapshotIngressWithFailoverThroughLocalGateway, 461 setup: nil, 462 }, 463 { 464 name: "ingress-with-tcp-chain-failover-through-local-gateway-triggered", 465 create: proxycfg.TestConfigSnapshotIngressWithFailoverThroughLocalGatewayTriggered, 466 setup: nil, 467 }, 468 { 469 name: "ingress-with-tcp-chain-double-failover-through-local-gateway", 470 create: proxycfg.TestConfigSnapshotIngressWithDoubleFailoverThroughLocalGateway, 471 setup: nil, 472 }, 473 { 474 name: "ingress-with-tcp-chain-double-failover-through-local-gateway-triggered", 475 create: proxycfg.TestConfigSnapshotIngressWithDoubleFailoverThroughLocalGatewayTriggered, 476 setup: nil, 477 }, 478 { 479 name: "ingress-splitter-with-resolver-redirect", 480 create: proxycfg.TestConfigSnapshotIngress_SplitterWithResolverRedirectMultiDC, 481 setup: nil, 482 }, 483 { 484 name: "ingress-lb-in-resolver", 485 create: proxycfg.TestConfigSnapshotIngressWithLB, 486 setup: nil, 487 }, 488 { 489 name: "terminating-gateway", 490 create: proxycfg.TestConfigSnapshotTerminatingGateway, 491 setup: nil, 492 }, 493 { 494 name: "terminating-gateway-no-services", 495 create: proxycfg.TestConfigSnapshotTerminatingGatewayNoServices, 496 setup: nil, 497 }, 498 { 499 name: "terminating-gateway-service-subsets", 500 create: proxycfg.TestConfigSnapshotTerminatingGateway, 501 setup: func(snap *proxycfg.ConfigSnapshot) { 502 snap.TerminatingGateway.ServiceResolvers = map[structs.ServiceName]*structs.ServiceResolverConfigEntry{ 503 structs.NewServiceName("web", nil): { 504 Kind: structs.ServiceResolver, 505 Name: "web", 506 Subsets: map[string]structs.ServiceResolverSubset{ 507 "v1": { 508 Filter: "Service.Meta.Version == 1", 509 }, 510 "v2": { 511 Filter: "Service.Meta.Version == 2", 512 OnlyPassing: true, 513 }, 514 }, 515 }, 516 structs.NewServiceName("cache", nil): { 517 Kind: structs.ServiceResolver, 518 Name: "cache", 519 Subsets: map[string]structs.ServiceResolverSubset{ 520 "prod": { 521 Filter: "Service.Meta.Env == prod", 522 }, 523 }, 524 }, 525 } 526 snap.TerminatingGateway.ServiceConfigs[structs.NewServiceName("web", nil)] = &structs.ServiceConfigResponse{ 527 ProxyConfig: map[string]interface{}{"protocol": "http"}, 528 } 529 snap.TerminatingGateway.ServiceConfigs[structs.NewServiceName("cache", nil)] = &structs.ServiceConfigResponse{ 530 ProxyConfig: map[string]interface{}{"protocol": "http"}, 531 } 532 }, 533 }, 534 { 535 name: "terminating-gateway-hostname-service-subsets", 536 create: proxycfg.TestConfigSnapshotTerminatingGateway, 537 setup: func(snap *proxycfg.ConfigSnapshot) { 538 snap.TerminatingGateway.ServiceResolvers = map[structs.ServiceName]*structs.ServiceResolverConfigEntry{ 539 structs.NewServiceName("api", nil): { 540 Kind: structs.ServiceResolver, 541 Name: "api", 542 Subsets: map[string]structs.ServiceResolverSubset{ 543 "alt": { 544 Filter: "Service.Meta.domain == alt", 545 }, 546 }, 547 }, 548 structs.NewServiceName("cache", nil): { 549 Kind: structs.ServiceResolver, 550 Name: "cache", 551 Subsets: map[string]structs.ServiceResolverSubset{ 552 "prod": { 553 Filter: "Service.Meta.Env == prod", 554 }, 555 }, 556 }, 557 } 558 snap.TerminatingGateway.ServiceConfigs[structs.NewServiceName("api", nil)] = &structs.ServiceConfigResponse{ 559 ProxyConfig: map[string]interface{}{"protocol": "http"}, 560 } 561 snap.TerminatingGateway.ServiceConfigs[structs.NewServiceName("cache", nil)] = &structs.ServiceConfigResponse{ 562 ProxyConfig: map[string]interface{}{"protocol": "http"}, 563 } 564 }, 565 }, 566 { 567 name: "terminating-gateway-ignore-extra-resolvers", 568 create: proxycfg.TestConfigSnapshotTerminatingGateway, 569 setup: func(snap *proxycfg.ConfigSnapshot) { 570 snap.TerminatingGateway.ServiceResolvers = map[structs.ServiceName]*structs.ServiceResolverConfigEntry{ 571 structs.NewServiceName("web", nil): { 572 Kind: structs.ServiceResolver, 573 Name: "web", 574 DefaultSubset: "v2", 575 Subsets: map[string]structs.ServiceResolverSubset{ 576 "v1": { 577 Filter: "Service.Meta.Version == 1", 578 }, 579 "v2": { 580 Filter: "Service.Meta.Version == 2", 581 OnlyPassing: true, 582 }, 583 }, 584 }, 585 structs.NewServiceName("notfound", nil): { 586 Kind: structs.ServiceResolver, 587 Name: "notfound", 588 DefaultSubset: "v2", 589 Subsets: map[string]structs.ServiceResolverSubset{ 590 "v1": { 591 Filter: "Service.Meta.Version == 1", 592 }, 593 "v2": { 594 Filter: "Service.Meta.Version == 2", 595 OnlyPassing: true, 596 }, 597 }, 598 }, 599 } 600 snap.TerminatingGateway.ServiceConfigs[structs.NewServiceName("web", nil)] = &structs.ServiceConfigResponse{ 601 ProxyConfig: map[string]interface{}{"protocol": "http"}, 602 } 603 }, 604 }, 605 { 606 name: "terminating-gateway-lb-config", 607 create: proxycfg.TestConfigSnapshotTerminatingGateway, 608 setup: func(snap *proxycfg.ConfigSnapshot) { 609 snap.TerminatingGateway.ServiceResolvers = map[structs.ServiceName]*structs.ServiceResolverConfigEntry{ 610 structs.NewServiceName("web", nil): { 611 Kind: structs.ServiceResolver, 612 Name: "web", 613 DefaultSubset: "v2", 614 Subsets: map[string]structs.ServiceResolverSubset{ 615 "v1": { 616 Filter: "Service.Meta.Version == 1", 617 }, 618 "v2": { 619 Filter: "Service.Meta.Version == 2", 620 OnlyPassing: true, 621 }, 622 }, 623 LoadBalancer: &structs.LoadBalancer{ 624 Policy: "ring_hash", 625 RingHashConfig: &structs.RingHashConfig{ 626 MinimumRingSize: 20, 627 MaximumRingSize: 50, 628 }, 629 }, 630 }, 631 } 632 snap.TerminatingGateway.ServiceConfigs[structs.NewServiceName("web", nil)] = &structs.ServiceConfigResponse{ 633 ProxyConfig: map[string]interface{}{"protocol": "http"}, 634 } 635 }, 636 }, 637 { 638 name: "ingress-multiple-listeners-duplicate-service", 639 create: proxycfg.TestConfigSnapshotIngress_MultipleListenersDuplicateService, 640 setup: nil, 641 }, 642 { 643 name: "transparent-proxy", 644 create: proxycfg.TestConfigSnapshot, 645 setup: func(snap *proxycfg.ConfigSnapshot) { 646 snap.Proxy.Mode = structs.ProxyModeTransparent 647 snap.ConnectProxy.MeshConfigSet = true 648 }, 649 }, 650 { 651 name: "transparent-proxy-catalog-destinations-only", 652 create: proxycfg.TestConfigSnapshot, 653 setup: func(snap *proxycfg.ConfigSnapshot) { 654 snap.Proxy.Mode = structs.ProxyModeTransparent 655 656 snap.ConnectProxy.MeshConfigSet = true 657 snap.ConnectProxy.MeshConfig = &structs.MeshConfigEntry{ 658 TransparentProxy: structs.TransparentProxyMeshConfig{ 659 MeshDestinationsOnly: true, 660 }, 661 } 662 }, 663 }, 664 { 665 name: "transparent-proxy-dial-instances-directly", 666 create: proxycfg.TestConfigSnapshot, 667 setup: func(snap *proxycfg.ConfigSnapshot) { 668 snap.Proxy.Mode = structs.ProxyModeTransparent 669 670 // We add a passthrough cluster for each upstream service name 671 snap.ConnectProxy.PassthroughUpstreams = map[string]proxycfg.ServicePassthroughAddrs{ 672 "default/kafka": { 673 SNI: "kafka.default.dc1.internal.e5b08d03-bfc3-c870-1833-baddb116e648.consul", 674 SpiffeID: connect.SpiffeIDService{ 675 Host: "e5b08d03-bfc3-c870-1833-baddb116e648.consul", 676 Namespace: "default", 677 Datacenter: "dc1", 678 Service: "kafka", 679 }, 680 Addrs: map[string]struct{}{ 681 "9.9.9.9": {}, 682 }, 683 }, 684 "default/mongo": { 685 SNI: "mongo.default.dc1.internal.e5b08d03-bfc3-c870-1833-baddb116e648.consul", 686 SpiffeID: connect.SpiffeIDService{ 687 Host: "e5b08d03-bfc3-c870-1833-baddb116e648.consul", 688 Namespace: "default", 689 Datacenter: "dc1", 690 Service: "mongo", 691 }, 692 Addrs: map[string]struct{}{ 693 "10.10.10.10": {}, 694 "10.10.10.12": {}, 695 }, 696 }, 697 } 698 699 // There should still be a cluster for non-passthrough requests 700 snap.ConnectProxy.DiscoveryChain["mongo"] = discoverychain.TestCompileConfigEntries( 701 t, "mongo", "default", "dc1", 702 connect.TestClusterID+".consul", "dc1", nil) 703 snap.ConnectProxy.WatchedUpstreamEndpoints["mongo"] = map[string]structs.CheckServiceNodes{ 704 "mongo.default.dc1": { 705 structs.CheckServiceNode{ 706 Node: &structs.Node{ 707 Datacenter: "dc1", 708 }, 709 Service: &structs.NodeService{ 710 Service: "mongo", 711 Address: "7.7.7.7", 712 Port: 27017, 713 TaggedAddresses: map[string]structs.ServiceAddress{ 714 "virtual": {Address: "6.6.6.6"}, 715 }, 716 }, 717 }, 718 }, 719 } 720 }, 721 }, 722 } 723 724 latestEnvoyVersion := proxysupport.EnvoyVersions[0] 725 latestEnvoyVersion_v2 := proxysupport.EnvoyVersionsV2[0] 726 for _, envoyVersion := range proxysupport.EnvoyVersions { 727 sf, err := determineSupportedProxyFeaturesFromString(envoyVersion) 728 require.NoError(t, err) 729 t.Run("envoy-"+envoyVersion, func(t *testing.T) { 730 for _, tt := range tests { 731 t.Run(tt.name, func(t *testing.T) { 732 // Sanity check default with no overrides first 733 snap := tt.create(t) 734 735 // We need to replace the TLS certs with deterministic ones to make golden 736 // files workable. Note we don't update these otherwise they'd change 737 // golder files for every test case and so not be any use! 738 setupTLSRootsAndLeaf(t, snap) 739 740 if tt.setup != nil { 741 tt.setup(snap) 742 } 743 744 // Need server just for logger dependency 745 g := newResourceGenerator(testutil.Logger(t), nil, nil, false) 746 g.ProxyFeatures = sf 747 748 clusters, err := g.clustersFromSnapshot(snap) 749 require.NoError(t, err) 750 751 sort.Slice(clusters, func(i, j int) bool { 752 return clusters[i].(*envoy_cluster_v3.Cluster).Name < clusters[j].(*envoy_cluster_v3.Cluster).Name 753 }) 754 755 r, err := createResponse(ClusterType, "00000001", "00000001", clusters) 756 require.NoError(t, err) 757 758 t.Run("current", func(t *testing.T) { 759 gotJSON := protoToJSON(t, r) 760 761 gName := tt.name 762 if tt.overrideGoldenName != "" { 763 gName = tt.overrideGoldenName 764 } 765 766 require.JSONEq(t, goldenEnvoy(t, filepath.Join("clusters", gName), envoyVersion, latestEnvoyVersion, gotJSON), gotJSON) 767 }) 768 769 t.Run("v2-compat", func(t *testing.T) { 770 if !stringslice.Contains(proxysupport.EnvoyVersionsV2, envoyVersion) { 771 t.Skip() 772 } 773 respV2, err := convertDiscoveryResponseToV2(r) 774 require.NoError(t, err) 775 776 gotJSON := protoToJSON(t, respV2) 777 778 gName := tt.name 779 if tt.overrideGoldenName != "" { 780 gName = tt.overrideGoldenName 781 } 782 783 gName += ".v2compat" 784 785 require.JSONEq(t, goldenEnvoy(t, filepath.Join("clusters", gName), envoyVersion, latestEnvoyVersion_v2, gotJSON), gotJSON) 786 }) 787 }) 788 } 789 }) 790 } 791} 792 793type customClusterJSONOptions struct { 794 Name string 795 TLSContext string 796} 797 798var customAppClusterJSONTpl = `{ 799 "@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster", 800 {{ if .TLSContext -}} 801 "transport_socket": { 802 "name": "tls", 803 "typed_config": { 804 "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext", 805 {{ .TLSContext }} 806 } 807 }, 808 {{- end }} 809 "name": "{{ .Name }}", 810 "connectTimeout": "15s", 811 "loadAssignment": { 812 "clusterName": "{{ .Name }}", 813 "endpoints": [ 814 { 815 "lbEndpoints": [ 816 { 817 "endpoint": { 818 "address": { 819 "socketAddress": { 820 "address": "127.0.0.1", 821 "portValue": 8080 822 } 823 } 824 } 825 } 826 ] 827 } 828 ] 829 } 830}` 831 832var customAppClusterJSONTemplate = template.Must(template.New("").Parse(customAppClusterJSONTpl)) 833 834func customAppClusterJSON(t *testing.T, opts customClusterJSONOptions) string { 835 t.Helper() 836 var buf bytes.Buffer 837 err := customAppClusterJSONTemplate.Execute(&buf, opts) 838 require.NoError(t, err) 839 return buf.String() 840} 841 842func setupTLSRootsAndLeaf(t *testing.T, snap *proxycfg.ConfigSnapshot) { 843 if snap.Leaf() != nil { 844 switch snap.Kind { 845 case structs.ServiceKindConnectProxy: 846 snap.ConnectProxy.Leaf.CertPEM = loadTestResource(t, "test-leaf-cert") 847 snap.ConnectProxy.Leaf.PrivateKeyPEM = loadTestResource(t, "test-leaf-key") 848 case structs.ServiceKindIngressGateway: 849 snap.IngressGateway.Leaf.CertPEM = loadTestResource(t, "test-leaf-cert") 850 snap.IngressGateway.Leaf.PrivateKeyPEM = loadTestResource(t, "test-leaf-key") 851 } 852 } 853 if snap.Roots != nil { 854 snap.Roots.Roots[0].RootCert = loadTestResource(t, "test-root-cert") 855 } 856} 857 858func TestEnvoyLBConfig_InjectToCluster(t *testing.T) { 859 var tests = []struct { 860 name string 861 lb *structs.LoadBalancer 862 expected *envoy_cluster_v3.Cluster 863 }{ 864 { 865 name: "skip empty", 866 lb: &structs.LoadBalancer{ 867 Policy: "", 868 }, 869 expected: &envoy_cluster_v3.Cluster{}, 870 }, 871 { 872 name: "round robin", 873 lb: &structs.LoadBalancer{ 874 Policy: structs.LBPolicyRoundRobin, 875 }, 876 expected: &envoy_cluster_v3.Cluster{LbPolicy: envoy_cluster_v3.Cluster_ROUND_ROBIN}, 877 }, 878 { 879 name: "random", 880 lb: &structs.LoadBalancer{ 881 Policy: structs.LBPolicyRandom, 882 }, 883 expected: &envoy_cluster_v3.Cluster{LbPolicy: envoy_cluster_v3.Cluster_RANDOM}, 884 }, 885 { 886 name: "maglev", 887 lb: &structs.LoadBalancer{ 888 Policy: structs.LBPolicyMaglev, 889 }, 890 expected: &envoy_cluster_v3.Cluster{LbPolicy: envoy_cluster_v3.Cluster_MAGLEV}, 891 }, 892 { 893 name: "ring_hash", 894 lb: &structs.LoadBalancer{ 895 Policy: structs.LBPolicyRingHash, 896 RingHashConfig: &structs.RingHashConfig{ 897 MinimumRingSize: 3, 898 MaximumRingSize: 7, 899 }, 900 }, 901 expected: &envoy_cluster_v3.Cluster{ 902 LbPolicy: envoy_cluster_v3.Cluster_RING_HASH, 903 LbConfig: &envoy_cluster_v3.Cluster_RingHashLbConfig_{ 904 RingHashLbConfig: &envoy_cluster_v3.Cluster_RingHashLbConfig{ 905 MinimumRingSize: &wrappers.UInt64Value{Value: 3}, 906 MaximumRingSize: &wrappers.UInt64Value{Value: 7}, 907 }, 908 }, 909 }, 910 }, 911 { 912 name: "least_request", 913 lb: &structs.LoadBalancer{ 914 Policy: "least_request", 915 LeastRequestConfig: &structs.LeastRequestConfig{ 916 ChoiceCount: 3, 917 }, 918 }, 919 expected: &envoy_cluster_v3.Cluster{ 920 LbPolicy: envoy_cluster_v3.Cluster_LEAST_REQUEST, 921 LbConfig: &envoy_cluster_v3.Cluster_LeastRequestLbConfig_{ 922 LeastRequestLbConfig: &envoy_cluster_v3.Cluster_LeastRequestLbConfig{ 923 ChoiceCount: &wrappers.UInt32Value{Value: 3}, 924 }, 925 }, 926 }, 927 }, 928 } 929 930 for _, tc := range tests { 931 t.Run(tc.name, func(t *testing.T) { 932 var c envoy_cluster_v3.Cluster 933 err := injectLBToCluster(tc.lb, &c) 934 require.NoError(t, err) 935 936 require.Equal(t, tc.expected, &c) 937 }) 938 } 939} 940