1// +build !providerless 2 3/* 4Copyright 2018 The Kubernetes Authors. 5 6Licensed under the Apache License, Version 2.0 (the "License"); 7you may not use this file except in compliance with the License. 8You may obtain a copy of the License at 9 10 http://www.apache.org/licenses/LICENSE-2.0 11 12Unless required by applicable law or agreed to in writing, software 13distributed under the License is distributed on an "AS IS" BASIS, 14WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15See the License for the specific language governing permissions and 16limitations under the License. 17*/ 18 19package azure 20 21import ( 22 "context" 23 "fmt" 24 "net" 25 "net/http" 26 "strings" 27 "testing" 28 29 "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-12-01/compute" 30 "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-06-01/network" 31 "github.com/Azure/go-autorest/autorest/to" 32 "github.com/golang/mock/gomock" 33 "github.com/stretchr/testify/assert" 34 35 v1 "k8s.io/api/core/v1" 36 "k8s.io/apimachinery/pkg/types" 37 cloudprovider "k8s.io/cloud-provider" 38 azcache "k8s.io/legacy-cloud-providers/azure/cache" 39 "k8s.io/legacy-cloud-providers/azure/clients/interfaceclient/mockinterfaceclient" 40 "k8s.io/legacy-cloud-providers/azure/clients/publicipclient/mockpublicipclient" 41 "k8s.io/legacy-cloud-providers/azure/clients/vmclient/mockvmclient" 42 "k8s.io/legacy-cloud-providers/azure/clients/vmssclient/mockvmssclient" 43 "k8s.io/legacy-cloud-providers/azure/clients/vmssvmclient/mockvmssvmclient" 44 "k8s.io/legacy-cloud-providers/azure/retry" 45) 46 47// setTestVirtualMachines sets test virtual machine with powerstate. 48func setTestVirtualMachines(c *Cloud, vmList map[string]string, isDataDisksFull bool) []compute.VirtualMachine { 49 expectedVMs := make([]compute.VirtualMachine, 0) 50 51 for nodeName, powerState := range vmList { 52 instanceID := fmt.Sprintf("/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Compute/virtualMachines/%s", nodeName) 53 vm := compute.VirtualMachine{ 54 Name: &nodeName, 55 ID: &instanceID, 56 Location: &c.Location, 57 } 58 status := []compute.InstanceViewStatus{ 59 { 60 Code: to.StringPtr(powerState), 61 }, 62 { 63 Code: to.StringPtr("ProvisioningState/succeeded"), 64 }, 65 } 66 vm.VirtualMachineProperties = &compute.VirtualMachineProperties{ 67 ProvisioningState: to.StringPtr(string(compute.ProvisioningStateSucceeded)), 68 HardwareProfile: &compute.HardwareProfile{ 69 VMSize: compute.VirtualMachineSizeTypesStandardA0, 70 }, 71 InstanceView: &compute.VirtualMachineInstanceView{ 72 Statuses: &status, 73 }, 74 StorageProfile: &compute.StorageProfile{ 75 DataDisks: &[]compute.DataDisk{}, 76 }, 77 } 78 if !isDataDisksFull { 79 vm.StorageProfile.DataDisks = &[]compute.DataDisk{{ 80 Lun: to.Int32Ptr(0), 81 Name: to.StringPtr("disk1"), 82 }} 83 } else { 84 dataDisks := make([]compute.DataDisk, maxLUN) 85 for i := 0; i < maxLUN; i++ { 86 dataDisks[i] = compute.DataDisk{Lun: to.Int32Ptr(int32(i))} 87 } 88 vm.StorageProfile.DataDisks = &dataDisks 89 } 90 91 expectedVMs = append(expectedVMs, vm) 92 } 93 94 return expectedVMs 95} 96 97func TestInstanceID(t *testing.T) { 98 ctrl := gomock.NewController(t) 99 defer ctrl.Finish() 100 101 cloud := GetTestCloud(ctrl) 102 103 testcases := []struct { 104 name string 105 vmList []string 106 nodeName string 107 vmssName string 108 metadataName string 109 metadataTemplate string 110 vmType string 111 expectedID string 112 useInstanceMetadata bool 113 useCustomImsCache bool 114 nilVMSet bool 115 expectedErrMsg error 116 }{ 117 { 118 name: "InstanceID should get instanceID if node's name are equal to metadataName", 119 vmList: []string{"vm1"}, 120 nodeName: "vm1", 121 metadataName: "vm1", 122 vmType: vmTypeStandard, 123 useInstanceMetadata: true, 124 expectedID: "/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Compute/virtualMachines/vm1", 125 }, 126 { 127 name: "InstanceID should get vmss instanceID from local if node's name are equal to metadataName and metadata.Compute.VMScaleSetName is not null", 128 vmList: []string{"vmss1_0"}, 129 vmssName: "vmss1", 130 nodeName: "vmss1_0", 131 metadataName: "vmss1_0", 132 vmType: vmTypeStandard, 133 useInstanceMetadata: true, 134 expectedID: "/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Compute/virtualMachineScaleSets/vmss1/virtualMachines/0", 135 }, 136 { 137 name: "InstanceID should get standard instanceID from local if node's name are equal to metadataName and format of nodeName is not compliance with vmss instance", 138 vmList: []string{"vmss1-0"}, 139 vmssName: "vmss1", 140 nodeName: "vmss1-0", 141 metadataName: "vmss1-0", 142 vmType: vmTypeStandard, 143 useInstanceMetadata: true, 144 expectedID: "/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Compute/virtualMachines/vmss1-0", 145 }, 146 { 147 name: "InstanceID should get instanceID from Azure API if node is not local instance", 148 vmList: []string{"vm2"}, 149 nodeName: "vm2", 150 metadataName: "vm1", 151 vmType: vmTypeStandard, 152 useInstanceMetadata: true, 153 expectedID: "/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Compute/virtualMachines/vm2", 154 }, 155 { 156 name: "InstanceID should get instanceID from Azure API if cloud.UseInstanceMetadata is false", 157 vmList: []string{"vm2"}, 158 nodeName: "vm2", 159 metadataName: "vm2", 160 vmType: vmTypeStandard, 161 expectedID: "/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Compute/virtualMachines/vm2", 162 }, 163 { 164 name: "InstanceID should report error if node doesn't exist", 165 vmList: []string{"vm1"}, 166 nodeName: "vm3", 167 vmType: vmTypeStandard, 168 useInstanceMetadata: true, 169 expectedErrMsg: fmt.Errorf("instance not found"), 170 }, 171 { 172 name: "InstanceID should report error if metadata.Compute is nil", 173 nodeName: "vm1", 174 metadataName: "vm1", 175 vmType: vmTypeStandard, 176 metadataTemplate: `{"network":{"interface":[]}}`, 177 useInstanceMetadata: true, 178 expectedErrMsg: fmt.Errorf("failure of getting instance metadata"), 179 }, 180 { 181 name: "NodeAddresses should report error if cloud.VMSet is nil", 182 nodeName: "vm1", 183 vmType: vmTypeStandard, 184 useInstanceMetadata: true, 185 nilVMSet: true, 186 expectedErrMsg: fmt.Errorf("no credentials provided for Azure cloud provider"), 187 }, 188 { 189 name: "NodeAddresses should report error if invoking GetMetadata returns error", 190 nodeName: "vm1", 191 metadataName: "vm1", 192 vmType: vmTypeStandard, 193 useCustomImsCache: true, 194 useInstanceMetadata: true, 195 expectedErrMsg: fmt.Errorf("getError"), 196 }, 197 } 198 199 for _, test := range testcases { 200 if test.nilVMSet { 201 cloud.VMSet = nil 202 } else { 203 cloud.VMSet = newAvailabilitySet(cloud) 204 } 205 cloud.Config.VMType = test.vmType 206 cloud.Config.UseInstanceMetadata = test.useInstanceMetadata 207 listener, err := net.Listen("tcp", "127.0.0.1:0") 208 if err != nil { 209 t.Errorf("Test [%s] unexpected error: %v", test.name, err) 210 } 211 212 mux := http.NewServeMux() 213 mux.Handle("/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 214 if test.metadataTemplate != "" { 215 fmt.Fprintf(w, test.metadataTemplate) 216 } else { 217 fmt.Fprintf(w, "{\"compute\":{\"name\":\"%s\",\"VMScaleSetName\":\"%s\",\"subscriptionId\":\"subscription\",\"resourceGroupName\":\"rg\"}}", test.metadataName, test.vmssName) 218 } 219 })) 220 go func() { 221 http.Serve(listener, mux) 222 }() 223 defer listener.Close() 224 225 cloud.metadata, err = NewInstanceMetadataService("http://" + listener.Addr().String() + "/") 226 if err != nil { 227 t.Errorf("Test [%s] unexpected error: %v", test.name, err) 228 } 229 if test.useCustomImsCache { 230 cloud.metadata.imsCache, err = azcache.NewTimedcache(metadataCacheTTL, func(key string) (interface{}, error) { 231 return nil, fmt.Errorf("getError") 232 }) 233 if err != nil { 234 t.Errorf("Test [%s] unexpected error: %v", test.name, err) 235 } 236 } 237 vmListWithPowerState := make(map[string]string) 238 for _, vm := range test.vmList { 239 vmListWithPowerState[vm] = "" 240 } 241 expectedVMs := setTestVirtualMachines(cloud, vmListWithPowerState, false) 242 mockVMsClient := cloud.VirtualMachinesClient.(*mockvmclient.MockInterface) 243 for _, vm := range expectedVMs { 244 mockVMsClient.EXPECT().Get(gomock.Any(), cloud.ResourceGroup, *vm.Name, gomock.Any()).Return(vm, nil).AnyTimes() 245 } 246 mockVMsClient.EXPECT().Get(gomock.Any(), cloud.ResourceGroup, "vm3", gomock.Any()).Return(compute.VirtualMachine{}, &retry.Error{HTTPStatusCode: http.StatusNotFound, RawError: cloudprovider.InstanceNotFound}).AnyTimes() 247 mockVMsClient.EXPECT().Update(gomock.Any(), cloud.ResourceGroup, gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() 248 249 instanceID, err := cloud.InstanceID(context.Background(), types.NodeName(test.nodeName)) 250 assert.Equal(t, test.expectedErrMsg, err, test.name) 251 assert.Equal(t, test.expectedID, instanceID, test.name) 252 } 253} 254 255func TestInstanceShutdownByProviderID(t *testing.T) { 256 testcases := []struct { 257 name string 258 vmList map[string]string 259 nodeName string 260 providerID string 261 provisioningState string 262 expected bool 263 expectedErrMsg error 264 }{ 265 { 266 name: "InstanceShutdownByProviderID should return false if the vm is in PowerState/Running status", 267 vmList: map[string]string{"vm1": "PowerState/Running"}, 268 nodeName: "vm1", 269 providerID: "azure:///subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Compute/virtualMachines/vm1", 270 expected: false, 271 }, 272 { 273 name: "InstanceShutdownByProviderID should return true if the vm is in PowerState/Deallocated status", 274 vmList: map[string]string{"vm2": "PowerState/Deallocated"}, 275 nodeName: "vm2", 276 providerID: "azure:///subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Compute/virtualMachines/vm2", 277 expected: true, 278 }, 279 { 280 name: "InstanceShutdownByProviderID should return false if the vm is in PowerState/Deallocating status", 281 vmList: map[string]string{"vm3": "PowerState/Deallocating"}, 282 nodeName: "vm3", 283 providerID: "azure:///subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Compute/virtualMachines/vm3", 284 expected: true, 285 }, 286 { 287 name: "InstanceShutdownByProviderID should return false if the vm is in PowerState/Starting status", 288 vmList: map[string]string{"vm4": "PowerState/Starting"}, 289 nodeName: "vm4", 290 providerID: "azure:///subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Compute/virtualMachines/vm4", 291 expected: false, 292 }, 293 { 294 name: "InstanceShutdownByProviderID should return true if the vm is in PowerState/Stopped status", 295 vmList: map[string]string{"vm5": "PowerState/Stopped"}, 296 nodeName: "vm5", 297 providerID: "azure:///subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Compute/virtualMachines/vm5", 298 expected: true, 299 }, 300 301 { 302 name: "InstanceShutdownByProviderID should return false if the vm is in PowerState/Stopping status", 303 vmList: map[string]string{"vm6": "PowerState/Stopping"}, 304 nodeName: "vm6", 305 providerID: "azure:///subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Compute/virtualMachines/vm6", 306 expected: false, 307 }, 308 { 309 name: "InstanceShutdownByProviderID should return false if the vm is in PowerState/Unknown status", 310 vmList: map[string]string{"vm7": "PowerState/Unknown"}, 311 nodeName: "vm7", 312 providerID: "azure:///subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Compute/virtualMachines/vm7", 313 expected: false, 314 }, 315 { 316 name: "InstanceShutdownByProviderID should return false if node doesn't exist", 317 vmList: map[string]string{"vm1": "PowerState/running"}, 318 nodeName: "vm8", 319 providerID: "azure:///subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Compute/virtualMachines/vm8", 320 expected: false, 321 }, 322 { 323 name: "InstanceShutdownByProviderID should return false if the vm is in PowerState/Stopped state with Creating provisioning state", 324 vmList: map[string]string{"vm9": "PowerState/Stopped"}, 325 nodeName: "vm9", 326 provisioningState: "Creating", 327 providerID: "azure:///subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Compute/virtualMachines/vm9", 328 expected: false, 329 }, 330 { 331 name: "InstanceShutdownByProviderID should report error if providerID is null", 332 nodeName: "vmm", 333 expected: false, 334 }, 335 { 336 name: "InstanceShutdownByProviderID should report error if providerID is invalid", 337 providerID: "azure:///subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Compute/VM/vm10", 338 nodeName: "vm10", 339 expected: false, 340 expectedErrMsg: fmt.Errorf("error splitting providerID"), 341 }, 342 } 343 344 ctrl := gomock.NewController(t) 345 defer ctrl.Finish() 346 for _, test := range testcases { 347 cloud := GetTestCloud(ctrl) 348 expectedVMs := setTestVirtualMachines(cloud, test.vmList, false) 349 if test.provisioningState != "" { 350 expectedVMs[0].ProvisioningState = to.StringPtr(test.provisioningState) 351 } 352 mockVMsClient := cloud.VirtualMachinesClient.(*mockvmclient.MockInterface) 353 for _, vm := range expectedVMs { 354 mockVMsClient.EXPECT().Get(gomock.Any(), cloud.ResourceGroup, *vm.Name, gomock.Any()).Return(vm, nil).AnyTimes() 355 } 356 mockVMsClient.EXPECT().Get(gomock.Any(), cloud.ResourceGroup, test.nodeName, gomock.Any()).Return(compute.VirtualMachine{}, &retry.Error{HTTPStatusCode: http.StatusNotFound, RawError: cloudprovider.InstanceNotFound}).AnyTimes() 357 358 hasShutdown, err := cloud.InstanceShutdownByProviderID(context.Background(), test.providerID) 359 assert.Equal(t, test.expectedErrMsg, err, test.name) 360 assert.Equal(t, test.expected, hasShutdown, test.name) 361 } 362} 363 364func TestNodeAddresses(t *testing.T) { 365 ctrl := gomock.NewController(t) 366 defer ctrl.Finish() 367 cloud := GetTestCloud(ctrl) 368 369 expectedVM := compute.VirtualMachine{ 370 VirtualMachineProperties: &compute.VirtualMachineProperties{ 371 NetworkProfile: &compute.NetworkProfile{ 372 NetworkInterfaces: &[]compute.NetworkInterfaceReference{ 373 { 374 NetworkInterfaceReferenceProperties: &compute.NetworkInterfaceReferenceProperties{ 375 Primary: to.BoolPtr(true), 376 }, 377 ID: to.StringPtr("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Network/networkInterfaces/nic"), 378 }, 379 }, 380 }, 381 }, 382 } 383 384 expectedPIP := network.PublicIPAddress{ 385 ID: to.StringPtr("/subscriptions/subscriptionID/resourceGroups/rg/providers/Microsoft.Network/publicIPAddresses/pip1"), 386 PublicIPAddressPropertiesFormat: &network.PublicIPAddressPropertiesFormat{ 387 IPAddress: to.StringPtr("192.168.1.12"), 388 }, 389 } 390 391 expectedInterface := network.Interface{ 392 InterfacePropertiesFormat: &network.InterfacePropertiesFormat{ 393 IPConfigurations: &[]network.InterfaceIPConfiguration{ 394 { 395 InterfaceIPConfigurationPropertiesFormat: &network.InterfaceIPConfigurationPropertiesFormat{ 396 PrivateIPAddress: to.StringPtr("172.1.0.3"), 397 PublicIPAddress: &expectedPIP, 398 }, 399 }, 400 }, 401 }, 402 } 403 404 expectedNodeAddress := []v1.NodeAddress{ 405 { 406 Type: v1.NodeInternalIP, 407 Address: "172.1.0.3", 408 }, 409 { 410 Type: v1.NodeHostName, 411 Address: "vm1", 412 }, 413 { 414 Type: v1.NodeExternalIP, 415 Address: "192.168.1.12", 416 }, 417 } 418 metadataTemplate := `{"compute":{"name":"%s"},"network":{"interface":[{"ipv4":{"ipAddress":[{"privateIpAddress":"%s","publicIpAddress":"%s"}]},"ipv6":{"ipAddress":[{"privateIpAddress":"%s","publicIpAddress":"%s"}]}}]}}` 419 loadbalancerTemplate := `{"loadbalancer": {"publicIpAddresses": [{"frontendIpAddress": "%s","privateIpAddress": "%s"},{"frontendIpAddress": "%s","privateIpAddress": "%s"}]}}` 420 testcases := []struct { 421 name string 422 nodeName string 423 metadataName string 424 metadataTemplate string 425 vmType string 426 ipV4 string 427 ipV6 string 428 ipV4Public string 429 ipV6Public string 430 loadBalancerSku string 431 expectedAddress []v1.NodeAddress 432 useInstanceMetadata bool 433 useCustomImsCache bool 434 nilVMSet bool 435 expectedErrMsg error 436 }{ 437 { 438 name: "NodeAddresses should report error if metadata.Network is nil", 439 metadataTemplate: `{"compute":{"name":"vm1"}}`, 440 useInstanceMetadata: true, 441 expectedErrMsg: fmt.Errorf("failure of getting instance metadata"), 442 }, 443 { 444 name: "NodeAddresses should report error if metadata.Compute is nil", 445 metadataTemplate: `{"network":{"interface":[]}}`, 446 useInstanceMetadata: true, 447 expectedErrMsg: fmt.Errorf("failure of getting instance metadata"), 448 }, 449 { 450 name: "NodeAddresses should report error if metadata.Network.Interface is nil", 451 nodeName: "vm1", 452 metadataName: "vm1", 453 vmType: vmTypeStandard, 454 metadataTemplate: `{"compute":{"name":"vm1"},"network":{}}`, 455 useInstanceMetadata: true, 456 expectedErrMsg: fmt.Errorf("no interface is found for the instance"), 457 }, 458 { 459 name: "NodeAddresses should report error when invoke GetMetadata", 460 nodeName: "vm1", 461 metadataName: "vm1", 462 vmType: vmTypeStandard, 463 useCustomImsCache: true, 464 useInstanceMetadata: true, 465 expectedErrMsg: fmt.Errorf("getError"), 466 }, 467 { 468 name: "NodeAddresses should report error if cloud.VMSet is nil", 469 nodeName: "vm1", 470 vmType: vmTypeStandard, 471 useInstanceMetadata: true, 472 nilVMSet: true, 473 expectedErrMsg: fmt.Errorf("no credentials provided for Azure cloud provider"), 474 }, 475 { 476 name: "NodeAddresses should report error when IPs are empty", 477 nodeName: "vm1", 478 metadataName: "vm1", 479 vmType: vmTypeStandard, 480 useInstanceMetadata: true, 481 expectedErrMsg: fmt.Errorf("get empty IP addresses from instance metadata service"), 482 }, 483 { 484 name: "NodeAddresses should report error if node don't exist", 485 nodeName: "vm2", 486 metadataName: "vm1", 487 vmType: vmTypeStandard, 488 useInstanceMetadata: true, 489 expectedErrMsg: fmt.Errorf("timed out waiting for the condition"), 490 }, 491 { 492 name: "NodeAddresses should get IP addresses from Azure API if node's name isn't equal to metadataName", 493 nodeName: "vm1", 494 vmType: vmTypeStandard, 495 useInstanceMetadata: true, 496 expectedAddress: expectedNodeAddress, 497 }, 498 { 499 name: "NodeAddresses should get IP addresses from Azure API if useInstanceMetadata is false", 500 nodeName: "vm1", 501 vmType: vmTypeStandard, 502 expectedAddress: expectedNodeAddress, 503 }, 504 { 505 name: "NodeAddresses should get IP addresses from local IMDS if node's name is equal to metadataName", 506 nodeName: "vm1", 507 metadataName: "vm1", 508 vmType: vmTypeStandard, 509 ipV4: "10.240.0.1", 510 ipV4Public: "192.168.1.12", 511 ipV6: "1111:11111:00:00:1111:1111:000:111", 512 ipV6Public: "2222:22221:00:00:2222:2222:000:111", 513 loadBalancerSku: "basic", 514 useInstanceMetadata: true, 515 expectedAddress: []v1.NodeAddress{ 516 { 517 Type: v1.NodeHostName, 518 Address: "vm1", 519 }, 520 { 521 Type: v1.NodeInternalIP, 522 Address: "10.240.0.1", 523 }, 524 { 525 Type: v1.NodeExternalIP, 526 Address: "192.168.1.12", 527 }, 528 { 529 Type: v1.NodeInternalIP, 530 Address: "1111:11111:00:00:1111:1111:000:111", 531 }, 532 { 533 Type: v1.NodeExternalIP, 534 Address: "2222:22221:00:00:2222:2222:000:111", 535 }, 536 }, 537 }, 538 { 539 name: "NodeAddresses should get IP addresses from local IMDS for standard LoadBalancer if node's name is equal to metadataName", 540 nodeName: "vm1", 541 metadataName: "vm1", 542 vmType: vmTypeStandard, 543 ipV4: "10.240.0.1", 544 ipV4Public: "192.168.1.12", 545 ipV6: "1111:11111:00:00:1111:1111:000:111", 546 ipV6Public: "2222:22221:00:00:2222:2222:000:111", 547 loadBalancerSku: "standard", 548 useInstanceMetadata: true, 549 expectedAddress: []v1.NodeAddress{ 550 { 551 Type: v1.NodeHostName, 552 Address: "vm1", 553 }, 554 { 555 Type: v1.NodeInternalIP, 556 Address: "10.240.0.1", 557 }, 558 { 559 Type: v1.NodeExternalIP, 560 Address: "192.168.1.12", 561 }, 562 { 563 Type: v1.NodeInternalIP, 564 Address: "1111:11111:00:00:1111:1111:000:111", 565 }, 566 { 567 Type: v1.NodeExternalIP, 568 Address: "2222:22221:00:00:2222:2222:000:111", 569 }, 570 }, 571 }, 572 } 573 574 for _, test := range testcases { 575 if test.nilVMSet { 576 cloud.VMSet = nil 577 } else { 578 cloud.VMSet = newAvailabilitySet(cloud) 579 } 580 cloud.Config.VMType = test.vmType 581 cloud.Config.UseInstanceMetadata = test.useInstanceMetadata 582 listener, err := net.Listen("tcp", "127.0.0.1:0") 583 if err != nil { 584 t.Errorf("Test [%s] unexpected error: %v", test.name, err) 585 } 586 587 mux := http.NewServeMux() 588 mux.Handle("/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 589 if strings.Contains(r.RequestURI, imdsLoadBalancerURI) { 590 fmt.Fprintf(w, loadbalancerTemplate, test.ipV4Public, test.ipV4, test.ipV6Public, test.ipV6) 591 return 592 } 593 594 if test.metadataTemplate != "" { 595 fmt.Fprintf(w, test.metadataTemplate) 596 } else { 597 if test.loadBalancerSku == "standard" { 598 fmt.Fprintf(w, metadataTemplate, test.metadataName, test.ipV4, "", test.ipV6, "") 599 } else { 600 fmt.Fprintf(w, metadataTemplate, test.metadataName, test.ipV4, test.ipV4Public, test.ipV6, test.ipV6Public) 601 } 602 } 603 })) 604 go func() { 605 http.Serve(listener, mux) 606 }() 607 defer listener.Close() 608 609 cloud.metadata, err = NewInstanceMetadataService("http://" + listener.Addr().String() + "/") 610 if err != nil { 611 t.Errorf("Test [%s] unexpected error: %v", test.name, err) 612 } 613 614 if test.useCustomImsCache { 615 cloud.metadata.imsCache, err = azcache.NewTimedcache(metadataCacheTTL, func(key string) (interface{}, error) { 616 return nil, fmt.Errorf("getError") 617 }) 618 if err != nil { 619 t.Errorf("Test [%s] unexpected error: %v", test.name, err) 620 } 621 } 622 mockVMClient := cloud.VirtualMachinesClient.(*mockvmclient.MockInterface) 623 mockVMClient.EXPECT().Get(gomock.Any(), cloud.ResourceGroup, "vm1", gomock.Any()).Return(expectedVM, nil).AnyTimes() 624 mockVMClient.EXPECT().Get(gomock.Any(), cloud.ResourceGroup, "vm2", gomock.Any()).Return(compute.VirtualMachine{}, &retry.Error{HTTPStatusCode: http.StatusNotFound, RawError: cloudprovider.InstanceNotFound}).AnyTimes() 625 626 mockPublicIPAddressesClient := cloud.PublicIPAddressesClient.(*mockpublicipclient.MockInterface) 627 mockPublicIPAddressesClient.EXPECT().Get(gomock.Any(), cloud.ResourceGroup, "pip1", gomock.Any()).Return(expectedPIP, nil).AnyTimes() 628 629 mockInterfaceClient := cloud.InterfacesClient.(*mockinterfaceclient.MockInterface) 630 mockInterfaceClient.EXPECT().Get(gomock.Any(), cloud.ResourceGroup, "nic", gomock.Any()).Return(expectedInterface, nil).AnyTimes() 631 632 ipAddresses, err := cloud.NodeAddresses(context.Background(), types.NodeName(test.nodeName)) 633 assert.Equal(t, test.expectedErrMsg, err, test.name) 634 assert.Equal(t, test.expectedAddress, ipAddresses, test.name) 635 } 636} 637 638func TestInstanceExistsByProviderID(t *testing.T) { 639 ctrl := gomock.NewController(t) 640 defer ctrl.Finish() 641 cloud := GetTestCloud(ctrl) 642 643 testcases := []struct { 644 name string 645 vmList []string 646 nodeName string 647 providerID string 648 expected bool 649 expectedErrMsg error 650 }{ 651 { 652 name: "InstanceExistsByProviderID should return true if node exists", 653 vmList: []string{"vm2"}, 654 nodeName: "vm2", 655 providerID: "azure:///subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Compute/virtualMachines/vm2", 656 expected: true, 657 }, 658 { 659 name: "InstanceExistsByProviderID should return true if node is unmanaged", 660 providerID: "/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Compute/virtualMachines/vm1", 661 expected: true, 662 }, 663 { 664 name: "InstanceExistsByProviderID should return false if node doesn't exist", 665 vmList: []string{"vm1"}, 666 nodeName: "vm3", 667 providerID: "azure:///subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Compute/virtualMachines/vm3", 668 expected: false, 669 }, 670 { 671 name: "InstanceExistsByProviderID should report error if providerID is invalid", 672 providerID: "azure:///subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Compute/virtualMachine/vm3", 673 expected: false, 674 expectedErrMsg: fmt.Errorf("error splitting providerID"), 675 }, 676 { 677 name: "InstanceExistsByProviderID should report error if providerID is null", 678 expected: false, 679 expectedErrMsg: fmt.Errorf("providerID is empty, the node is not initialized yet"), 680 }, 681 } 682 683 for _, test := range testcases { 684 vmListWithPowerState := make(map[string]string) 685 for _, vm := range test.vmList { 686 vmListWithPowerState[vm] = "" 687 } 688 expectedVMs := setTestVirtualMachines(cloud, vmListWithPowerState, false) 689 mockVMsClient := cloud.VirtualMachinesClient.(*mockvmclient.MockInterface) 690 for _, vm := range expectedVMs { 691 mockVMsClient.EXPECT().Get(gomock.Any(), cloud.ResourceGroup, *vm.Name, gomock.Any()).Return(vm, nil).AnyTimes() 692 } 693 mockVMsClient.EXPECT().Get(gomock.Any(), cloud.ResourceGroup, "vm3", gomock.Any()).Return(compute.VirtualMachine{}, &retry.Error{HTTPStatusCode: http.StatusNotFound, RawError: cloudprovider.InstanceNotFound}).AnyTimes() 694 mockVMsClient.EXPECT().Update(gomock.Any(), cloud.ResourceGroup, gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() 695 696 exist, err := cloud.InstanceExistsByProviderID(context.Background(), test.providerID) 697 assert.Equal(t, test.expectedErrMsg, err, test.name) 698 assert.Equal(t, test.expected, exist, test.name) 699 } 700 701 vmssTestCases := []struct { 702 name string 703 providerID string 704 scaleSet string 705 vmList []string 706 expected bool 707 rerr *retry.Error 708 }{ 709 { 710 name: "InstanceExistsByProviderID should return true if VMSS and VM exist", 711 providerID: "azure:///subscriptions/script/resourceGroups/rg/providers/Microsoft.Compute/virtualMachineScaleSets/vmssee6c2/virtualMachines/0", 712 scaleSet: "vmssee6c2", 713 vmList: []string{"vmssee6c2000000"}, 714 expected: true, 715 }, 716 { 717 name: "InstanceExistsByProviderID should return false if VMSS exist but VM doesn't", 718 providerID: "azure:///subscriptions/script/resourceGroups/rg/providers/Microsoft.Compute/virtualMachineScaleSets/vmssee6c2/virtualMachines/0", 719 scaleSet: "vmssee6c2", 720 expected: false, 721 }, 722 { 723 name: "InstanceExistsByProviderID should return false if VMSS doesn't exist", 724 providerID: "azure:///subscriptions/script/resourceGroups/rg/providers/Microsoft.Compute/virtualMachineScaleSets/missing-vmss/virtualMachines/0", 725 rerr: &retry.Error{HTTPStatusCode: 404}, 726 expected: false, 727 }, 728 } 729 730 for _, test := range vmssTestCases { 731 ss, err := newTestScaleSet(ctrl) 732 assert.NoError(t, err, test.name) 733 cloud.VMSet = ss 734 735 mockVMSSClient := mockvmssclient.NewMockInterface(ctrl) 736 mockVMSSVMClient := mockvmssvmclient.NewMockInterface(ctrl) 737 ss.cloud.VirtualMachineScaleSetsClient = mockVMSSClient 738 ss.cloud.VirtualMachineScaleSetVMsClient = mockVMSSVMClient 739 740 expectedScaleSet := buildTestVMSS(test.scaleSet, test.scaleSet) 741 mockVMSSClient.EXPECT().List(gomock.Any(), gomock.Any()).Return([]compute.VirtualMachineScaleSet{expectedScaleSet}, test.rerr).AnyTimes() 742 743 expectedVMs, _, _ := buildTestVirtualMachineEnv(ss.cloud, test.scaleSet, "", 0, test.vmList, "succeeded", false) 744 mockVMSSVMClient.EXPECT().List(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(expectedVMs, test.rerr).AnyTimes() 745 746 mockVMsClient := ss.cloud.VirtualMachinesClient.(*mockvmclient.MockInterface) 747 mockVMsClient.EXPECT().List(gomock.Any(), gomock.Any()).Return([]compute.VirtualMachine{}, nil).AnyTimes() 748 749 exist, _ := cloud.InstanceExistsByProviderID(context.Background(), test.providerID) 750 assert.Equal(t, test.expected, exist, test.name) 751 } 752} 753 754func TestNodeAddressesByProviderID(t *testing.T) { 755 ctrl := gomock.NewController(t) 756 defer ctrl.Finish() 757 cloud := GetTestCloud(ctrl) 758 cloud.Config.UseInstanceMetadata = true 759 metadataTemplate := `{"compute":{"name":"%s"},"network":{"interface":[{"ipv4":{"ipAddress":[{"privateIpAddress":"%s","publicIpAddress":"%s"}]},"ipv6":{"ipAddress":[{"privateIpAddress":"%s","publicIpAddress":"%s"}]}}]}}` 760 761 testcases := []struct { 762 name string 763 nodeName string 764 ipV4 string 765 ipV6 string 766 ipV4Public string 767 ipV6Public string 768 providerID string 769 expectedAddress []v1.NodeAddress 770 expectedErrMsg error 771 }{ 772 { 773 name: "NodeAddressesByProviderID should get both ipV4 and ipV6 private addresses", 774 nodeName: "vm1", 775 providerID: "azure:///subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Compute/virtualMachines/vm1", 776 ipV4: "10.240.0.1", 777 ipV6: "1111:11111:00:00:1111:1111:000:111", 778 expectedAddress: []v1.NodeAddress{ 779 { 780 Type: v1.NodeHostName, 781 Address: "vm1", 782 }, 783 { 784 Type: v1.NodeInternalIP, 785 Address: "10.240.0.1", 786 }, 787 { 788 Type: v1.NodeInternalIP, 789 Address: "1111:11111:00:00:1111:1111:000:111", 790 }, 791 }, 792 }, 793 { 794 name: "NodeAddressesByProviderID should report error when IPs are empty", 795 nodeName: "vm1", 796 providerID: "azure:///subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Compute/virtualMachines/vm1", 797 expectedErrMsg: fmt.Errorf("get empty IP addresses from instance metadata service"), 798 }, 799 { 800 name: "NodeAddressesByProviderID should return nil if node is unmanaged", 801 providerID: "/subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Compute/virtualMachines/vm1", 802 }, 803 { 804 name: "NodeAddressesByProviderID should report error if providerID is invalid", 805 providerID: "azure:///subscriptions/subscription/resourceGroups/rg/providers/Microsoft.Compute/virtualMachine/vm3", 806 expectedErrMsg: fmt.Errorf("error splitting providerID"), 807 }, 808 { 809 name: "NodeAddressesByProviderID should report error if providerID is null", 810 expectedErrMsg: fmt.Errorf("providerID is empty, the node is not initialized yet"), 811 }, 812 } 813 814 for _, test := range testcases { 815 listener, err := net.Listen("tcp", "127.0.0.1:0") 816 if err != nil { 817 t.Errorf("Test [%s] unexpected error: %v", test.name, err) 818 } 819 820 mux := http.NewServeMux() 821 mux.Handle("/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 822 fmt.Fprintf(w, metadataTemplate, test.nodeName, test.ipV4, test.ipV4Public, test.ipV6, test.ipV6Public) 823 })) 824 go func() { 825 http.Serve(listener, mux) 826 }() 827 defer listener.Close() 828 829 cloud.metadata, err = NewInstanceMetadataService("http://" + listener.Addr().String() + "/") 830 if err != nil { 831 t.Errorf("Test [%s] unexpected error: %v", test.name, err) 832 } 833 834 ipAddresses, err := cloud.NodeAddressesByProviderID(context.Background(), test.providerID) 835 assert.Equal(t, test.expectedErrMsg, err, test.name) 836 assert.Equal(t, test.expectedAddress, ipAddresses, test.name) 837 } 838} 839 840func TestCurrentNodeName(t *testing.T) { 841 ctrl := gomock.NewController(t) 842 defer ctrl.Finish() 843 cloud := GetTestCloud(ctrl) 844 845 hostname := "testvm" 846 nodeName, err := cloud.CurrentNodeName(context.Background(), hostname) 847 assert.Equal(t, types.NodeName(hostname), nodeName) 848 assert.NoError(t, err) 849} 850