1/* 2Copyright 2017 The Kubernetes Authors. 3 4Licensed under the Apache License, Version 2.0 (the "License"); 5you may not use this file except in compliance with the License. 6You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10Unless required by applicable law or agreed to in writing, software 11distributed under the License is distributed on an "AS IS" BASIS, 12WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13See the License for the specific language governing permissions and 14limitations under the License. 15*/ 16 17package apps 18 19import ( 20 "context" 21 "github.com/onsi/ginkgo" 22 23 appsv1 "k8s.io/api/apps/v1" 24 v1 "k8s.io/api/core/v1" 25 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 26 "k8s.io/apimachinery/pkg/labels" 27 "k8s.io/apimachinery/pkg/util/wait" 28 "k8s.io/kubernetes/pkg/controller" 29 "k8s.io/kubernetes/test/e2e/framework" 30 "k8s.io/kubernetes/test/e2e/upgrades" 31) 32 33// DaemonSetUpgradeTest tests that a DaemonSet is running before and after 34// a cluster upgrade. 35type DaemonSetUpgradeTest struct { 36 daemonSet *appsv1.DaemonSet 37} 38 39// Name returns the tracking name of the test. 40func (DaemonSetUpgradeTest) Name() string { return "[sig-apps] daemonset-upgrade" } 41 42// Setup creates a DaemonSet and verifies that it's running 43func (t *DaemonSetUpgradeTest) Setup(f *framework.Framework) { 44 daemonSetName := "ds1" 45 labelSet := map[string]string{"ds-name": daemonSetName} 46 image := framework.ServeHostnameImage 47 48 ns := f.Namespace 49 50 t.daemonSet = &appsv1.DaemonSet{ 51 ObjectMeta: metav1.ObjectMeta{ 52 Namespace: ns.Name, 53 Name: daemonSetName, 54 }, 55 Spec: appsv1.DaemonSetSpec{ 56 Selector: &metav1.LabelSelector{ 57 MatchLabels: labelSet, 58 }, 59 Template: v1.PodTemplateSpec{ 60 ObjectMeta: metav1.ObjectMeta{ 61 Labels: labelSet, 62 }, 63 Spec: v1.PodSpec{ 64 Tolerations: []v1.Toleration{ 65 {Operator: v1.TolerationOpExists}, 66 }, 67 Containers: []v1.Container{ 68 { 69 Name: daemonSetName, 70 Image: image, 71 Args: []string{"serve-hostname"}, 72 Ports: []v1.ContainerPort{{ContainerPort: 9376}}, 73 SecurityContext: &v1.SecurityContext{}, 74 }, 75 }, 76 }, 77 }, 78 }, 79 } 80 81 ginkgo.By("Creating a DaemonSet") 82 var err error 83 if t.daemonSet, err = f.ClientSet.AppsV1().DaemonSets(ns.Name).Create(context.TODO(), t.daemonSet, metav1.CreateOptions{}); err != nil { 84 framework.Failf("unable to create test DaemonSet %s: %v", t.daemonSet.Name, err) 85 } 86 87 ginkgo.By("Waiting for DaemonSet pods to become ready") 88 err = wait.Poll(framework.Poll, framework.PodStartTimeout, func() (bool, error) { 89 return checkRunningOnAllNodes(f, t.daemonSet.Namespace, t.daemonSet.Labels) 90 }) 91 framework.ExpectNoError(err) 92 93 ginkgo.By("Validating the DaemonSet after creation") 94 t.validateRunningDaemonSet(f) 95} 96 97// Test waits until the upgrade has completed and then verifies that the DaemonSet 98// is still running 99func (t *DaemonSetUpgradeTest) Test(f *framework.Framework, done <-chan struct{}, upgrade upgrades.UpgradeType) { 100 ginkgo.By("Waiting for upgradet to complete before re-validating DaemonSet") 101 <-done 102 103 ginkgo.By("validating the DaemonSet is still running after upgrade") 104 t.validateRunningDaemonSet(f) 105} 106 107// Teardown cleans up any remaining resources. 108func (t *DaemonSetUpgradeTest) Teardown(f *framework.Framework) { 109 // rely on the namespace deletion to clean up everything 110} 111 112func (t *DaemonSetUpgradeTest) validateRunningDaemonSet(f *framework.Framework) { 113 ginkgo.By("confirming the DaemonSet pods are running on all expected nodes") 114 res, err := checkRunningOnAllNodes(f, t.daemonSet.Namespace, t.daemonSet.Labels) 115 framework.ExpectNoError(err) 116 if !res { 117 framework.Failf("expected DaemonSet pod to be running on all nodes, it was not") 118 } 119 120 // DaemonSet resource itself should be good 121 ginkgo.By("confirming the DaemonSet resource is in a good state") 122 res, err = checkDaemonStatus(f, t.daemonSet.Namespace, t.daemonSet.Name) 123 framework.ExpectNoError(err) 124 if !res { 125 framework.Failf("expected DaemonSet to be in a good state, it was not") 126 } 127} 128 129func checkRunningOnAllNodes(f *framework.Framework, namespace string, selector map[string]string) (bool, error) { 130 nodeList, err := f.ClientSet.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{}) 131 if err != nil { 132 return false, err 133 } 134 135 nodeNames := make([]string, 0) 136 for _, node := range nodeList.Items { 137 if len(node.Spec.Taints) != 0 { 138 framework.Logf("Ignore taints %v on Node %v for DaemonSet Pod.", node.Spec.Taints, node.Name) 139 } 140 // DaemonSet Pods are expected to run on all the nodes in e2e. 141 nodeNames = append(nodeNames, node.Name) 142 } 143 144 return checkDaemonPodOnNodes(f, namespace, selector, nodeNames) 145} 146 147func checkDaemonPodOnNodes(f *framework.Framework, namespace string, labelSet map[string]string, nodeNames []string) (bool, error) { 148 selector := labels.Set(labelSet).AsSelector() 149 options := metav1.ListOptions{LabelSelector: selector.String()} 150 podList, err := f.ClientSet.CoreV1().Pods(namespace).List(context.TODO(), options) 151 if err != nil { 152 return false, err 153 } 154 pods := podList.Items 155 156 nodesToPodCount := make(map[string]int) 157 for _, pod := range pods { 158 if controller.IsPodActive(&pod) { 159 framework.Logf("Pod name: %v\t Node Name: %v", pod.Name, pod.Spec.NodeName) 160 nodesToPodCount[pod.Spec.NodeName]++ 161 } 162 } 163 framework.Logf("nodesToPodCount: %v", nodesToPodCount) 164 165 // Ensure that exactly 1 pod is running on all nodes in nodeNames. 166 for _, nodeName := range nodeNames { 167 if nodesToPodCount[nodeName] != 1 { 168 return false, nil 169 } 170 } 171 172 // Ensure that sizes of the lists are the same. We've verified that every element of nodeNames is in 173 // nodesToPodCount, so verifying the lengths are equal ensures that there aren't pods running on any 174 // other nodes. 175 return len(nodesToPodCount) == len(nodeNames), nil 176} 177 178func checkDaemonStatus(f *framework.Framework, namespace string, dsName string) (bool, error) { 179 ds, err := f.ClientSet.AppsV1().DaemonSets(namespace).Get(context.TODO(), dsName, metav1.GetOptions{}) 180 if err != nil { 181 return false, err 182 } 183 184 desired, scheduled, ready := ds.Status.DesiredNumberScheduled, ds.Status.CurrentNumberScheduled, ds.Status.NumberReady 185 if desired != scheduled && desired != ready { 186 return false, nil 187 } 188 189 return true, nil 190} 191