1/*
2Copyright 2017 The Kubernetes Authors.
3
4Licensed under the Apache License, Version 2.0 (the "License");
5you may not use this file except in compliance with the License.
6You may obtain a copy of the License at
7
8    http://www.apache.org/licenses/LICENSE-2.0
9
10Unless required by applicable law or agreed to in writing, software
11distributed under the License is distributed on an "AS IS" BASIS,
12WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13See the License for the specific language governing permissions and
14limitations under the License.
15*/
16
17package apps
18
19import (
20	"context"
21	"github.com/onsi/ginkgo"
22
23	appsv1 "k8s.io/api/apps/v1"
24	v1 "k8s.io/api/core/v1"
25	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
26	"k8s.io/apimachinery/pkg/labels"
27	"k8s.io/apimachinery/pkg/util/wait"
28	"k8s.io/kubernetes/pkg/controller"
29	"k8s.io/kubernetes/test/e2e/framework"
30	"k8s.io/kubernetes/test/e2e/upgrades"
31)
32
33// DaemonSetUpgradeTest tests that a DaemonSet is running before and after
34// a cluster upgrade.
35type DaemonSetUpgradeTest struct {
36	daemonSet *appsv1.DaemonSet
37}
38
39// Name returns the tracking name of the test.
40func (DaemonSetUpgradeTest) Name() string { return "[sig-apps] daemonset-upgrade" }
41
42// Setup creates a DaemonSet and verifies that it's running
43func (t *DaemonSetUpgradeTest) Setup(f *framework.Framework) {
44	daemonSetName := "ds1"
45	labelSet := map[string]string{"ds-name": daemonSetName}
46	image := framework.ServeHostnameImage
47
48	ns := f.Namespace
49
50	t.daemonSet = &appsv1.DaemonSet{
51		ObjectMeta: metav1.ObjectMeta{
52			Namespace: ns.Name,
53			Name:      daemonSetName,
54		},
55		Spec: appsv1.DaemonSetSpec{
56			Selector: &metav1.LabelSelector{
57				MatchLabels: labelSet,
58			},
59			Template: v1.PodTemplateSpec{
60				ObjectMeta: metav1.ObjectMeta{
61					Labels: labelSet,
62				},
63				Spec: v1.PodSpec{
64					Tolerations: []v1.Toleration{
65						{Operator: v1.TolerationOpExists},
66					},
67					Containers: []v1.Container{
68						{
69							Name:            daemonSetName,
70							Image:           image,
71							Args:            []string{"serve-hostname"},
72							Ports:           []v1.ContainerPort{{ContainerPort: 9376}},
73							SecurityContext: &v1.SecurityContext{},
74						},
75					},
76				},
77			},
78		},
79	}
80
81	ginkgo.By("Creating a DaemonSet")
82	var err error
83	if t.daemonSet, err = f.ClientSet.AppsV1().DaemonSets(ns.Name).Create(context.TODO(), t.daemonSet, metav1.CreateOptions{}); err != nil {
84		framework.Failf("unable to create test DaemonSet %s: %v", t.daemonSet.Name, err)
85	}
86
87	ginkgo.By("Waiting for DaemonSet pods to become ready")
88	err = wait.Poll(framework.Poll, framework.PodStartTimeout, func() (bool, error) {
89		return checkRunningOnAllNodes(f, t.daemonSet.Namespace, t.daemonSet.Labels)
90	})
91	framework.ExpectNoError(err)
92
93	ginkgo.By("Validating the DaemonSet after creation")
94	t.validateRunningDaemonSet(f)
95}
96
97// Test waits until the upgrade has completed and then verifies that the DaemonSet
98// is still running
99func (t *DaemonSetUpgradeTest) Test(f *framework.Framework, done <-chan struct{}, upgrade upgrades.UpgradeType) {
100	ginkgo.By("Waiting for upgradet to complete before re-validating DaemonSet")
101	<-done
102
103	ginkgo.By("validating the DaemonSet is still running after upgrade")
104	t.validateRunningDaemonSet(f)
105}
106
107// Teardown cleans up any remaining resources.
108func (t *DaemonSetUpgradeTest) Teardown(f *framework.Framework) {
109	// rely on the namespace deletion to clean up everything
110}
111
112func (t *DaemonSetUpgradeTest) validateRunningDaemonSet(f *framework.Framework) {
113	ginkgo.By("confirming the DaemonSet pods are running on all expected nodes")
114	res, err := checkRunningOnAllNodes(f, t.daemonSet.Namespace, t.daemonSet.Labels)
115	framework.ExpectNoError(err)
116	if !res {
117		framework.Failf("expected DaemonSet pod to be running on all nodes, it was not")
118	}
119
120	// DaemonSet resource itself should be good
121	ginkgo.By("confirming the DaemonSet resource is in a good state")
122	res, err = checkDaemonStatus(f, t.daemonSet.Namespace, t.daemonSet.Name)
123	framework.ExpectNoError(err)
124	if !res {
125		framework.Failf("expected DaemonSet to be in a good state, it was not")
126	}
127}
128
129func checkRunningOnAllNodes(f *framework.Framework, namespace string, selector map[string]string) (bool, error) {
130	nodeList, err := f.ClientSet.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{})
131	if err != nil {
132		return false, err
133	}
134
135	nodeNames := make([]string, 0)
136	for _, node := range nodeList.Items {
137		if len(node.Spec.Taints) != 0 {
138			framework.Logf("Ignore taints %v on Node %v for DaemonSet Pod.", node.Spec.Taints, node.Name)
139		}
140		// DaemonSet Pods are expected to run on all the nodes in e2e.
141		nodeNames = append(nodeNames, node.Name)
142	}
143
144	return checkDaemonPodOnNodes(f, namespace, selector, nodeNames)
145}
146
147func checkDaemonPodOnNodes(f *framework.Framework, namespace string, labelSet map[string]string, nodeNames []string) (bool, error) {
148	selector := labels.Set(labelSet).AsSelector()
149	options := metav1.ListOptions{LabelSelector: selector.String()}
150	podList, err := f.ClientSet.CoreV1().Pods(namespace).List(context.TODO(), options)
151	if err != nil {
152		return false, err
153	}
154	pods := podList.Items
155
156	nodesToPodCount := make(map[string]int)
157	for _, pod := range pods {
158		if controller.IsPodActive(&pod) {
159			framework.Logf("Pod name: %v\t Node Name: %v", pod.Name, pod.Spec.NodeName)
160			nodesToPodCount[pod.Spec.NodeName]++
161		}
162	}
163	framework.Logf("nodesToPodCount: %v", nodesToPodCount)
164
165	// Ensure that exactly 1 pod is running on all nodes in nodeNames.
166	for _, nodeName := range nodeNames {
167		if nodesToPodCount[nodeName] != 1 {
168			return false, nil
169		}
170	}
171
172	// Ensure that sizes of the lists are the same. We've verified that every element of nodeNames is in
173	// nodesToPodCount, so verifying the lengths are equal ensures that there aren't pods running on any
174	// other nodes.
175	return len(nodesToPodCount) == len(nodeNames), nil
176}
177
178func checkDaemonStatus(f *framework.Framework, namespace string, dsName string) (bool, error) {
179	ds, err := f.ClientSet.AppsV1().DaemonSets(namespace).Get(context.TODO(), dsName, metav1.GetOptions{})
180	if err != nil {
181		return false, err
182	}
183
184	desired, scheduled, ready := ds.Status.DesiredNumberScheduled, ds.Status.CurrentNumberScheduled, ds.Status.NumberReady
185	if desired != scheduled && desired != ready {
186		return false, nil
187	}
188
189	return true, nil
190}
191