1/* 2Copyright 2018 The Kubernetes Authors. 3 4Licensed under the Apache License, Version 2.0 (the "License"); 5you may not use this file except in compliance with the License. 6You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10Unless required by applicable law or agreed to in writing, software 11distributed under the License is distributed on an "AS IS" BASIS, 12WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13See the License for the specific language governing permissions and 14limitations under the License. 15*/ 16 17package node 18 19import ( 20 "context" 21 "fmt" 22 "time" 23 24 v1 "k8s.io/api/core/v1" 25 nodev1 "k8s.io/api/node/v1" 26 apierrors "k8s.io/apimachinery/pkg/api/errors" 27 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 28 "k8s.io/apimachinery/pkg/fields" 29 types "k8s.io/apimachinery/pkg/types" 30 "k8s.io/apimachinery/pkg/util/wait" 31 "k8s.io/apimachinery/pkg/watch" 32 "k8s.io/kubernetes/pkg/kubelet/events" 33 runtimeclasstest "k8s.io/kubernetes/pkg/kubelet/runtimeclass/testing" 34 "k8s.io/kubernetes/test/e2e/framework" 35 e2eevents "k8s.io/kubernetes/test/e2e/framework/events" 36 e2enode "k8s.io/kubernetes/test/e2e/framework/node" 37 e2epod "k8s.io/kubernetes/test/e2e/framework/pod" 38 39 "github.com/onsi/ginkgo" 40) 41 42var _ = SIGDescribe("RuntimeClass", func() { 43 f := framework.NewDefaultFramework("runtimeclass") 44 45 ginkgo.It("should reject a Pod requesting a non-existent RuntimeClass [NodeFeature:RuntimeHandler]", func() { 46 rcName := f.Namespace.Name + "-nonexistent" 47 expectPodRejection(f, e2enode.NewRuntimeClassPod(rcName)) 48 }) 49 50 ginkgo.It("should reject a Pod requesting a RuntimeClass with an unconfigured handler [NodeFeature:RuntimeHandler]", func() { 51 handler := f.Namespace.Name + "-handler" 52 rcName := createRuntimeClass(f, "unconfigured-handler", handler) 53 defer deleteRuntimeClass(f, rcName) 54 pod := f.PodClient().Create(e2enode.NewRuntimeClassPod(rcName)) 55 eventSelector := fields.Set{ 56 "involvedObject.kind": "Pod", 57 "involvedObject.name": pod.Name, 58 "involvedObject.namespace": f.Namespace.Name, 59 "reason": events.FailedCreatePodSandBox, 60 }.AsSelector().String() 61 // Events are unreliable, don't depend on the event. It's used only to speed up the test. 62 err := e2eevents.WaitTimeoutForEvent(f.ClientSet, f.Namespace.Name, eventSelector, handler, framework.PodEventTimeout) 63 if err != nil { 64 framework.Logf("Warning: did not get event about FailedCreatePodSandBox. Err: %v", err) 65 } 66 // Check the pod is still not running 67 p, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Get(context.TODO(), pod.Name, metav1.GetOptions{}) 68 framework.ExpectNoError(err, "could not re-read the pod after event (or timeout)") 69 framework.ExpectEqual(p.Status.Phase, v1.PodPending, "Pod phase isn't pending") 70 }) 71 72 // This test requires that the PreconfiguredRuntimeHandler has already been set up on nodes. 73 ginkgo.It("should run a Pod requesting a RuntimeClass with a configured handler [NodeFeature:RuntimeHandler]", func() { 74 // The built-in docker runtime does not support configuring runtime handlers. 75 handler := e2enode.PreconfiguredRuntimeClassHandler(framework.TestContext.ContainerRuntime) 76 77 rcName := createRuntimeClass(f, "preconfigured-handler", handler) 78 defer deleteRuntimeClass(f, rcName) 79 pod := f.PodClient().Create(e2enode.NewRuntimeClassPod(rcName)) 80 expectPodSuccess(f, pod) 81 }) 82 83 ginkgo.It("should reject a Pod requesting a deleted RuntimeClass [NodeFeature:RuntimeHandler]", func() { 84 rcName := createRuntimeClass(f, "delete-me", "runc") 85 rcClient := f.ClientSet.NodeV1().RuntimeClasses() 86 87 ginkgo.By("Deleting RuntimeClass "+rcName, func() { 88 err := rcClient.Delete(context.TODO(), rcName, metav1.DeleteOptions{}) 89 framework.ExpectNoError(err, "failed to delete RuntimeClass %s", rcName) 90 91 ginkgo.By("Waiting for the RuntimeClass to disappear") 92 framework.ExpectNoError(wait.PollImmediate(framework.Poll, time.Minute, func() (bool, error) { 93 _, err := rcClient.Get(context.TODO(), rcName, metav1.GetOptions{}) 94 if apierrors.IsNotFound(err) { 95 return true, nil // done 96 } 97 if err != nil { 98 return true, err // stop wait with error 99 } 100 return false, nil 101 })) 102 }) 103 104 expectPodRejection(f, e2enode.NewRuntimeClassPod(rcName)) 105 }) 106 107 /* 108 Release: v1.20 109 Testname: RuntimeClass API 110 Description: 111 The node.k8s.io API group MUST exist in the /apis discovery document. 112 The node.k8s.io/v1 API group/version MUST exist in the /apis/mode.k8s.io discovery document. 113 The runtimeclasses resource MUST exist in the /apis/node.k8s.io/v1 discovery document. 114 The runtimeclasses resource must support create, get, list, watch, update, patch, delete, and deletecollection. 115 */ 116 framework.ConformanceIt(" should support RuntimeClasses API operations", func() { 117 // Setup 118 rcVersion := "v1" 119 rcClient := f.ClientSet.NodeV1().RuntimeClasses() 120 121 // This is a conformance test that must configure opaque handlers to validate CRUD operations. 122 // Test should not use any existing handler like gVisor or runc 123 // 124 // All CRUD operations in this test are limited to the objects with the label test=f.UniqueName 125 rc := runtimeclasstest.NewRuntimeClass(f.UniqueName+"-handler", f.UniqueName+"-conformance-runtime-class") 126 rc.SetLabels(map[string]string{"test": f.UniqueName}) 127 rc2 := runtimeclasstest.NewRuntimeClass(f.UniqueName+"-handler2", f.UniqueName+"-conformance-runtime-class2") 128 rc2.SetLabels(map[string]string{"test": f.UniqueName}) 129 rc3 := runtimeclasstest.NewRuntimeClass(f.UniqueName+"-handler3", f.UniqueName+"-conformance-runtime-class3") 130 rc3.SetLabels(map[string]string{"test": f.UniqueName}) 131 132 // Discovery 133 134 ginkgo.By("getting /apis") 135 { 136 discoveryGroups, err := f.ClientSet.Discovery().ServerGroups() 137 framework.ExpectNoError(err) 138 found := false 139 for _, group := range discoveryGroups.Groups { 140 if group.Name == nodev1.GroupName { 141 for _, version := range group.Versions { 142 if version.Version == rcVersion { 143 found = true 144 break 145 } 146 } 147 } 148 } 149 framework.ExpectEqual(found, true, fmt.Sprintf("expected RuntimeClass API group/version, got %#v", discoveryGroups.Groups)) 150 } 151 152 ginkgo.By("getting /apis/node.k8s.io") 153 { 154 group := &metav1.APIGroup{} 155 err := f.ClientSet.Discovery().RESTClient().Get().AbsPath("/apis/node.k8s.io").Do(context.TODO()).Into(group) 156 framework.ExpectNoError(err) 157 found := false 158 for _, version := range group.Versions { 159 if version.Version == rcVersion { 160 found = true 161 break 162 } 163 } 164 framework.ExpectEqual(found, true, fmt.Sprintf("expected RuntimeClass API version, got %#v", group.Versions)) 165 } 166 167 ginkgo.By("getting /apis/node.k8s.io/" + rcVersion) 168 { 169 resources, err := f.ClientSet.Discovery().ServerResourcesForGroupVersion(nodev1.SchemeGroupVersion.String()) 170 framework.ExpectNoError(err) 171 found := false 172 for _, resource := range resources.APIResources { 173 switch resource.Name { 174 case "runtimeclasses": 175 found = true 176 } 177 } 178 framework.ExpectEqual(found, true, fmt.Sprintf("expected runtimeclasses, got %#v", resources.APIResources)) 179 } 180 181 // Main resource create/read/update/watch operations 182 183 ginkgo.By("creating") 184 createdRC, err := rcClient.Create(context.TODO(), rc, metav1.CreateOptions{}) 185 framework.ExpectNoError(err) 186 _, err = rcClient.Create(context.TODO(), rc, metav1.CreateOptions{}) 187 framework.ExpectEqual(apierrors.IsAlreadyExists(err), true, fmt.Sprintf("expected 409, got %#v", err)) 188 _, err = rcClient.Create(context.TODO(), rc2, metav1.CreateOptions{}) 189 framework.ExpectNoError(err) 190 191 ginkgo.By("watching") 192 framework.Logf("starting watch") 193 rcWatch, err := rcClient.Watch(context.TODO(), metav1.ListOptions{LabelSelector: "test=" + f.UniqueName}) 194 framework.ExpectNoError(err) 195 196 // added for a watch 197 _, err = rcClient.Create(context.TODO(), rc3, metav1.CreateOptions{}) 198 framework.ExpectNoError(err) 199 200 ginkgo.By("getting") 201 gottenRC, err := rcClient.Get(context.TODO(), rc.Name, metav1.GetOptions{}) 202 framework.ExpectNoError(err) 203 framework.ExpectEqual(gottenRC.UID, createdRC.UID) 204 205 ginkgo.By("listing") 206 rcs, err := rcClient.List(context.TODO(), metav1.ListOptions{LabelSelector: "test=" + f.UniqueName}) 207 framework.ExpectNoError(err) 208 framework.ExpectEqual(len(rcs.Items), 3, "filtered list should have 3 items") 209 210 ginkgo.By("patching") 211 patchedRC, err := rcClient.Patch(context.TODO(), createdRC.Name, types.MergePatchType, []byte(`{"metadata":{"annotations":{"patched":"true"}}}`), metav1.PatchOptions{}) 212 framework.ExpectNoError(err) 213 framework.ExpectEqual(patchedRC.Annotations["patched"], "true", "patched object should have the applied annotation") 214 215 ginkgo.By("updating") 216 csrToUpdate := patchedRC.DeepCopy() 217 csrToUpdate.Annotations["updated"] = "true" 218 updatedRC, err := rcClient.Update(context.TODO(), csrToUpdate, metav1.UpdateOptions{}) 219 framework.ExpectNoError(err) 220 framework.ExpectEqual(updatedRC.Annotations["updated"], "true", "updated object should have the applied annotation") 221 222 framework.Logf("waiting for watch events with expected annotations") 223 for sawAdded, sawPatched, sawUpdated := false, false, false; !sawAdded && !sawPatched && !sawUpdated; { 224 select { 225 case evt, ok := <-rcWatch.ResultChan(): 226 framework.ExpectEqual(ok, true, "watch channel should not close") 227 if evt.Type == watch.Modified { 228 watchedRC, isRC := evt.Object.(*nodev1.RuntimeClass) 229 framework.ExpectEqual(isRC, true, fmt.Sprintf("expected RC, got %T", evt.Object)) 230 if watchedRC.Annotations["patched"] == "true" { 231 framework.Logf("saw patched annotations") 232 sawPatched = true 233 } else if watchedRC.Annotations["updated"] == "true" { 234 framework.Logf("saw updated annotations") 235 sawUpdated = true 236 } else { 237 framework.Logf("missing expected annotations, waiting: %#v", watchedRC.Annotations) 238 } 239 } else if evt.Type == watch.Added { 240 _, isRC := evt.Object.(*nodev1.RuntimeClass) 241 framework.ExpectEqual(isRC, true, fmt.Sprintf("expected RC, got %T", evt.Object)) 242 sawAdded = true 243 } 244 245 case <-time.After(wait.ForeverTestTimeout): 246 framework.Fail("timed out waiting for watch event") 247 } 248 } 249 rcWatch.Stop() 250 251 // main resource delete operations 252 253 ginkgo.By("deleting") 254 err = rcClient.Delete(context.TODO(), createdRC.Name, metav1.DeleteOptions{}) 255 framework.ExpectNoError(err) 256 _, err = rcClient.Get(context.TODO(), createdRC.Name, metav1.GetOptions{}) 257 framework.ExpectEqual(apierrors.IsNotFound(err), true, fmt.Sprintf("expected 404, got %#v", err)) 258 rcs, err = rcClient.List(context.TODO(), metav1.ListOptions{LabelSelector: "test=" + f.UniqueName}) 259 framework.ExpectNoError(err) 260 framework.ExpectEqual(len(rcs.Items), 2, "filtered list should have 2 items") 261 262 ginkgo.By("deleting a collection") 263 err = rcClient.DeleteCollection(context.TODO(), metav1.DeleteOptions{}, metav1.ListOptions{LabelSelector: "test=" + f.UniqueName}) 264 framework.ExpectNoError(err) 265 rcs, err = rcClient.List(context.TODO(), metav1.ListOptions{LabelSelector: "test=" + f.UniqueName}) 266 framework.ExpectNoError(err) 267 framework.ExpectEqual(len(rcs.Items), 0, "filtered list should have 0 items") 268 }) 269}) 270 271func deleteRuntimeClass(f *framework.Framework, name string) { 272 err := f.ClientSet.NodeV1().RuntimeClasses().Delete(context.TODO(), name, metav1.DeleteOptions{}) 273 framework.ExpectNoError(err, "failed to delete RuntimeClass resource") 274} 275 276// createRuntimeClass generates a RuntimeClass with the desired handler and a "namespaced" name, 277// synchronously creates it, and returns the generated name. 278func createRuntimeClass(f *framework.Framework, name, handler string) string { 279 uniqueName := fmt.Sprintf("%s-%s", f.Namespace.Name, name) 280 rc := runtimeclasstest.NewRuntimeClass(uniqueName, handler) 281 rc, err := f.ClientSet.NodeV1().RuntimeClasses().Create(context.TODO(), rc, metav1.CreateOptions{}) 282 framework.ExpectNoError(err, "failed to create RuntimeClass resource") 283 return rc.GetName() 284} 285 286func expectPodRejection(f *framework.Framework, pod *v1.Pod) { 287 _, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(context.TODO(), pod, metav1.CreateOptions{}) 288 framework.ExpectError(err, "should be forbidden") 289 framework.ExpectEqual(apierrors.IsForbidden(err), true, "should be forbidden error") 290} 291 292// expectPodSuccess waits for the given pod to terminate successfully. 293func expectPodSuccess(f *framework.Framework, pod *v1.Pod) { 294 framework.ExpectNoError(e2epod.WaitForPodSuccessInNamespace( 295 f.ClientSet, pod.Name, f.Namespace.Name)) 296} 297