1/* 2Copyright 2016 The Kubernetes Authors. 3 4Licensed under the Apache License, Version 2.0 (the "License"); 5you may not use this file except in compliance with the License. 6You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10Unless required by applicable law or agreed to in writing, software 11distributed under the License is distributed on an "AS IS" BASIS, 12WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13See the License for the specific language governing permissions and 14limitations under the License. 15*/ 16 17package node 18 19import ( 20 "bytes" 21 "context" 22 "encoding/json" 23 "fmt" 24 "io" 25 "k8s.io/client-go/util/retry" 26 "runtime/debug" 27 "strconv" 28 "strings" 29 "time" 30 31 "golang.org/x/net/websocket" 32 33 v1 "k8s.io/api/core/v1" 34 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 35 "k8s.io/apimachinery/pkg/labels" 36 "k8s.io/apimachinery/pkg/runtime" 37 "k8s.io/apimachinery/pkg/runtime/schema" 38 "k8s.io/apimachinery/pkg/types" 39 "k8s.io/apimachinery/pkg/util/intstr" 40 "k8s.io/apimachinery/pkg/util/uuid" 41 "k8s.io/apimachinery/pkg/util/wait" 42 "k8s.io/apimachinery/pkg/watch" 43 "k8s.io/client-go/dynamic" 44 "k8s.io/client-go/tools/cache" 45 watchtools "k8s.io/client-go/tools/watch" 46 "k8s.io/kubectl/pkg/util/podutils" 47 podutil "k8s.io/kubernetes/pkg/api/v1/pod" 48 "k8s.io/kubernetes/pkg/kubelet" 49 "k8s.io/kubernetes/test/e2e/framework" 50 e2epod "k8s.io/kubernetes/test/e2e/framework/pod" 51 e2ewebsocket "k8s.io/kubernetes/test/e2e/framework/websocket" 52 imageutils "k8s.io/kubernetes/test/utils/image" 53 54 "github.com/onsi/ginkgo" 55 "github.com/onsi/gomega" 56) 57 58const ( 59 buildBackOffDuration = time.Minute 60 syncLoopFrequency = 10 * time.Second 61 maxBackOffTolerance = time.Duration(1.3 * float64(kubelet.MaxContainerBackOff)) 62 podRetryPeriod = 1 * time.Second 63 podRetryTimeout = 1 * time.Minute 64) 65 66// testHostIP tests that a pod gets a host IP 67func testHostIP(podClient *framework.PodClient, pod *v1.Pod) { 68 ginkgo.By("creating pod") 69 podClient.CreateSync(pod) 70 71 // Try to make sure we get a hostIP for each pod. 72 hostIPTimeout := 2 * time.Minute 73 t := time.Now() 74 for { 75 p, err := podClient.Get(context.TODO(), pod.Name, metav1.GetOptions{}) 76 framework.ExpectNoError(err, "Failed to get pod %q", pod.Name) 77 if p.Status.HostIP != "" { 78 framework.Logf("Pod %s has hostIP: %s", p.Name, p.Status.HostIP) 79 break 80 } 81 if time.Since(t) >= hostIPTimeout { 82 framework.Failf("Gave up waiting for hostIP of pod %s after %v seconds", 83 p.Name, time.Since(t).Seconds()) 84 } 85 framework.Logf("Retrying to get the hostIP of pod %s", p.Name) 86 time.Sleep(5 * time.Second) 87 } 88} 89 90func startPodAndGetBackOffs(podClient *framework.PodClient, pod *v1.Pod, sleepAmount time.Duration) (time.Duration, time.Duration) { 91 podClient.CreateSync(pod) 92 time.Sleep(sleepAmount) 93 gomega.Expect(pod.Spec.Containers).NotTo(gomega.BeEmpty()) 94 podName := pod.Name 95 containerName := pod.Spec.Containers[0].Name 96 97 ginkgo.By("getting restart delay-0") 98 _, err := getRestartDelay(podClient, podName, containerName) 99 if err != nil { 100 framework.Failf("timed out waiting for container restart in pod=%s/%s", podName, containerName) 101 } 102 103 ginkgo.By("getting restart delay-1") 104 delay1, err := getRestartDelay(podClient, podName, containerName) 105 if err != nil { 106 framework.Failf("timed out waiting for container restart in pod=%s/%s", podName, containerName) 107 } 108 109 ginkgo.By("getting restart delay-2") 110 delay2, err := getRestartDelay(podClient, podName, containerName) 111 if err != nil { 112 framework.Failf("timed out waiting for container restart in pod=%s/%s", podName, containerName) 113 } 114 return delay1, delay2 115} 116 117func getRestartDelay(podClient *framework.PodClient, podName string, containerName string) (time.Duration, error) { 118 beginTime := time.Now() 119 var previousRestartCount int32 = -1 120 var previousFinishedAt time.Time 121 for time.Since(beginTime) < (2 * maxBackOffTolerance) { // may just miss the 1st MaxContainerBackOff delay 122 time.Sleep(time.Second) 123 pod, err := podClient.Get(context.TODO(), podName, metav1.GetOptions{}) 124 framework.ExpectNoError(err, fmt.Sprintf("getting pod %s", podName)) 125 status, ok := podutil.GetContainerStatus(pod.Status.ContainerStatuses, containerName) 126 if !ok { 127 framework.Logf("getRestartDelay: status missing") 128 continue 129 } 130 131 // the only case this happens is if this is the first time the Pod is running and there is no "Last State". 132 if status.LastTerminationState.Terminated == nil { 133 framework.Logf("Container's last state is not \"Terminated\".") 134 continue 135 } 136 137 if previousRestartCount == -1 { 138 if status.State.Running != nil { 139 // container is still Running, there is no "FinishedAt" time. 140 continue 141 } else if status.State.Terminated != nil { 142 previousFinishedAt = status.State.Terminated.FinishedAt.Time 143 } else { 144 previousFinishedAt = status.LastTerminationState.Terminated.FinishedAt.Time 145 } 146 previousRestartCount = status.RestartCount 147 } 148 149 // when the RestartCount is changed, the Containers will be in one of the following states: 150 //Running, Terminated, Waiting (it already is waiting for the backoff period to expire, and the last state details have been stored into status.LastTerminationState). 151 if status.RestartCount > previousRestartCount { 152 var startedAt time.Time 153 if status.State.Running != nil { 154 startedAt = status.State.Running.StartedAt.Time 155 } else if status.State.Terminated != nil { 156 startedAt = status.State.Terminated.StartedAt.Time 157 } else { 158 startedAt = status.LastTerminationState.Terminated.StartedAt.Time 159 } 160 framework.Logf("getRestartDelay: restartCount = %d, finishedAt=%s restartedAt=%s (%s)", status.RestartCount, previousFinishedAt, startedAt, startedAt.Sub(previousFinishedAt)) 161 return startedAt.Sub(previousFinishedAt), nil 162 } 163 } 164 return 0, fmt.Errorf("timeout getting pod restart delay") 165} 166 167// expectNoErrorWithRetries checks if an error occurs with the given retry count. 168func expectNoErrorWithRetries(fn func() error, maxRetries int, explain ...interface{}) { 169 var err error 170 for i := 0; i < maxRetries; i++ { 171 err = fn() 172 if err == nil { 173 return 174 } 175 framework.Logf("(Attempt %d of %d) Unexpected error occurred: %v", i+1, maxRetries, err) 176 } 177 if err != nil { 178 debug.PrintStack() 179 } 180 gomega.ExpectWithOffset(1, err).NotTo(gomega.HaveOccurred(), explain...) 181} 182 183var _ = SIGDescribe("Pods", func() { 184 f := framework.NewDefaultFramework("pods") 185 var podClient *framework.PodClient 186 var dc dynamic.Interface 187 188 ginkgo.BeforeEach(func() { 189 podClient = f.PodClient() 190 dc = f.DynamicClient 191 }) 192 193 /* 194 Release: v1.9 195 Testname: Pods, assigned hostip 196 Description: Create a Pod. Pod status MUST return successfully and contains a valid IP address. 197 */ 198 framework.ConformanceIt("should get a host IP [NodeConformance]", func() { 199 name := "pod-hostip-" + string(uuid.NewUUID()) 200 testHostIP(podClient, &v1.Pod{ 201 ObjectMeta: metav1.ObjectMeta{ 202 Name: name, 203 }, 204 Spec: v1.PodSpec{ 205 Containers: []v1.Container{ 206 { 207 Name: "test", 208 Image: imageutils.GetPauseImageName(), 209 }, 210 }, 211 }, 212 }) 213 }) 214 215 /* 216 Release: v1.9 217 Testname: Pods, lifecycle 218 Description: A Pod is created with a unique label. Pod MUST be accessible when queried using the label selector upon creation. Add a watch, check if the Pod is running. Pod then deleted, The pod deletion timestamp is observed. The watch MUST return the pod deleted event. Query with the original selector for the Pod MUST return empty list. 219 */ 220 framework.ConformanceIt("should be submitted and removed [NodeConformance]", func() { 221 ginkgo.By("creating the pod") 222 name := "pod-submit-remove-" + string(uuid.NewUUID()) 223 value := strconv.Itoa(time.Now().Nanosecond()) 224 pod := &v1.Pod{ 225 ObjectMeta: metav1.ObjectMeta{ 226 Name: name, 227 Labels: map[string]string{ 228 "name": "foo", 229 "time": value, 230 }, 231 }, 232 Spec: v1.PodSpec{ 233 Containers: []v1.Container{ 234 { 235 Name: "nginx", 236 Image: imageutils.GetE2EImage(imageutils.Nginx), 237 }, 238 }, 239 }, 240 } 241 242 ginkgo.By("setting up watch") 243 selector := labels.SelectorFromSet(labels.Set(map[string]string{"time": value})) 244 options := metav1.ListOptions{LabelSelector: selector.String()} 245 pods, err := podClient.List(context.TODO(), options) 246 framework.ExpectNoError(err, "failed to query for pods") 247 framework.ExpectEqual(len(pods.Items), 0) 248 249 listCompleted := make(chan bool, 1) 250 lw := &cache.ListWatch{ 251 ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { 252 options.LabelSelector = selector.String() 253 podList, err := podClient.List(context.TODO(), options) 254 if err == nil { 255 select { 256 case listCompleted <- true: 257 framework.Logf("observed the pod list") 258 return podList, err 259 default: 260 framework.Logf("channel blocked") 261 } 262 } 263 return podList, err 264 }, 265 WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { 266 options.LabelSelector = selector.String() 267 return podClient.Watch(context.TODO(), options) 268 }, 269 } 270 _, _, w, _ := watchtools.NewIndexerInformerWatcher(lw, &v1.Pod{}) 271 defer w.Stop() 272 273 ginkgo.By("submitting the pod to kubernetes") 274 podClient.Create(pod) 275 276 ginkgo.By("verifying the pod is in kubernetes") 277 selector = labels.SelectorFromSet(labels.Set(map[string]string{"time": value})) 278 options = metav1.ListOptions{LabelSelector: selector.String()} 279 pods, err = podClient.List(context.TODO(), options) 280 framework.ExpectNoError(err, "failed to query for pods") 281 framework.ExpectEqual(len(pods.Items), 1) 282 283 ginkgo.By("verifying pod creation was observed") 284 select { 285 case <-listCompleted: 286 select { 287 case event := <-w.ResultChan(): 288 if event.Type != watch.Added { 289 framework.Failf("Failed to observe pod creation: %v", event) 290 } 291 case <-time.After(framework.PodStartTimeout): 292 framework.Failf("Timeout while waiting for pod creation") 293 } 294 case <-time.After(10 * time.Second): 295 framework.Failf("Timeout while waiting to observe pod list") 296 } 297 298 // We need to wait for the pod to be running, otherwise the deletion 299 // may be carried out immediately rather than gracefully. 300 framework.ExpectNoError(e2epod.WaitForPodNameRunningInNamespace(f.ClientSet, pod.Name, f.Namespace.Name)) 301 // save the running pod 302 pod, err = podClient.Get(context.TODO(), pod.Name, metav1.GetOptions{}) 303 framework.ExpectNoError(err, "failed to GET scheduled pod") 304 305 ginkgo.By("deleting the pod gracefully") 306 err = podClient.Delete(context.TODO(), pod.Name, *metav1.NewDeleteOptions(30)) 307 framework.ExpectNoError(err, "failed to delete pod") 308 309 ginkgo.By("verifying pod deletion was observed") 310 deleted := false 311 var lastPod *v1.Pod 312 timer := time.After(framework.DefaultPodDeletionTimeout) 313 for !deleted { 314 select { 315 case event := <-w.ResultChan(): 316 switch event.Type { 317 case watch.Deleted: 318 lastPod = event.Object.(*v1.Pod) 319 deleted = true 320 case watch.Error: 321 framework.Logf("received a watch error: %v", event.Object) 322 framework.Failf("watch closed with error") 323 } 324 case <-timer: 325 framework.Failf("timed out waiting for pod deletion") 326 } 327 } 328 if !deleted { 329 framework.Failf("Failed to observe pod deletion") 330 } 331 332 gomega.Expect(lastPod.DeletionTimestamp).ToNot(gomega.BeNil()) 333 gomega.Expect(lastPod.Spec.TerminationGracePeriodSeconds).ToNot(gomega.BeZero()) 334 335 selector = labels.SelectorFromSet(labels.Set(map[string]string{"time": value})) 336 options = metav1.ListOptions{LabelSelector: selector.String()} 337 pods, err = podClient.List(context.TODO(), options) 338 framework.ExpectNoError(err, "failed to query for pods") 339 framework.ExpectEqual(len(pods.Items), 0) 340 }) 341 342 /* 343 Release: v1.9 344 Testname: Pods, update 345 Description: Create a Pod with a unique label. Query for the Pod with the label as selector MUST be successful. Update the pod to change the value of the Label. Query for the Pod with the new value for the label MUST be successful. 346 */ 347 framework.ConformanceIt("should be updated [NodeConformance]", func() { 348 ginkgo.By("creating the pod") 349 name := "pod-update-" + string(uuid.NewUUID()) 350 value := strconv.Itoa(time.Now().Nanosecond()) 351 pod := &v1.Pod{ 352 ObjectMeta: metav1.ObjectMeta{ 353 Name: name, 354 Labels: map[string]string{ 355 "name": "foo", 356 "time": value, 357 }, 358 }, 359 Spec: v1.PodSpec{ 360 Containers: []v1.Container{ 361 { 362 Name: "nginx", 363 Image: imageutils.GetE2EImage(imageutils.Nginx), 364 }, 365 }, 366 }, 367 } 368 369 ginkgo.By("submitting the pod to kubernetes") 370 pod = podClient.CreateSync(pod) 371 372 ginkgo.By("verifying the pod is in kubernetes") 373 selector := labels.SelectorFromSet(labels.Set(map[string]string{"time": value})) 374 options := metav1.ListOptions{LabelSelector: selector.String()} 375 pods, err := podClient.List(context.TODO(), options) 376 framework.ExpectNoError(err, "failed to query for pods") 377 framework.ExpectEqual(len(pods.Items), 1) 378 379 ginkgo.By("updating the pod") 380 podClient.Update(name, func(pod *v1.Pod) { 381 value = strconv.Itoa(time.Now().Nanosecond()) 382 pod.Labels["time"] = value 383 }) 384 385 framework.ExpectNoError(e2epod.WaitForPodNameRunningInNamespace(f.ClientSet, pod.Name, f.Namespace.Name)) 386 387 ginkgo.By("verifying the updated pod is in kubernetes") 388 selector = labels.SelectorFromSet(labels.Set(map[string]string{"time": value})) 389 options = metav1.ListOptions{LabelSelector: selector.String()} 390 pods, err = podClient.List(context.TODO(), options) 391 framework.ExpectNoError(err, "failed to query for pods") 392 framework.ExpectEqual(len(pods.Items), 1) 393 framework.Logf("Pod update OK") 394 }) 395 396 /* 397 Release: v1.9 398 Testname: Pods, ActiveDeadlineSeconds 399 Description: Create a Pod with a unique label. Query for the Pod with the label as selector MUST be successful. The Pod is updated with ActiveDeadlineSeconds set on the Pod spec. Pod MUST terminate of the specified time elapses. 400 */ 401 framework.ConformanceIt("should allow activeDeadlineSeconds to be updated [NodeConformance]", func() { 402 ginkgo.By("creating the pod") 403 name := "pod-update-activedeadlineseconds-" + string(uuid.NewUUID()) 404 value := strconv.Itoa(time.Now().Nanosecond()) 405 pod := &v1.Pod{ 406 ObjectMeta: metav1.ObjectMeta{ 407 Name: name, 408 Labels: map[string]string{ 409 "name": "foo", 410 "time": value, 411 }, 412 }, 413 Spec: v1.PodSpec{ 414 Containers: []v1.Container{ 415 { 416 Name: "nginx", 417 Image: imageutils.GetE2EImage(imageutils.Nginx), 418 }, 419 }, 420 }, 421 } 422 423 ginkgo.By("submitting the pod to kubernetes") 424 podClient.CreateSync(pod) 425 426 ginkgo.By("verifying the pod is in kubernetes") 427 selector := labels.SelectorFromSet(labels.Set(map[string]string{"time": value})) 428 options := metav1.ListOptions{LabelSelector: selector.String()} 429 pods, err := podClient.List(context.TODO(), options) 430 framework.ExpectNoError(err, "failed to query for pods") 431 framework.ExpectEqual(len(pods.Items), 1) 432 433 ginkgo.By("updating the pod") 434 podClient.Update(name, func(pod *v1.Pod) { 435 newDeadline := int64(5) 436 pod.Spec.ActiveDeadlineSeconds = &newDeadline 437 }) 438 439 framework.ExpectNoError(e2epod.WaitForPodTerminatedInNamespace(f.ClientSet, pod.Name, "DeadlineExceeded", f.Namespace.Name)) 440 }) 441 442 /* 443 Release: v1.9 444 Testname: Pods, service environment variables 445 Description: Create a server Pod listening on port 9376. A Service called fooservice is created for the server Pod listening on port 8765 targeting port 8080. If a new Pod is created in the cluster then the Pod MUST have the fooservice environment variables available from this new Pod. The new create Pod MUST have environment variables such as FOOSERVICE_SERVICE_HOST, FOOSERVICE_SERVICE_PORT, FOOSERVICE_PORT, FOOSERVICE_PORT_8765_TCP_PORT, FOOSERVICE_PORT_8765_TCP_PROTO, FOOSERVICE_PORT_8765_TCP and FOOSERVICE_PORT_8765_TCP_ADDR that are populated with proper values. 446 */ 447 framework.ConformanceIt("should contain environment variables for services [NodeConformance]", func() { 448 // Make a pod that will be a service. 449 // This pod serves its hostname via HTTP. 450 serverName := "server-envvars-" + string(uuid.NewUUID()) 451 serverPod := &v1.Pod{ 452 ObjectMeta: metav1.ObjectMeta{ 453 Name: serverName, 454 Labels: map[string]string{"name": serverName}, 455 }, 456 Spec: v1.PodSpec{ 457 Containers: []v1.Container{ 458 { 459 Name: "srv", 460 Image: framework.ServeHostnameImage, 461 Ports: []v1.ContainerPort{{ContainerPort: 9376}}, 462 }, 463 }, 464 }, 465 } 466 podClient.CreateSync(serverPod) 467 468 // This service exposes port 8080 of the test pod as a service on port 8765 469 // TODO(filbranden): We would like to use a unique service name such as: 470 // svcName := "svc-envvars-" + randomSuffix() 471 // However, that affects the name of the environment variables which are the capitalized 472 // service name, so that breaks this test. One possibility is to tweak the variable names 473 // to match the service. Another is to rethink environment variable names and possibly 474 // allow overriding the prefix in the service manifest. 475 svcName := "fooservice" 476 svc := &v1.Service{ 477 ObjectMeta: metav1.ObjectMeta{ 478 Name: svcName, 479 Labels: map[string]string{ 480 "name": svcName, 481 }, 482 }, 483 Spec: v1.ServiceSpec{ 484 Ports: []v1.ServicePort{{ 485 Port: 8765, 486 TargetPort: intstr.FromInt(8080), 487 }}, 488 Selector: map[string]string{ 489 "name": serverName, 490 }, 491 }, 492 } 493 _, err := f.ClientSet.CoreV1().Services(f.Namespace.Name).Create(context.TODO(), svc, metav1.CreateOptions{}) 494 framework.ExpectNoError(err, "failed to create service") 495 496 // Make a client pod that verifies that it has the service environment variables. 497 podName := "client-envvars-" + string(uuid.NewUUID()) 498 const containerName = "env3cont" 499 pod := &v1.Pod{ 500 ObjectMeta: metav1.ObjectMeta{ 501 Name: podName, 502 Labels: map[string]string{"name": podName}, 503 }, 504 Spec: v1.PodSpec{ 505 Containers: []v1.Container{ 506 { 507 Name: containerName, 508 Image: imageutils.GetE2EImage(imageutils.BusyBox), 509 Command: []string{"sh", "-c", "env"}, 510 }, 511 }, 512 RestartPolicy: v1.RestartPolicyNever, 513 }, 514 } 515 516 // It's possible for the Pod to be created before the Kubelet is updated with the new 517 // service. In that case, we just retry. 518 const maxRetries = 3 519 expectedVars := []string{ 520 "FOOSERVICE_SERVICE_HOST=", 521 "FOOSERVICE_SERVICE_PORT=", 522 "FOOSERVICE_PORT=", 523 "FOOSERVICE_PORT_8765_TCP_PORT=", 524 "FOOSERVICE_PORT_8765_TCP_PROTO=", 525 "FOOSERVICE_PORT_8765_TCP=", 526 "FOOSERVICE_PORT_8765_TCP_ADDR=", 527 } 528 expectNoErrorWithRetries(func() error { 529 return f.MatchContainerOutput(pod, containerName, expectedVars, gomega.ContainSubstring) 530 }, maxRetries, "Container should have service environment variables set") 531 }) 532 533 /* 534 Release: v1.13 535 Testname: Pods, remote command execution over websocket 536 Description: A Pod is created. Websocket is created to retrieve exec command output from this pod. 537 Message retrieved form Websocket MUST match with expected exec command output. 538 */ 539 framework.ConformanceIt("should support remote command execution over websockets [NodeConformance]", func() { 540 config, err := framework.LoadConfig() 541 framework.ExpectNoError(err, "unable to get base config") 542 543 ginkgo.By("creating the pod") 544 name := "pod-exec-websocket-" + string(uuid.NewUUID()) 545 pod := &v1.Pod{ 546 ObjectMeta: metav1.ObjectMeta{ 547 Name: name, 548 }, 549 Spec: v1.PodSpec{ 550 Containers: []v1.Container{ 551 { 552 Name: "main", 553 Image: imageutils.GetE2EImage(imageutils.BusyBox), 554 Command: []string{"/bin/sh", "-c", "echo container is alive; sleep 600"}, 555 }, 556 }, 557 }, 558 } 559 560 ginkgo.By("submitting the pod to kubernetes") 561 pod = podClient.CreateSync(pod) 562 563 req := f.ClientSet.CoreV1().RESTClient().Get(). 564 Namespace(f.Namespace.Name). 565 Resource("pods"). 566 Name(pod.Name). 567 Suffix("exec"). 568 Param("stderr", "1"). 569 Param("stdout", "1"). 570 Param("container", pod.Spec.Containers[0].Name). 571 Param("command", "echo"). 572 Param("command", "remote execution test") 573 574 url := req.URL() 575 ws, err := e2ewebsocket.OpenWebSocketForURL(url, config, []string{"channel.k8s.io"}) 576 if err != nil { 577 framework.Failf("Failed to open websocket to %s: %v", url.String(), err) 578 } 579 defer ws.Close() 580 581 buf := &bytes.Buffer{} 582 gomega.Eventually(func() error { 583 for { 584 var msg []byte 585 if err := websocket.Message.Receive(ws, &msg); err != nil { 586 if err == io.EOF { 587 break 588 } 589 framework.Failf("Failed to read completely from websocket %s: %v", url.String(), err) 590 } 591 if len(msg) == 0 { 592 continue 593 } 594 if msg[0] != 1 { 595 if len(msg) == 1 { 596 // skip an empty message on stream other than stdout 597 continue 598 } else { 599 framework.Failf("Got message from server that didn't start with channel 1 (STDOUT): %v", msg) 600 } 601 602 } 603 buf.Write(msg[1:]) 604 } 605 if buf.Len() == 0 { 606 return fmt.Errorf("unexpected output from server") 607 } 608 if !strings.Contains(buf.String(), "remote execution test") { 609 return fmt.Errorf("expected to find 'remote execution test' in %q", buf.String()) 610 } 611 return nil 612 }, time.Minute, 10*time.Second).Should(gomega.BeNil()) 613 }) 614 615 /* 616 Release: v1.13 617 Testname: Pods, logs from websockets 618 Description: A Pod is created. Websocket is created to retrieve log of a container from this pod. 619 Message retrieved form Websocket MUST match with container's output. 620 */ 621 framework.ConformanceIt("should support retrieving logs from the container over websockets [NodeConformance]", func() { 622 config, err := framework.LoadConfig() 623 framework.ExpectNoError(err, "unable to get base config") 624 625 ginkgo.By("creating the pod") 626 name := "pod-logs-websocket-" + string(uuid.NewUUID()) 627 pod := &v1.Pod{ 628 ObjectMeta: metav1.ObjectMeta{ 629 Name: name, 630 }, 631 Spec: v1.PodSpec{ 632 Containers: []v1.Container{ 633 { 634 Name: "main", 635 Image: imageutils.GetE2EImage(imageutils.BusyBox), 636 Command: []string{"/bin/sh", "-c", "echo container is alive; sleep 10000"}, 637 }, 638 }, 639 }, 640 } 641 642 ginkgo.By("submitting the pod to kubernetes") 643 podClient.CreateSync(pod) 644 645 req := f.ClientSet.CoreV1().RESTClient().Get(). 646 Namespace(f.Namespace.Name). 647 Resource("pods"). 648 Name(pod.Name). 649 Suffix("log"). 650 Param("container", pod.Spec.Containers[0].Name) 651 652 url := req.URL() 653 654 ws, err := e2ewebsocket.OpenWebSocketForURL(url, config, []string{"binary.k8s.io"}) 655 if err != nil { 656 framework.Failf("Failed to open websocket to %s: %v", url.String(), err) 657 } 658 defer ws.Close() 659 buf := &bytes.Buffer{} 660 for { 661 var msg []byte 662 if err := websocket.Message.Receive(ws, &msg); err != nil { 663 if err == io.EOF { 664 break 665 } 666 framework.Failf("Failed to read completely from websocket %s: %v", url.String(), err) 667 } 668 if len(strings.TrimSpace(string(msg))) == 0 { 669 continue 670 } 671 buf.Write(msg) 672 } 673 if buf.String() != "container is alive\n" { 674 framework.Failf("Unexpected websocket logs:\n%s", buf.String()) 675 } 676 }) 677 678 // Slow (~7 mins) 679 ginkgo.It("should have their auto-restart back-off timer reset on image update [Slow][NodeConformance]", func() { 680 podName := "pod-back-off-image" 681 containerName := "back-off" 682 pod := &v1.Pod{ 683 ObjectMeta: metav1.ObjectMeta{ 684 Name: podName, 685 Labels: map[string]string{"test": "back-off-image"}, 686 }, 687 Spec: v1.PodSpec{ 688 Containers: []v1.Container{ 689 { 690 Name: containerName, 691 Image: imageutils.GetE2EImage(imageutils.BusyBox), 692 Command: []string{"/bin/sh", "-c", "sleep 5", "/crash/missing"}, 693 }, 694 }, 695 }, 696 } 697 698 delay1, delay2 := startPodAndGetBackOffs(podClient, pod, buildBackOffDuration) 699 700 ginkgo.By("updating the image") 701 podClient.Update(podName, func(pod *v1.Pod) { 702 pod.Spec.Containers[0].Image = imageutils.GetE2EImage(imageutils.Nginx) 703 }) 704 705 time.Sleep(syncLoopFrequency) 706 framework.ExpectNoError(e2epod.WaitForPodNameRunningInNamespace(f.ClientSet, pod.Name, f.Namespace.Name)) 707 708 ginkgo.By("get restart delay after image update") 709 delayAfterUpdate, err := getRestartDelay(podClient, podName, containerName) 710 if err != nil { 711 framework.Failf("timed out waiting for container restart in pod=%s/%s", podName, containerName) 712 } 713 714 if delayAfterUpdate > 2*delay2 || delayAfterUpdate > 2*delay1 { 715 framework.Failf("updating image did not reset the back-off value in pod=%s/%s d3=%s d2=%s d1=%s", podName, containerName, delayAfterUpdate, delay1, delay2) 716 } 717 }) 718 719 // Slow by design (~27 mins) issue #19027 720 ginkgo.It("should cap back-off at MaxContainerBackOff [Slow][NodeConformance]", func() { 721 podName := "back-off-cap" 722 containerName := "back-off-cap" 723 pod := &v1.Pod{ 724 ObjectMeta: metav1.ObjectMeta{ 725 Name: podName, 726 Labels: map[string]string{"test": "liveness"}, 727 }, 728 Spec: v1.PodSpec{ 729 Containers: []v1.Container{ 730 { 731 Name: containerName, 732 Image: imageutils.GetE2EImage(imageutils.BusyBox), 733 Command: []string{"/bin/sh", "-c", "sleep 5", "/crash/missing"}, 734 }, 735 }, 736 }, 737 } 738 739 podClient.CreateSync(pod) 740 time.Sleep(2 * kubelet.MaxContainerBackOff) // it takes slightly more than 2*x to get to a back-off of x 741 742 // wait for a delay == capped delay of MaxContainerBackOff 743 ginkgo.By("getting restart delay when capped") 744 var ( 745 delay1 time.Duration 746 err error 747 ) 748 for i := 0; i < 3; i++ { 749 delay1, err = getRestartDelay(podClient, podName, containerName) 750 if err != nil { 751 framework.Failf("timed out waiting for container restart in pod=%s/%s", podName, containerName) 752 } 753 754 if delay1 < kubelet.MaxContainerBackOff { 755 continue 756 } 757 } 758 759 if (delay1 < kubelet.MaxContainerBackOff) || (delay1 > maxBackOffTolerance) { 760 framework.Failf("expected %s back-off got=%s in delay1", kubelet.MaxContainerBackOff, delay1) 761 } 762 763 ginkgo.By("getting restart delay after a capped delay") 764 delay2, err := getRestartDelay(podClient, podName, containerName) 765 if err != nil { 766 framework.Failf("timed out waiting for container restart in pod=%s/%s", podName, containerName) 767 } 768 769 if delay2 < kubelet.MaxContainerBackOff || delay2 > maxBackOffTolerance { // syncloop cumulative drift 770 framework.Failf("expected %s back-off got=%s on delay2", kubelet.MaxContainerBackOff, delay2) 771 } 772 }) 773 774 // TODO(freehan): label the test to be [NodeConformance] after tests are proven to be stable. 775 ginkgo.It("should support pod readiness gates [NodeFeature:PodReadinessGate]", func() { 776 podName := "pod-ready" 777 readinessGate1 := "k8s.io/test-condition1" 778 readinessGate2 := "k8s.io/test-condition2" 779 patchStatusFmt := `{"status":{"conditions":[{"type":%q, "status":%q}]}}` 780 pod := &v1.Pod{ 781 ObjectMeta: metav1.ObjectMeta{ 782 Name: podName, 783 Labels: map[string]string{"test": "pod-readiness-gate"}, 784 }, 785 Spec: v1.PodSpec{ 786 Containers: []v1.Container{ 787 { 788 Name: "pod-readiness-gate", 789 Image: imageutils.GetE2EImage(imageutils.BusyBox), 790 Command: []string{"/bin/sh", "-c", "echo container is alive; sleep 10000"}, 791 }, 792 }, 793 ReadinessGates: []v1.PodReadinessGate{ 794 {ConditionType: v1.PodConditionType(readinessGate1)}, 795 {ConditionType: v1.PodConditionType(readinessGate2)}, 796 }, 797 }, 798 } 799 800 validatePodReadiness := func(expectReady bool) { 801 err := wait.Poll(time.Second, time.Minute, func() (bool, error) { 802 pod, err := podClient.Get(context.TODO(), podName, metav1.GetOptions{}) 803 framework.ExpectNoError(err) 804 podReady := podutils.IsPodReady(pod) 805 res := expectReady == podReady 806 if !res { 807 framework.Logf("Expect the Ready condition of pod %q to be %v, but got %v (pod status %#v)", podName, expectReady, podReady, pod.Status) 808 } 809 return res, nil 810 }) 811 framework.ExpectNoError(err) 812 } 813 814 ginkgo.By("submitting the pod to kubernetes") 815 f.PodClient().Create(pod) 816 e2epod.WaitForPodNameRunningInNamespace(f.ClientSet, pod.Name, f.Namespace.Name) 817 framework.ExpectEqual(podClient.PodIsReady(podName), false, "Expect pod's Ready condition to be false initially.") 818 819 ginkgo.By(fmt.Sprintf("patching pod status with condition %q to true", readinessGate1)) 820 _, err := podClient.Patch(context.TODO(), podName, types.StrategicMergePatchType, []byte(fmt.Sprintf(patchStatusFmt, readinessGate1, "True")), metav1.PatchOptions{}, "status") 821 framework.ExpectNoError(err) 822 // Sleep for 10 seconds. 823 time.Sleep(syncLoopFrequency) 824 // Verify the pod is still not ready 825 framework.ExpectEqual(podClient.PodIsReady(podName), false, "Expect pod's Ready condition to be false with only one condition in readinessGates equal to True") 826 827 ginkgo.By(fmt.Sprintf("patching pod status with condition %q to true", readinessGate2)) 828 _, err = podClient.Patch(context.TODO(), podName, types.StrategicMergePatchType, []byte(fmt.Sprintf(patchStatusFmt, readinessGate2, "True")), metav1.PatchOptions{}, "status") 829 framework.ExpectNoError(err) 830 validatePodReadiness(true) 831 832 ginkgo.By(fmt.Sprintf("patching pod status with condition %q to false", readinessGate1)) 833 _, err = podClient.Patch(context.TODO(), podName, types.StrategicMergePatchType, []byte(fmt.Sprintf(patchStatusFmt, readinessGate1, "False")), metav1.PatchOptions{}, "status") 834 framework.ExpectNoError(err) 835 validatePodReadiness(false) 836 837 }) 838 839 /* 840 Release: v1.19 841 Testname: Pods, delete a collection 842 Description: A set of pods is created with a label selector which MUST be found when listed. 843 The set of pods is deleted and MUST NOT show up when listed by its label selector. 844 */ 845 framework.ConformanceIt("should delete a collection of pods", func() { 846 podTestNames := []string{"test-pod-1", "test-pod-2", "test-pod-3"} 847 848 one := int64(1) 849 850 ginkgo.By("Create set of pods") 851 // create a set of pods in test namespace 852 for _, podTestName := range podTestNames { 853 _, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(context.TODO(), &v1.Pod{ 854 ObjectMeta: metav1.ObjectMeta{ 855 Name: podTestName, 856 Labels: map[string]string{ 857 "type": "Testing"}, 858 }, 859 Spec: v1.PodSpec{ 860 TerminationGracePeriodSeconds: &one, 861 Containers: []v1.Container{{ 862 Image: imageutils.GetE2EImage(imageutils.Agnhost), 863 Name: "token-test", 864 }}, 865 RestartPolicy: v1.RestartPolicyNever, 866 }}, metav1.CreateOptions{}) 867 framework.ExpectNoError(err, "failed to create pod") 868 framework.Logf("created %v", podTestName) 869 } 870 871 // wait as required for all 3 pods to be found 872 ginkgo.By("waiting for all 3 pods to be located") 873 err := wait.PollImmediate(podRetryPeriod, podRetryTimeout, checkPodListQuantity(f, "type=Testing", 3)) 874 framework.ExpectNoError(err, "3 pods not found") 875 876 // delete Collection of pods with a label in the current namespace 877 err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).DeleteCollection(context.TODO(), metav1.DeleteOptions{GracePeriodSeconds: &one}, metav1.ListOptions{ 878 LabelSelector: "type=Testing"}) 879 framework.ExpectNoError(err, "failed to delete collection of pods") 880 881 // wait for all pods to be deleted 882 ginkgo.By("waiting for all pods to be deleted") 883 err = wait.PollImmediate(podRetryPeriod, podRetryTimeout, checkPodListQuantity(f, "type=Testing", 0)) 884 framework.ExpectNoError(err, "found a pod(s)") 885 }) 886 887 /* 888 Release: v1.20 889 Testname: Pods, completes the lifecycle of a Pod and the PodStatus 890 Description: A Pod is created with a static label which MUST succeed. It MUST succeed when 891 patching the label and the pod data. When checking and replacing the PodStatus it MUST 892 succeed. It MUST succeed when deleting the Pod. 893 */ 894 framework.ConformanceIt("should run through the lifecycle of Pods and PodStatus", func() { 895 podResource := schema.GroupVersionResource{Group: "", Version: "v1", Resource: "pods"} 896 testNamespaceName := f.Namespace.Name 897 testPodName := "pod-test" 898 testPodImage := imageutils.GetE2EImage(imageutils.Agnhost) 899 testPodImage2 := imageutils.GetE2EImage(imageutils.Httpd) 900 testPodLabels := map[string]string{"test-pod-static": "true"} 901 testPodLabelsFlat := "test-pod-static=true" 902 one := int64(1) 903 904 w := &cache.ListWatch{ 905 WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { 906 options.LabelSelector = testPodLabelsFlat 907 return f.ClientSet.CoreV1().Pods(testNamespaceName).Watch(context.TODO(), options) 908 }, 909 } 910 podsList, err := f.ClientSet.CoreV1().Pods("").List(context.TODO(), metav1.ListOptions{LabelSelector: testPodLabelsFlat}) 911 framework.ExpectNoError(err, "failed to list Pods") 912 913 testPod := v1.Pod{ 914 ObjectMeta: metav1.ObjectMeta{ 915 Name: testPodName, 916 Labels: testPodLabels, 917 }, 918 Spec: v1.PodSpec{ 919 TerminationGracePeriodSeconds: &one, 920 Containers: []v1.Container{ 921 { 922 Name: testPodName, 923 Image: testPodImage, 924 }, 925 }, 926 }, 927 } 928 ginkgo.By("creating a Pod with a static label") 929 _, err = f.ClientSet.CoreV1().Pods(testNamespaceName).Create(context.TODO(), &testPod, metav1.CreateOptions{}) 930 framework.ExpectNoError(err, "failed to create Pod %v in namespace %v", testPod.ObjectMeta.Name, testNamespaceName) 931 932 ginkgo.By("watching for Pod to be ready") 933 ctx, cancel := context.WithTimeout(context.Background(), f.Timeouts.PodStart) 934 defer cancel() 935 _, err = watchtools.Until(ctx, podsList.ResourceVersion, w, func(event watch.Event) (bool, error) { 936 if pod, ok := event.Object.(*v1.Pod); ok { 937 found := pod.ObjectMeta.Name == testPod.ObjectMeta.Name && 938 pod.ObjectMeta.Namespace == testNamespaceName && 939 pod.Labels["test-pod-static"] == "true" && 940 pod.Status.Phase == v1.PodRunning 941 if !found { 942 framework.Logf("observed Pod %v in namespace %v in phase %v with labels: %v & conditions %v", pod.ObjectMeta.Name, pod.ObjectMeta.Namespace, pod.Status.Phase, pod.Labels, pod.Status.Conditions) 943 return false, nil 944 } 945 framework.Logf("Found Pod %v in namespace %v in phase %v with labels: %v & conditions %v", pod.ObjectMeta.Name, pod.ObjectMeta.Namespace, pod.Status.Phase, pod.Labels, pod.Status.Conditions) 946 return found, nil 947 } 948 framework.Logf("Observed event: %+v", event.Object) 949 return false, nil 950 }) 951 if err != nil { 952 p, _ := f.ClientSet.CoreV1().Pods(testNamespaceName).Get(context.TODO(), testPodName, metav1.GetOptions{}) 953 framework.Logf("Pod: %+v", p) 954 } 955 framework.ExpectNoError(err, "failed to see Pod %v in namespace %v running", testPod.ObjectMeta.Name, testNamespaceName) 956 957 ginkgo.By("patching the Pod with a new Label and updated data") 958 podPatch, err := json.Marshal(v1.Pod{ 959 ObjectMeta: metav1.ObjectMeta{ 960 Labels: map[string]string{"test-pod": "patched"}, 961 }, 962 Spec: v1.PodSpec{ 963 TerminationGracePeriodSeconds: &one, 964 Containers: []v1.Container{{ 965 Name: testPodName, 966 Image: testPodImage2, 967 }}, 968 }, 969 }) 970 framework.ExpectNoError(err, "failed to marshal JSON patch for Pod") 971 _, err = f.ClientSet.CoreV1().Pods(testNamespaceName).Patch(context.TODO(), testPodName, types.StrategicMergePatchType, []byte(podPatch), metav1.PatchOptions{}) 972 framework.ExpectNoError(err, "failed to patch Pod %s in namespace %s", testPodName, testNamespaceName) 973 ctx, cancel = context.WithTimeout(context.Background(), 30*time.Second) 974 defer cancel() 975 _, err = watchtools.Until(ctx, podsList.ResourceVersion, w, func(event watch.Event) (bool, error) { 976 switch event.Type { 977 case watch.Modified: 978 if pod, ok := event.Object.(*v1.Pod); ok { 979 found := pod.ObjectMeta.Name == pod.Name && 980 pod.Labels["test-pod-static"] == "true" 981 return found, nil 982 } 983 default: 984 framework.Logf("observed event type %v", event.Type) 985 } 986 return false, nil 987 }) 988 framework.ExpectNoError(err, "failed to see %v event", watch.Modified) 989 990 ginkgo.By("getting the Pod and ensuring that it's patched") 991 pod, err := f.ClientSet.CoreV1().Pods(testNamespaceName).Get(context.TODO(), testPodName, metav1.GetOptions{}) 992 framework.ExpectNoError(err, "failed to fetch Pod %s in namespace %s", testPodName, testNamespaceName) 993 framework.ExpectEqual(pod.ObjectMeta.Labels["test-pod"], "patched", "failed to patch Pod - missing label") 994 framework.ExpectEqual(pod.Spec.Containers[0].Image, testPodImage2, "failed to patch Pod - wrong image") 995 996 ginkgo.By("replacing the Pod's status Ready condition to False") 997 var podStatusUpdate *v1.Pod 998 999 err = retry.RetryOnConflict(retry.DefaultRetry, func() error { 1000 podStatusUnstructured, err := dc.Resource(podResource).Namespace(testNamespaceName).Get(context.TODO(), testPodName, metav1.GetOptions{}, "status") 1001 framework.ExpectNoError(err, "failed to fetch PodStatus of Pod %s in namespace %s", testPodName, testNamespaceName) 1002 podStatusBytes, err := json.Marshal(podStatusUnstructured) 1003 framework.ExpectNoError(err, "failed to marshal unstructured response") 1004 var podStatus v1.Pod 1005 err = json.Unmarshal(podStatusBytes, &podStatus) 1006 framework.ExpectNoError(err, "failed to unmarshal JSON bytes to a Pod object type") 1007 podStatusUpdated := podStatus 1008 podStatusFieldPatchCount := 0 1009 podStatusFieldPatchCountTotal := 2 1010 for pos, cond := range podStatusUpdated.Status.Conditions { 1011 if (cond.Type == v1.PodReady && cond.Status == v1.ConditionTrue) || (cond.Type == v1.ContainersReady && cond.Status == v1.ConditionTrue) { 1012 podStatusUpdated.Status.Conditions[pos].Status = v1.ConditionFalse 1013 podStatusFieldPatchCount++ 1014 } 1015 } 1016 framework.ExpectEqual(podStatusFieldPatchCount, podStatusFieldPatchCountTotal, "failed to patch all relevant Pod conditions") 1017 podStatusUpdate, err = f.ClientSet.CoreV1().Pods(testNamespaceName).UpdateStatus(context.TODO(), &podStatusUpdated, metav1.UpdateOptions{}) 1018 return err 1019 }) 1020 framework.ExpectNoError(err, "failed to update PodStatus of Pod %s in namespace %s", testPodName, testNamespaceName) 1021 1022 ginkgo.By("check the Pod again to ensure its Ready conditions are False") 1023 podStatusFieldPatchCount := 0 1024 podStatusFieldPatchCountTotal := 2 1025 for _, cond := range podStatusUpdate.Status.Conditions { 1026 if (cond.Type == v1.PodReady && cond.Status == v1.ConditionFalse) || (cond.Type == v1.ContainersReady && cond.Status == v1.ConditionFalse) { 1027 podStatusFieldPatchCount++ 1028 } 1029 } 1030 framework.ExpectEqual(podStatusFieldPatchCount, podStatusFieldPatchCountTotal, "failed to update PodStatus - field patch count doesn't match the total") 1031 1032 ginkgo.By("deleting the Pod via a Collection with a LabelSelector") 1033 err = f.ClientSet.CoreV1().Pods(testNamespaceName).DeleteCollection(context.TODO(), metav1.DeleteOptions{GracePeriodSeconds: &one}, metav1.ListOptions{LabelSelector: testPodLabelsFlat}) 1034 framework.ExpectNoError(err, "failed to delete Pod by collection") 1035 1036 ginkgo.By("watching for the Pod to be deleted") 1037 ctx, cancel = context.WithTimeout(context.Background(), 1*time.Minute) 1038 defer cancel() 1039 _, err = watchtools.Until(ctx, podsList.ResourceVersion, w, func(event watch.Event) (bool, error) { 1040 switch event.Type { 1041 case watch.Deleted: 1042 if pod, ok := event.Object.(*v1.Pod); ok { 1043 found := pod.ObjectMeta.Name == pod.Name && 1044 pod.Labels["test-pod-static"] == "true" 1045 return found, nil 1046 } 1047 default: 1048 framework.Logf("observed event type %v", event.Type) 1049 } 1050 return false, nil 1051 }) 1052 framework.ExpectNoError(err, "failed to see %v event", watch.Deleted) 1053 }) 1054}) 1055 1056func checkPodListQuantity(f *framework.Framework, label string, quantity int) func() (bool, error) { 1057 return func() (bool, error) { 1058 var err error 1059 1060 list, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).List(context.TODO(), metav1.ListOptions{ 1061 LabelSelector: label}) 1062 1063 if err != nil { 1064 return false, err 1065 } 1066 1067 if len(list.Items) != quantity { 1068 framework.Logf("Pod quantity %d is different from expected quantity %d", len(list.Items), quantity) 1069 return false, err 1070 } 1071 return true, nil 1072 } 1073} 1074