1/* 2 * 3 * Copyright 2019 gRPC authors. 4 * 5 * Licensed under the Apache License, Version 2.0 (the "License"); 6 * you may not use this file except in compliance with the License. 7 * You may obtain a copy of the License at 8 * 9 * http://www.apache.org/licenses/LICENSE-2.0 10 * 11 * Unless required by applicable law or agreed to in writing, software 12 * distributed under the License is distributed on an "AS IS" BASIS, 13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 * See the License for the specific language governing permissions and 15 * limitations under the License. 16 * 17 */ 18 19// This file contains tests related to the following proposals: 20// https://github.com/grpc/proposal/blob/master/A8-client-side-keepalive.md 21// https://github.com/grpc/proposal/blob/master/A9-server-side-conn-mgt.md 22// https://github.com/grpc/proposal/blob/master/A18-tcp-user-timeout.md 23package transport 24 25import ( 26 "context" 27 "io" 28 "net" 29 "testing" 30 "time" 31 32 "golang.org/x/net/http2" 33 "google.golang.org/grpc/internal/syscall" 34 "google.golang.org/grpc/keepalive" 35) 36 37// TestMaxConnectionIdle tests that a server will send GoAway to an idle 38// client. An idle client is one who doesn't make any RPC calls for a duration 39// of MaxConnectionIdle time. 40func TestMaxConnectionIdle(t *testing.T) { 41 serverConfig := &ServerConfig{ 42 KeepaliveParams: keepalive.ServerParameters{ 43 MaxConnectionIdle: 2 * time.Second, 44 }, 45 } 46 server, client, cancel := setUpWithOptions(t, 0, serverConfig, suspended, ConnectOptions{}) 47 defer func() { 48 client.Close() 49 server.stop() 50 cancel() 51 }() 52 53 stream, err := client.NewStream(context.Background(), &CallHdr{}) 54 if err != nil { 55 t.Fatalf("client.NewStream() failed: %v", err) 56 } 57 client.CloseStream(stream, io.EOF) 58 59 // Wait for the server's MaxConnectionIdle timeout to kick in, and for it 60 // to send a GoAway. 61 timeout := time.NewTimer(time.Second * 4) 62 select { 63 case <-client.Error(): 64 if !timeout.Stop() { 65 <-timeout.C 66 } 67 if reason := client.GetGoAwayReason(); reason != GoAwayNoReason { 68 t.Fatalf("GoAwayReason is %v, want %v", reason, GoAwayNoReason) 69 } 70 case <-timeout.C: 71 t.Fatalf("MaxConnectionIdle timeout expired, expected a GoAway from the server.") 72 } 73} 74 75// TestMaxConenctionIdleBusyClient tests that a server will not send GoAway to 76// a busy client. 77func TestMaxConnectionIdleBusyClient(t *testing.T) { 78 serverConfig := &ServerConfig{ 79 KeepaliveParams: keepalive.ServerParameters{ 80 MaxConnectionIdle: 2 * time.Second, 81 }, 82 } 83 server, client, cancel := setUpWithOptions(t, 0, serverConfig, suspended, ConnectOptions{}) 84 defer func() { 85 client.Close() 86 server.stop() 87 cancel() 88 }() 89 90 _, err := client.NewStream(context.Background(), &CallHdr{}) 91 if err != nil { 92 t.Fatalf("client.NewStream() failed: %v", err) 93 } 94 95 // Wait for double the MaxConnectionIdle time to make sure the server does 96 // not send a GoAway, as the client has an open stream. 97 timeout := time.NewTimer(time.Second * 4) 98 select { 99 case <-client.GoAway(): 100 if !timeout.Stop() { 101 <-timeout.C 102 } 103 t.Fatalf("A non-idle client received a GoAway.") 104 case <-timeout.C: 105 } 106} 107 108// TestMaxConnectionAge tests that a server will send GoAway after a duration 109// of MaxConnectionAge. 110func TestMaxConnectionAge(t *testing.T) { 111 serverConfig := &ServerConfig{ 112 KeepaliveParams: keepalive.ServerParameters{ 113 MaxConnectionAge: 1 * time.Second, 114 MaxConnectionAgeGrace: 1 * time.Second, 115 }, 116 } 117 server, client, cancel := setUpWithOptions(t, 0, serverConfig, suspended, ConnectOptions{}) 118 defer func() { 119 client.Close() 120 server.stop() 121 cancel() 122 }() 123 124 _, err := client.NewStream(context.Background(), &CallHdr{}) 125 if err != nil { 126 t.Fatalf("client.NewStream() failed: %v", err) 127 } 128 129 // Wait for the server's MaxConnectionAge timeout to kick in, and for it 130 // to send a GoAway. 131 timeout := time.NewTimer(4 * time.Second) 132 select { 133 case <-client.Error(): 134 if !timeout.Stop() { 135 <-timeout.C 136 } 137 if reason := client.GetGoAwayReason(); reason != GoAwayNoReason { 138 t.Fatalf("GoAwayReason is %v, want %v", reason, GoAwayNoReason) 139 } 140 case <-timeout.C: 141 t.Fatalf("MaxConnectionAge timeout expired, expected a GoAway from the server.") 142 } 143} 144 145const ( 146 defaultWriteBufSize = 32 * 1024 147 defaultReadBufSize = 32 * 1024 148) 149 150// TestKeepaliveServerClosesUnresponsiveClient tests that a server closes 151// the connection with a client that doesn't respond to keepalive pings. 152// 153// This test creates a regular net.Conn connection to the server and sends the 154// clientPreface and the initial Settings frame, and then remains unresponsive. 155func TestKeepaliveServerClosesUnresponsiveClient(t *testing.T) { 156 serverConfig := &ServerConfig{ 157 KeepaliveParams: keepalive.ServerParameters{ 158 Time: 1 * time.Second, 159 Timeout: 1 * time.Second, 160 }, 161 } 162 server, client, cancel := setUpWithOptions(t, 0, serverConfig, suspended, ConnectOptions{}) 163 defer func() { 164 client.Close() 165 server.stop() 166 cancel() 167 }() 168 169 addr := server.addr() 170 conn, err := net.Dial("tcp", addr) 171 if err != nil { 172 t.Fatalf("net.Dial(tcp, %v) failed: %v", addr, err) 173 } 174 defer conn.Close() 175 176 if n, err := conn.Write(clientPreface); err != nil || n != len(clientPreface) { 177 t.Fatalf("conn.Write(clientPreface) failed: n=%v, err=%v", n, err) 178 } 179 framer := newFramer(conn, defaultWriteBufSize, defaultReadBufSize, 0) 180 if err := framer.fr.WriteSettings(http2.Setting{}); err != nil { 181 t.Fatal("framer.WriteSettings(http2.Setting{}) failed:", err) 182 } 183 framer.writer.Flush() 184 185 // We read from the net.Conn till we get an error, which is expected when 186 // the server closes the connection as part of the keepalive logic. 187 errCh := make(chan error) 188 go func() { 189 b := make([]byte, 24) 190 for { 191 if _, err = conn.Read(b); err != nil { 192 errCh <- err 193 return 194 } 195 } 196 }() 197 198 // Server waits for KeepaliveParams.Time seconds before sending out a ping, 199 // and then waits for KeepaliveParams.Timeout for a ping ack. 200 timeout := time.NewTimer(4 * time.Second) 201 select { 202 case err := <-errCh: 203 if err != io.EOF { 204 t.Fatalf("client.Read(_) = _,%v, want io.EOF", err) 205 206 } 207 case <-timeout.C: 208 t.Fatalf("keepalive timeout expired, server should have closed the connection.") 209 } 210} 211 212// TestKeepaliveServerWithResponsiveClient tests that a server doesn't close 213// the connection with a client that responds to keepalive pings. 214func TestKeepaliveServerWithResponsiveClient(t *testing.T) { 215 serverConfig := &ServerConfig{ 216 KeepaliveParams: keepalive.ServerParameters{ 217 Time: 1 * time.Second, 218 Timeout: 1 * time.Second, 219 }, 220 } 221 server, client, cancel := setUpWithOptions(t, 0, serverConfig, suspended, ConnectOptions{}) 222 defer func() { 223 client.Close() 224 server.stop() 225 cancel() 226 }() 227 228 // Give keepalive logic some time by sleeping. 229 time.Sleep(4 * time.Second) 230 231 // Make sure the client transport is healthy. 232 if _, err := client.NewStream(context.Background(), &CallHdr{}); err != nil { 233 t.Fatalf("client.NewStream() failed: %v", err) 234 } 235} 236 237// TestKeepaliveClientClosesUnresponsiveServer creates a server which does not 238// respond to keepalive pings, and makes sure that the client closes the 239// transport once the keepalive logic kicks in. Here, we set the 240// `PermitWithoutStream` parameter to true which ensures that the keepalive 241// logic is running even without any active streams. 242func TestKeepaliveClientClosesUnresponsiveServer(t *testing.T) { 243 connCh := make(chan net.Conn, 1) 244 client, cancel := setUpWithNoPingServer(t, ConnectOptions{KeepaliveParams: keepalive.ClientParameters{ 245 Time: 1 * time.Second, 246 Timeout: 1 * time.Second, 247 PermitWithoutStream: true, 248 }}, connCh) 249 defer cancel() 250 defer client.Close() 251 252 conn, ok := <-connCh 253 if !ok { 254 t.Fatalf("Server didn't return connection object") 255 } 256 defer conn.Close() 257 258 // Sleep for keepalive to close the connection. 259 time.Sleep(4 * time.Second) 260 261 // Make sure the client transport is not healthy. 262 if _, err := client.NewStream(context.Background(), &CallHdr{}); err == nil { 263 t.Fatal("client.NewStream() should have failed, but succeeded") 264 } 265} 266 267// TestKeepaliveClientOpenWithUnresponsiveServer creates a server which does 268// not respond to keepalive pings, and makes sure that the client does not 269// close the transport. Here, we do not set the `PermitWithoutStream` parameter 270// to true which ensures that the keepalive logic is turned off without any 271// active streams, and therefore the transport stays open. 272func TestKeepaliveClientOpenWithUnresponsiveServer(t *testing.T) { 273 connCh := make(chan net.Conn, 1) 274 client, cancel := setUpWithNoPingServer(t, ConnectOptions{KeepaliveParams: keepalive.ClientParameters{ 275 Time: 1 * time.Second, 276 Timeout: 1 * time.Second, 277 }}, connCh) 278 defer cancel() 279 defer client.Close() 280 281 conn, ok := <-connCh 282 if !ok { 283 t.Fatalf("Server didn't return connection object") 284 } 285 defer conn.Close() 286 287 // Give keepalive some time. 288 time.Sleep(4 * time.Second) 289 290 // Make sure the client transport is healthy. 291 if _, err := client.NewStream(context.Background(), &CallHdr{}); err != nil { 292 t.Fatalf("client.NewStream() failed: %v", err) 293 } 294} 295 296// TestKeepaliveClientClosesWithActiveStreams creates a server which does not 297// respond to keepalive pings, and makes sure that the client closes the 298// transport even when there is an active stream. 299func TestKeepaliveClientClosesWithActiveStreams(t *testing.T) { 300 connCh := make(chan net.Conn, 1) 301 client, cancel := setUpWithNoPingServer(t, ConnectOptions{KeepaliveParams: keepalive.ClientParameters{ 302 Time: 1 * time.Second, 303 Timeout: 1 * time.Second, 304 }}, connCh) 305 defer cancel() 306 defer client.Close() 307 308 conn, ok := <-connCh 309 if !ok { 310 t.Fatalf("Server didn't return connection object") 311 } 312 defer conn.Close() 313 314 // Create a stream, but send no data on it. 315 if _, err := client.NewStream(context.Background(), &CallHdr{}); err != nil { 316 t.Fatalf("client.NewStream() failed: %v", err) 317 } 318 319 // Give keepalive some time. 320 time.Sleep(4 * time.Second) 321 322 // Make sure the client transport is not healthy. 323 if _, err := client.NewStream(context.Background(), &CallHdr{}); err == nil { 324 t.Fatal("client.NewStream() should have failed, but succeeded") 325 } 326} 327 328// TestKeepaliveClientStaysHealthyWithResponsiveServer creates a server which 329// responds to keepalive pings, and makes sure than a client transport stays 330// healthy without any active streams. 331func TestKeepaliveClientStaysHealthyWithResponsiveServer(t *testing.T) { 332 server, client, cancel := setUpWithOptions(t, 0, &ServerConfig{}, normal, ConnectOptions{ 333 KeepaliveParams: keepalive.ClientParameters{ 334 Time: 1 * time.Second, 335 Timeout: 1 * time.Second, 336 PermitWithoutStream: true, 337 }}) 338 defer func() { 339 client.Close() 340 server.stop() 341 cancel() 342 }() 343 344 // Give keepalive some time. 345 time.Sleep(4 * time.Second) 346 347 // Make sure the client transport is healthy. 348 if _, err := client.NewStream(context.Background(), &CallHdr{}); err != nil { 349 t.Fatalf("client.NewStream() failed: %v", err) 350 } 351} 352 353// TestKeepaliveClientFrequency creates a server which expects at most 1 client 354// ping for every 1.2 seconds, while the client is configured to send a ping 355// every 1 second. So, this configuration should end up with the client 356// transport being closed. But we had a bug wherein the client was sending one 357// ping every [Time+Timeout] instead of every [Time] period, and this test 358// explicitly makes sure the fix works and the client sends a ping every [Time] 359// period. 360func TestKeepaliveClientFrequency(t *testing.T) { 361 serverConfig := &ServerConfig{ 362 KeepalivePolicy: keepalive.EnforcementPolicy{ 363 MinTime: 1200 * time.Millisecond, // 1.2 seconds 364 PermitWithoutStream: true, 365 }, 366 } 367 clientOptions := ConnectOptions{ 368 KeepaliveParams: keepalive.ClientParameters{ 369 Time: 1 * time.Second, 370 Timeout: 2 * time.Second, 371 PermitWithoutStream: true, 372 }, 373 } 374 server, client, cancel := setUpWithOptions(t, 0, serverConfig, normal, clientOptions) 375 defer func() { 376 client.Close() 377 server.stop() 378 cancel() 379 }() 380 381 timeout := time.NewTimer(6 * time.Second) 382 select { 383 case <-client.Error(): 384 if !timeout.Stop() { 385 <-timeout.C 386 } 387 if reason := client.GetGoAwayReason(); reason != GoAwayTooManyPings { 388 t.Fatalf("GoAwayReason is %v, want %v", reason, GoAwayTooManyPings) 389 } 390 case <-timeout.C: 391 t.Fatalf("client transport still healthy; expected GoAway from the server.") 392 } 393 394 // Make sure the client transport is not healthy. 395 if _, err := client.NewStream(context.Background(), &CallHdr{}); err == nil { 396 t.Fatal("client.NewStream() should have failed, but succeeded") 397 } 398} 399 400// TestKeepaliveServerEnforcementWithAbusiveClientNoRPC verifies that the 401// server closes a client transport when it sends too many keepalive pings 402// (when there are no active streams), based on the configured 403// EnforcementPolicy. 404func TestKeepaliveServerEnforcementWithAbusiveClientNoRPC(t *testing.T) { 405 serverConfig := &ServerConfig{ 406 KeepalivePolicy: keepalive.EnforcementPolicy{ 407 MinTime: 2 * time.Second, 408 }, 409 } 410 clientOptions := ConnectOptions{ 411 KeepaliveParams: keepalive.ClientParameters{ 412 Time: 50 * time.Millisecond, 413 Timeout: 1 * time.Second, 414 PermitWithoutStream: true, 415 }, 416 } 417 server, client, cancel := setUpWithOptions(t, 0, serverConfig, normal, clientOptions) 418 defer func() { 419 client.Close() 420 server.stop() 421 cancel() 422 }() 423 424 timeout := time.NewTimer(4 * time.Second) 425 select { 426 case <-client.Error(): 427 if !timeout.Stop() { 428 <-timeout.C 429 } 430 if reason := client.GetGoAwayReason(); reason != GoAwayTooManyPings { 431 t.Fatalf("GoAwayReason is %v, want %v", reason, GoAwayTooManyPings) 432 } 433 case <-timeout.C: 434 t.Fatalf("client transport still healthy; expected GoAway from the server.") 435 } 436 437 // Make sure the client transport is not healthy. 438 if _, err := client.NewStream(context.Background(), &CallHdr{}); err == nil { 439 t.Fatal("client.NewStream() should have failed, but succeeded") 440 } 441} 442 443// TestKeepaliveServerEnforcementWithAbusiveClientWithRPC verifies that the 444// server closes a client transport when it sends too many keepalive pings 445// (even when there is an active stream), based on the configured 446// EnforcementPolicy. 447func TestKeepaliveServerEnforcementWithAbusiveClientWithRPC(t *testing.T) { 448 serverConfig := &ServerConfig{ 449 KeepalivePolicy: keepalive.EnforcementPolicy{ 450 MinTime: 2 * time.Second, 451 }, 452 } 453 clientOptions := ConnectOptions{ 454 KeepaliveParams: keepalive.ClientParameters{ 455 Time: 50 * time.Millisecond, 456 Timeout: 1 * time.Second, 457 }, 458 } 459 server, client, cancel := setUpWithOptions(t, 0, serverConfig, suspended, clientOptions) 460 defer func() { 461 client.Close() 462 server.stop() 463 cancel() 464 }() 465 466 if _, err := client.NewStream(context.Background(), &CallHdr{}); err != nil { 467 t.Fatalf("client.NewStream() failed: %v", err) 468 } 469 470 timeout := time.NewTimer(4 * time.Second) 471 select { 472 case <-client.Error(): 473 if !timeout.Stop() { 474 <-timeout.C 475 } 476 if reason := client.GetGoAwayReason(); reason != GoAwayTooManyPings { 477 t.Fatalf("GoAwayReason is %v, want %v", reason, GoAwayTooManyPings) 478 } 479 case <-timeout.C: 480 t.Fatalf("client transport still healthy; expected GoAway from the server.") 481 } 482 483 // Make sure the client transport is not healthy. 484 if _, err := client.NewStream(context.Background(), &CallHdr{}); err == nil { 485 t.Fatal("client.NewStream() should have failed, but succeeded") 486 } 487} 488 489// TestKeepaliveServerEnforcementWithObeyingClientNoRPC verifies that the 490// server does not close a client transport (with no active streams) which 491// sends keepalive pings in accordance to the configured keepalive 492// EnforcementPolicy. 493func TestKeepaliveServerEnforcementWithObeyingClientNoRPC(t *testing.T) { 494 serverConfig := &ServerConfig{ 495 KeepalivePolicy: keepalive.EnforcementPolicy{ 496 MinTime: 100 * time.Millisecond, 497 PermitWithoutStream: true, 498 }, 499 } 500 clientOptions := ConnectOptions{ 501 KeepaliveParams: keepalive.ClientParameters{ 502 Time: 101 * time.Millisecond, 503 Timeout: 1 * time.Second, 504 PermitWithoutStream: true, 505 }, 506 } 507 server, client, cancel := setUpWithOptions(t, 0, serverConfig, normal, clientOptions) 508 defer func() { 509 client.Close() 510 server.stop() 511 cancel() 512 }() 513 514 // Give keepalive enough time. 515 time.Sleep(3 * time.Second) 516 517 // Make sure the client transport is healthy. 518 if _, err := client.NewStream(context.Background(), &CallHdr{}); err != nil { 519 t.Fatalf("client.NewStream() failed: %v", err) 520 } 521} 522 523// TestKeepaliveServerEnforcementWithObeyingClientWithRPC verifies that the 524// server does not close a client transport (with active streams) which 525// sends keepalive pings in accordance to the configured keepalive 526// EnforcementPolicy. 527func TestKeepaliveServerEnforcementWithObeyingClientWithRPC(t *testing.T) { 528 serverConfig := &ServerConfig{ 529 KeepalivePolicy: keepalive.EnforcementPolicy{ 530 MinTime: 100 * time.Millisecond, 531 }, 532 } 533 clientOptions := ConnectOptions{ 534 KeepaliveParams: keepalive.ClientParameters{ 535 Time: 101 * time.Millisecond, 536 Timeout: 1 * time.Second, 537 }, 538 } 539 server, client, cancel := setUpWithOptions(t, 0, serverConfig, suspended, clientOptions) 540 defer func() { 541 client.Close() 542 server.stop() 543 cancel() 544 }() 545 546 if _, err := client.NewStream(context.Background(), &CallHdr{}); err != nil { 547 t.Fatalf("client.NewStream() failed: %v", err) 548 } 549 550 // Give keepalive enough time. 551 time.Sleep(3 * time.Second) 552 553 // Make sure the client transport is healthy. 554 if _, err := client.NewStream(context.Background(), &CallHdr{}); err != nil { 555 t.Fatalf("client.NewStream() failed: %v", err) 556 } 557} 558 559// TestKeepaliveServerEnforcementWithDormantKeepaliveOnClient verifies that the 560// server does not closes a client transport, which has been configured to send 561// more pings than allowed by the server's EnforcementPolicy. This client 562// transport does not have any active streams and `PermitWithoutStream` is set 563// to false. This should ensure that the keepalive functionality on the client 564// side enters a dormant state. 565func TestKeepaliveServerEnforcementWithDormantKeepaliveOnClient(t *testing.T) { 566 serverConfig := &ServerConfig{ 567 KeepalivePolicy: keepalive.EnforcementPolicy{ 568 MinTime: 2 * time.Second, 569 }, 570 } 571 clientOptions := ConnectOptions{ 572 KeepaliveParams: keepalive.ClientParameters{ 573 Time: 50 * time.Millisecond, 574 Timeout: 1 * time.Second, 575 }, 576 } 577 server, client, cancel := setUpWithOptions(t, 0, serverConfig, normal, clientOptions) 578 defer func() { 579 client.Close() 580 server.stop() 581 cancel() 582 }() 583 584 // No active streams on the client. Give keepalive enough time. 585 time.Sleep(5 * time.Second) 586 587 // Make sure the client transport is healthy. 588 if _, err := client.NewStream(context.Background(), &CallHdr{}); err != nil { 589 t.Fatalf("client.NewStream() failed: %v", err) 590 } 591} 592 593// TestTCPUserTimeout tests that the TCP_USER_TIMEOUT socket option is set to 594// the keepalive timeout, as detailed in proposal A18. 595func TestTCPUserTimeout(t *testing.T) { 596 tests := []struct { 597 time time.Duration 598 timeout time.Duration 599 wantTimeout time.Duration 600 }{ 601 { 602 10 * time.Second, 603 10 * time.Second, 604 10 * 1000 * time.Millisecond, 605 }, 606 { 607 0, 608 0, 609 0, 610 }, 611 } 612 for _, tt := range tests { 613 server, client, cancel := setUpWithOptions( 614 t, 615 0, 616 &ServerConfig{ 617 KeepaliveParams: keepalive.ServerParameters{ 618 Time: tt.timeout, 619 Timeout: tt.timeout, 620 }, 621 }, 622 normal, 623 ConnectOptions{ 624 KeepaliveParams: keepalive.ClientParameters{ 625 Time: tt.time, 626 Timeout: tt.timeout, 627 }, 628 }, 629 ) 630 defer func() { 631 client.Close() 632 server.stop() 633 cancel() 634 }() 635 636 stream, err := client.NewStream(context.Background(), &CallHdr{}) 637 if err != nil { 638 t.Fatalf("client.NewStream() failed: %v", err) 639 } 640 client.CloseStream(stream, io.EOF) 641 642 opt, err := syscall.GetTCPUserTimeout(client.conn) 643 if err != nil { 644 t.Fatalf("syscall.GetTCPUserTimeout() failed: %v", err) 645 } 646 if opt < 0 { 647 t.Skipf("skipping test on unsupported environment") 648 } 649 if gotTimeout := time.Duration(opt) * time.Millisecond; gotTimeout != tt.wantTimeout { 650 t.Fatalf("syscall.GetTCPUserTimeout() = %d, want %d", gotTimeout, tt.wantTimeout) 651 } 652 } 653} 654