1package libnetwork 2 3import ( 4 "encoding/json" 5 "fmt" 6 "net" 7 "strings" 8 "sync" 9 10 "github.com/docker/libnetwork/datastore" 11 "github.com/docker/libnetwork/ipamapi" 12 "github.com/docker/libnetwork/netlabel" 13 "github.com/docker/libnetwork/options" 14 "github.com/docker/libnetwork/types" 15 "github.com/sirupsen/logrus" 16) 17 18// Endpoint represents a logical connection between a network and a sandbox. 19type Endpoint interface { 20 // A system generated id for this endpoint. 21 ID() string 22 23 // Name returns the name of this endpoint. 24 Name() string 25 26 // Network returns the name of the network to which this endpoint is attached. 27 Network() string 28 29 // Join joins the sandbox to the endpoint and populates into the sandbox 30 // the network resources allocated for the endpoint. 31 Join(sandbox Sandbox, options ...EndpointOption) error 32 33 // Leave detaches the network resources populated in the sandbox. 34 Leave(sandbox Sandbox, options ...EndpointOption) error 35 36 // Return certain operational data belonging to this endpoint 37 Info() EndpointInfo 38 39 // DriverInfo returns a collection of driver operational data related to this endpoint retrieved from the driver 40 DriverInfo() (map[string]interface{}, error) 41 42 // Delete and detaches this endpoint from the network. 43 Delete(force bool) error 44} 45 46// EndpointOption is an option setter function type used to pass various options to Network 47// and Endpoint interfaces methods. The various setter functions of type EndpointOption are 48// provided by libnetwork, they look like <Create|Join|Leave>Option[...](...) 49type EndpointOption func(ep *endpoint) 50 51type endpoint struct { 52 name string 53 id string 54 network *network 55 iface *endpointInterface 56 joinInfo *endpointJoinInfo 57 sandboxID string 58 locator string 59 exposedPorts []types.TransportPort 60 anonymous bool 61 disableResolution bool 62 generic map[string]interface{} 63 joinLeaveDone chan struct{} 64 prefAddress net.IP 65 prefAddressV6 net.IP 66 ipamOptions map[string]string 67 aliases map[string]string 68 myAliases []string 69 svcID string 70 svcName string 71 virtualIP net.IP 72 svcAliases []string 73 ingressPorts []*PortConfig 74 dbIndex uint64 75 dbExists bool 76 serviceEnabled bool 77 loadBalancer bool 78 sync.Mutex 79} 80 81func (ep *endpoint) MarshalJSON() ([]byte, error) { 82 ep.Lock() 83 defer ep.Unlock() 84 85 epMap := make(map[string]interface{}) 86 epMap["name"] = ep.name 87 epMap["id"] = ep.id 88 epMap["ep_iface"] = ep.iface 89 epMap["joinInfo"] = ep.joinInfo 90 epMap["exposed_ports"] = ep.exposedPorts 91 if ep.generic != nil { 92 epMap["generic"] = ep.generic 93 } 94 epMap["sandbox"] = ep.sandboxID 95 epMap["locator"] = ep.locator 96 epMap["anonymous"] = ep.anonymous 97 epMap["disableResolution"] = ep.disableResolution 98 epMap["myAliases"] = ep.myAliases 99 epMap["svcName"] = ep.svcName 100 epMap["svcID"] = ep.svcID 101 epMap["virtualIP"] = ep.virtualIP.String() 102 epMap["ingressPorts"] = ep.ingressPorts 103 epMap["svcAliases"] = ep.svcAliases 104 epMap["loadBalancer"] = ep.loadBalancer 105 106 return json.Marshal(epMap) 107} 108 109func (ep *endpoint) UnmarshalJSON(b []byte) (err error) { 110 ep.Lock() 111 defer ep.Unlock() 112 113 var epMap map[string]interface{} 114 if err := json.Unmarshal(b, &epMap); err != nil { 115 return err 116 } 117 ep.name = epMap["name"].(string) 118 ep.id = epMap["id"].(string) 119 120 ib, _ := json.Marshal(epMap["ep_iface"]) 121 json.Unmarshal(ib, &ep.iface) 122 123 jb, _ := json.Marshal(epMap["joinInfo"]) 124 json.Unmarshal(jb, &ep.joinInfo) 125 126 tb, _ := json.Marshal(epMap["exposed_ports"]) 127 var tPorts []types.TransportPort 128 json.Unmarshal(tb, &tPorts) 129 ep.exposedPorts = tPorts 130 131 cb, _ := json.Marshal(epMap["sandbox"]) 132 json.Unmarshal(cb, &ep.sandboxID) 133 134 if v, ok := epMap["generic"]; ok { 135 ep.generic = v.(map[string]interface{}) 136 137 if opt, ok := ep.generic[netlabel.PortMap]; ok { 138 pblist := []types.PortBinding{} 139 140 for i := 0; i < len(opt.([]interface{})); i++ { 141 pb := types.PortBinding{} 142 tmp := opt.([]interface{})[i].(map[string]interface{}) 143 144 bytes, err := json.Marshal(tmp) 145 if err != nil { 146 logrus.Error(err) 147 break 148 } 149 err = json.Unmarshal(bytes, &pb) 150 if err != nil { 151 logrus.Error(err) 152 break 153 } 154 pblist = append(pblist, pb) 155 } 156 ep.generic[netlabel.PortMap] = pblist 157 } 158 159 if opt, ok := ep.generic[netlabel.ExposedPorts]; ok { 160 tplist := []types.TransportPort{} 161 162 for i := 0; i < len(opt.([]interface{})); i++ { 163 tp := types.TransportPort{} 164 tmp := opt.([]interface{})[i].(map[string]interface{}) 165 166 bytes, err := json.Marshal(tmp) 167 if err != nil { 168 logrus.Error(err) 169 break 170 } 171 err = json.Unmarshal(bytes, &tp) 172 if err != nil { 173 logrus.Error(err) 174 break 175 } 176 tplist = append(tplist, tp) 177 } 178 ep.generic[netlabel.ExposedPorts] = tplist 179 180 } 181 } 182 183 if v, ok := epMap["anonymous"]; ok { 184 ep.anonymous = v.(bool) 185 } 186 if v, ok := epMap["disableResolution"]; ok { 187 ep.disableResolution = v.(bool) 188 } 189 if l, ok := epMap["locator"]; ok { 190 ep.locator = l.(string) 191 } 192 193 if sn, ok := epMap["svcName"]; ok { 194 ep.svcName = sn.(string) 195 } 196 197 if si, ok := epMap["svcID"]; ok { 198 ep.svcID = si.(string) 199 } 200 201 if vip, ok := epMap["virtualIP"]; ok { 202 ep.virtualIP = net.ParseIP(vip.(string)) 203 } 204 205 if v, ok := epMap["loadBalancer"]; ok { 206 ep.loadBalancer = v.(bool) 207 } 208 209 sal, _ := json.Marshal(epMap["svcAliases"]) 210 var svcAliases []string 211 json.Unmarshal(sal, &svcAliases) 212 ep.svcAliases = svcAliases 213 214 pc, _ := json.Marshal(epMap["ingressPorts"]) 215 var ingressPorts []*PortConfig 216 json.Unmarshal(pc, &ingressPorts) 217 ep.ingressPorts = ingressPorts 218 219 ma, _ := json.Marshal(epMap["myAliases"]) 220 var myAliases []string 221 json.Unmarshal(ma, &myAliases) 222 ep.myAliases = myAliases 223 return nil 224} 225 226func (ep *endpoint) New() datastore.KVObject { 227 return &endpoint{network: ep.getNetwork()} 228} 229 230func (ep *endpoint) CopyTo(o datastore.KVObject) error { 231 ep.Lock() 232 defer ep.Unlock() 233 234 dstEp := o.(*endpoint) 235 dstEp.name = ep.name 236 dstEp.id = ep.id 237 dstEp.sandboxID = ep.sandboxID 238 dstEp.locator = ep.locator 239 dstEp.dbIndex = ep.dbIndex 240 dstEp.dbExists = ep.dbExists 241 dstEp.anonymous = ep.anonymous 242 dstEp.disableResolution = ep.disableResolution 243 dstEp.svcName = ep.svcName 244 dstEp.svcID = ep.svcID 245 dstEp.virtualIP = ep.virtualIP 246 dstEp.loadBalancer = ep.loadBalancer 247 248 dstEp.svcAliases = make([]string, len(ep.svcAliases)) 249 copy(dstEp.svcAliases, ep.svcAliases) 250 251 dstEp.ingressPorts = make([]*PortConfig, len(ep.ingressPorts)) 252 copy(dstEp.ingressPorts, ep.ingressPorts) 253 254 if ep.iface != nil { 255 dstEp.iface = &endpointInterface{} 256 ep.iface.CopyTo(dstEp.iface) 257 } 258 259 if ep.joinInfo != nil { 260 dstEp.joinInfo = &endpointJoinInfo{} 261 ep.joinInfo.CopyTo(dstEp.joinInfo) 262 } 263 264 dstEp.exposedPorts = make([]types.TransportPort, len(ep.exposedPorts)) 265 copy(dstEp.exposedPorts, ep.exposedPorts) 266 267 dstEp.myAliases = make([]string, len(ep.myAliases)) 268 copy(dstEp.myAliases, ep.myAliases) 269 270 dstEp.generic = options.Generic{} 271 for k, v := range ep.generic { 272 dstEp.generic[k] = v 273 } 274 275 return nil 276} 277 278func (ep *endpoint) ID() string { 279 ep.Lock() 280 defer ep.Unlock() 281 282 return ep.id 283} 284 285func (ep *endpoint) Name() string { 286 ep.Lock() 287 defer ep.Unlock() 288 289 return ep.name 290} 291 292func (ep *endpoint) MyAliases() []string { 293 ep.Lock() 294 defer ep.Unlock() 295 296 return ep.myAliases 297} 298 299func (ep *endpoint) Network() string { 300 if ep.network == nil { 301 return "" 302 } 303 304 return ep.network.name 305} 306 307func (ep *endpoint) isAnonymous() bool { 308 ep.Lock() 309 defer ep.Unlock() 310 return ep.anonymous 311} 312 313// isServiceEnabled check if service is enabled on the endpoint 314func (ep *endpoint) isServiceEnabled() bool { 315 ep.Lock() 316 defer ep.Unlock() 317 return ep.serviceEnabled 318} 319 320// enableService sets service enabled on the endpoint 321func (ep *endpoint) enableService() { 322 ep.Lock() 323 defer ep.Unlock() 324 ep.serviceEnabled = true 325} 326 327// disableService disables service on the endpoint 328func (ep *endpoint) disableService() { 329 ep.Lock() 330 defer ep.Unlock() 331 ep.serviceEnabled = false 332} 333 334func (ep *endpoint) needResolver() bool { 335 ep.Lock() 336 defer ep.Unlock() 337 return !ep.disableResolution 338} 339 340// endpoint Key structure : endpoint/network-id/endpoint-id 341func (ep *endpoint) Key() []string { 342 if ep.network == nil { 343 return nil 344 } 345 346 return []string{datastore.EndpointKeyPrefix, ep.network.id, ep.id} 347} 348 349func (ep *endpoint) KeyPrefix() []string { 350 if ep.network == nil { 351 return nil 352 } 353 354 return []string{datastore.EndpointKeyPrefix, ep.network.id} 355} 356 357func (ep *endpoint) networkIDFromKey(key string) (string, error) { 358 // endpoint Key structure : docker/libnetwork/endpoint/${network-id}/${endpoint-id} 359 // it's an invalid key if the key doesn't have all the 5 key elements above 360 keyElements := strings.Split(key, "/") 361 if !strings.HasPrefix(key, datastore.Key(datastore.EndpointKeyPrefix)) || len(keyElements) < 5 { 362 return "", fmt.Errorf("invalid endpoint key : %v", key) 363 } 364 // network-id is placed at index=3. pls refer to endpoint.Key() method 365 return strings.Split(key, "/")[3], nil 366} 367 368func (ep *endpoint) Value() []byte { 369 b, err := json.Marshal(ep) 370 if err != nil { 371 return nil 372 } 373 return b 374} 375 376func (ep *endpoint) SetValue(value []byte) error { 377 return json.Unmarshal(value, ep) 378} 379 380func (ep *endpoint) Index() uint64 { 381 ep.Lock() 382 defer ep.Unlock() 383 return ep.dbIndex 384} 385 386func (ep *endpoint) SetIndex(index uint64) { 387 ep.Lock() 388 defer ep.Unlock() 389 ep.dbIndex = index 390 ep.dbExists = true 391} 392 393func (ep *endpoint) Exists() bool { 394 ep.Lock() 395 defer ep.Unlock() 396 return ep.dbExists 397} 398 399func (ep *endpoint) Skip() bool { 400 return ep.getNetwork().Skip() 401} 402 403func (ep *endpoint) processOptions(options ...EndpointOption) { 404 ep.Lock() 405 defer ep.Unlock() 406 407 for _, opt := range options { 408 if opt != nil { 409 opt(ep) 410 } 411 } 412} 413 414func (ep *endpoint) getNetwork() *network { 415 ep.Lock() 416 defer ep.Unlock() 417 418 return ep.network 419} 420 421func (ep *endpoint) getNetworkFromStore() (*network, error) { 422 if ep.network == nil { 423 return nil, fmt.Errorf("invalid network object in endpoint %s", ep.Name()) 424 } 425 426 return ep.network.getController().getNetworkFromStore(ep.network.id) 427} 428 429func (ep *endpoint) Join(sbox Sandbox, options ...EndpointOption) error { 430 if sbox == nil { 431 return types.BadRequestErrorf("endpoint cannot be joined by nil container") 432 } 433 434 sb, ok := sbox.(*sandbox) 435 if !ok { 436 return types.BadRequestErrorf("not a valid Sandbox interface") 437 } 438 439 sb.joinLeaveStart() 440 defer sb.joinLeaveEnd() 441 442 return ep.sbJoin(sb, options...) 443} 444 445func (ep *endpoint) sbJoin(sb *sandbox, options ...EndpointOption) (err error) { 446 n, err := ep.getNetworkFromStore() 447 if err != nil { 448 return fmt.Errorf("failed to get network from store during join: %v", err) 449 } 450 451 ep, err = n.getEndpointFromStore(ep.ID()) 452 if err != nil { 453 return fmt.Errorf("failed to get endpoint from store during join: %v", err) 454 } 455 456 ep.Lock() 457 if ep.sandboxID != "" { 458 ep.Unlock() 459 return types.ForbiddenErrorf("another container is attached to the same network endpoint") 460 } 461 ep.network = n 462 ep.sandboxID = sb.ID() 463 ep.joinInfo = &endpointJoinInfo{} 464 epid := ep.id 465 ep.Unlock() 466 defer func() { 467 if err != nil { 468 ep.Lock() 469 ep.sandboxID = "" 470 ep.Unlock() 471 } 472 }() 473 474 nid := n.ID() 475 476 ep.processOptions(options...) 477 478 d, err := n.driver(true) 479 if err != nil { 480 return fmt.Errorf("failed to get driver during join: %v", err) 481 } 482 483 err = d.Join(nid, epid, sb.Key(), ep, sb.Labels()) 484 if err != nil { 485 return err 486 } 487 defer func() { 488 if err != nil { 489 if e := d.Leave(nid, epid); e != nil { 490 logrus.Warnf("driver leave failed while rolling back join: %v", e) 491 } 492 } 493 }() 494 495 // Watch for service records 496 if !n.getController().isAgent() { 497 n.getController().watchSvcRecord(ep) 498 } 499 500 if doUpdateHostsFile(n, sb) { 501 address := "" 502 if ip := ep.getFirstInterfaceAddress(); ip != nil { 503 address = ip.String() 504 } 505 if err = sb.updateHostsFile(address); err != nil { 506 return err 507 } 508 } 509 if err = sb.updateDNS(n.enableIPv6); err != nil { 510 return err 511 } 512 513 // Current endpoint providing external connectivity for the sandbox 514 extEp := sb.getGatewayEndpoint() 515 516 sb.addEndpoint(ep) 517 defer func() { 518 if err != nil { 519 sb.removeEndpoint(ep) 520 } 521 }() 522 523 if err = sb.populateNetworkResources(ep); err != nil { 524 return err 525 } 526 527 if err = n.getController().updateToStore(ep); err != nil { 528 return err 529 } 530 531 if err = ep.addDriverInfoToCluster(); err != nil { 532 return err 533 } 534 535 defer func() { 536 if err != nil { 537 if e := ep.deleteDriverInfoFromCluster(); e != nil { 538 logrus.Errorf("Could not delete endpoint state for endpoint %s from cluster on join failure: %v", ep.Name(), e) 539 } 540 } 541 }() 542 543 // Load balancing endpoints should never have a default gateway nor 544 // should they alter the status of a network's default gateway 545 if ep.loadBalancer && !sb.ingress { 546 return nil 547 } 548 549 if sb.needDefaultGW() && sb.getEndpointInGWNetwork() == nil { 550 return sb.setupDefaultGW() 551 } 552 553 moveExtConn := sb.getGatewayEndpoint() != extEp 554 555 if moveExtConn { 556 if extEp != nil { 557 logrus.Debugf("Revoking external connectivity on endpoint %s (%s)", extEp.Name(), extEp.ID()) 558 extN, err := extEp.getNetworkFromStore() 559 if err != nil { 560 return fmt.Errorf("failed to get network from store for revoking external connectivity during join: %v", err) 561 } 562 extD, err := extN.driver(true) 563 if err != nil { 564 return fmt.Errorf("failed to get driver for revoking external connectivity during join: %v", err) 565 } 566 if err = extD.RevokeExternalConnectivity(extEp.network.ID(), extEp.ID()); err != nil { 567 return types.InternalErrorf( 568 "driver failed revoking external connectivity on endpoint %s (%s): %v", 569 extEp.Name(), extEp.ID(), err) 570 } 571 defer func() { 572 if err != nil { 573 if e := extD.ProgramExternalConnectivity(extEp.network.ID(), extEp.ID(), sb.Labels()); e != nil { 574 logrus.Warnf("Failed to roll-back external connectivity on endpoint %s (%s): %v", 575 extEp.Name(), extEp.ID(), e) 576 } 577 } 578 }() 579 } 580 if !n.internal { 581 logrus.Debugf("Programming external connectivity on endpoint %s (%s)", ep.Name(), ep.ID()) 582 if err = d.ProgramExternalConnectivity(n.ID(), ep.ID(), sb.Labels()); err != nil { 583 return types.InternalErrorf( 584 "driver failed programming external connectivity on endpoint %s (%s): %v", 585 ep.Name(), ep.ID(), err) 586 } 587 } 588 589 } 590 591 if !sb.needDefaultGW() { 592 if e := sb.clearDefaultGW(); e != nil { 593 logrus.Warnf("Failure while disconnecting sandbox %s (%s) from gateway network: %v", 594 sb.ID(), sb.ContainerID(), e) 595 } 596 } 597 598 return nil 599} 600 601func doUpdateHostsFile(n *network, sb *sandbox) bool { 602 return !n.ingress && n.Name() != libnGWNetwork 603} 604 605func (ep *endpoint) rename(name string) error { 606 var ( 607 err error 608 netWatch *netWatch 609 ok bool 610 ) 611 612 n := ep.getNetwork() 613 if n == nil { 614 return fmt.Errorf("network not connected for ep %q", ep.name) 615 } 616 617 c := n.getController() 618 619 sb, ok := ep.getSandbox() 620 if !ok { 621 logrus.Warnf("rename for %s aborted, sandbox %s is not anymore present", ep.ID(), ep.sandboxID) 622 return nil 623 } 624 625 if c.isAgent() { 626 if err = ep.deleteServiceInfoFromCluster(sb, true, "rename"); err != nil { 627 return types.InternalErrorf("Could not delete service state for endpoint %s from cluster on rename: %v", ep.Name(), err) 628 } 629 } else { 630 c.Lock() 631 netWatch, ok = c.nmap[n.ID()] 632 c.Unlock() 633 if !ok { 634 return fmt.Errorf("watch null for network %q", n.Name()) 635 } 636 n.updateSvcRecord(ep, c.getLocalEps(netWatch), false) 637 } 638 639 oldName := ep.name 640 oldAnonymous := ep.anonymous 641 ep.name = name 642 ep.anonymous = false 643 644 if c.isAgent() { 645 if err = ep.addServiceInfoToCluster(sb); err != nil { 646 return types.InternalErrorf("Could not add service state for endpoint %s to cluster on rename: %v", ep.Name(), err) 647 } 648 defer func() { 649 if err != nil { 650 ep.deleteServiceInfoFromCluster(sb, true, "rename") 651 ep.name = oldName 652 ep.anonymous = oldAnonymous 653 ep.addServiceInfoToCluster(sb) 654 } 655 }() 656 } else { 657 n.updateSvcRecord(ep, c.getLocalEps(netWatch), true) 658 defer func() { 659 if err != nil { 660 n.updateSvcRecord(ep, c.getLocalEps(netWatch), false) 661 ep.name = oldName 662 ep.anonymous = oldAnonymous 663 n.updateSvcRecord(ep, c.getLocalEps(netWatch), true) 664 } 665 }() 666 } 667 668 // Update the store with the updated name 669 if err = c.updateToStore(ep); err != nil { 670 return err 671 } 672 // After the name change do a dummy endpoint count update to 673 // trigger the service record update in the peer nodes 674 675 // Ignore the error because updateStore fail for EpCnt is a 676 // benign error. Besides there is no meaningful recovery that 677 // we can do. When the cluster recovers subsequent EpCnt update 678 // will force the peers to get the correct EP name. 679 n.getEpCnt().updateStore() 680 681 return err 682} 683 684func (ep *endpoint) hasInterface(iName string) bool { 685 ep.Lock() 686 defer ep.Unlock() 687 688 return ep.iface != nil && ep.iface.srcName == iName 689} 690 691func (ep *endpoint) Leave(sbox Sandbox, options ...EndpointOption) error { 692 if sbox == nil || sbox.ID() == "" || sbox.Key() == "" { 693 return types.BadRequestErrorf("invalid Sandbox passed to endpoint leave: %v", sbox) 694 } 695 696 sb, ok := sbox.(*sandbox) 697 if !ok { 698 return types.BadRequestErrorf("not a valid Sandbox interface") 699 } 700 701 sb.joinLeaveStart() 702 defer sb.joinLeaveEnd() 703 704 return ep.sbLeave(sb, false, options...) 705} 706 707func (ep *endpoint) sbLeave(sb *sandbox, force bool, options ...EndpointOption) error { 708 n, err := ep.getNetworkFromStore() 709 if err != nil { 710 return fmt.Errorf("failed to get network from store during leave: %v", err) 711 } 712 713 ep, err = n.getEndpointFromStore(ep.ID()) 714 if err != nil { 715 return fmt.Errorf("failed to get endpoint from store during leave: %v", err) 716 } 717 718 ep.Lock() 719 sid := ep.sandboxID 720 ep.Unlock() 721 722 if sid == "" { 723 return types.ForbiddenErrorf("cannot leave endpoint with no attached sandbox") 724 } 725 if sid != sb.ID() { 726 return types.ForbiddenErrorf("unexpected sandbox ID in leave request. Expected %s. Got %s", ep.sandboxID, sb.ID()) 727 } 728 729 ep.processOptions(options...) 730 731 d, err := n.driver(!force) 732 if err != nil { 733 return fmt.Errorf("failed to get driver during endpoint leave: %v", err) 734 } 735 736 ep.Lock() 737 ep.sandboxID = "" 738 ep.network = n 739 ep.Unlock() 740 741 // Current endpoint providing external connectivity to the sandbox 742 extEp := sb.getGatewayEndpoint() 743 moveExtConn := extEp != nil && (extEp.ID() == ep.ID()) 744 745 if d != nil { 746 if moveExtConn { 747 logrus.Debugf("Revoking external connectivity on endpoint %s (%s)", ep.Name(), ep.ID()) 748 if err := d.RevokeExternalConnectivity(n.id, ep.id); err != nil { 749 logrus.Warnf("driver failed revoking external connectivity on endpoint %s (%s): %v", 750 ep.Name(), ep.ID(), err) 751 } 752 } 753 754 if err := d.Leave(n.id, ep.id); err != nil { 755 if _, ok := err.(types.MaskableError); !ok { 756 logrus.Warnf("driver error disconnecting container %s : %v", ep.name, err) 757 } 758 } 759 } 760 761 if err := ep.deleteServiceInfoFromCluster(sb, true, "sbLeave"); err != nil { 762 logrus.Warnf("Failed to clean up service info on container %s disconnect: %v", ep.name, err) 763 } 764 765 if err := sb.clearNetworkResources(ep); err != nil { 766 logrus.Warnf("Failed to clean up network resources on container %s disconnect: %v", ep.name, err) 767 } 768 769 // Update the store about the sandbox detach only after we 770 // have completed sb.clearNetworkresources above to avoid 771 // spurious logs when cleaning up the sandbox when the daemon 772 // ungracefully exits and restarts before completing sandbox 773 // detach but after store has been updated. 774 if err := n.getController().updateToStore(ep); err != nil { 775 return err 776 } 777 778 if e := ep.deleteDriverInfoFromCluster(); e != nil { 779 logrus.Errorf("Failed to delete endpoint state for endpoint %s from cluster: %v", ep.Name(), e) 780 } 781 782 sb.deleteHostsEntries(n.getSvcRecords(ep)) 783 if !sb.inDelete && sb.needDefaultGW() && sb.getEndpointInGWNetwork() == nil { 784 return sb.setupDefaultGW() 785 } 786 787 // New endpoint providing external connectivity for the sandbox 788 extEp = sb.getGatewayEndpoint() 789 if moveExtConn && extEp != nil { 790 logrus.Debugf("Programming external connectivity on endpoint %s (%s)", extEp.Name(), extEp.ID()) 791 extN, err := extEp.getNetworkFromStore() 792 if err != nil { 793 return fmt.Errorf("failed to get network from store for programming external connectivity during leave: %v", err) 794 } 795 extD, err := extN.driver(true) 796 if err != nil { 797 return fmt.Errorf("failed to get driver for programming external connectivity during leave: %v", err) 798 } 799 if err := extD.ProgramExternalConnectivity(extEp.network.ID(), extEp.ID(), sb.Labels()); err != nil { 800 logrus.Warnf("driver failed programming external connectivity on endpoint %s: (%s) %v", 801 extEp.Name(), extEp.ID(), err) 802 } 803 } 804 805 if !sb.needDefaultGW() { 806 if err := sb.clearDefaultGW(); err != nil { 807 logrus.Warnf("Failure while disconnecting sandbox %s (%s) from gateway network: %v", 808 sb.ID(), sb.ContainerID(), err) 809 } 810 } 811 812 return nil 813} 814 815func (ep *endpoint) Delete(force bool) error { 816 var err error 817 n, err := ep.getNetworkFromStore() 818 if err != nil { 819 return fmt.Errorf("failed to get network during Delete: %v", err) 820 } 821 822 ep, err = n.getEndpointFromStore(ep.ID()) 823 if err != nil { 824 return fmt.Errorf("failed to get endpoint from store during Delete: %v", err) 825 } 826 827 ep.Lock() 828 epid := ep.id 829 name := ep.name 830 sbid := ep.sandboxID 831 ep.Unlock() 832 833 sb, _ := n.getController().SandboxByID(sbid) 834 if sb != nil && !force { 835 return &ActiveContainerError{name: name, id: epid} 836 } 837 838 if sb != nil { 839 if e := ep.sbLeave(sb.(*sandbox), force); e != nil { 840 logrus.Warnf("failed to leave sandbox for endpoint %s : %v", name, e) 841 } 842 } 843 844 if err = n.getController().deleteFromStore(ep); err != nil { 845 return err 846 } 847 848 defer func() { 849 if err != nil && !force { 850 ep.dbExists = false 851 if e := n.getController().updateToStore(ep); e != nil { 852 logrus.Warnf("failed to recreate endpoint in store %s : %v", name, e) 853 } 854 } 855 }() 856 857 // unwatch for service records 858 n.getController().unWatchSvcRecord(ep) 859 860 if err = ep.deleteEndpoint(force); err != nil && !force { 861 return err 862 } 863 864 ep.releaseAddress() 865 866 if err := n.getEpCnt().DecEndpointCnt(); err != nil { 867 logrus.Warnf("failed to decrement endpoint count for ep %s: %v", ep.ID(), err) 868 } 869 870 return nil 871} 872 873func (ep *endpoint) deleteEndpoint(force bool) error { 874 ep.Lock() 875 n := ep.network 876 name := ep.name 877 epid := ep.id 878 ep.Unlock() 879 880 driver, err := n.driver(!force) 881 if err != nil { 882 return fmt.Errorf("failed to delete endpoint: %v", err) 883 } 884 885 if driver == nil { 886 return nil 887 } 888 889 if err := driver.DeleteEndpoint(n.id, epid); err != nil { 890 if _, ok := err.(types.ForbiddenError); ok { 891 return err 892 } 893 894 if _, ok := err.(types.MaskableError); !ok { 895 logrus.Warnf("driver error deleting endpoint %s : %v", name, err) 896 } 897 } 898 899 return nil 900} 901 902func (ep *endpoint) getSandbox() (*sandbox, bool) { 903 c := ep.network.getController() 904 ep.Lock() 905 sid := ep.sandboxID 906 ep.Unlock() 907 908 c.Lock() 909 ps, ok := c.sandboxes[sid] 910 c.Unlock() 911 912 return ps, ok 913} 914 915func (ep *endpoint) getFirstInterfaceAddress() net.IP { 916 ep.Lock() 917 defer ep.Unlock() 918 919 if ep.iface.addr != nil { 920 return ep.iface.addr.IP 921 } 922 923 return nil 924} 925 926// EndpointOptionGeneric function returns an option setter for a Generic option defined 927// in a Dictionary of Key-Value pair 928func EndpointOptionGeneric(generic map[string]interface{}) EndpointOption { 929 return func(ep *endpoint) { 930 for k, v := range generic { 931 ep.generic[k] = v 932 } 933 } 934} 935 936var ( 937 linkLocalMask = net.CIDRMask(16, 32) 938 linkLocalMaskIPv6 = net.CIDRMask(64, 128) 939) 940 941// CreateOptionIpam function returns an option setter for the ipam configuration for this endpoint 942func CreateOptionIpam(ipV4, ipV6 net.IP, llIPs []net.IP, ipamOptions map[string]string) EndpointOption { 943 return func(ep *endpoint) { 944 ep.prefAddress = ipV4 945 ep.prefAddressV6 = ipV6 946 if len(llIPs) != 0 { 947 for _, ip := range llIPs { 948 nw := &net.IPNet{IP: ip, Mask: linkLocalMask} 949 if ip.To4() == nil { 950 nw.Mask = linkLocalMaskIPv6 951 } 952 ep.iface.llAddrs = append(ep.iface.llAddrs, nw) 953 } 954 } 955 ep.ipamOptions = ipamOptions 956 } 957} 958 959// CreateOptionExposedPorts function returns an option setter for the container exposed 960// ports option to be passed to network.CreateEndpoint() method. 961func CreateOptionExposedPorts(exposedPorts []types.TransportPort) EndpointOption { 962 return func(ep *endpoint) { 963 // Defensive copy 964 eps := make([]types.TransportPort, len(exposedPorts)) 965 copy(eps, exposedPorts) 966 // Store endpoint label and in generic because driver needs it 967 ep.exposedPorts = eps 968 ep.generic[netlabel.ExposedPorts] = eps 969 } 970} 971 972// CreateOptionPortMapping function returns an option setter for the mapping 973// ports option to be passed to network.CreateEndpoint() method. 974func CreateOptionPortMapping(portBindings []types.PortBinding) EndpointOption { 975 return func(ep *endpoint) { 976 // Store a copy of the bindings as generic data to pass to the driver 977 pbs := make([]types.PortBinding, len(portBindings)) 978 copy(pbs, portBindings) 979 ep.generic[netlabel.PortMap] = pbs 980 } 981} 982 983// CreateOptionDNS function returns an option setter for dns entry option to 984// be passed to container Create method. 985func CreateOptionDNS(dns []string) EndpointOption { 986 return func(ep *endpoint) { 987 ep.generic[netlabel.DNSServers] = dns 988 } 989} 990 991// CreateOptionAnonymous function returns an option setter for setting 992// this endpoint as anonymous 993func CreateOptionAnonymous() EndpointOption { 994 return func(ep *endpoint) { 995 ep.anonymous = true 996 } 997} 998 999// CreateOptionDisableResolution function returns an option setter to indicate 1000// this endpoint doesn't want embedded DNS server functionality 1001func CreateOptionDisableResolution() EndpointOption { 1002 return func(ep *endpoint) { 1003 ep.disableResolution = true 1004 } 1005} 1006 1007// CreateOptionAlias function returns an option setter for setting endpoint alias 1008func CreateOptionAlias(name string, alias string) EndpointOption { 1009 return func(ep *endpoint) { 1010 if ep.aliases == nil { 1011 ep.aliases = make(map[string]string) 1012 } 1013 ep.aliases[alias] = name 1014 } 1015} 1016 1017// CreateOptionService function returns an option setter for setting service binding configuration 1018func CreateOptionService(name, id string, vip net.IP, ingressPorts []*PortConfig, aliases []string) EndpointOption { 1019 return func(ep *endpoint) { 1020 ep.svcName = name 1021 ep.svcID = id 1022 ep.virtualIP = vip 1023 ep.ingressPorts = ingressPorts 1024 ep.svcAliases = aliases 1025 } 1026} 1027 1028// CreateOptionMyAlias function returns an option setter for setting endpoint's self alias 1029func CreateOptionMyAlias(alias string) EndpointOption { 1030 return func(ep *endpoint) { 1031 ep.myAliases = append(ep.myAliases, alias) 1032 } 1033} 1034 1035// CreateOptionLoadBalancer function returns an option setter for denoting the endpoint is a load balancer for a network 1036func CreateOptionLoadBalancer() EndpointOption { 1037 return func(ep *endpoint) { 1038 ep.loadBalancer = true 1039 } 1040} 1041 1042// JoinOptionPriority function returns an option setter for priority option to 1043// be passed to the endpoint.Join() method. 1044func JoinOptionPriority(ep Endpoint, prio int) EndpointOption { 1045 return func(ep *endpoint) { 1046 // ep lock already acquired 1047 c := ep.network.getController() 1048 c.Lock() 1049 sb, ok := c.sandboxes[ep.sandboxID] 1050 c.Unlock() 1051 if !ok { 1052 logrus.Errorf("Could not set endpoint priority value during Join to endpoint %s: No sandbox id present in endpoint", ep.id) 1053 return 1054 } 1055 sb.epPriority[ep.id] = prio 1056 } 1057} 1058 1059func (ep *endpoint) DataScope() string { 1060 return ep.getNetwork().DataScope() 1061} 1062 1063func (ep *endpoint) assignAddress(ipam ipamapi.Ipam, assignIPv4, assignIPv6 bool) error { 1064 var err error 1065 1066 n := ep.getNetwork() 1067 if n.hasSpecialDriver() { 1068 return nil 1069 } 1070 1071 logrus.Debugf("Assigning addresses for endpoint %s's interface on network %s", ep.Name(), n.Name()) 1072 1073 if assignIPv4 { 1074 if err = ep.assignAddressVersion(4, ipam); err != nil { 1075 return err 1076 } 1077 } 1078 1079 if assignIPv6 { 1080 err = ep.assignAddressVersion(6, ipam) 1081 } 1082 1083 return err 1084} 1085 1086func (ep *endpoint) assignAddressVersion(ipVer int, ipam ipamapi.Ipam) error { 1087 var ( 1088 poolID *string 1089 address **net.IPNet 1090 prefAdd net.IP 1091 progAdd net.IP 1092 ) 1093 1094 n := ep.getNetwork() 1095 switch ipVer { 1096 case 4: 1097 poolID = &ep.iface.v4PoolID 1098 address = &ep.iface.addr 1099 prefAdd = ep.prefAddress 1100 case 6: 1101 poolID = &ep.iface.v6PoolID 1102 address = &ep.iface.addrv6 1103 prefAdd = ep.prefAddressV6 1104 default: 1105 return types.InternalErrorf("incorrect ip version number passed: %d", ipVer) 1106 } 1107 1108 ipInfo := n.getIPInfo(ipVer) 1109 1110 // ipv6 address is not mandatory 1111 if len(ipInfo) == 0 && ipVer == 6 { 1112 return nil 1113 } 1114 1115 // The address to program may be chosen by the user or by the network driver in one specific 1116 // case to support backward compatibility with `docker daemon --fixed-cidrv6` use case 1117 if prefAdd != nil { 1118 progAdd = prefAdd 1119 } else if *address != nil { 1120 progAdd = (*address).IP 1121 } 1122 1123 for _, d := range ipInfo { 1124 if progAdd != nil && !d.Pool.Contains(progAdd) { 1125 continue 1126 } 1127 addr, _, err := ipam.RequestAddress(d.PoolID, progAdd, ep.ipamOptions) 1128 if err == nil { 1129 ep.Lock() 1130 *address = addr 1131 *poolID = d.PoolID 1132 ep.Unlock() 1133 return nil 1134 } 1135 if err != ipamapi.ErrNoAvailableIPs || progAdd != nil { 1136 return err 1137 } 1138 } 1139 if progAdd != nil { 1140 return types.BadRequestErrorf("Invalid address %s: It does not belong to any of this network's subnets", prefAdd) 1141 } 1142 return fmt.Errorf("no available IPv%d addresses on this network's address pools: %s (%s)", ipVer, n.Name(), n.ID()) 1143} 1144 1145func (ep *endpoint) releaseAddress() { 1146 n := ep.getNetwork() 1147 if n.hasSpecialDriver() { 1148 return 1149 } 1150 1151 logrus.Debugf("Releasing addresses for endpoint %s's interface on network %s", ep.Name(), n.Name()) 1152 1153 ipam, _, err := n.getController().getIPAMDriver(n.ipamType) 1154 if err != nil { 1155 logrus.Warnf("Failed to retrieve ipam driver to release interface address on delete of endpoint %s (%s): %v", ep.Name(), ep.ID(), err) 1156 return 1157 } 1158 1159 if ep.iface.addr != nil { 1160 if err := ipam.ReleaseAddress(ep.iface.v4PoolID, ep.iface.addr.IP); err != nil { 1161 logrus.Warnf("Failed to release ip address %s on delete of endpoint %s (%s): %v", ep.iface.addr.IP, ep.Name(), ep.ID(), err) 1162 } 1163 } 1164 1165 if ep.iface.addrv6 != nil && ep.iface.addrv6.IP.IsGlobalUnicast() { 1166 if err := ipam.ReleaseAddress(ep.iface.v6PoolID, ep.iface.addrv6.IP); err != nil { 1167 logrus.Warnf("Failed to release ip address %s on delete of endpoint %s (%s): %v", ep.iface.addrv6.IP, ep.Name(), ep.ID(), err) 1168 } 1169 } 1170} 1171 1172func (c *controller) cleanupLocalEndpoints() { 1173 // Get used endpoints 1174 eps := make(map[string]interface{}) 1175 for _, sb := range c.sandboxes { 1176 for _, ep := range sb.endpoints { 1177 eps[ep.id] = true 1178 } 1179 } 1180 nl, err := c.getNetworksForScope(datastore.LocalScope) 1181 if err != nil { 1182 logrus.Warnf("Could not get list of networks during endpoint cleanup: %v", err) 1183 return 1184 } 1185 1186 for _, n := range nl { 1187 if n.ConfigOnly() { 1188 continue 1189 } 1190 epl, err := n.getEndpointsFromStore() 1191 if err != nil { 1192 logrus.Warnf("Could not get list of endpoints in network %s during endpoint cleanup: %v", n.name, err) 1193 continue 1194 } 1195 1196 for _, ep := range epl { 1197 if _, ok := eps[ep.id]; ok { 1198 continue 1199 } 1200 logrus.Infof("Removing stale endpoint %s (%s)", ep.name, ep.id) 1201 if err := ep.Delete(true); err != nil { 1202 logrus.Warnf("Could not delete local endpoint %s during endpoint cleanup: %v", ep.name, err) 1203 } 1204 } 1205 1206 epl, err = n.getEndpointsFromStore() 1207 if err != nil { 1208 logrus.Warnf("Could not get list of endpoints in network %s for count update: %v", n.name, err) 1209 continue 1210 } 1211 1212 epCnt := n.getEpCnt().EndpointCnt() 1213 if epCnt != uint64(len(epl)) { 1214 logrus.Infof("Fixing inconsistent endpoint_cnt for network %s. Expected=%d, Actual=%d", n.name, len(epl), epCnt) 1215 n.getEpCnt().setCnt(uint64(len(epl))) 1216 } 1217 } 1218} 1219