1/* 2Package libnetwork provides the basic functionality and extension points to 3create network namespaces and allocate interfaces for containers to use. 4 5 networkType := "bridge" 6 7 // Create a new controller instance 8 driverOptions := options.Generic{} 9 genericOption := make(map[string]interface{}) 10 genericOption[netlabel.GenericData] = driverOptions 11 controller, err := libnetwork.New(config.OptionDriverConfig(networkType, genericOption)) 12 if err != nil { 13 return 14 } 15 16 // Create a network for containers to join. 17 // NewNetwork accepts Variadic optional arguments that libnetwork and Drivers can make use of 18 network, err := controller.NewNetwork(networkType, "network1", "") 19 if err != nil { 20 return 21 } 22 23 // For each new container: allocate IP and interfaces. The returned network 24 // settings will be used for container infos (inspect and such), as well as 25 // iptables rules for port publishing. This info is contained or accessible 26 // from the returned endpoint. 27 ep, err := network.CreateEndpoint("Endpoint1") 28 if err != nil { 29 return 30 } 31 32 // Create the sandbox for the container. 33 // NewSandbox accepts Variadic optional arguments which libnetwork can use. 34 sbx, err := controller.NewSandbox("container1", 35 libnetwork.OptionHostname("test"), 36 libnetwork.OptionDomainname("docker.io")) 37 38 // A sandbox can join the endpoint via the join api. 39 err = ep.Join(sbx) 40 if err != nil { 41 return 42 } 43*/ 44package libnetwork 45 46import ( 47 "fmt" 48 "net" 49 "path/filepath" 50 "runtime" 51 "strings" 52 "sync" 53 "time" 54 55 "github.com/docker/docker/pkg/discovery" 56 "github.com/docker/docker/pkg/locker" 57 "github.com/docker/docker/pkg/plugingetter" 58 "github.com/docker/docker/pkg/plugins" 59 "github.com/docker/docker/pkg/stringid" 60 "github.com/docker/libnetwork/cluster" 61 "github.com/docker/libnetwork/config" 62 "github.com/docker/libnetwork/datastore" 63 "github.com/docker/libnetwork/diagnostic" 64 "github.com/docker/libnetwork/discoverapi" 65 "github.com/docker/libnetwork/driverapi" 66 "github.com/docker/libnetwork/drvregistry" 67 "github.com/docker/libnetwork/hostdiscovery" 68 "github.com/docker/libnetwork/ipamapi" 69 "github.com/docker/libnetwork/netlabel" 70 "github.com/docker/libnetwork/osl" 71 "github.com/docker/libnetwork/types" 72 "github.com/pkg/errors" 73 "github.com/sirupsen/logrus" 74) 75 76// NetworkController provides the interface for controller instance which manages 77// networks. 78type NetworkController interface { 79 // ID provides a unique identity for the controller 80 ID() string 81 82 // BuiltinDrivers returns list of builtin drivers 83 BuiltinDrivers() []string 84 85 // BuiltinIPAMDrivers returns list of builtin ipam drivers 86 BuiltinIPAMDrivers() []string 87 88 // Config method returns the bootup configuration for the controller 89 Config() config.Config 90 91 // Create a new network. The options parameter carries network specific options. 92 NewNetwork(networkType, name string, id string, options ...NetworkOption) (Network, error) 93 94 // Networks returns the list of Network(s) managed by this controller. 95 Networks() []Network 96 97 // WalkNetworks uses the provided function to walk the Network(s) managed by this controller. 98 WalkNetworks(walker NetworkWalker) 99 100 // NetworkByName returns the Network which has the passed name. If not found, the error ErrNoSuchNetwork is returned. 101 NetworkByName(name string) (Network, error) 102 103 // NetworkByID returns the Network which has the passed id. If not found, the error ErrNoSuchNetwork is returned. 104 NetworkByID(id string) (Network, error) 105 106 // NewSandbox creates a new network sandbox for the passed container id 107 NewSandbox(containerID string, options ...SandboxOption) (Sandbox, error) 108 109 // Sandboxes returns the list of Sandbox(s) managed by this controller. 110 Sandboxes() []Sandbox 111 112 // WalkSandboxes uses the provided function to walk the Sandbox(s) managed by this controller. 113 WalkSandboxes(walker SandboxWalker) 114 115 // SandboxByID returns the Sandbox which has the passed id. If not found, a types.NotFoundError is returned. 116 SandboxByID(id string) (Sandbox, error) 117 118 // SandboxDestroy destroys a sandbox given a container ID 119 SandboxDestroy(id string) error 120 121 // Stop network controller 122 Stop() 123 124 // ReloadConfiguration updates the controller configuration 125 ReloadConfiguration(cfgOptions ...config.Option) error 126 127 // SetClusterProvider sets cluster provider 128 SetClusterProvider(provider cluster.Provider) 129 130 // Wait for agent initialization complete in libnetwork controller 131 AgentInitWait() 132 133 // Wait for agent to stop if running 134 AgentStopWait() 135 136 // SetKeys configures the encryption key for gossip and overlay data path 137 SetKeys(keys []*types.EncryptionKey) error 138 139 // StartDiagnostic start the network diagnostic mode 140 StartDiagnostic(port int) 141 // StopDiagnostic start the network diagnostic mode 142 StopDiagnostic() 143 // IsDiagnosticEnabled returns true if the diagnostic is enabled 144 IsDiagnosticEnabled() bool 145} 146 147// NetworkWalker is a client provided function which will be used to walk the Networks. 148// When the function returns true, the walk will stop. 149type NetworkWalker func(nw Network) bool 150 151// SandboxWalker is a client provided function which will be used to walk the Sandboxes. 152// When the function returns true, the walk will stop. 153type SandboxWalker func(sb Sandbox) bool 154 155type sandboxTable map[string]*sandbox 156 157type controller struct { 158 id string 159 drvRegistry *drvregistry.DrvRegistry 160 sandboxes sandboxTable 161 cfg *config.Config 162 stores []datastore.DataStore 163 discovery hostdiscovery.HostDiscovery 164 extKeyListener net.Listener 165 watchCh chan *endpoint 166 unWatchCh chan *endpoint 167 svcRecords map[string]svcInfo 168 nmap map[string]*netWatch 169 serviceBindings map[serviceKey]*service 170 defOsSbox osl.Sandbox 171 ingressSandbox *sandbox 172 sboxOnce sync.Once 173 agent *agent 174 networkLocker *locker.Locker 175 agentInitDone chan struct{} 176 agentStopDone chan struct{} 177 keys []*types.EncryptionKey 178 clusterConfigAvailable bool 179 DiagnosticServer *diagnostic.Server 180 sync.Mutex 181} 182 183type initializer struct { 184 fn drvregistry.InitFunc 185 ntype string 186} 187 188// New creates a new instance of network controller. 189func New(cfgOptions ...config.Option) (NetworkController, error) { 190 c := &controller{ 191 id: stringid.GenerateRandomID(), 192 cfg: config.ParseConfigOptions(cfgOptions...), 193 sandboxes: sandboxTable{}, 194 svcRecords: make(map[string]svcInfo), 195 serviceBindings: make(map[serviceKey]*service), 196 agentInitDone: make(chan struct{}), 197 networkLocker: locker.New(), 198 DiagnosticServer: diagnostic.New(), 199 } 200 c.DiagnosticServer.Init() 201 202 if err := c.initStores(); err != nil { 203 return nil, err 204 } 205 206 drvRegistry, err := drvregistry.New(c.getStore(datastore.LocalScope), c.getStore(datastore.GlobalScope), c.RegisterDriver, nil, c.cfg.PluginGetter) 207 if err != nil { 208 return nil, err 209 } 210 211 for _, i := range getInitializers(c.cfg.Daemon.Experimental) { 212 var dcfg map[string]interface{} 213 214 // External plugins don't need config passed through daemon. They can 215 // bootstrap themselves 216 if i.ntype != "remote" { 217 dcfg = c.makeDriverConfig(i.ntype) 218 } 219 220 if err := drvRegistry.AddDriver(i.ntype, i.fn, dcfg); err != nil { 221 return nil, err 222 } 223 } 224 225 if err = initIPAMDrivers(drvRegistry, nil, c.getStore(datastore.GlobalScope), c.cfg.Daemon.DefaultAddressPool); err != nil { 226 return nil, err 227 } 228 229 c.drvRegistry = drvRegistry 230 231 if c.cfg != nil && c.cfg.Cluster.Watcher != nil { 232 if err := c.initDiscovery(c.cfg.Cluster.Watcher); err != nil { 233 // Failing to initialize discovery is a bad situation to be in. 234 // But it cannot fail creating the Controller 235 logrus.Errorf("Failed to Initialize Discovery : %v", err) 236 } 237 } 238 239 c.WalkNetworks(populateSpecial) 240 241 // Reserve pools first before doing cleanup. Otherwise the 242 // cleanups of endpoint/network and sandbox below will 243 // generate many unnecessary warnings 244 c.reservePools() 245 246 // Cleanup resources 247 c.sandboxCleanup(c.cfg.ActiveSandboxes) 248 c.cleanupLocalEndpoints() 249 c.networkCleanup() 250 251 if err := c.startExternalKeyListener(); err != nil { 252 return nil, err 253 } 254 255 return c, nil 256} 257 258func (c *controller) SetClusterProvider(provider cluster.Provider) { 259 var sameProvider bool 260 c.Lock() 261 // Avoids to spawn multiple goroutine for the same cluster provider 262 if c.cfg.Daemon.ClusterProvider == provider { 263 // If the cluster provider is already set, there is already a go routine spawned 264 // that is listening for events, so nothing to do here 265 sameProvider = true 266 } else { 267 c.cfg.Daemon.ClusterProvider = provider 268 } 269 c.Unlock() 270 271 if provider == nil || sameProvider { 272 return 273 } 274 // We don't want to spawn a new go routine if the previous one did not exit yet 275 c.AgentStopWait() 276 go c.clusterAgentInit() 277} 278 279func isValidClusteringIP(addr string) bool { 280 return addr != "" && !net.ParseIP(addr).IsLoopback() && !net.ParseIP(addr).IsUnspecified() 281} 282 283// libnetwork side of agent depends on the keys. On the first receipt of 284// keys setup the agent. For subsequent key set handle the key change 285func (c *controller) SetKeys(keys []*types.EncryptionKey) error { 286 subsysKeys := make(map[string]int) 287 for _, key := range keys { 288 if key.Subsystem != subsysGossip && 289 key.Subsystem != subsysIPSec { 290 return fmt.Errorf("key received for unrecognized subsystem") 291 } 292 subsysKeys[key.Subsystem]++ 293 } 294 for s, count := range subsysKeys { 295 if count != keyringSize { 296 return fmt.Errorf("incorrect number of keys for subsystem %v", s) 297 } 298 } 299 300 agent := c.getAgent() 301 302 if agent == nil { 303 c.Lock() 304 c.keys = keys 305 c.Unlock() 306 return nil 307 } 308 return c.handleKeyChange(keys) 309} 310 311func (c *controller) getAgent() *agent { 312 c.Lock() 313 defer c.Unlock() 314 return c.agent 315} 316 317func (c *controller) clusterAgentInit() { 318 clusterProvider := c.cfg.Daemon.ClusterProvider 319 var keysAvailable bool 320 for { 321 eventType := <-clusterProvider.ListenClusterEvents() 322 // The events: EventSocketChange, EventNodeReady and EventNetworkKeysAvailable are not ordered 323 // when all the condition for the agent initialization are met then proceed with it 324 switch eventType { 325 case cluster.EventNetworkKeysAvailable: 326 // Validates that the keys are actually available before starting the initialization 327 // This will handle old spurious messages left on the channel 328 c.Lock() 329 keysAvailable = c.keys != nil 330 c.Unlock() 331 fallthrough 332 case cluster.EventSocketChange, cluster.EventNodeReady: 333 if keysAvailable && !c.isDistributedControl() { 334 c.agentOperationStart() 335 if err := c.agentSetup(clusterProvider); err != nil { 336 c.agentStopComplete() 337 } else { 338 c.agentInitComplete() 339 } 340 } 341 case cluster.EventNodeLeave: 342 keysAvailable = false 343 c.agentOperationStart() 344 c.Lock() 345 c.keys = nil 346 c.Unlock() 347 348 // We are leaving the cluster. Make sure we 349 // close the gossip so that we stop all 350 // incoming gossip updates before cleaning up 351 // any remaining service bindings. But before 352 // deleting the networks since the networks 353 // should still be present when cleaning up 354 // service bindings 355 c.agentClose() 356 c.cleanupServiceDiscovery("") 357 c.cleanupServiceBindings("") 358 359 c.agentStopComplete() 360 361 return 362 } 363 } 364} 365 366// AgentInitWait waits for agent initialization to be completed in the controller. 367func (c *controller) AgentInitWait() { 368 c.Lock() 369 agentInitDone := c.agentInitDone 370 c.Unlock() 371 372 if agentInitDone != nil { 373 <-agentInitDone 374 } 375} 376 377// AgentStopWait waits for the Agent stop to be completed in the controller 378func (c *controller) AgentStopWait() { 379 c.Lock() 380 agentStopDone := c.agentStopDone 381 c.Unlock() 382 if agentStopDone != nil { 383 <-agentStopDone 384 } 385} 386 387// agentOperationStart marks the start of an Agent Init or Agent Stop 388func (c *controller) agentOperationStart() { 389 c.Lock() 390 if c.agentInitDone == nil { 391 c.agentInitDone = make(chan struct{}) 392 } 393 if c.agentStopDone == nil { 394 c.agentStopDone = make(chan struct{}) 395 } 396 c.Unlock() 397} 398 399// agentInitComplete notifies the successful completion of the Agent initialization 400func (c *controller) agentInitComplete() { 401 c.Lock() 402 if c.agentInitDone != nil { 403 close(c.agentInitDone) 404 c.agentInitDone = nil 405 } 406 c.Unlock() 407} 408 409// agentStopComplete notifies the successful completion of the Agent stop 410func (c *controller) agentStopComplete() { 411 c.Lock() 412 if c.agentStopDone != nil { 413 close(c.agentStopDone) 414 c.agentStopDone = nil 415 } 416 c.Unlock() 417} 418 419func (c *controller) makeDriverConfig(ntype string) map[string]interface{} { 420 if c.cfg == nil { 421 return nil 422 } 423 424 config := make(map[string]interface{}) 425 426 for _, label := range c.cfg.Daemon.Labels { 427 if !strings.HasPrefix(netlabel.Key(label), netlabel.DriverPrefix+"."+ntype) { 428 continue 429 } 430 431 config[netlabel.Key(label)] = netlabel.Value(label) 432 } 433 434 drvCfg, ok := c.cfg.Daemon.DriverCfg[ntype] 435 if ok { 436 for k, v := range drvCfg.(map[string]interface{}) { 437 config[k] = v 438 } 439 } 440 441 for k, v := range c.cfg.Scopes { 442 if !v.IsValid() { 443 continue 444 } 445 config[netlabel.MakeKVClient(k)] = discoverapi.DatastoreConfigData{ 446 Scope: k, 447 Provider: v.Client.Provider, 448 Address: v.Client.Address, 449 Config: v.Client.Config, 450 } 451 } 452 453 return config 454} 455 456var procReloadConfig = make(chan (bool), 1) 457 458func (c *controller) ReloadConfiguration(cfgOptions ...config.Option) error { 459 procReloadConfig <- true 460 defer func() { <-procReloadConfig }() 461 462 // For now we accept the configuration reload only as a mean to provide a global store config after boot. 463 // Refuse the configuration if it alters an existing datastore client configuration. 464 update := false 465 cfg := config.ParseConfigOptions(cfgOptions...) 466 467 for s := range c.cfg.Scopes { 468 if _, ok := cfg.Scopes[s]; !ok { 469 return types.ForbiddenErrorf("cannot accept new configuration because it removes an existing datastore client") 470 } 471 } 472 for s, nSCfg := range cfg.Scopes { 473 if eSCfg, ok := c.cfg.Scopes[s]; ok { 474 if eSCfg.Client.Provider != nSCfg.Client.Provider || 475 eSCfg.Client.Address != nSCfg.Client.Address { 476 return types.ForbiddenErrorf("cannot accept new configuration because it modifies an existing datastore client") 477 } 478 } else { 479 if err := c.initScopedStore(s, nSCfg); err != nil { 480 return err 481 } 482 update = true 483 } 484 } 485 if !update { 486 return nil 487 } 488 489 c.Lock() 490 c.cfg = cfg 491 c.Unlock() 492 493 var dsConfig *discoverapi.DatastoreConfigData 494 for scope, sCfg := range cfg.Scopes { 495 if scope == datastore.LocalScope || !sCfg.IsValid() { 496 continue 497 } 498 dsConfig = &discoverapi.DatastoreConfigData{ 499 Scope: scope, 500 Provider: sCfg.Client.Provider, 501 Address: sCfg.Client.Address, 502 Config: sCfg.Client.Config, 503 } 504 break 505 } 506 if dsConfig == nil { 507 return nil 508 } 509 510 c.drvRegistry.WalkIPAMs(func(name string, driver ipamapi.Ipam, cap *ipamapi.Capability) bool { 511 err := driver.DiscoverNew(discoverapi.DatastoreConfig, *dsConfig) 512 if err != nil { 513 logrus.Errorf("Failed to set datastore in driver %s: %v", name, err) 514 } 515 return false 516 }) 517 518 c.drvRegistry.WalkDrivers(func(name string, driver driverapi.Driver, capability driverapi.Capability) bool { 519 err := driver.DiscoverNew(discoverapi.DatastoreConfig, *dsConfig) 520 if err != nil { 521 logrus.Errorf("Failed to set datastore in driver %s: %v", name, err) 522 } 523 return false 524 }) 525 526 if c.discovery == nil && c.cfg.Cluster.Watcher != nil { 527 if err := c.initDiscovery(c.cfg.Cluster.Watcher); err != nil { 528 logrus.Errorf("Failed to Initialize Discovery after configuration update: %v", err) 529 } 530 } 531 532 return nil 533} 534 535func (c *controller) ID() string { 536 return c.id 537} 538 539func (c *controller) BuiltinDrivers() []string { 540 drivers := []string{} 541 c.drvRegistry.WalkDrivers(func(name string, driver driverapi.Driver, capability driverapi.Capability) bool { 542 if driver.IsBuiltIn() { 543 drivers = append(drivers, name) 544 } 545 return false 546 }) 547 return drivers 548} 549 550func (c *controller) BuiltinIPAMDrivers() []string { 551 drivers := []string{} 552 c.drvRegistry.WalkIPAMs(func(name string, driver ipamapi.Ipam, cap *ipamapi.Capability) bool { 553 if driver.IsBuiltIn() { 554 drivers = append(drivers, name) 555 } 556 return false 557 }) 558 return drivers 559} 560 561func (c *controller) validateHostDiscoveryConfig() bool { 562 if c.cfg == nil || c.cfg.Cluster.Discovery == "" || c.cfg.Cluster.Address == "" { 563 return false 564 } 565 return true 566} 567 568func (c *controller) clusterHostID() string { 569 c.Lock() 570 defer c.Unlock() 571 if c.cfg == nil || c.cfg.Cluster.Address == "" { 572 return "" 573 } 574 addr := strings.Split(c.cfg.Cluster.Address, ":") 575 return addr[0] 576} 577 578func (c *controller) isNodeAlive(node string) bool { 579 if c.discovery == nil { 580 return false 581 } 582 583 nodes := c.discovery.Fetch() 584 for _, n := range nodes { 585 if n.String() == node { 586 return true 587 } 588 } 589 590 return false 591} 592 593func (c *controller) initDiscovery(watcher discovery.Watcher) error { 594 if c.cfg == nil { 595 return fmt.Errorf("discovery initialization requires a valid configuration") 596 } 597 598 c.discovery = hostdiscovery.NewHostDiscovery(watcher) 599 return c.discovery.Watch(c.activeCallback, c.hostJoinCallback, c.hostLeaveCallback) 600} 601 602func (c *controller) activeCallback() { 603 ds := c.getStore(datastore.GlobalScope) 604 if ds != nil && !ds.Active() { 605 ds.RestartWatch() 606 } 607} 608 609func (c *controller) hostJoinCallback(nodes []net.IP) { 610 c.processNodeDiscovery(nodes, true) 611} 612 613func (c *controller) hostLeaveCallback(nodes []net.IP) { 614 c.processNodeDiscovery(nodes, false) 615} 616 617func (c *controller) processNodeDiscovery(nodes []net.IP, add bool) { 618 c.drvRegistry.WalkDrivers(func(name string, driver driverapi.Driver, capability driverapi.Capability) bool { 619 c.pushNodeDiscovery(driver, capability, nodes, add) 620 return false 621 }) 622} 623 624func (c *controller) pushNodeDiscovery(d driverapi.Driver, cap driverapi.Capability, nodes []net.IP, add bool) { 625 var self net.IP 626 if c.cfg != nil { 627 addr := strings.Split(c.cfg.Cluster.Address, ":") 628 self = net.ParseIP(addr[0]) 629 // if external kvstore is not configured, try swarm-mode config 630 if self == nil { 631 if agent := c.getAgent(); agent != nil { 632 self = net.ParseIP(agent.advertiseAddr) 633 } 634 } 635 } 636 637 if d == nil || cap.ConnectivityScope != datastore.GlobalScope || nodes == nil { 638 return 639 } 640 641 for _, node := range nodes { 642 nodeData := discoverapi.NodeDiscoveryData{Address: node.String(), Self: node.Equal(self)} 643 var err error 644 if add { 645 err = d.DiscoverNew(discoverapi.NodeDiscovery, nodeData) 646 } else { 647 err = d.DiscoverDelete(discoverapi.NodeDiscovery, nodeData) 648 } 649 if err != nil { 650 logrus.Debugf("discovery notification error: %v", err) 651 } 652 } 653} 654 655func (c *controller) Config() config.Config { 656 c.Lock() 657 defer c.Unlock() 658 if c.cfg == nil { 659 return config.Config{} 660 } 661 return *c.cfg 662} 663 664func (c *controller) isManager() bool { 665 c.Lock() 666 defer c.Unlock() 667 if c.cfg == nil || c.cfg.Daemon.ClusterProvider == nil { 668 return false 669 } 670 return c.cfg.Daemon.ClusterProvider.IsManager() 671} 672 673func (c *controller) isAgent() bool { 674 c.Lock() 675 defer c.Unlock() 676 if c.cfg == nil || c.cfg.Daemon.ClusterProvider == nil { 677 return false 678 } 679 return c.cfg.Daemon.ClusterProvider.IsAgent() 680} 681 682func (c *controller) isDistributedControl() bool { 683 return !c.isManager() && !c.isAgent() 684} 685 686func (c *controller) GetPluginGetter() plugingetter.PluginGetter { 687 return c.drvRegistry.GetPluginGetter() 688} 689 690func (c *controller) RegisterDriver(networkType string, driver driverapi.Driver, capability driverapi.Capability) error { 691 c.Lock() 692 hd := c.discovery 693 c.Unlock() 694 695 if hd != nil { 696 c.pushNodeDiscovery(driver, capability, hd.Fetch(), true) 697 } 698 699 c.agentDriverNotify(driver) 700 return nil 701} 702 703// NewNetwork creates a new network of the specified network type. The options 704// are network specific and modeled in a generic way. 705func (c *controller) NewNetwork(networkType, name string, id string, options ...NetworkOption) (Network, error) { 706 if id != "" { 707 c.networkLocker.Lock(id) 708 defer c.networkLocker.Unlock(id) 709 710 if _, err := c.NetworkByID(id); err == nil { 711 return nil, NetworkNameError(id) 712 } 713 } 714 715 if !config.IsValidName(name) { 716 return nil, ErrInvalidName(name) 717 } 718 719 if id == "" { 720 id = stringid.GenerateRandomID() 721 } 722 723 defaultIpam := defaultIpamForNetworkType(networkType) 724 // Construct the network object 725 network := &network{ 726 name: name, 727 networkType: networkType, 728 generic: map[string]interface{}{netlabel.GenericData: make(map[string]string)}, 729 ipamType: defaultIpam, 730 id: id, 731 created: time.Now(), 732 ctrlr: c, 733 persist: true, 734 drvOnce: &sync.Once{}, 735 } 736 737 network.processOptions(options...) 738 if err := network.validateConfiguration(); err != nil { 739 return nil, err 740 } 741 742 var ( 743 cap *driverapi.Capability 744 err error 745 ) 746 747 // Reset network types, force local scope and skip allocation and 748 // plumbing for configuration networks. Reset of the config-only 749 // network drivers is needed so that this special network is not 750 // usable by old engine versions. 751 if network.configOnly { 752 network.scope = datastore.LocalScope 753 network.networkType = "null" 754 goto addToStore 755 } 756 757 _, cap, err = network.resolveDriver(network.networkType, true) 758 if err != nil { 759 return nil, err 760 } 761 762 if network.scope == datastore.LocalScope && cap.DataScope == datastore.GlobalScope { 763 return nil, types.ForbiddenErrorf("cannot downgrade network scope for %s networks", networkType) 764 765 } 766 if network.ingress && cap.DataScope != datastore.GlobalScope { 767 return nil, types.ForbiddenErrorf("Ingress network can only be global scope network") 768 } 769 770 // At this point the network scope is still unknown if not set by user 771 if (cap.DataScope == datastore.GlobalScope || network.scope == datastore.SwarmScope) && 772 !c.isDistributedControl() && !network.dynamic { 773 if c.isManager() { 774 // For non-distributed controlled environment, globalscoped non-dynamic networks are redirected to Manager 775 return nil, ManagerRedirectError(name) 776 } 777 return nil, types.ForbiddenErrorf("Cannot create a multi-host network from a worker node. Please create the network from a manager node.") 778 } 779 780 if network.scope == datastore.SwarmScope && c.isDistributedControl() { 781 return nil, types.ForbiddenErrorf("cannot create a swarm scoped network when swarm is not active") 782 } 783 784 // Make sure we have a driver available for this network type 785 // before we allocate anything. 786 if _, err := network.driver(true); err != nil { 787 return nil, err 788 } 789 790 // From this point on, we need the network specific configuration, 791 // which may come from a configuration-only network 792 if network.configFrom != "" { 793 t, err := c.getConfigNetwork(network.configFrom) 794 if err != nil { 795 return nil, types.NotFoundErrorf("configuration network %q does not exist", network.configFrom) 796 } 797 if err := t.applyConfigurationTo(network); err != nil { 798 return nil, types.InternalErrorf("Failed to apply configuration: %v", err) 799 } 800 defer func() { 801 if err == nil { 802 if err := t.getEpCnt().IncEndpointCnt(); err != nil { 803 logrus.Warnf("Failed to update reference count for configuration network %q on creation of network %q: %v", 804 t.Name(), network.Name(), err) 805 } 806 } 807 }() 808 } 809 810 err = network.ipamAllocate() 811 if err != nil { 812 return nil, err 813 } 814 defer func() { 815 if err != nil { 816 network.ipamRelease() 817 } 818 }() 819 820 err = c.addNetwork(network) 821 if err != nil { 822 return nil, err 823 } 824 defer func() { 825 if err != nil { 826 if e := network.deleteNetwork(); e != nil { 827 logrus.Warnf("couldn't roll back driver network on network %s creation failure: %v", network.name, err) 828 } 829 } 830 }() 831 832addToStore: 833 // First store the endpoint count, then the network. To avoid to 834 // end up with a datastore containing a network and not an epCnt, 835 // in case of an ungraceful shutdown during this function call. 836 epCnt := &endpointCnt{n: network} 837 if err = c.updateToStore(epCnt); err != nil { 838 return nil, err 839 } 840 defer func() { 841 if err != nil { 842 if e := c.deleteFromStore(epCnt); e != nil { 843 logrus.Warnf("could not rollback from store, epCnt %v on failure (%v): %v", epCnt, err, e) 844 } 845 } 846 }() 847 848 network.epCnt = epCnt 849 if err = c.updateToStore(network); err != nil { 850 return nil, err 851 } 852 defer func() { 853 if err != nil { 854 if e := c.deleteFromStore(network); e != nil { 855 logrus.Warnf("could not rollback from store, network %v on failure (%v): %v", network, err, e) 856 } 857 } 858 }() 859 860 if network.configOnly { 861 return network, nil 862 } 863 864 joinCluster(network) 865 defer func() { 866 if err != nil { 867 network.cancelDriverWatches() 868 if e := network.leaveCluster(); e != nil { 869 logrus.Warnf("Failed to leave agent cluster on network %s on failure (%v): %v", network.name, err, e) 870 } 871 } 872 }() 873 874 if network.hasLoadBalancerEndpoint() { 875 if err = network.createLoadBalancerSandbox(); err != nil { 876 return nil, err 877 } 878 } 879 880 if !c.isDistributedControl() { 881 c.Lock() 882 arrangeIngressFilterRule() 883 c.Unlock() 884 } 885 886 c.arrangeUserFilterRule() 887 888 return network, nil 889} 890 891var joinCluster NetworkWalker = func(nw Network) bool { 892 n := nw.(*network) 893 if n.configOnly { 894 return false 895 } 896 if err := n.joinCluster(); err != nil { 897 logrus.Errorf("Failed to join network %s (%s) into agent cluster: %v", n.Name(), n.ID(), err) 898 } 899 n.addDriverWatches() 900 return false 901} 902 903func (c *controller) reservePools() { 904 networks, err := c.getNetworksForScope(datastore.LocalScope) 905 if err != nil { 906 logrus.Warnf("Could not retrieve networks from local store during ipam allocation for existing networks: %v", err) 907 return 908 } 909 910 for _, n := range networks { 911 if n.configOnly { 912 continue 913 } 914 if !doReplayPoolReserve(n) { 915 continue 916 } 917 // Construct pseudo configs for the auto IP case 918 autoIPv4 := (len(n.ipamV4Config) == 0 || (len(n.ipamV4Config) == 1 && n.ipamV4Config[0].PreferredPool == "")) && len(n.ipamV4Info) > 0 919 autoIPv6 := (len(n.ipamV6Config) == 0 || (len(n.ipamV6Config) == 1 && n.ipamV6Config[0].PreferredPool == "")) && len(n.ipamV6Info) > 0 920 if autoIPv4 { 921 n.ipamV4Config = []*IpamConf{{PreferredPool: n.ipamV4Info[0].Pool.String()}} 922 } 923 if n.enableIPv6 && autoIPv6 { 924 n.ipamV6Config = []*IpamConf{{PreferredPool: n.ipamV6Info[0].Pool.String()}} 925 } 926 // Account current network gateways 927 for i, c := range n.ipamV4Config { 928 if c.Gateway == "" && n.ipamV4Info[i].Gateway != nil { 929 c.Gateway = n.ipamV4Info[i].Gateway.IP.String() 930 } 931 } 932 if n.enableIPv6 { 933 for i, c := range n.ipamV6Config { 934 if c.Gateway == "" && n.ipamV6Info[i].Gateway != nil { 935 c.Gateway = n.ipamV6Info[i].Gateway.IP.String() 936 } 937 } 938 } 939 // Reserve pools 940 if err := n.ipamAllocate(); err != nil { 941 logrus.Warnf("Failed to allocate ipam pool(s) for network %q (%s): %v", n.Name(), n.ID(), err) 942 } 943 // Reserve existing endpoints' addresses 944 ipam, _, err := n.getController().getIPAMDriver(n.ipamType) 945 if err != nil { 946 logrus.Warnf("Failed to retrieve ipam driver for network %q (%s) during address reservation", n.Name(), n.ID()) 947 continue 948 } 949 epl, err := n.getEndpointsFromStore() 950 if err != nil { 951 logrus.Warnf("Failed to retrieve list of current endpoints on network %q (%s)", n.Name(), n.ID()) 952 continue 953 } 954 for _, ep := range epl { 955 if err := ep.assignAddress(ipam, true, ep.Iface().AddressIPv6() != nil); err != nil { 956 logrus.Warnf("Failed to reserve current address for endpoint %q (%s) on network %q (%s)", 957 ep.Name(), ep.ID(), n.Name(), n.ID()) 958 } 959 } 960 } 961} 962 963func doReplayPoolReserve(n *network) bool { 964 _, caps, err := n.getController().getIPAMDriver(n.ipamType) 965 if err != nil { 966 logrus.Warnf("Failed to retrieve ipam driver for network %q (%s): %v", n.Name(), n.ID(), err) 967 return false 968 } 969 return caps.RequiresRequestReplay 970} 971 972func (c *controller) addNetwork(n *network) error { 973 d, err := n.driver(true) 974 if err != nil { 975 return err 976 } 977 978 // Create the network 979 if err := d.CreateNetwork(n.id, n.generic, n, n.getIPData(4), n.getIPData(6)); err != nil { 980 return err 981 } 982 983 n.startResolver() 984 985 return nil 986} 987 988func (c *controller) Networks() []Network { 989 var list []Network 990 991 networks, err := c.getNetworksFromStore() 992 if err != nil { 993 logrus.Error(err) 994 } 995 996 for _, n := range networks { 997 if n.inDelete { 998 continue 999 } 1000 list = append(list, n) 1001 } 1002 1003 return list 1004} 1005 1006func (c *controller) WalkNetworks(walker NetworkWalker) { 1007 for _, n := range c.Networks() { 1008 if walker(n) { 1009 return 1010 } 1011 } 1012} 1013 1014func (c *controller) NetworkByName(name string) (Network, error) { 1015 if name == "" { 1016 return nil, ErrInvalidName(name) 1017 } 1018 var n Network 1019 1020 s := func(current Network) bool { 1021 if current.Name() == name { 1022 n = current 1023 return true 1024 } 1025 return false 1026 } 1027 1028 c.WalkNetworks(s) 1029 1030 if n == nil { 1031 return nil, ErrNoSuchNetwork(name) 1032 } 1033 1034 return n, nil 1035} 1036 1037func (c *controller) NetworkByID(id string) (Network, error) { 1038 if id == "" { 1039 return nil, ErrInvalidID(id) 1040 } 1041 1042 n, err := c.getNetworkFromStore(id) 1043 if err != nil { 1044 return nil, ErrNoSuchNetwork(id) 1045 } 1046 1047 return n, nil 1048} 1049 1050// NewSandbox creates a new sandbox for the passed container id 1051func (c *controller) NewSandbox(containerID string, options ...SandboxOption) (Sandbox, error) { 1052 if containerID == "" { 1053 return nil, types.BadRequestErrorf("invalid container ID") 1054 } 1055 1056 var sb *sandbox 1057 c.Lock() 1058 for _, s := range c.sandboxes { 1059 if s.containerID == containerID { 1060 // If not a stub, then we already have a complete sandbox. 1061 if !s.isStub { 1062 sbID := s.ID() 1063 c.Unlock() 1064 return nil, types.ForbiddenErrorf("container %s is already present in sandbox %s", containerID, sbID) 1065 } 1066 1067 // We already have a stub sandbox from the 1068 // store. Make use of it so that we don't lose 1069 // the endpoints from store but reset the 1070 // isStub flag. 1071 sb = s 1072 sb.isStub = false 1073 break 1074 } 1075 } 1076 c.Unlock() 1077 1078 sandboxID := stringid.GenerateRandomID() 1079 if runtime.GOOS == "windows" { 1080 sandboxID = containerID 1081 } 1082 1083 // Create sandbox and process options first. Key generation depends on an option 1084 if sb == nil { 1085 sb = &sandbox{ 1086 id: sandboxID, 1087 containerID: containerID, 1088 endpoints: []*endpoint{}, 1089 epPriority: map[string]int{}, 1090 populatedEndpoints: map[string]struct{}{}, 1091 config: containerConfig{}, 1092 controller: c, 1093 extDNS: []extDNSEntry{}, 1094 } 1095 } 1096 1097 sb.processOptions(options...) 1098 1099 c.Lock() 1100 if sb.ingress && c.ingressSandbox != nil { 1101 c.Unlock() 1102 return nil, types.ForbiddenErrorf("ingress sandbox already present") 1103 } 1104 1105 if sb.ingress { 1106 c.ingressSandbox = sb 1107 sb.config.hostsPath = filepath.Join(c.cfg.Daemon.DataDir, "/network/files/hosts") 1108 sb.config.resolvConfPath = filepath.Join(c.cfg.Daemon.DataDir, "/network/files/resolv.conf") 1109 sb.id = "ingress_sbox" 1110 } else if sb.loadBalancerNID != "" { 1111 sb.id = "lb_" + sb.loadBalancerNID 1112 } 1113 c.Unlock() 1114 1115 var err error 1116 defer func() { 1117 if err != nil { 1118 c.Lock() 1119 if sb.ingress { 1120 c.ingressSandbox = nil 1121 } 1122 c.Unlock() 1123 } 1124 }() 1125 1126 if err = sb.setupResolutionFiles(); err != nil { 1127 return nil, err 1128 } 1129 1130 if sb.config.useDefaultSandBox { 1131 c.sboxOnce.Do(func() { 1132 c.defOsSbox, err = osl.NewSandbox(sb.Key(), false, false) 1133 }) 1134 1135 if err != nil { 1136 c.sboxOnce = sync.Once{} 1137 return nil, fmt.Errorf("failed to create default sandbox: %v", err) 1138 } 1139 1140 sb.osSbox = c.defOsSbox 1141 } 1142 1143 if sb.osSbox == nil && !sb.config.useExternalKey { 1144 if sb.osSbox, err = osl.NewSandbox(sb.Key(), !sb.config.useDefaultSandBox, false); err != nil { 1145 return nil, fmt.Errorf("failed to create new osl sandbox: %v", err) 1146 } 1147 } 1148 1149 if sb.osSbox != nil { 1150 // Apply operating specific knobs on the load balancer sandbox 1151 sb.osSbox.ApplyOSTweaks(sb.oslTypes) 1152 } 1153 1154 c.Lock() 1155 c.sandboxes[sb.id] = sb 1156 c.Unlock() 1157 defer func() { 1158 if err != nil { 1159 c.Lock() 1160 delete(c.sandboxes, sb.id) 1161 c.Unlock() 1162 } 1163 }() 1164 1165 err = sb.storeUpdate() 1166 if err != nil { 1167 return nil, fmt.Errorf("failed to update the store state of sandbox: %v", err) 1168 } 1169 1170 return sb, nil 1171} 1172 1173func (c *controller) Sandboxes() []Sandbox { 1174 c.Lock() 1175 defer c.Unlock() 1176 1177 list := make([]Sandbox, 0, len(c.sandboxes)) 1178 for _, s := range c.sandboxes { 1179 // Hide stub sandboxes from libnetwork users 1180 if s.isStub { 1181 continue 1182 } 1183 1184 list = append(list, s) 1185 } 1186 1187 return list 1188} 1189 1190func (c *controller) WalkSandboxes(walker SandboxWalker) { 1191 for _, sb := range c.Sandboxes() { 1192 if walker(sb) { 1193 return 1194 } 1195 } 1196} 1197 1198func (c *controller) SandboxByID(id string) (Sandbox, error) { 1199 if id == "" { 1200 return nil, ErrInvalidID(id) 1201 } 1202 c.Lock() 1203 s, ok := c.sandboxes[id] 1204 c.Unlock() 1205 if !ok { 1206 return nil, types.NotFoundErrorf("sandbox %s not found", id) 1207 } 1208 return s, nil 1209} 1210 1211// SandboxDestroy destroys a sandbox given a container ID 1212func (c *controller) SandboxDestroy(id string) error { 1213 var sb *sandbox 1214 c.Lock() 1215 for _, s := range c.sandboxes { 1216 if s.containerID == id { 1217 sb = s 1218 break 1219 } 1220 } 1221 c.Unlock() 1222 1223 // It is not an error if sandbox is not available 1224 if sb == nil { 1225 return nil 1226 } 1227 1228 return sb.Delete() 1229} 1230 1231// SandboxContainerWalker returns a Sandbox Walker function which looks for an existing Sandbox with the passed containerID 1232func SandboxContainerWalker(out *Sandbox, containerID string) SandboxWalker { 1233 return func(sb Sandbox) bool { 1234 if sb.ContainerID() == containerID { 1235 *out = sb 1236 return true 1237 } 1238 return false 1239 } 1240} 1241 1242// SandboxKeyWalker returns a Sandbox Walker function which looks for an existing Sandbox with the passed key 1243func SandboxKeyWalker(out *Sandbox, key string) SandboxWalker { 1244 return func(sb Sandbox) bool { 1245 if sb.Key() == key { 1246 *out = sb 1247 return true 1248 } 1249 return false 1250 } 1251} 1252 1253func (c *controller) loadDriver(networkType string) error { 1254 var err error 1255 1256 if pg := c.GetPluginGetter(); pg != nil { 1257 _, err = pg.Get(networkType, driverapi.NetworkPluginEndpointType, plugingetter.Lookup) 1258 } else { 1259 _, err = plugins.Get(networkType, driverapi.NetworkPluginEndpointType) 1260 } 1261 1262 if err != nil { 1263 if errors.Cause(err) == plugins.ErrNotFound { 1264 return types.NotFoundErrorf(err.Error()) 1265 } 1266 return err 1267 } 1268 1269 return nil 1270} 1271 1272func (c *controller) loadIPAMDriver(name string) error { 1273 var err error 1274 1275 if pg := c.GetPluginGetter(); pg != nil { 1276 _, err = pg.Get(name, ipamapi.PluginEndpointType, plugingetter.Lookup) 1277 } else { 1278 _, err = plugins.Get(name, ipamapi.PluginEndpointType) 1279 } 1280 1281 if err != nil { 1282 if err == plugins.ErrNotFound { 1283 return types.NotFoundErrorf(err.Error()) 1284 } 1285 return err 1286 } 1287 1288 return nil 1289} 1290 1291func (c *controller) getIPAMDriver(name string) (ipamapi.Ipam, *ipamapi.Capability, error) { 1292 id, cap := c.drvRegistry.IPAM(name) 1293 if id == nil { 1294 // Might be a plugin name. Try loading it 1295 if err := c.loadIPAMDriver(name); err != nil { 1296 return nil, nil, err 1297 } 1298 1299 // Now that we resolved the plugin, try again looking up the registry 1300 id, cap = c.drvRegistry.IPAM(name) 1301 if id == nil { 1302 return nil, nil, types.BadRequestErrorf("invalid ipam driver: %q", name) 1303 } 1304 } 1305 1306 return id, cap, nil 1307} 1308 1309func (c *controller) Stop() { 1310 c.closeStores() 1311 c.stopExternalKeyListener() 1312 osl.GC() 1313} 1314 1315// StartDiagnostic start the network dias mode 1316func (c *controller) StartDiagnostic(port int) { 1317 c.Lock() 1318 if !c.DiagnosticServer.IsDiagnosticEnabled() { 1319 c.DiagnosticServer.EnableDiagnostic("127.0.0.1", port) 1320 } 1321 c.Unlock() 1322} 1323 1324// StopDiagnostic start the network dias mode 1325func (c *controller) StopDiagnostic() { 1326 c.Lock() 1327 if c.DiagnosticServer.IsDiagnosticEnabled() { 1328 c.DiagnosticServer.DisableDiagnostic() 1329 } 1330 c.Unlock() 1331} 1332 1333// IsDiagnosticEnabled returns true if the dias is enabled 1334func (c *controller) IsDiagnosticEnabled() bool { 1335 c.Lock() 1336 defer c.Unlock() 1337 return c.DiagnosticServer.IsDiagnosticEnabled() 1338} 1339