1/* 2Package libnetwork provides the basic functionality and extension points to 3create network namespaces and allocate interfaces for containers to use. 4 5 networkType := "bridge" 6 7 // Create a new controller instance 8 driverOptions := options.Generic{} 9 genericOption := make(map[string]interface{}) 10 genericOption[netlabel.GenericData] = driverOptions 11 controller, err := libnetwork.New(config.OptionDriverConfig(networkType, genericOption)) 12 if err != nil { 13 return 14 } 15 16 // Create a network for containers to join. 17 // NewNetwork accepts Variadic optional arguments that libnetwork and Drivers can make use of 18 network, err := controller.NewNetwork(networkType, "network1", "") 19 if err != nil { 20 return 21 } 22 23 // For each new container: allocate IP and interfaces. The returned network 24 // settings will be used for container infos (inspect and such), as well as 25 // iptables rules for port publishing. This info is contained or accessible 26 // from the returned endpoint. 27 ep, err := network.CreateEndpoint("Endpoint1") 28 if err != nil { 29 return 30 } 31 32 // Create the sandbox for the container. 33 // NewSandbox accepts Variadic optional arguments which libnetwork can use. 34 sbx, err := controller.NewSandbox("container1", 35 libnetwork.OptionHostname("test"), 36 libnetwork.OptionDomainname("docker.io")) 37 38 // A sandbox can join the endpoint via the join api. 39 err = ep.Join(sbx) 40 if err != nil { 41 return 42 } 43*/ 44package libnetwork 45 46import ( 47 "container/heap" 48 "fmt" 49 "net" 50 "path/filepath" 51 "strings" 52 "sync" 53 "time" 54 55 "github.com/docker/docker/pkg/discovery" 56 "github.com/docker/docker/pkg/locker" 57 "github.com/docker/docker/pkg/plugingetter" 58 "github.com/docker/docker/pkg/plugins" 59 "github.com/docker/docker/pkg/stringid" 60 "github.com/docker/libnetwork/cluster" 61 "github.com/docker/libnetwork/config" 62 "github.com/docker/libnetwork/datastore" 63 "github.com/docker/libnetwork/diagnose" 64 "github.com/docker/libnetwork/discoverapi" 65 "github.com/docker/libnetwork/driverapi" 66 "github.com/docker/libnetwork/drvregistry" 67 "github.com/docker/libnetwork/hostdiscovery" 68 "github.com/docker/libnetwork/ipamapi" 69 "github.com/docker/libnetwork/netlabel" 70 "github.com/docker/libnetwork/osl" 71 "github.com/docker/libnetwork/types" 72 "github.com/sirupsen/logrus" 73) 74 75// NetworkController provides the interface for controller instance which manages 76// networks. 77type NetworkController interface { 78 // ID provides a unique identity for the controller 79 ID() string 80 81 // BuiltinDrivers returns list of builtin drivers 82 BuiltinDrivers() []string 83 84 // BuiltinIPAMDrivers returns list of builtin ipam drivers 85 BuiltinIPAMDrivers() []string 86 87 // Config method returns the bootup configuration for the controller 88 Config() config.Config 89 90 // Create a new network. The options parameter carries network specific options. 91 NewNetwork(networkType, name string, id string, options ...NetworkOption) (Network, error) 92 93 // Networks returns the list of Network(s) managed by this controller. 94 Networks() []Network 95 96 // WalkNetworks uses the provided function to walk the Network(s) managed by this controller. 97 WalkNetworks(walker NetworkWalker) 98 99 // NetworkByName returns the Network which has the passed name. If not found, the error ErrNoSuchNetwork is returned. 100 NetworkByName(name string) (Network, error) 101 102 // NetworkByID returns the Network which has the passed id. If not found, the error ErrNoSuchNetwork is returned. 103 NetworkByID(id string) (Network, error) 104 105 // NewSandbox creates a new network sandbox for the passed container id 106 NewSandbox(containerID string, options ...SandboxOption) (Sandbox, error) 107 108 // Sandboxes returns the list of Sandbox(s) managed by this controller. 109 Sandboxes() []Sandbox 110 111 // WalkSandboxes uses the provided function to walk the Sandbox(s) managed by this controller. 112 WalkSandboxes(walker SandboxWalker) 113 114 // SandboxByID returns the Sandbox which has the passed id. If not found, a types.NotFoundError is returned. 115 SandboxByID(id string) (Sandbox, error) 116 117 // SandboxDestroy destroys a sandbox given a container ID 118 SandboxDestroy(id string) error 119 120 // Stop network controller 121 Stop() 122 123 // ReloadCondfiguration updates the controller configuration 124 ReloadConfiguration(cfgOptions ...config.Option) error 125 126 // SetClusterProvider sets cluster provider 127 SetClusterProvider(provider cluster.Provider) 128 129 // Wait for agent initialization complete in libnetwork controller 130 AgentInitWait() 131 132 // Wait for agent to stop if running 133 AgentStopWait() 134 135 // SetKeys configures the encryption key for gossip and overlay data path 136 SetKeys(keys []*types.EncryptionKey) error 137 138 // StartDiagnose start the network diagnose mode 139 StartDiagnose(port int) 140 // StopDiagnose start the network diagnose mode 141 StopDiagnose() 142 // IsDiagnoseEnabled returns true if the diagnose is enabled 143 IsDiagnoseEnabled() bool 144} 145 146// NetworkWalker is a client provided function which will be used to walk the Networks. 147// When the function returns true, the walk will stop. 148type NetworkWalker func(nw Network) bool 149 150// SandboxWalker is a client provided function which will be used to walk the Sandboxes. 151// When the function returns true, the walk will stop. 152type SandboxWalker func(sb Sandbox) bool 153 154type sandboxTable map[string]*sandbox 155 156type controller struct { 157 id string 158 drvRegistry *drvregistry.DrvRegistry 159 sandboxes sandboxTable 160 cfg *config.Config 161 stores []datastore.DataStore 162 discovery hostdiscovery.HostDiscovery 163 extKeyListener net.Listener 164 watchCh chan *endpoint 165 unWatchCh chan *endpoint 166 svcRecords map[string]svcInfo 167 nmap map[string]*netWatch 168 serviceBindings map[serviceKey]*service 169 defOsSbox osl.Sandbox 170 ingressSandbox *sandbox 171 sboxOnce sync.Once 172 agent *agent 173 networkLocker *locker.Locker 174 agentInitDone chan struct{} 175 agentStopDone chan struct{} 176 keys []*types.EncryptionKey 177 clusterConfigAvailable bool 178 DiagnoseServer *diagnose.Server 179 sync.Mutex 180} 181 182type initializer struct { 183 fn drvregistry.InitFunc 184 ntype string 185} 186 187// New creates a new instance of network controller. 188func New(cfgOptions ...config.Option) (NetworkController, error) { 189 c := &controller{ 190 id: stringid.GenerateRandomID(), 191 cfg: config.ParseConfigOptions(cfgOptions...), 192 sandboxes: sandboxTable{}, 193 svcRecords: make(map[string]svcInfo), 194 serviceBindings: make(map[serviceKey]*service), 195 agentInitDone: make(chan struct{}), 196 networkLocker: locker.New(), 197 DiagnoseServer: diagnose.New(), 198 } 199 c.DiagnoseServer.Init() 200 201 if err := c.initStores(); err != nil { 202 return nil, err 203 } 204 205 drvRegistry, err := drvregistry.New(c.getStore(datastore.LocalScope), c.getStore(datastore.GlobalScope), c.RegisterDriver, nil, c.cfg.PluginGetter) 206 if err != nil { 207 return nil, err 208 } 209 210 for _, i := range getInitializers(c.cfg.Daemon.Experimental) { 211 var dcfg map[string]interface{} 212 213 // External plugins don't need config passed through daemon. They can 214 // bootstrap themselves 215 if i.ntype != "remote" { 216 dcfg = c.makeDriverConfig(i.ntype) 217 } 218 219 if err := drvRegistry.AddDriver(i.ntype, i.fn, dcfg); err != nil { 220 return nil, err 221 } 222 } 223 224 if err = initIPAMDrivers(drvRegistry, nil, c.getStore(datastore.GlobalScope)); err != nil { 225 return nil, err 226 } 227 228 c.drvRegistry = drvRegistry 229 230 if c.cfg != nil && c.cfg.Cluster.Watcher != nil { 231 if err := c.initDiscovery(c.cfg.Cluster.Watcher); err != nil { 232 // Failing to initialize discovery is a bad situation to be in. 233 // But it cannot fail creating the Controller 234 logrus.Errorf("Failed to Initialize Discovery : %v", err) 235 } 236 } 237 238 c.WalkNetworks(populateSpecial) 239 240 // Reserve pools first before doing cleanup. Otherwise the 241 // cleanups of endpoint/network and sandbox below will 242 // generate many unnecessary warnings 243 c.reservePools() 244 245 // Cleanup resources 246 c.sandboxCleanup(c.cfg.ActiveSandboxes) 247 c.cleanupLocalEndpoints() 248 c.networkCleanup() 249 250 if err := c.startExternalKeyListener(); err != nil { 251 return nil, err 252 } 253 254 return c, nil 255} 256 257func (c *controller) SetClusterProvider(provider cluster.Provider) { 258 var sameProvider bool 259 c.Lock() 260 // Avoids to spawn multiple goroutine for the same cluster provider 261 if c.cfg.Daemon.ClusterProvider == provider { 262 // If the cluster provider is already set, there is already a go routine spawned 263 // that is listening for events, so nothing to do here 264 sameProvider = true 265 } else { 266 c.cfg.Daemon.ClusterProvider = provider 267 } 268 c.Unlock() 269 270 if provider == nil || sameProvider { 271 return 272 } 273 // We don't want to spawn a new go routine if the previous one did not exit yet 274 c.AgentStopWait() 275 go c.clusterAgentInit() 276} 277 278func isValidClusteringIP(addr string) bool { 279 return addr != "" && !net.ParseIP(addr).IsLoopback() && !net.ParseIP(addr).IsUnspecified() 280} 281 282// libnetwork side of agent depends on the keys. On the first receipt of 283// keys setup the agent. For subsequent key set handle the key change 284func (c *controller) SetKeys(keys []*types.EncryptionKey) error { 285 subsysKeys := make(map[string]int) 286 for _, key := range keys { 287 if key.Subsystem != subsysGossip && 288 key.Subsystem != subsysIPSec { 289 return fmt.Errorf("key received for unrecognized subsystem") 290 } 291 subsysKeys[key.Subsystem]++ 292 } 293 for s, count := range subsysKeys { 294 if count != keyringSize { 295 return fmt.Errorf("incorrect number of keys for subsystem %v", s) 296 } 297 } 298 299 agent := c.getAgent() 300 301 if agent == nil { 302 c.Lock() 303 c.keys = keys 304 c.Unlock() 305 return nil 306 } 307 return c.handleKeyChange(keys) 308} 309 310func (c *controller) getAgent() *agent { 311 c.Lock() 312 defer c.Unlock() 313 return c.agent 314} 315 316func (c *controller) clusterAgentInit() { 317 clusterProvider := c.cfg.Daemon.ClusterProvider 318 var keysAvailable bool 319 for { 320 eventType := <-clusterProvider.ListenClusterEvents() 321 // The events: EventSocketChange, EventNodeReady and EventNetworkKeysAvailable are not ordered 322 // when all the condition for the agent initialization are met then proceed with it 323 switch eventType { 324 case cluster.EventNetworkKeysAvailable: 325 // Validates that the keys are actually available before starting the initialization 326 // This will handle old spurious messages left on the channel 327 c.Lock() 328 keysAvailable = c.keys != nil 329 c.Unlock() 330 fallthrough 331 case cluster.EventSocketChange, cluster.EventNodeReady: 332 if keysAvailable && !c.isDistributedControl() { 333 c.agentOperationStart() 334 if err := c.agentSetup(clusterProvider); err != nil { 335 c.agentStopComplete() 336 } else { 337 c.agentInitComplete() 338 } 339 } 340 case cluster.EventNodeLeave: 341 keysAvailable = false 342 c.agentOperationStart() 343 c.Lock() 344 c.keys = nil 345 c.Unlock() 346 347 // We are leaving the cluster. Make sure we 348 // close the gossip so that we stop all 349 // incoming gossip updates before cleaning up 350 // any remaining service bindings. But before 351 // deleting the networks since the networks 352 // should still be present when cleaning up 353 // service bindings 354 c.agentClose() 355 c.cleanupServiceDiscovery("") 356 c.cleanupServiceBindings("") 357 358 c.agentStopComplete() 359 360 return 361 } 362 } 363} 364 365// AgentInitWait waits for agent initialization to be completed in the controller. 366func (c *controller) AgentInitWait() { 367 c.Lock() 368 agentInitDone := c.agentInitDone 369 c.Unlock() 370 371 if agentInitDone != nil { 372 <-agentInitDone 373 } 374} 375 376// AgentStopWait waits for the Agent stop to be completed in the controller 377func (c *controller) AgentStopWait() { 378 c.Lock() 379 agentStopDone := c.agentStopDone 380 c.Unlock() 381 if agentStopDone != nil { 382 <-agentStopDone 383 } 384} 385 386// agentOperationStart marks the start of an Agent Init or Agent Stop 387func (c *controller) agentOperationStart() { 388 c.Lock() 389 if c.agentInitDone == nil { 390 c.agentInitDone = make(chan struct{}) 391 } 392 if c.agentStopDone == nil { 393 c.agentStopDone = make(chan struct{}) 394 } 395 c.Unlock() 396} 397 398// agentInitComplete notifies the successful completion of the Agent initialization 399func (c *controller) agentInitComplete() { 400 c.Lock() 401 if c.agentInitDone != nil { 402 close(c.agentInitDone) 403 c.agentInitDone = nil 404 } 405 c.Unlock() 406} 407 408// agentStopComplete notifies the successful completion of the Agent stop 409func (c *controller) agentStopComplete() { 410 c.Lock() 411 if c.agentStopDone != nil { 412 close(c.agentStopDone) 413 c.agentStopDone = nil 414 } 415 c.Unlock() 416} 417 418func (c *controller) makeDriverConfig(ntype string) map[string]interface{} { 419 if c.cfg == nil { 420 return nil 421 } 422 423 config := make(map[string]interface{}) 424 425 for _, label := range c.cfg.Daemon.Labels { 426 if !strings.HasPrefix(netlabel.Key(label), netlabel.DriverPrefix+"."+ntype) { 427 continue 428 } 429 430 config[netlabel.Key(label)] = netlabel.Value(label) 431 } 432 433 drvCfg, ok := c.cfg.Daemon.DriverCfg[ntype] 434 if ok { 435 for k, v := range drvCfg.(map[string]interface{}) { 436 config[k] = v 437 } 438 } 439 440 for k, v := range c.cfg.Scopes { 441 if !v.IsValid() { 442 continue 443 } 444 config[netlabel.MakeKVClient(k)] = discoverapi.DatastoreConfigData{ 445 Scope: k, 446 Provider: v.Client.Provider, 447 Address: v.Client.Address, 448 Config: v.Client.Config, 449 } 450 } 451 452 return config 453} 454 455var procReloadConfig = make(chan (bool), 1) 456 457func (c *controller) ReloadConfiguration(cfgOptions ...config.Option) error { 458 procReloadConfig <- true 459 defer func() { <-procReloadConfig }() 460 461 // For now we accept the configuration reload only as a mean to provide a global store config after boot. 462 // Refuse the configuration if it alters an existing datastore client configuration. 463 update := false 464 cfg := config.ParseConfigOptions(cfgOptions...) 465 466 for s := range c.cfg.Scopes { 467 if _, ok := cfg.Scopes[s]; !ok { 468 return types.ForbiddenErrorf("cannot accept new configuration because it removes an existing datastore client") 469 } 470 } 471 for s, nSCfg := range cfg.Scopes { 472 if eSCfg, ok := c.cfg.Scopes[s]; ok { 473 if eSCfg.Client.Provider != nSCfg.Client.Provider || 474 eSCfg.Client.Address != nSCfg.Client.Address { 475 return types.ForbiddenErrorf("cannot accept new configuration because it modifies an existing datastore client") 476 } 477 } else { 478 if err := c.initScopedStore(s, nSCfg); err != nil { 479 return err 480 } 481 update = true 482 } 483 } 484 if !update { 485 return nil 486 } 487 488 c.Lock() 489 c.cfg = cfg 490 c.Unlock() 491 492 var dsConfig *discoverapi.DatastoreConfigData 493 for scope, sCfg := range cfg.Scopes { 494 if scope == datastore.LocalScope || !sCfg.IsValid() { 495 continue 496 } 497 dsConfig = &discoverapi.DatastoreConfigData{ 498 Scope: scope, 499 Provider: sCfg.Client.Provider, 500 Address: sCfg.Client.Address, 501 Config: sCfg.Client.Config, 502 } 503 break 504 } 505 if dsConfig == nil { 506 return nil 507 } 508 509 c.drvRegistry.WalkIPAMs(func(name string, driver ipamapi.Ipam, cap *ipamapi.Capability) bool { 510 err := driver.DiscoverNew(discoverapi.DatastoreConfig, *dsConfig) 511 if err != nil { 512 logrus.Errorf("Failed to set datastore in driver %s: %v", name, err) 513 } 514 return false 515 }) 516 517 c.drvRegistry.WalkDrivers(func(name string, driver driverapi.Driver, capability driverapi.Capability) bool { 518 err := driver.DiscoverNew(discoverapi.DatastoreConfig, *dsConfig) 519 if err != nil { 520 logrus.Errorf("Failed to set datastore in driver %s: %v", name, err) 521 } 522 return false 523 }) 524 525 if c.discovery == nil && c.cfg.Cluster.Watcher != nil { 526 if err := c.initDiscovery(c.cfg.Cluster.Watcher); err != nil { 527 logrus.Errorf("Failed to Initialize Discovery after configuration update: %v", err) 528 } 529 } 530 531 return nil 532} 533 534func (c *controller) ID() string { 535 return c.id 536} 537 538func (c *controller) BuiltinDrivers() []string { 539 drivers := []string{} 540 c.drvRegistry.WalkDrivers(func(name string, driver driverapi.Driver, capability driverapi.Capability) bool { 541 if driver.IsBuiltIn() { 542 drivers = append(drivers, name) 543 } 544 return false 545 }) 546 return drivers 547} 548 549func (c *controller) BuiltinIPAMDrivers() []string { 550 drivers := []string{} 551 c.drvRegistry.WalkIPAMs(func(name string, driver ipamapi.Ipam, cap *ipamapi.Capability) bool { 552 if driver.IsBuiltIn() { 553 drivers = append(drivers, name) 554 } 555 return false 556 }) 557 return drivers 558} 559 560func (c *controller) validateHostDiscoveryConfig() bool { 561 if c.cfg == nil || c.cfg.Cluster.Discovery == "" || c.cfg.Cluster.Address == "" { 562 return false 563 } 564 return true 565} 566 567func (c *controller) clusterHostID() string { 568 c.Lock() 569 defer c.Unlock() 570 if c.cfg == nil || c.cfg.Cluster.Address == "" { 571 return "" 572 } 573 addr := strings.Split(c.cfg.Cluster.Address, ":") 574 return addr[0] 575} 576 577func (c *controller) isNodeAlive(node string) bool { 578 if c.discovery == nil { 579 return false 580 } 581 582 nodes := c.discovery.Fetch() 583 for _, n := range nodes { 584 if n.String() == node { 585 return true 586 } 587 } 588 589 return false 590} 591 592func (c *controller) initDiscovery(watcher discovery.Watcher) error { 593 if c.cfg == nil { 594 return fmt.Errorf("discovery initialization requires a valid configuration") 595 } 596 597 c.discovery = hostdiscovery.NewHostDiscovery(watcher) 598 return c.discovery.Watch(c.activeCallback, c.hostJoinCallback, c.hostLeaveCallback) 599} 600 601func (c *controller) activeCallback() { 602 ds := c.getStore(datastore.GlobalScope) 603 if ds != nil && !ds.Active() { 604 ds.RestartWatch() 605 } 606} 607 608func (c *controller) hostJoinCallback(nodes []net.IP) { 609 c.processNodeDiscovery(nodes, true) 610} 611 612func (c *controller) hostLeaveCallback(nodes []net.IP) { 613 c.processNodeDiscovery(nodes, false) 614} 615 616func (c *controller) processNodeDiscovery(nodes []net.IP, add bool) { 617 c.drvRegistry.WalkDrivers(func(name string, driver driverapi.Driver, capability driverapi.Capability) bool { 618 c.pushNodeDiscovery(driver, capability, nodes, add) 619 return false 620 }) 621} 622 623func (c *controller) pushNodeDiscovery(d driverapi.Driver, cap driverapi.Capability, nodes []net.IP, add bool) { 624 var self net.IP 625 if c.cfg != nil { 626 addr := strings.Split(c.cfg.Cluster.Address, ":") 627 self = net.ParseIP(addr[0]) 628 // if external kvstore is not configured, try swarm-mode config 629 if self == nil { 630 if agent := c.getAgent(); agent != nil { 631 self = net.ParseIP(agent.advertiseAddr) 632 } 633 } 634 } 635 636 if d == nil || cap.ConnectivityScope != datastore.GlobalScope || nodes == nil { 637 return 638 } 639 640 for _, node := range nodes { 641 nodeData := discoverapi.NodeDiscoveryData{Address: node.String(), Self: node.Equal(self)} 642 var err error 643 if add { 644 err = d.DiscoverNew(discoverapi.NodeDiscovery, nodeData) 645 } else { 646 err = d.DiscoverDelete(discoverapi.NodeDiscovery, nodeData) 647 } 648 if err != nil { 649 logrus.Debugf("discovery notification error: %v", err) 650 } 651 } 652} 653 654func (c *controller) Config() config.Config { 655 c.Lock() 656 defer c.Unlock() 657 if c.cfg == nil { 658 return config.Config{} 659 } 660 return *c.cfg 661} 662 663func (c *controller) isManager() bool { 664 c.Lock() 665 defer c.Unlock() 666 if c.cfg == nil || c.cfg.Daemon.ClusterProvider == nil { 667 return false 668 } 669 return c.cfg.Daemon.ClusterProvider.IsManager() 670} 671 672func (c *controller) isAgent() bool { 673 c.Lock() 674 defer c.Unlock() 675 if c.cfg == nil || c.cfg.Daemon.ClusterProvider == nil { 676 return false 677 } 678 return c.cfg.Daemon.ClusterProvider.IsAgent() 679} 680 681func (c *controller) isDistributedControl() bool { 682 return !c.isManager() && !c.isAgent() 683} 684 685func (c *controller) GetPluginGetter() plugingetter.PluginGetter { 686 return c.drvRegistry.GetPluginGetter() 687} 688 689func (c *controller) RegisterDriver(networkType string, driver driverapi.Driver, capability driverapi.Capability) error { 690 c.Lock() 691 hd := c.discovery 692 c.Unlock() 693 694 if hd != nil { 695 c.pushNodeDiscovery(driver, capability, hd.Fetch(), true) 696 } 697 698 c.agentDriverNotify(driver) 699 return nil 700} 701 702// NewNetwork creates a new network of the specified network type. The options 703// are network specific and modeled in a generic way. 704func (c *controller) NewNetwork(networkType, name string, id string, options ...NetworkOption) (Network, error) { 705 if id != "" { 706 c.networkLocker.Lock(id) 707 defer c.networkLocker.Unlock(id) 708 709 if _, err := c.NetworkByID(id); err == nil { 710 return nil, NetworkNameError(id) 711 } 712 } 713 714 if !config.IsValidName(name) { 715 return nil, ErrInvalidName(name) 716 } 717 718 if id == "" { 719 id = stringid.GenerateRandomID() 720 } 721 722 defaultIpam := defaultIpamForNetworkType(networkType) 723 // Construct the network object 724 network := &network{ 725 name: name, 726 networkType: networkType, 727 generic: map[string]interface{}{netlabel.GenericData: make(map[string]string)}, 728 ipamType: defaultIpam, 729 id: id, 730 created: time.Now(), 731 ctrlr: c, 732 persist: true, 733 drvOnce: &sync.Once{}, 734 } 735 736 network.processOptions(options...) 737 if err := network.validateConfiguration(); err != nil { 738 return nil, err 739 } 740 741 var ( 742 cap *driverapi.Capability 743 err error 744 ) 745 746 // Reset network types, force local scope and skip allocation and 747 // plumbing for configuration networks. Reset of the config-only 748 // network drivers is needed so that this special network is not 749 // usable by old engine versions. 750 if network.configOnly { 751 network.scope = datastore.LocalScope 752 network.networkType = "null" 753 goto addToStore 754 } 755 756 _, cap, err = network.resolveDriver(network.networkType, true) 757 if err != nil { 758 return nil, err 759 } 760 761 if network.scope == datastore.LocalScope && cap.DataScope == datastore.GlobalScope { 762 return nil, types.ForbiddenErrorf("cannot downgrade network scope for %s networks", networkType) 763 764 } 765 if network.ingress && cap.DataScope != datastore.GlobalScope { 766 return nil, types.ForbiddenErrorf("Ingress network can only be global scope network") 767 } 768 769 // At this point the network scope is still unknown if not set by user 770 if (cap.DataScope == datastore.GlobalScope || network.scope == datastore.SwarmScope) && 771 !c.isDistributedControl() && !network.dynamic { 772 if c.isManager() { 773 // For non-distributed controlled environment, globalscoped non-dynamic networks are redirected to Manager 774 return nil, ManagerRedirectError(name) 775 } 776 return nil, types.ForbiddenErrorf("Cannot create a multi-host network from a worker node. Please create the network from a manager node.") 777 } 778 779 if network.scope == datastore.SwarmScope && c.isDistributedControl() { 780 return nil, types.ForbiddenErrorf("cannot create a swarm scoped network when swarm is not active") 781 } 782 783 // Make sure we have a driver available for this network type 784 // before we allocate anything. 785 if _, err := network.driver(true); err != nil { 786 return nil, err 787 } 788 789 // From this point on, we need the network specific configuration, 790 // which may come from a configuration-only network 791 if network.configFrom != "" { 792 t, err := c.getConfigNetwork(network.configFrom) 793 if err != nil { 794 return nil, types.NotFoundErrorf("configuration network %q does not exist", network.configFrom) 795 } 796 if err := t.applyConfigurationTo(network); err != nil { 797 return nil, types.InternalErrorf("Failed to apply configuration: %v", err) 798 } 799 defer func() { 800 if err == nil { 801 if err := t.getEpCnt().IncEndpointCnt(); err != nil { 802 logrus.Warnf("Failed to update reference count for configuration network %q on creation of network %q: %v", 803 t.Name(), network.Name(), err) 804 } 805 } 806 }() 807 } 808 809 err = network.ipamAllocate() 810 if err != nil { 811 return nil, err 812 } 813 defer func() { 814 if err != nil { 815 network.ipamRelease() 816 } 817 }() 818 819 err = c.addNetwork(network) 820 if err != nil { 821 return nil, err 822 } 823 defer func() { 824 if err != nil { 825 if e := network.deleteNetwork(); e != nil { 826 logrus.Warnf("couldn't roll back driver network on network %s creation failure: %v", network.name, err) 827 } 828 } 829 }() 830 831addToStore: 832 // First store the endpoint count, then the network. To avoid to 833 // end up with a datastore containing a network and not an epCnt, 834 // in case of an ungraceful shutdown during this function call. 835 epCnt := &endpointCnt{n: network} 836 if err = c.updateToStore(epCnt); err != nil { 837 return nil, err 838 } 839 defer func() { 840 if err != nil { 841 if e := c.deleteFromStore(epCnt); e != nil { 842 logrus.Warnf("could not rollback from store, epCnt %v on failure (%v): %v", epCnt, err, e) 843 } 844 } 845 }() 846 847 network.epCnt = epCnt 848 if err = c.updateToStore(network); err != nil { 849 return nil, err 850 } 851 defer func() { 852 if err != nil { 853 if e := c.deleteFromStore(network); e != nil { 854 logrus.Warnf("could not rollback from store, network %v on failure (%v): %v", network, err, e) 855 } 856 } 857 }() 858 859 if network.configOnly { 860 return network, nil 861 } 862 863 joinCluster(network) 864 defer func() { 865 if err != nil { 866 network.cancelDriverWatches() 867 if e := network.leaveCluster(); e != nil { 868 logrus.Warnf("Failed to leave agent cluster on network %s on failure (%v): %v", network.name, err, e) 869 } 870 } 871 }() 872 873 if len(network.loadBalancerIP) != 0 { 874 if err = network.createLoadBalancerSandbox(); err != nil { 875 return nil, err 876 } 877 } 878 879 if !c.isDistributedControl() { 880 c.Lock() 881 arrangeIngressFilterRule() 882 c.Unlock() 883 } 884 885 c.Lock() 886 arrangeUserFilterRule() 887 c.Unlock() 888 889 return network, nil 890} 891 892var joinCluster NetworkWalker = func(nw Network) bool { 893 n := nw.(*network) 894 if n.configOnly { 895 return false 896 } 897 if err := n.joinCluster(); err != nil { 898 logrus.Errorf("Failed to join network %s (%s) into agent cluster: %v", n.Name(), n.ID(), err) 899 } 900 n.addDriverWatches() 901 return false 902} 903 904func (c *controller) reservePools() { 905 networks, err := c.getNetworksForScope(datastore.LocalScope) 906 if err != nil { 907 logrus.Warnf("Could not retrieve networks from local store during ipam allocation for existing networks: %v", err) 908 return 909 } 910 911 for _, n := range networks { 912 if n.configOnly { 913 continue 914 } 915 if !doReplayPoolReserve(n) { 916 continue 917 } 918 // Construct pseudo configs for the auto IP case 919 autoIPv4 := (len(n.ipamV4Config) == 0 || (len(n.ipamV4Config) == 1 && n.ipamV4Config[0].PreferredPool == "")) && len(n.ipamV4Info) > 0 920 autoIPv6 := (len(n.ipamV6Config) == 0 || (len(n.ipamV6Config) == 1 && n.ipamV6Config[0].PreferredPool == "")) && len(n.ipamV6Info) > 0 921 if autoIPv4 { 922 n.ipamV4Config = []*IpamConf{{PreferredPool: n.ipamV4Info[0].Pool.String()}} 923 } 924 if n.enableIPv6 && autoIPv6 { 925 n.ipamV6Config = []*IpamConf{{PreferredPool: n.ipamV6Info[0].Pool.String()}} 926 } 927 // Account current network gateways 928 for i, c := range n.ipamV4Config { 929 if c.Gateway == "" && n.ipamV4Info[i].Gateway != nil { 930 c.Gateway = n.ipamV4Info[i].Gateway.IP.String() 931 } 932 } 933 if n.enableIPv6 { 934 for i, c := range n.ipamV6Config { 935 if c.Gateway == "" && n.ipamV6Info[i].Gateway != nil { 936 c.Gateway = n.ipamV6Info[i].Gateway.IP.String() 937 } 938 } 939 } 940 // Reserve pools 941 if err := n.ipamAllocate(); err != nil { 942 logrus.Warnf("Failed to allocate ipam pool(s) for network %q (%s): %v", n.Name(), n.ID(), err) 943 } 944 // Reserve existing endpoints' addresses 945 ipam, _, err := n.getController().getIPAMDriver(n.ipamType) 946 if err != nil { 947 logrus.Warnf("Failed to retrieve ipam driver for network %q (%s) during address reservation", n.Name(), n.ID()) 948 continue 949 } 950 epl, err := n.getEndpointsFromStore() 951 if err != nil { 952 logrus.Warnf("Failed to retrieve list of current endpoints on network %q (%s)", n.Name(), n.ID()) 953 continue 954 } 955 for _, ep := range epl { 956 if err := ep.assignAddress(ipam, true, ep.Iface().AddressIPv6() != nil); err != nil { 957 logrus.Warnf("Failed to reserve current address for endpoint %q (%s) on network %q (%s)", 958 ep.Name(), ep.ID(), n.Name(), n.ID()) 959 } 960 } 961 } 962} 963 964func doReplayPoolReserve(n *network) bool { 965 _, caps, err := n.getController().getIPAMDriver(n.ipamType) 966 if err != nil { 967 logrus.Warnf("Failed to retrieve ipam driver for network %q (%s): %v", n.Name(), n.ID(), err) 968 return false 969 } 970 return caps.RequiresRequestReplay 971} 972 973func (c *controller) addNetwork(n *network) error { 974 d, err := n.driver(true) 975 if err != nil { 976 return err 977 } 978 979 // Create the network 980 if err := d.CreateNetwork(n.id, n.generic, n, n.getIPData(4), n.getIPData(6)); err != nil { 981 return err 982 } 983 984 n.startResolver() 985 986 return nil 987} 988 989func (c *controller) Networks() []Network { 990 var list []Network 991 992 networks, err := c.getNetworksFromStore() 993 if err != nil { 994 logrus.Error(err) 995 } 996 997 for _, n := range networks { 998 if n.inDelete { 999 continue 1000 } 1001 list = append(list, n) 1002 } 1003 1004 return list 1005} 1006 1007func (c *controller) WalkNetworks(walker NetworkWalker) { 1008 for _, n := range c.Networks() { 1009 if walker(n) { 1010 return 1011 } 1012 } 1013} 1014 1015func (c *controller) NetworkByName(name string) (Network, error) { 1016 if name == "" { 1017 return nil, ErrInvalidName(name) 1018 } 1019 var n Network 1020 1021 s := func(current Network) bool { 1022 if current.Name() == name { 1023 n = current 1024 return true 1025 } 1026 return false 1027 } 1028 1029 c.WalkNetworks(s) 1030 1031 if n == nil { 1032 return nil, ErrNoSuchNetwork(name) 1033 } 1034 1035 return n, nil 1036} 1037 1038func (c *controller) NetworkByID(id string) (Network, error) { 1039 if id == "" { 1040 return nil, ErrInvalidID(id) 1041 } 1042 1043 n, err := c.getNetworkFromStore(id) 1044 if err != nil { 1045 return nil, ErrNoSuchNetwork(id) 1046 } 1047 1048 return n, nil 1049} 1050 1051// NewSandbox creates a new sandbox for the passed container id 1052func (c *controller) NewSandbox(containerID string, options ...SandboxOption) (Sandbox, error) { 1053 if containerID == "" { 1054 return nil, types.BadRequestErrorf("invalid container ID") 1055 } 1056 1057 var sb *sandbox 1058 c.Lock() 1059 for _, s := range c.sandboxes { 1060 if s.containerID == containerID { 1061 // If not a stub, then we already have a complete sandbox. 1062 if !s.isStub { 1063 sbID := s.ID() 1064 c.Unlock() 1065 return nil, types.ForbiddenErrorf("container %s is already present in sandbox %s", containerID, sbID) 1066 } 1067 1068 // We already have a stub sandbox from the 1069 // store. Make use of it so that we don't lose 1070 // the endpoints from store but reset the 1071 // isStub flag. 1072 sb = s 1073 sb.isStub = false 1074 break 1075 } 1076 } 1077 c.Unlock() 1078 1079 // Create sandbox and process options first. Key generation depends on an option 1080 if sb == nil { 1081 sb = &sandbox{ 1082 id: stringid.GenerateRandomID(), 1083 containerID: containerID, 1084 endpoints: epHeap{}, 1085 epPriority: map[string]int{}, 1086 populatedEndpoints: map[string]struct{}{}, 1087 config: containerConfig{}, 1088 controller: c, 1089 extDNS: []extDNSEntry{}, 1090 } 1091 } 1092 1093 heap.Init(&sb.endpoints) 1094 1095 sb.processOptions(options...) 1096 1097 c.Lock() 1098 if sb.ingress && c.ingressSandbox != nil { 1099 c.Unlock() 1100 return nil, types.ForbiddenErrorf("ingress sandbox already present") 1101 } 1102 1103 if sb.ingress { 1104 c.ingressSandbox = sb 1105 sb.config.hostsPath = filepath.Join(c.cfg.Daemon.DataDir, "/network/files/hosts") 1106 sb.config.resolvConfPath = filepath.Join(c.cfg.Daemon.DataDir, "/network/files/resolv.conf") 1107 sb.id = "ingress_sbox" 1108 } 1109 c.Unlock() 1110 1111 var err error 1112 defer func() { 1113 if err != nil { 1114 c.Lock() 1115 if sb.ingress { 1116 c.ingressSandbox = nil 1117 } 1118 c.Unlock() 1119 } 1120 }() 1121 1122 if err = sb.setupResolutionFiles(); err != nil { 1123 return nil, err 1124 } 1125 1126 if sb.config.useDefaultSandBox { 1127 c.sboxOnce.Do(func() { 1128 c.defOsSbox, err = osl.NewSandbox(sb.Key(), false, false) 1129 }) 1130 1131 if err != nil { 1132 c.sboxOnce = sync.Once{} 1133 return nil, fmt.Errorf("failed to create default sandbox: %v", err) 1134 } 1135 1136 sb.osSbox = c.defOsSbox 1137 } 1138 1139 if sb.osSbox == nil && !sb.config.useExternalKey { 1140 if sb.osSbox, err = osl.NewSandbox(sb.Key(), !sb.config.useDefaultSandBox, false); err != nil { 1141 return nil, fmt.Errorf("failed to create new osl sandbox: %v", err) 1142 } 1143 } 1144 1145 c.Lock() 1146 c.sandboxes[sb.id] = sb 1147 c.Unlock() 1148 defer func() { 1149 if err != nil { 1150 c.Lock() 1151 delete(c.sandboxes, sb.id) 1152 c.Unlock() 1153 } 1154 }() 1155 1156 err = sb.storeUpdate() 1157 if err != nil { 1158 return nil, fmt.Errorf("failed to update the store state of sandbox: %v", err) 1159 } 1160 1161 return sb, nil 1162} 1163 1164func (c *controller) Sandboxes() []Sandbox { 1165 c.Lock() 1166 defer c.Unlock() 1167 1168 list := make([]Sandbox, 0, len(c.sandboxes)) 1169 for _, s := range c.sandboxes { 1170 // Hide stub sandboxes from libnetwork users 1171 if s.isStub { 1172 continue 1173 } 1174 1175 list = append(list, s) 1176 } 1177 1178 return list 1179} 1180 1181func (c *controller) WalkSandboxes(walker SandboxWalker) { 1182 for _, sb := range c.Sandboxes() { 1183 if walker(sb) { 1184 return 1185 } 1186 } 1187} 1188 1189func (c *controller) SandboxByID(id string) (Sandbox, error) { 1190 if id == "" { 1191 return nil, ErrInvalidID(id) 1192 } 1193 c.Lock() 1194 s, ok := c.sandboxes[id] 1195 c.Unlock() 1196 if !ok { 1197 return nil, types.NotFoundErrorf("sandbox %s not found", id) 1198 } 1199 return s, nil 1200} 1201 1202// SandboxDestroy destroys a sandbox given a container ID 1203func (c *controller) SandboxDestroy(id string) error { 1204 var sb *sandbox 1205 c.Lock() 1206 for _, s := range c.sandboxes { 1207 if s.containerID == id { 1208 sb = s 1209 break 1210 } 1211 } 1212 c.Unlock() 1213 1214 // It is not an error if sandbox is not available 1215 if sb == nil { 1216 return nil 1217 } 1218 1219 return sb.Delete() 1220} 1221 1222// SandboxContainerWalker returns a Sandbox Walker function which looks for an existing Sandbox with the passed containerID 1223func SandboxContainerWalker(out *Sandbox, containerID string) SandboxWalker { 1224 return func(sb Sandbox) bool { 1225 if sb.ContainerID() == containerID { 1226 *out = sb 1227 return true 1228 } 1229 return false 1230 } 1231} 1232 1233// SandboxKeyWalker returns a Sandbox Walker function which looks for an existing Sandbox with the passed key 1234func SandboxKeyWalker(out *Sandbox, key string) SandboxWalker { 1235 return func(sb Sandbox) bool { 1236 if sb.Key() == key { 1237 *out = sb 1238 return true 1239 } 1240 return false 1241 } 1242} 1243 1244func (c *controller) loadDriver(networkType string) error { 1245 var err error 1246 1247 if pg := c.GetPluginGetter(); pg != nil { 1248 _, err = pg.Get(networkType, driverapi.NetworkPluginEndpointType, plugingetter.Lookup) 1249 } else { 1250 _, err = plugins.Get(networkType, driverapi.NetworkPluginEndpointType) 1251 } 1252 1253 if err != nil { 1254 if err == plugins.ErrNotFound { 1255 return types.NotFoundErrorf(err.Error()) 1256 } 1257 return err 1258 } 1259 1260 return nil 1261} 1262 1263func (c *controller) loadIPAMDriver(name string) error { 1264 var err error 1265 1266 if pg := c.GetPluginGetter(); pg != nil { 1267 _, err = pg.Get(name, ipamapi.PluginEndpointType, plugingetter.Lookup) 1268 } else { 1269 _, err = plugins.Get(name, ipamapi.PluginEndpointType) 1270 } 1271 1272 if err != nil { 1273 if err == plugins.ErrNotFound { 1274 return types.NotFoundErrorf(err.Error()) 1275 } 1276 return err 1277 } 1278 1279 return nil 1280} 1281 1282func (c *controller) getIPAMDriver(name string) (ipamapi.Ipam, *ipamapi.Capability, error) { 1283 id, cap := c.drvRegistry.IPAM(name) 1284 if id == nil { 1285 // Might be a plugin name. Try loading it 1286 if err := c.loadIPAMDriver(name); err != nil { 1287 return nil, nil, err 1288 } 1289 1290 // Now that we resolved the plugin, try again looking up the registry 1291 id, cap = c.drvRegistry.IPAM(name) 1292 if id == nil { 1293 return nil, nil, types.BadRequestErrorf("invalid ipam driver: %q", name) 1294 } 1295 } 1296 1297 return id, cap, nil 1298} 1299 1300func (c *controller) Stop() { 1301 c.closeStores() 1302 c.stopExternalKeyListener() 1303 osl.GC() 1304} 1305 1306// StartDiagnose start the network diagnose mode 1307func (c *controller) StartDiagnose(port int) { 1308 c.Lock() 1309 if !c.DiagnoseServer.IsDebugEnable() { 1310 c.DiagnoseServer.EnableDebug("127.0.0.1", port) 1311 } 1312 c.Unlock() 1313} 1314 1315// StopDiagnose start the network diagnose mode 1316func (c *controller) StopDiagnose() { 1317 c.Lock() 1318 if c.DiagnoseServer.IsDebugEnable() { 1319 c.DiagnoseServer.DisableDebug() 1320 } 1321 c.Unlock() 1322} 1323 1324// IsDiagnoseEnabled returns true if the diagnose is enabled 1325func (c *controller) IsDiagnoseEnabled() bool { 1326 c.Lock() 1327 defer c.Unlock() 1328 return c.DiagnoseServer.IsDebugEnable() 1329} 1330