1package overlay 2 3//go:generate protoc -I.:../../Godeps/_workspace/src/github.com/gogo/protobuf --gogo_out=import_path=github.com/docker/libnetwork/drivers/overlay,Mgogoproto/gogo.proto=github.com/gogo/protobuf/gogoproto:. overlay.proto 4 5import ( 6 "fmt" 7 "net" 8 "sync" 9 10 "github.com/sirupsen/logrus" 11 "github.com/docker/libnetwork/datastore" 12 "github.com/docker/libnetwork/discoverapi" 13 "github.com/docker/libnetwork/driverapi" 14 "github.com/docker/libnetwork/idm" 15 "github.com/docker/libnetwork/netlabel" 16 "github.com/docker/libnetwork/osl" 17 "github.com/docker/libnetwork/types" 18 "github.com/hashicorp/serf/serf" 19) 20 21// XXX OVERLAY_SOLARIS 22// Might need changes for names/constant values in solaris 23const ( 24 networkType = "overlay" 25 vethPrefix = "veth" 26 vethLen = 7 27 vxlanIDStart = 256 28 vxlanIDEnd = (1 << 24) - 1 29 vxlanPort = 4789 30 vxlanEncap = 50 31 secureOption = "encrypted" 32) 33 34var initVxlanIdm = make(chan (bool), 1) 35 36type driver struct { 37 eventCh chan serf.Event 38 notifyCh chan ovNotify 39 exitCh chan chan struct{} 40 bindAddress string 41 advertiseAddress string 42 neighIP string 43 config map[string]interface{} 44 peerDb peerNetworkMap 45 secMap *encrMap 46 serfInstance *serf.Serf 47 networks networkTable 48 store datastore.DataStore 49 localStore datastore.DataStore 50 vxlanIdm *idm.Idm 51 once sync.Once 52 joinOnce sync.Once 53 keys []*key 54 sync.Mutex 55} 56 57// Init registers a new instance of overlay driver 58func Init(dc driverapi.DriverCallback, config map[string]interface{}) error { 59 c := driverapi.Capability{ 60 DataScope: datastore.GlobalScope, 61 } 62 d := &driver{ 63 networks: networkTable{}, 64 peerDb: peerNetworkMap{ 65 mp: map[string]*peerMap{}, 66 }, 67 secMap: &encrMap{nodes: map[string][]*spi{}}, 68 config: config, 69 } 70 71 if data, ok := config[netlabel.GlobalKVClient]; ok { 72 var err error 73 dsc, ok := data.(discoverapi.DatastoreConfigData) 74 if !ok { 75 return types.InternalErrorf("incorrect data in datastore configuration: %v", data) 76 } 77 d.store, err = datastore.NewDataStoreFromConfig(dsc) 78 if err != nil { 79 return types.InternalErrorf("failed to initialize data store: %v", err) 80 } 81 } 82 83 if data, ok := config[netlabel.LocalKVClient]; ok { 84 var err error 85 dsc, ok := data.(discoverapi.DatastoreConfigData) 86 if !ok { 87 return types.InternalErrorf("incorrect data in datastore configuration: %v", data) 88 } 89 d.localStore, err = datastore.NewDataStoreFromConfig(dsc) 90 if err != nil { 91 return types.InternalErrorf("failed to initialize local data store: %v", err) 92 } 93 } 94 95 d.restoreEndpoints() 96 97 return dc.RegisterDriver(networkType, d, c) 98} 99 100// Endpoints are stored in the local store. Restore them and reconstruct the overlay sandbox 101func (d *driver) restoreEndpoints() error { 102 if d.localStore == nil { 103 logrus.Warnf("Cannot restore overlay endpoints because local datastore is missing") 104 return nil 105 } 106 kvol, err := d.localStore.List(datastore.Key(overlayEndpointPrefix), &endpoint{}) 107 if err != nil && err != datastore.ErrKeyNotFound { 108 return fmt.Errorf("failed to read overlay endpoint from store: %v", err) 109 } 110 111 if err == datastore.ErrKeyNotFound { 112 return nil 113 } 114 for _, kvo := range kvol { 115 ep := kvo.(*endpoint) 116 n := d.network(ep.nid) 117 if n == nil { 118 logrus.Debugf("Network (%s) not found for restored endpoint (%s)", ep.nid[0:7], ep.id[0:7]) 119 logrus.Debugf("Deleting stale overlay endpoint (%s) from store", ep.id[0:7]) 120 if err := d.deleteEndpointFromStore(ep); err != nil { 121 logrus.Debugf("Failed to delete stale overlay endpoint (%s) from store", ep.id[0:7]) 122 } 123 continue 124 } 125 n.addEndpoint(ep) 126 127 s := n.getSubnetforIP(ep.addr) 128 if s == nil { 129 return fmt.Errorf("could not find subnet for endpoint %s", ep.id) 130 } 131 132 if err := n.joinSandbox(true); err != nil { 133 return fmt.Errorf("restore network sandbox failed: %v", err) 134 } 135 136 if err := n.joinSubnetSandbox(s, true); err != nil { 137 return fmt.Errorf("restore subnet sandbox failed for %q: %v", s.subnetIP.String(), err) 138 } 139 140 Ifaces := make(map[string][]osl.IfaceOption) 141 vethIfaceOption := make([]osl.IfaceOption, 1) 142 vethIfaceOption = append(vethIfaceOption, n.sbox.InterfaceOptions().Master(s.brName)) 143 Ifaces[fmt.Sprintf("%s+%s", "veth", "veth")] = vethIfaceOption 144 145 err := n.sbox.Restore(Ifaces, nil, nil, nil) 146 if err != nil { 147 return fmt.Errorf("failed to restore overlay sandbox: %v", err) 148 } 149 150 n.incEndpointCount() 151 d.peerDbAdd(ep.nid, ep.id, ep.addr.IP, ep.addr.Mask, ep.mac, net.ParseIP(d.advertiseAddress), true) 152 } 153 return nil 154} 155 156// Fini cleans up the driver resources 157func Fini(drv driverapi.Driver) { 158 d := drv.(*driver) 159 160 if d.exitCh != nil { 161 waitCh := make(chan struct{}) 162 163 d.exitCh <- waitCh 164 165 <-waitCh 166 } 167} 168 169func (d *driver) configure() error { 170 if d.store == nil { 171 return nil 172 } 173 174 if d.vxlanIdm == nil { 175 return d.initializeVxlanIdm() 176 } 177 178 return nil 179} 180 181func (d *driver) initializeVxlanIdm() error { 182 var err error 183 184 initVxlanIdm <- true 185 defer func() { <-initVxlanIdm }() 186 187 if d.vxlanIdm != nil { 188 return nil 189 } 190 191 d.vxlanIdm, err = idm.New(d.store, "vxlan-id", vxlanIDStart, vxlanIDEnd) 192 if err != nil { 193 return fmt.Errorf("failed to initialize vxlan id manager: %v", err) 194 } 195 196 return nil 197} 198 199func (d *driver) Type() string { 200 return networkType 201} 202 203func validateSelf(node string) error { 204 advIP := net.ParseIP(node) 205 if advIP == nil { 206 return fmt.Errorf("invalid self address (%s)", node) 207 } 208 209 addrs, err := net.InterfaceAddrs() 210 if err != nil { 211 return fmt.Errorf("Unable to get interface addresses %v", err) 212 } 213 for _, addr := range addrs { 214 ip, _, err := net.ParseCIDR(addr.String()) 215 if err == nil && ip.Equal(advIP) { 216 return nil 217 } 218 } 219 return fmt.Errorf("Multi-Host overlay networking requires cluster-advertise(%s) to be configured with a local ip-address that is reachable within the cluster", advIP.String()) 220} 221 222func (d *driver) nodeJoin(advertiseAddress, bindAddress string, self bool) { 223 if self && !d.isSerfAlive() { 224 d.Lock() 225 d.advertiseAddress = advertiseAddress 226 d.bindAddress = bindAddress 227 d.Unlock() 228 229 // If there is no cluster store there is no need to start serf. 230 if d.store != nil { 231 if err := validateSelf(advertiseAddress); err != nil { 232 logrus.Warnf("%s", err.Error()) 233 } 234 err := d.serfInit() 235 if err != nil { 236 logrus.Errorf("initializing serf instance failed: %v", err) 237 d.Lock() 238 d.advertiseAddress = "" 239 d.bindAddress = "" 240 d.Unlock() 241 return 242 } 243 } 244 } 245 246 d.Lock() 247 if !self { 248 d.neighIP = advertiseAddress 249 } 250 neighIP := d.neighIP 251 d.Unlock() 252 253 if d.serfInstance != nil && neighIP != "" { 254 var err error 255 d.joinOnce.Do(func() { 256 err = d.serfJoin(neighIP) 257 if err == nil { 258 d.pushLocalDb() 259 } 260 }) 261 if err != nil { 262 logrus.Errorf("joining serf neighbor %s failed: %v", advertiseAddress, err) 263 d.Lock() 264 d.joinOnce = sync.Once{} 265 d.Unlock() 266 return 267 } 268 } 269} 270 271func (d *driver) pushLocalEndpointEvent(action, nid, eid string) { 272 n := d.network(nid) 273 if n == nil { 274 logrus.Debugf("Error pushing local endpoint event for network %s", nid) 275 return 276 } 277 ep := n.endpoint(eid) 278 if ep == nil { 279 logrus.Debugf("Error pushing local endpoint event for ep %s / %s", nid, eid) 280 return 281 } 282 283 if !d.isSerfAlive() { 284 return 285 } 286 d.notifyCh <- ovNotify{ 287 action: "join", 288 nw: n, 289 ep: ep, 290 } 291} 292 293// DiscoverNew is a notification for a new discovery event, such as a new node joining a cluster 294func (d *driver) DiscoverNew(dType discoverapi.DiscoveryType, data interface{}) error { 295 var err error 296 switch dType { 297 case discoverapi.NodeDiscovery: 298 nodeData, ok := data.(discoverapi.NodeDiscoveryData) 299 if !ok || nodeData.Address == "" { 300 return fmt.Errorf("invalid discovery data") 301 } 302 d.nodeJoin(nodeData.Address, nodeData.BindAddress, nodeData.Self) 303 case discoverapi.DatastoreConfig: 304 if d.store != nil { 305 return types.ForbiddenErrorf("cannot accept datastore configuration: Overlay driver has a datastore configured already") 306 } 307 dsc, ok := data.(discoverapi.DatastoreConfigData) 308 if !ok { 309 return types.InternalErrorf("incorrect data in datastore configuration: %v", data) 310 } 311 d.store, err = datastore.NewDataStoreFromConfig(dsc) 312 if err != nil { 313 return types.InternalErrorf("failed to initialize data store: %v", err) 314 } 315 case discoverapi.EncryptionKeysConfig: 316 encrData, ok := data.(discoverapi.DriverEncryptionConfig) 317 if !ok { 318 return fmt.Errorf("invalid encryption key notification data") 319 } 320 keys := make([]*key, 0, len(encrData.Keys)) 321 for i := 0; i < len(encrData.Keys); i++ { 322 k := &key{ 323 value: encrData.Keys[i], 324 tag: uint32(encrData.Tags[i]), 325 } 326 keys = append(keys, k) 327 } 328 d.setKeys(keys) 329 case discoverapi.EncryptionKeysUpdate: 330 var newKey, delKey, priKey *key 331 encrData, ok := data.(discoverapi.DriverEncryptionUpdate) 332 if !ok { 333 return fmt.Errorf("invalid encryption key notification data") 334 } 335 if encrData.Key != nil { 336 newKey = &key{ 337 value: encrData.Key, 338 tag: uint32(encrData.Tag), 339 } 340 } 341 if encrData.Primary != nil { 342 priKey = &key{ 343 value: encrData.Primary, 344 tag: uint32(encrData.PrimaryTag), 345 } 346 } 347 if encrData.Prune != nil { 348 delKey = &key{ 349 value: encrData.Prune, 350 tag: uint32(encrData.PruneTag), 351 } 352 } 353 d.updateKeys(newKey, priKey, delKey) 354 default: 355 } 356 return nil 357} 358 359// DiscoverDelete is a notification for a discovery delete event, such as a node leaving a cluster 360func (d *driver) DiscoverDelete(dType discoverapi.DiscoveryType, data interface{}) error { 361 return nil 362} 363