1package overlay
2
3//go:generate protoc -I.:../../Godeps/_workspace/src/github.com/gogo/protobuf  --gogo_out=import_path=github.com/docker/libnetwork/drivers/overlay,Mgogoproto/gogo.proto=github.com/gogo/protobuf/gogoproto:. overlay.proto
4
5import (
6	"fmt"
7	"net"
8	"sync"
9
10	"github.com/sirupsen/logrus"
11	"github.com/docker/libnetwork/datastore"
12	"github.com/docker/libnetwork/discoverapi"
13	"github.com/docker/libnetwork/driverapi"
14	"github.com/docker/libnetwork/idm"
15	"github.com/docker/libnetwork/netlabel"
16	"github.com/docker/libnetwork/osl"
17	"github.com/docker/libnetwork/types"
18	"github.com/hashicorp/serf/serf"
19)
20
21// XXX OVERLAY_SOLARIS
22// Might need changes for names/constant values in solaris
23const (
24	networkType  = "overlay"
25	vethPrefix   = "veth"
26	vethLen      = 7
27	vxlanIDStart = 256
28	vxlanIDEnd   = (1 << 24) - 1
29	vxlanPort    = 4789
30	vxlanEncap   = 50
31	secureOption = "encrypted"
32)
33
34var initVxlanIdm = make(chan (bool), 1)
35
36type driver struct {
37	eventCh          chan serf.Event
38	notifyCh         chan ovNotify
39	exitCh           chan chan struct{}
40	bindAddress      string
41	advertiseAddress string
42	neighIP          string
43	config           map[string]interface{}
44	peerDb           peerNetworkMap
45	secMap           *encrMap
46	serfInstance     *serf.Serf
47	networks         networkTable
48	store            datastore.DataStore
49	localStore       datastore.DataStore
50	vxlanIdm         *idm.Idm
51	once             sync.Once
52	joinOnce         sync.Once
53	keys             []*key
54	sync.Mutex
55}
56
57// Init registers a new instance of overlay driver
58func Init(dc driverapi.DriverCallback, config map[string]interface{}) error {
59	c := driverapi.Capability{
60		DataScope: datastore.GlobalScope,
61	}
62	d := &driver{
63		networks: networkTable{},
64		peerDb: peerNetworkMap{
65			mp: map[string]*peerMap{},
66		},
67		secMap: &encrMap{nodes: map[string][]*spi{}},
68		config: config,
69	}
70
71	if data, ok := config[netlabel.GlobalKVClient]; ok {
72		var err error
73		dsc, ok := data.(discoverapi.DatastoreConfigData)
74		if !ok {
75			return types.InternalErrorf("incorrect data in datastore configuration: %v", data)
76		}
77		d.store, err = datastore.NewDataStoreFromConfig(dsc)
78		if err != nil {
79			return types.InternalErrorf("failed to initialize data store: %v", err)
80		}
81	}
82
83	if data, ok := config[netlabel.LocalKVClient]; ok {
84		var err error
85		dsc, ok := data.(discoverapi.DatastoreConfigData)
86		if !ok {
87			return types.InternalErrorf("incorrect data in datastore configuration: %v", data)
88		}
89		d.localStore, err = datastore.NewDataStoreFromConfig(dsc)
90		if err != nil {
91			return types.InternalErrorf("failed to initialize local data store: %v", err)
92		}
93	}
94
95	d.restoreEndpoints()
96
97	return dc.RegisterDriver(networkType, d, c)
98}
99
100// Endpoints are stored in the local store. Restore them and reconstruct the overlay sandbox
101func (d *driver) restoreEndpoints() error {
102	if d.localStore == nil {
103		logrus.Warnf("Cannot restore overlay endpoints because local datastore is missing")
104		return nil
105	}
106	kvol, err := d.localStore.List(datastore.Key(overlayEndpointPrefix), &endpoint{})
107	if err != nil && err != datastore.ErrKeyNotFound {
108		return fmt.Errorf("failed to read overlay endpoint from store: %v", err)
109	}
110
111	if err == datastore.ErrKeyNotFound {
112		return nil
113	}
114	for _, kvo := range kvol {
115		ep := kvo.(*endpoint)
116		n := d.network(ep.nid)
117		if n == nil {
118			logrus.Debugf("Network (%s) not found for restored endpoint (%s)", ep.nid[0:7], ep.id[0:7])
119			logrus.Debugf("Deleting stale overlay endpoint (%s) from store", ep.id[0:7])
120			if err := d.deleteEndpointFromStore(ep); err != nil {
121				logrus.Debugf("Failed to delete stale overlay endpoint (%s) from store", ep.id[0:7])
122			}
123			continue
124		}
125		n.addEndpoint(ep)
126
127		s := n.getSubnetforIP(ep.addr)
128		if s == nil {
129			return fmt.Errorf("could not find subnet for endpoint %s", ep.id)
130		}
131
132		if err := n.joinSandbox(true); err != nil {
133			return fmt.Errorf("restore network sandbox failed: %v", err)
134		}
135
136		if err := n.joinSubnetSandbox(s, true); err != nil {
137			return fmt.Errorf("restore subnet sandbox failed for %q: %v", s.subnetIP.String(), err)
138		}
139
140		Ifaces := make(map[string][]osl.IfaceOption)
141		vethIfaceOption := make([]osl.IfaceOption, 1)
142		vethIfaceOption = append(vethIfaceOption, n.sbox.InterfaceOptions().Master(s.brName))
143		Ifaces[fmt.Sprintf("%s+%s", "veth", "veth")] = vethIfaceOption
144
145		err := n.sbox.Restore(Ifaces, nil, nil, nil)
146		if err != nil {
147			return fmt.Errorf("failed to restore overlay sandbox: %v", err)
148		}
149
150		n.incEndpointCount()
151		d.peerDbAdd(ep.nid, ep.id, ep.addr.IP, ep.addr.Mask, ep.mac, net.ParseIP(d.advertiseAddress), true)
152	}
153	return nil
154}
155
156// Fini cleans up the driver resources
157func Fini(drv driverapi.Driver) {
158	d := drv.(*driver)
159
160	if d.exitCh != nil {
161		waitCh := make(chan struct{})
162
163		d.exitCh <- waitCh
164
165		<-waitCh
166	}
167}
168
169func (d *driver) configure() error {
170	if d.store == nil {
171		return nil
172	}
173
174	if d.vxlanIdm == nil {
175		return d.initializeVxlanIdm()
176	}
177
178	return nil
179}
180
181func (d *driver) initializeVxlanIdm() error {
182	var err error
183
184	initVxlanIdm <- true
185	defer func() { <-initVxlanIdm }()
186
187	if d.vxlanIdm != nil {
188		return nil
189	}
190
191	d.vxlanIdm, err = idm.New(d.store, "vxlan-id", vxlanIDStart, vxlanIDEnd)
192	if err != nil {
193		return fmt.Errorf("failed to initialize vxlan id manager: %v", err)
194	}
195
196	return nil
197}
198
199func (d *driver) Type() string {
200	return networkType
201}
202
203func validateSelf(node string) error {
204	advIP := net.ParseIP(node)
205	if advIP == nil {
206		return fmt.Errorf("invalid self address (%s)", node)
207	}
208
209	addrs, err := net.InterfaceAddrs()
210	if err != nil {
211		return fmt.Errorf("Unable to get interface addresses %v", err)
212	}
213	for _, addr := range addrs {
214		ip, _, err := net.ParseCIDR(addr.String())
215		if err == nil && ip.Equal(advIP) {
216			return nil
217		}
218	}
219	return fmt.Errorf("Multi-Host overlay networking requires cluster-advertise(%s) to be configured with a local ip-address that is reachable within the cluster", advIP.String())
220}
221
222func (d *driver) nodeJoin(advertiseAddress, bindAddress string, self bool) {
223	if self && !d.isSerfAlive() {
224		d.Lock()
225		d.advertiseAddress = advertiseAddress
226		d.bindAddress = bindAddress
227		d.Unlock()
228
229		// If there is no cluster store there is no need to start serf.
230		if d.store != nil {
231			if err := validateSelf(advertiseAddress); err != nil {
232				logrus.Warnf("%s", err.Error())
233			}
234			err := d.serfInit()
235			if err != nil {
236				logrus.Errorf("initializing serf instance failed: %v", err)
237				d.Lock()
238				d.advertiseAddress = ""
239				d.bindAddress = ""
240				d.Unlock()
241				return
242			}
243		}
244	}
245
246	d.Lock()
247	if !self {
248		d.neighIP = advertiseAddress
249	}
250	neighIP := d.neighIP
251	d.Unlock()
252
253	if d.serfInstance != nil && neighIP != "" {
254		var err error
255		d.joinOnce.Do(func() {
256			err = d.serfJoin(neighIP)
257			if err == nil {
258				d.pushLocalDb()
259			}
260		})
261		if err != nil {
262			logrus.Errorf("joining serf neighbor %s failed: %v", advertiseAddress, err)
263			d.Lock()
264			d.joinOnce = sync.Once{}
265			d.Unlock()
266			return
267		}
268	}
269}
270
271func (d *driver) pushLocalEndpointEvent(action, nid, eid string) {
272	n := d.network(nid)
273	if n == nil {
274		logrus.Debugf("Error pushing local endpoint event for network %s", nid)
275		return
276	}
277	ep := n.endpoint(eid)
278	if ep == nil {
279		logrus.Debugf("Error pushing local endpoint event for ep %s / %s", nid, eid)
280		return
281	}
282
283	if !d.isSerfAlive() {
284		return
285	}
286	d.notifyCh <- ovNotify{
287		action: "join",
288		nw:     n,
289		ep:     ep,
290	}
291}
292
293// DiscoverNew is a notification for a new discovery event, such as a new node joining a cluster
294func (d *driver) DiscoverNew(dType discoverapi.DiscoveryType, data interface{}) error {
295	var err error
296	switch dType {
297	case discoverapi.NodeDiscovery:
298		nodeData, ok := data.(discoverapi.NodeDiscoveryData)
299		if !ok || nodeData.Address == "" {
300			return fmt.Errorf("invalid discovery data")
301		}
302		d.nodeJoin(nodeData.Address, nodeData.BindAddress, nodeData.Self)
303	case discoverapi.DatastoreConfig:
304		if d.store != nil {
305			return types.ForbiddenErrorf("cannot accept datastore configuration: Overlay driver has a datastore configured already")
306		}
307		dsc, ok := data.(discoverapi.DatastoreConfigData)
308		if !ok {
309			return types.InternalErrorf("incorrect data in datastore configuration: %v", data)
310		}
311		d.store, err = datastore.NewDataStoreFromConfig(dsc)
312		if err != nil {
313			return types.InternalErrorf("failed to initialize data store: %v", err)
314		}
315	case discoverapi.EncryptionKeysConfig:
316		encrData, ok := data.(discoverapi.DriverEncryptionConfig)
317		if !ok {
318			return fmt.Errorf("invalid encryption key notification data")
319		}
320		keys := make([]*key, 0, len(encrData.Keys))
321		for i := 0; i < len(encrData.Keys); i++ {
322			k := &key{
323				value: encrData.Keys[i],
324				tag:   uint32(encrData.Tags[i]),
325			}
326			keys = append(keys, k)
327		}
328		d.setKeys(keys)
329	case discoverapi.EncryptionKeysUpdate:
330		var newKey, delKey, priKey *key
331		encrData, ok := data.(discoverapi.DriverEncryptionUpdate)
332		if !ok {
333			return fmt.Errorf("invalid encryption key notification data")
334		}
335		if encrData.Key != nil {
336			newKey = &key{
337				value: encrData.Key,
338				tag:   uint32(encrData.Tag),
339			}
340		}
341		if encrData.Primary != nil {
342			priKey = &key{
343				value: encrData.Primary,
344				tag:   uint32(encrData.PrimaryTag),
345			}
346		}
347		if encrData.Prune != nil {
348			delKey = &key{
349				value: encrData.Prune,
350				tag:   uint32(encrData.PruneTag),
351			}
352		}
353		d.updateKeys(newKey, priKey, delKey)
354	default:
355	}
356	return nil
357}
358
359// DiscoverDelete is a notification for a discovery delete event, such as a node leaving a cluster
360func (d *driver) DiscoverDelete(dType discoverapi.DiscoveryType, data interface{}) error {
361	return nil
362}
363