1package overlay
2
3//go:generate protoc -I.:../../Godeps/_workspace/src/github.com/gogo/protobuf  --gogo_out=import_path=github.com/docker/libnetwork/drivers/overlay,Mgogoproto/gogo.proto=github.com/gogo/protobuf/gogoproto:. overlay.proto
4
5import (
6	"context"
7	"fmt"
8	"net"
9	"sync"
10
11	"github.com/docker/libnetwork/datastore"
12	"github.com/docker/libnetwork/discoverapi"
13	"github.com/docker/libnetwork/driverapi"
14	"github.com/docker/libnetwork/idm"
15	"github.com/docker/libnetwork/netlabel"
16	"github.com/docker/libnetwork/osl"
17	"github.com/docker/libnetwork/types"
18	"github.com/hashicorp/serf/serf"
19	"github.com/sirupsen/logrus"
20)
21
22const (
23	networkType  = "overlay"
24	vethPrefix   = "veth"
25	vethLen      = 7
26	vxlanIDStart = 256
27	vxlanIDEnd   = (1 << 24) - 1
28	vxlanEncap   = 50
29	secureOption = "encrypted"
30)
31
32var initVxlanIdm = make(chan (bool), 1)
33
34type driver struct {
35	eventCh          chan serf.Event
36	notifyCh         chan ovNotify
37	exitCh           chan chan struct{}
38	bindAddress      string
39	advertiseAddress string
40	neighIP          string
41	config           map[string]interface{}
42	peerDb           peerNetworkMap
43	secMap           *encrMap
44	serfInstance     *serf.Serf
45	networks         networkTable
46	store            datastore.DataStore
47	localStore       datastore.DataStore
48	vxlanIdm         *idm.Idm
49	initOS           sync.Once
50	joinOnce         sync.Once
51	localJoinOnce    sync.Once
52	keys             []*key
53	peerOpCh         chan *peerOperation
54	peerOpCancel     context.CancelFunc
55	sync.Mutex
56}
57
58// Init registers a new instance of overlay driver
59func Init(dc driverapi.DriverCallback, config map[string]interface{}) error {
60	c := driverapi.Capability{
61		DataScope:         datastore.GlobalScope,
62		ConnectivityScope: datastore.GlobalScope,
63	}
64	d := &driver{
65		networks: networkTable{},
66		peerDb: peerNetworkMap{
67			mp: map[string]*peerMap{},
68		},
69		secMap:   &encrMap{nodes: map[string][]*spi{}},
70		config:   config,
71		peerOpCh: make(chan *peerOperation),
72	}
73
74	// Launch the go routine for processing peer operations
75	ctx, cancel := context.WithCancel(context.Background())
76	d.peerOpCancel = cancel
77	go d.peerOpRoutine(ctx, d.peerOpCh)
78
79	if data, ok := config[netlabel.GlobalKVClient]; ok {
80		var err error
81		dsc, ok := data.(discoverapi.DatastoreConfigData)
82		if !ok {
83			return types.InternalErrorf("incorrect data in datastore configuration: %v", data)
84		}
85		d.store, err = datastore.NewDataStoreFromConfig(dsc)
86		if err != nil {
87			return types.InternalErrorf("failed to initialize data store: %v", err)
88		}
89	}
90
91	if data, ok := config[netlabel.LocalKVClient]; ok {
92		var err error
93		dsc, ok := data.(discoverapi.DatastoreConfigData)
94		if !ok {
95			return types.InternalErrorf("incorrect data in datastore configuration: %v", data)
96		}
97		d.localStore, err = datastore.NewDataStoreFromConfig(dsc)
98		if err != nil {
99			return types.InternalErrorf("failed to initialize local data store: %v", err)
100		}
101	}
102
103	if err := d.restoreEndpoints(); err != nil {
104		logrus.Warnf("Failure during overlay endpoints restore: %v", err)
105	}
106
107	return dc.RegisterDriver(networkType, d, c)
108}
109
110// Endpoints are stored in the local store. Restore them and reconstruct the overlay sandbox
111func (d *driver) restoreEndpoints() error {
112	if d.localStore == nil {
113		logrus.Warn("Cannot restore overlay endpoints because local datastore is missing")
114		return nil
115	}
116	kvol, err := d.localStore.List(datastore.Key(overlayEndpointPrefix), &endpoint{})
117	if err != nil && err != datastore.ErrKeyNotFound {
118		return fmt.Errorf("failed to read overlay endpoint from store: %v", err)
119	}
120
121	if err == datastore.ErrKeyNotFound {
122		return nil
123	}
124	for _, kvo := range kvol {
125		ep := kvo.(*endpoint)
126		n := d.network(ep.nid)
127		if n == nil {
128			logrus.Debugf("Network (%.7s) not found for restored endpoint (%.7s)", ep.nid, ep.id)
129			logrus.Debugf("Deleting stale overlay endpoint (%.7s) from store", ep.id)
130			if err := d.deleteEndpointFromStore(ep); err != nil {
131				logrus.Debugf("Failed to delete stale overlay endpoint (%.7s) from store", ep.id)
132			}
133			continue
134		}
135		n.addEndpoint(ep)
136
137		s := n.getSubnetforIP(ep.addr)
138		if s == nil {
139			return fmt.Errorf("could not find subnet for endpoint %s", ep.id)
140		}
141
142		if err := n.joinSandbox(s, true, true); err != nil {
143			return fmt.Errorf("restore network sandbox failed: %v", err)
144		}
145
146		Ifaces := make(map[string][]osl.IfaceOption)
147		vethIfaceOption := make([]osl.IfaceOption, 1)
148		vethIfaceOption = append(vethIfaceOption, n.sbox.InterfaceOptions().Master(s.brName))
149		Ifaces["veth+veth"] = vethIfaceOption
150
151		err := n.sbox.Restore(Ifaces, nil, nil, nil)
152		if err != nil {
153			n.leaveSandbox()
154			return fmt.Errorf("failed to restore overlay sandbox: %v", err)
155		}
156
157		d.peerAdd(ep.nid, ep.id, ep.addr.IP, ep.addr.Mask, ep.mac, net.ParseIP(d.advertiseAddress), false, false, true)
158	}
159	return nil
160}
161
162// Fini cleans up the driver resources
163func Fini(drv driverapi.Driver) {
164	d := drv.(*driver)
165
166	// Notify the peer go routine to return
167	if d.peerOpCancel != nil {
168		d.peerOpCancel()
169	}
170
171	if d.exitCh != nil {
172		waitCh := make(chan struct{})
173
174		d.exitCh <- waitCh
175
176		<-waitCh
177	}
178}
179
180func (d *driver) configure() error {
181
182	// Apply OS specific kernel configs if needed
183	d.initOS.Do(applyOStweaks)
184
185	if d.store == nil {
186		return nil
187	}
188
189	if d.vxlanIdm == nil {
190		return d.initializeVxlanIdm()
191	}
192
193	return nil
194}
195
196func (d *driver) initializeVxlanIdm() error {
197	var err error
198
199	initVxlanIdm <- true
200	defer func() { <-initVxlanIdm }()
201
202	if d.vxlanIdm != nil {
203		return nil
204	}
205
206	d.vxlanIdm, err = idm.New(d.store, "vxlan-id", vxlanIDStart, vxlanIDEnd)
207	if err != nil {
208		return fmt.Errorf("failed to initialize vxlan id manager: %v", err)
209	}
210
211	return nil
212}
213
214func (d *driver) Type() string {
215	return networkType
216}
217
218func (d *driver) IsBuiltIn() bool {
219	return true
220}
221
222func validateSelf(node string) error {
223	advIP := net.ParseIP(node)
224	if advIP == nil {
225		return fmt.Errorf("invalid self address (%s)", node)
226	}
227
228	addrs, err := net.InterfaceAddrs()
229	if err != nil {
230		return fmt.Errorf("Unable to get interface addresses %v", err)
231	}
232	for _, addr := range addrs {
233		ip, _, err := net.ParseCIDR(addr.String())
234		if err == nil && ip.Equal(advIP) {
235			return nil
236		}
237	}
238	return fmt.Errorf("Multi-Host overlay networking requires cluster-advertise(%s) to be configured with a local ip-address that is reachable within the cluster", advIP.String())
239}
240
241func (d *driver) nodeJoin(advertiseAddress, bindAddress string, self bool) {
242	if self && !d.isSerfAlive() {
243		d.Lock()
244		d.advertiseAddress = advertiseAddress
245		d.bindAddress = bindAddress
246		d.Unlock()
247
248		// If containers are already running on this network update the
249		// advertise address in the peerDB
250		d.localJoinOnce.Do(func() {
251			d.peerDBUpdateSelf()
252		})
253
254		// If there is no cluster store there is no need to start serf.
255		if d.store != nil {
256			if err := validateSelf(advertiseAddress); err != nil {
257				logrus.Warn(err.Error())
258			}
259			err := d.serfInit()
260			if err != nil {
261				logrus.Errorf("initializing serf instance failed: %v", err)
262				d.Lock()
263				d.advertiseAddress = ""
264				d.bindAddress = ""
265				d.Unlock()
266				return
267			}
268		}
269	}
270
271	d.Lock()
272	if !self {
273		d.neighIP = advertiseAddress
274	}
275	neighIP := d.neighIP
276	d.Unlock()
277
278	if d.serfInstance != nil && neighIP != "" {
279		var err error
280		d.joinOnce.Do(func() {
281			err = d.serfJoin(neighIP)
282			if err == nil {
283				d.pushLocalDb()
284			}
285		})
286		if err != nil {
287			logrus.Errorf("joining serf neighbor %s failed: %v", advertiseAddress, err)
288			d.Lock()
289			d.joinOnce = sync.Once{}
290			d.Unlock()
291			return
292		}
293	}
294}
295
296func (d *driver) pushLocalEndpointEvent(action, nid, eid string) {
297	n := d.network(nid)
298	if n == nil {
299		logrus.Debugf("Error pushing local endpoint event for network %s", nid)
300		return
301	}
302	ep := n.endpoint(eid)
303	if ep == nil {
304		logrus.Debugf("Error pushing local endpoint event for ep %s / %s", nid, eid)
305		return
306	}
307
308	if !d.isSerfAlive() {
309		return
310	}
311	d.notifyCh <- ovNotify{
312		action: "join",
313		nw:     n,
314		ep:     ep,
315	}
316}
317
318// DiscoverNew is a notification for a new discovery event, such as a new node joining a cluster
319func (d *driver) DiscoverNew(dType discoverapi.DiscoveryType, data interface{}) error {
320	var err error
321	switch dType {
322	case discoverapi.NodeDiscovery:
323		nodeData, ok := data.(discoverapi.NodeDiscoveryData)
324		if !ok || nodeData.Address == "" {
325			return fmt.Errorf("invalid discovery data")
326		}
327		d.nodeJoin(nodeData.Address, nodeData.BindAddress, nodeData.Self)
328	case discoverapi.DatastoreConfig:
329		if d.store != nil {
330			return types.ForbiddenErrorf("cannot accept datastore configuration: Overlay driver has a datastore configured already")
331		}
332		dsc, ok := data.(discoverapi.DatastoreConfigData)
333		if !ok {
334			return types.InternalErrorf("incorrect data in datastore configuration: %v", data)
335		}
336		d.store, err = datastore.NewDataStoreFromConfig(dsc)
337		if err != nil {
338			return types.InternalErrorf("failed to initialize data store: %v", err)
339		}
340	case discoverapi.EncryptionKeysConfig:
341		encrData, ok := data.(discoverapi.DriverEncryptionConfig)
342		if !ok {
343			return fmt.Errorf("invalid encryption key notification data")
344		}
345		keys := make([]*key, 0, len(encrData.Keys))
346		for i := 0; i < len(encrData.Keys); i++ {
347			k := &key{
348				value: encrData.Keys[i],
349				tag:   uint32(encrData.Tags[i]),
350			}
351			keys = append(keys, k)
352		}
353		if err := d.setKeys(keys); err != nil {
354			logrus.Warn(err)
355		}
356	case discoverapi.EncryptionKeysUpdate:
357		var newKey, delKey, priKey *key
358		encrData, ok := data.(discoverapi.DriverEncryptionUpdate)
359		if !ok {
360			return fmt.Errorf("invalid encryption key notification data")
361		}
362		if encrData.Key != nil {
363			newKey = &key{
364				value: encrData.Key,
365				tag:   uint32(encrData.Tag),
366			}
367		}
368		if encrData.Primary != nil {
369			priKey = &key{
370				value: encrData.Primary,
371				tag:   uint32(encrData.PrimaryTag),
372			}
373		}
374		if encrData.Prune != nil {
375			delKey = &key{
376				value: encrData.Prune,
377				tag:   uint32(encrData.PruneTag),
378			}
379		}
380		if err := d.updateKeys(newKey, priKey, delKey); err != nil {
381			logrus.Warn(err)
382		}
383	default:
384	}
385	return nil
386}
387
388// DiscoverDelete is a notification for a discovery delete event, such as a node leaving a cluster
389func (d *driver) DiscoverDelete(dType discoverapi.DiscoveryType, data interface{}) error {
390	return nil
391}
392