1package overlay
2
3import (
4	"encoding/json"
5	"fmt"
6	"net"
7	"os"
8	"path/filepath"
9	"strconv"
10	"strings"
11	"sync"
12	"syscall"
13
14	"github.com/sirupsen/logrus"
15	"github.com/docker/libnetwork/datastore"
16	"github.com/docker/libnetwork/driverapi"
17	"github.com/docker/libnetwork/netlabel"
18	"github.com/docker/libnetwork/netutils"
19	"github.com/docker/libnetwork/ns"
20	"github.com/docker/libnetwork/osl"
21	"github.com/docker/libnetwork/resolvconf"
22	"github.com/docker/libnetwork/types"
23	"github.com/vishvananda/netlink"
24	"github.com/vishvananda/netlink/nl"
25	"github.com/vishvananda/netns"
26)
27
28var (
29	hostMode    bool
30	networkOnce sync.Once
31	networkMu   sync.Mutex
32	vniTbl      = make(map[uint32]string)
33)
34
35type networkTable map[string]*network
36
37type subnet struct {
38	once      *sync.Once
39	vxlanName string
40	brName    string
41	vni       uint32
42	initErr   error
43	subnetIP  *net.IPNet
44	gwIP      *net.IPNet
45}
46
47type subnetJSON struct {
48	SubnetIP string
49	GwIP     string
50	Vni      uint32
51}
52
53type network struct {
54	id        string
55	dbIndex   uint64
56	dbExists  bool
57	sbox      osl.Sandbox
58	endpoints endpointTable
59	driver    *driver
60	joinCnt   int
61	once      *sync.Once
62	initEpoch int
63	initErr   error
64	subnets   []*subnet
65	secure    bool
66	mtu       int
67	sync.Mutex
68}
69
70func (d *driver) NetworkAllocate(id string, option map[string]string, ipV4Data, ipV6Data []driverapi.IPAMData) (map[string]string, error) {
71	return nil, types.NotImplementedErrorf("not implemented")
72}
73
74func (d *driver) NetworkFree(id string) error {
75	return types.NotImplementedErrorf("not implemented")
76}
77
78func (d *driver) CreateNetwork(id string, option map[string]interface{}, nInfo driverapi.NetworkInfo, ipV4Data, ipV6Data []driverapi.IPAMData) error {
79	if id == "" {
80		return fmt.Errorf("invalid network id")
81	}
82	if len(ipV4Data) == 0 || ipV4Data[0].Pool.String() == "0.0.0.0/0" {
83		return types.BadRequestErrorf("ipv4 pool is empty")
84	}
85
86	// Since we perform lazy configuration make sure we try
87	// configuring the driver when we enter CreateNetwork
88	if err := d.configure(); err != nil {
89		return err
90	}
91
92	n := &network{
93		id:        id,
94		driver:    d,
95		endpoints: endpointTable{},
96		once:      &sync.Once{},
97		subnets:   []*subnet{},
98	}
99
100	vnis := make([]uint32, 0, len(ipV4Data))
101	if gval, ok := option[netlabel.GenericData]; ok {
102		optMap := gval.(map[string]string)
103		if val, ok := optMap[netlabel.OverlayVxlanIDList]; ok {
104			logrus.Debugf("overlay: Received vxlan IDs: %s", val)
105			vniStrings := strings.Split(val, ",")
106			for _, vniStr := range vniStrings {
107				vni, err := strconv.Atoi(vniStr)
108				if err != nil {
109					return fmt.Errorf("invalid vxlan id value %q passed", vniStr)
110				}
111
112				vnis = append(vnis, uint32(vni))
113			}
114		}
115		if _, ok := optMap[secureOption]; ok {
116			n.secure = true
117		}
118		if val, ok := optMap[netlabel.DriverMTU]; ok {
119			var err error
120			if n.mtu, err = strconv.Atoi(val); err != nil {
121				return fmt.Errorf("failed to parse %v: %v", val, err)
122			}
123			if n.mtu < 0 {
124				return fmt.Errorf("invalid MTU value: %v", n.mtu)
125			}
126		}
127	}
128
129	// If we are getting vnis from libnetwork, either we get for
130	// all subnets or none.
131	if len(vnis) != 0 && len(vnis) < len(ipV4Data) {
132		return fmt.Errorf("insufficient vnis(%d) passed to overlay", len(vnis))
133	}
134
135	for i, ipd := range ipV4Data {
136		s := &subnet{
137			subnetIP: ipd.Pool,
138			gwIP:     ipd.Gateway,
139			once:     &sync.Once{},
140		}
141
142		if len(vnis) != 0 {
143			s.vni = vnis[i]
144		}
145
146		n.subnets = append(n.subnets, s)
147	}
148
149	if err := n.writeToStore(); err != nil {
150		return fmt.Errorf("failed to update data store for network %v: %v", n.id, err)
151	}
152
153	// Make sure no rule is on the way from any stale secure network
154	if !n.secure {
155		for _, vni := range vnis {
156			programMangle(vni, false)
157		}
158	}
159
160	if nInfo != nil {
161		if err := nInfo.TableEventRegister(ovPeerTable); err != nil {
162			return err
163		}
164	}
165
166	d.addNetwork(n)
167	return nil
168}
169
170func (d *driver) DeleteNetwork(nid string) error {
171	if nid == "" {
172		return fmt.Errorf("invalid network id")
173	}
174
175	// Make sure driver resources are initialized before proceeding
176	if err := d.configure(); err != nil {
177		return err
178	}
179
180	n := d.network(nid)
181	if n == nil {
182		return fmt.Errorf("could not find network with id %s", nid)
183	}
184
185	for _, ep := range n.endpoints {
186		if ep.ifName != "" {
187			if link, err := ns.NlHandle().LinkByName(ep.ifName); err != nil {
188				ns.NlHandle().LinkDel(link)
189			}
190		}
191
192		if err := d.deleteEndpointFromStore(ep); err != nil {
193			logrus.Warnf("Failed to delete overlay endpoint %s from local store: %v", ep.id[0:7], err)
194		}
195
196	}
197	d.deleteNetwork(nid)
198
199	vnis, err := n.releaseVxlanID()
200	if err != nil {
201		return err
202	}
203
204	if n.secure {
205		for _, vni := range vnis {
206			programMangle(vni, false)
207		}
208	}
209
210	return nil
211}
212
213func (d *driver) ProgramExternalConnectivity(nid, eid string, options map[string]interface{}) error {
214	return nil
215}
216
217func (d *driver) RevokeExternalConnectivity(nid, eid string) error {
218	return nil
219}
220
221func (n *network) incEndpointCount() {
222	n.Lock()
223	defer n.Unlock()
224	n.joinCnt++
225}
226
227func (n *network) joinSandbox(restore bool) error {
228	// If there is a race between two go routines here only one will win
229	// the other will wait.
230	n.once.Do(func() {
231		// save the error status of initSandbox in n.initErr so that
232		// all the racing go routines are able to know the status.
233		n.initErr = n.initSandbox(restore)
234	})
235
236	return n.initErr
237}
238
239func (n *network) joinSubnetSandbox(s *subnet, restore bool) error {
240	s.once.Do(func() {
241		s.initErr = n.initSubnetSandbox(s, restore)
242	})
243	return s.initErr
244}
245
246func (n *network) leaveSandbox() {
247	n.Lock()
248	defer n.Unlock()
249	n.joinCnt--
250	if n.joinCnt != 0 {
251		return
252	}
253
254	// We are about to destroy sandbox since the container is leaving the network
255	// Reinitialize the once variable so that we will be able to trigger one time
256	// sandbox initialization(again) when another container joins subsequently.
257	n.once = &sync.Once{}
258	for _, s := range n.subnets {
259		s.once = &sync.Once{}
260	}
261
262	n.destroySandbox()
263}
264
265// to be called while holding network lock
266func (n *network) destroySandbox() {
267	if n.sbox != nil {
268		for _, iface := range n.sbox.Info().Interfaces() {
269			if err := iface.Remove(); err != nil {
270				logrus.Debugf("Remove interface %s failed: %v", iface.SrcName(), err)
271			}
272		}
273
274		for _, s := range n.subnets {
275			if hostMode {
276				if err := removeFilters(n.id[:12], s.brName); err != nil {
277					logrus.Warnf("Could not remove overlay filters: %v", err)
278				}
279			}
280
281			if s.vxlanName != "" {
282				err := deleteInterface(s.vxlanName)
283				if err != nil {
284					logrus.Warnf("could not cleanup sandbox properly: %v", err)
285				}
286			}
287		}
288
289		if hostMode {
290			if err := removeNetworkChain(n.id[:12]); err != nil {
291				logrus.Warnf("could not remove network chain: %v", err)
292			}
293		}
294
295		n.sbox.Destroy()
296		n.sbox = nil
297	}
298}
299
300func populateVNITbl() {
301	filepath.Walk(filepath.Dir(osl.GenerateKey("walk")),
302		func(path string, info os.FileInfo, err error) error {
303			_, fname := filepath.Split(path)
304
305			if len(strings.Split(fname, "-")) <= 1 {
306				return nil
307			}
308
309			ns, err := netns.GetFromPath(path)
310			if err != nil {
311				logrus.Errorf("Could not open namespace path %s during vni population: %v", path, err)
312				return nil
313			}
314			defer ns.Close()
315
316			nlh, err := netlink.NewHandleAt(ns, syscall.NETLINK_ROUTE)
317			if err != nil {
318				logrus.Errorf("Could not open netlink handle during vni population for ns %s: %v", path, err)
319				return nil
320			}
321			defer nlh.Delete()
322
323			err = nlh.SetSocketTimeout(soTimeout)
324			if err != nil {
325				logrus.Warnf("Failed to set the timeout on the netlink handle sockets for vni table population: %v", err)
326			}
327
328			links, err := nlh.LinkList()
329			if err != nil {
330				logrus.Errorf("Failed to list interfaces during vni population for ns %s: %v", path, err)
331				return nil
332			}
333
334			for _, l := range links {
335				if l.Type() == "vxlan" {
336					vniTbl[uint32(l.(*netlink.Vxlan).VxlanId)] = path
337				}
338			}
339
340			return nil
341		})
342}
343
344func networkOnceInit() {
345	populateVNITbl()
346
347	if os.Getenv("_OVERLAY_HOST_MODE") != "" {
348		hostMode = true
349		return
350	}
351
352	err := createVxlan("testvxlan", 1, 0)
353	if err != nil {
354		logrus.Errorf("Failed to create testvxlan interface: %v", err)
355		return
356	}
357
358	defer deleteInterface("testvxlan")
359
360	path := "/proc/self/ns/net"
361	hNs, err := netns.GetFromPath(path)
362	if err != nil {
363		logrus.Errorf("Failed to get network namespace from path %s while setting host mode: %v", path, err)
364		return
365	}
366	defer hNs.Close()
367
368	nlh := ns.NlHandle()
369
370	iface, err := nlh.LinkByName("testvxlan")
371	if err != nil {
372		logrus.Errorf("Failed to get link testvxlan while setting host mode: %v", err)
373		return
374	}
375
376	// If we are not able to move the vxlan interface to a namespace
377	// then fallback to host mode
378	if err := nlh.LinkSetNsFd(iface, int(hNs)); err != nil {
379		hostMode = true
380	}
381}
382
383func (n *network) generateVxlanName(s *subnet) string {
384	id := n.id
385	if len(n.id) > 5 {
386		id = n.id[:5]
387	}
388
389	return "vx-" + fmt.Sprintf("%06x", n.vxlanID(s)) + "-" + id
390}
391
392func (n *network) generateBridgeName(s *subnet) string {
393	id := n.id
394	if len(n.id) > 5 {
395		id = n.id[:5]
396	}
397
398	return n.getBridgeNamePrefix(s) + "-" + id
399}
400
401func (n *network) getBridgeNamePrefix(s *subnet) string {
402	return "ov-" + fmt.Sprintf("%06x", n.vxlanID(s))
403}
404
405func isOverlap(nw *net.IPNet) bool {
406	var nameservers []string
407
408	if rc, err := resolvconf.Get(); err == nil {
409		nameservers = resolvconf.GetNameserversAsCIDR(rc.Content)
410	}
411
412	if err := netutils.CheckNameserverOverlaps(nameservers, nw); err != nil {
413		return true
414	}
415
416	if err := netutils.CheckRouteOverlaps(nw); err != nil {
417		return true
418	}
419
420	return false
421}
422
423func (n *network) restoreSubnetSandbox(s *subnet, brName, vxlanName string) error {
424	sbox := n.sandbox()
425
426	// restore overlay osl sandbox
427	Ifaces := make(map[string][]osl.IfaceOption)
428	brIfaceOption := make([]osl.IfaceOption, 2)
429	brIfaceOption = append(brIfaceOption, sbox.InterfaceOptions().Address(s.gwIP))
430	brIfaceOption = append(brIfaceOption, sbox.InterfaceOptions().Bridge(true))
431	Ifaces[fmt.Sprintf("%s+%s", brName, "br")] = brIfaceOption
432
433	err := sbox.Restore(Ifaces, nil, nil, nil)
434	if err != nil {
435		return err
436	}
437
438	Ifaces = make(map[string][]osl.IfaceOption)
439	vxlanIfaceOption := make([]osl.IfaceOption, 1)
440	vxlanIfaceOption = append(vxlanIfaceOption, sbox.InterfaceOptions().Master(brName))
441	Ifaces[fmt.Sprintf("%s+%s", vxlanName, "vxlan")] = vxlanIfaceOption
442	err = sbox.Restore(Ifaces, nil, nil, nil)
443	if err != nil {
444		return err
445	}
446	return nil
447}
448
449func (n *network) setupSubnetSandbox(s *subnet, brName, vxlanName string) error {
450
451	if hostMode {
452		// Try to delete stale bridge interface if it exists
453		if err := deleteInterface(brName); err != nil {
454			deleteInterfaceBySubnet(n.getBridgeNamePrefix(s), s)
455		}
456		// Try to delete the vxlan interface by vni if already present
457		deleteVxlanByVNI("", n.vxlanID(s))
458
459		if isOverlap(s.subnetIP) {
460			return fmt.Errorf("overlay subnet %s has conflicts in the host while running in host mode", s.subnetIP.String())
461		}
462	}
463
464	if !hostMode {
465		// Try to find this subnet's vni is being used in some
466		// other namespace by looking at vniTbl that we just
467		// populated in the once init. If a hit is found then
468		// it must a stale namespace from previous
469		// life. Destroy it completely and reclaim resourced.
470		networkMu.Lock()
471		path, ok := vniTbl[n.vxlanID(s)]
472		networkMu.Unlock()
473
474		if ok {
475			deleteVxlanByVNI(path, n.vxlanID(s))
476			if err := syscall.Unmount(path, syscall.MNT_FORCE); err != nil {
477				logrus.Errorf("unmount of %s failed: %v", path, err)
478			}
479			os.Remove(path)
480
481			networkMu.Lock()
482			delete(vniTbl, n.vxlanID(s))
483			networkMu.Unlock()
484		}
485	}
486
487	// create a bridge and vxlan device for this subnet and move it to the sandbox
488	sbox := n.sandbox()
489
490	if err := sbox.AddInterface(brName, "br",
491		sbox.InterfaceOptions().Address(s.gwIP),
492		sbox.InterfaceOptions().Bridge(true)); err != nil {
493		return fmt.Errorf("bridge creation in sandbox failed for subnet %q: %v", s.subnetIP.String(), err)
494	}
495
496	err := createVxlan(vxlanName, n.vxlanID(s), n.maxMTU())
497	if err != nil {
498		return err
499	}
500
501	if err := sbox.AddInterface(vxlanName, "vxlan",
502		sbox.InterfaceOptions().Master(brName)); err != nil {
503		return fmt.Errorf("vxlan interface creation failed for subnet %q: %v", s.subnetIP.String(), err)
504	}
505
506	if hostMode {
507		if err := addFilters(n.id[:12], brName); err != nil {
508			return err
509		}
510	}
511
512	return nil
513}
514
515func (n *network) initSubnetSandbox(s *subnet, restore bool) error {
516	brName := n.generateBridgeName(s)
517	vxlanName := n.generateVxlanName(s)
518
519	if restore {
520		if err := n.restoreSubnetSandbox(s, brName, vxlanName); err != nil {
521			return err
522		}
523	} else {
524		if err := n.setupSubnetSandbox(s, brName, vxlanName); err != nil {
525			return err
526		}
527	}
528
529	n.Lock()
530	s.vxlanName = vxlanName
531	s.brName = brName
532	n.Unlock()
533
534	return nil
535}
536
537func (n *network) cleanupStaleSandboxes() {
538	filepath.Walk(filepath.Dir(osl.GenerateKey("walk")),
539		func(path string, info os.FileInfo, err error) error {
540			_, fname := filepath.Split(path)
541
542			pList := strings.Split(fname, "-")
543			if len(pList) <= 1 {
544				return nil
545			}
546
547			pattern := pList[1]
548			if strings.Contains(n.id, pattern) {
549				// Delete all vnis
550				deleteVxlanByVNI(path, 0)
551				syscall.Unmount(path, syscall.MNT_DETACH)
552				os.Remove(path)
553
554				// Now that we have destroyed this
555				// sandbox, remove all references to
556				// it in vniTbl so that we don't
557				// inadvertently destroy the sandbox
558				// created in this life.
559				networkMu.Lock()
560				for vni, tblPath := range vniTbl {
561					if tblPath == path {
562						delete(vniTbl, vni)
563					}
564				}
565				networkMu.Unlock()
566			}
567
568			return nil
569		})
570}
571
572func (n *network) initSandbox(restore bool) error {
573	n.Lock()
574	n.initEpoch++
575	n.Unlock()
576
577	networkOnce.Do(networkOnceInit)
578
579	if !restore {
580		if hostMode {
581			if err := addNetworkChain(n.id[:12]); err != nil {
582				return err
583			}
584		}
585
586		// If there are any stale sandboxes related to this network
587		// from previous daemon life clean it up here
588		n.cleanupStaleSandboxes()
589	}
590
591	// In the restore case network sandbox already exist; but we don't know
592	// what epoch number it was created with. It has to be retrieved by
593	// searching the net namespaces.
594	key := ""
595	if restore {
596		key = osl.GenerateKey("-" + n.id)
597	} else {
598		key = osl.GenerateKey(fmt.Sprintf("%d-", n.initEpoch) + n.id)
599	}
600
601	sbox, err := osl.NewSandbox(key, !hostMode, restore)
602	if err != nil {
603		return fmt.Errorf("could not get network sandbox (oper %t): %v", restore, err)
604	}
605
606	n.setSandbox(sbox)
607
608	if !restore {
609		n.driver.peerDbUpdateSandbox(n.id)
610	}
611
612	var nlSock *nl.NetlinkSocket
613	sbox.InvokeFunc(func() {
614		nlSock, err = nl.Subscribe(syscall.NETLINK_ROUTE, syscall.RTNLGRP_NEIGH)
615		if err != nil {
616			err = fmt.Errorf("failed to subscribe to neighbor group netlink messages")
617		}
618	})
619
620	if nlSock != nil {
621		go n.watchMiss(nlSock)
622	}
623
624	return nil
625}
626
627func (n *network) watchMiss(nlSock *nl.NetlinkSocket) {
628	for {
629		msgs, err := nlSock.Receive()
630		if err != nil {
631			logrus.Errorf("Failed to receive from netlink: %v ", err)
632			continue
633		}
634
635		for _, msg := range msgs {
636			if msg.Header.Type != syscall.RTM_GETNEIGH && msg.Header.Type != syscall.RTM_NEWNEIGH {
637				continue
638			}
639
640			neigh, err := netlink.NeighDeserialize(msg.Data)
641			if err != nil {
642				logrus.Errorf("Failed to deserialize netlink ndmsg: %v", err)
643				continue
644			}
645
646			if neigh.IP.To4() == nil {
647				continue
648			}
649
650			// Not any of the network's subnets. Ignore.
651			if !n.contains(neigh.IP) {
652				continue
653			}
654
655			logrus.Debugf("miss notification for dest IP, %v", neigh.IP.String())
656
657			if neigh.State&(netlink.NUD_STALE|netlink.NUD_INCOMPLETE) == 0 {
658				continue
659			}
660
661			if !n.driver.isSerfAlive() {
662				continue
663			}
664
665			mac, IPmask, vtep, err := n.driver.resolvePeer(n.id, neigh.IP)
666			if err != nil {
667				logrus.Errorf("could not resolve peer %q: %v", neigh.IP, err)
668				continue
669			}
670
671			if err := n.driver.peerAdd(n.id, "dummy", neigh.IP, IPmask, mac, vtep, true); err != nil {
672				logrus.Errorf("could not add neighbor entry for missed peer %q: %v", neigh.IP, err)
673			}
674		}
675	}
676}
677
678func (d *driver) addNetwork(n *network) {
679	d.Lock()
680	d.networks[n.id] = n
681	d.Unlock()
682}
683
684func (d *driver) deleteNetwork(nid string) {
685	d.Lock()
686	delete(d.networks, nid)
687	d.Unlock()
688}
689
690func (d *driver) network(nid string) *network {
691	d.Lock()
692	n, ok := d.networks[nid]
693	d.Unlock()
694	if !ok {
695		n = d.getNetworkFromStore(nid)
696		if n != nil {
697			n.driver = d
698			n.endpoints = endpointTable{}
699			n.once = &sync.Once{}
700			d.Lock()
701			d.networks[nid] = n
702			d.Unlock()
703		}
704	}
705
706	return n
707}
708
709func (d *driver) getNetworkFromStore(nid string) *network {
710	if d.store == nil {
711		return nil
712	}
713
714	n := &network{id: nid}
715	if err := d.store.GetObject(datastore.Key(n.Key()...), n); err != nil {
716		return nil
717	}
718
719	return n
720}
721
722func (n *network) sandbox() osl.Sandbox {
723	n.Lock()
724	defer n.Unlock()
725
726	return n.sbox
727}
728
729func (n *network) setSandbox(sbox osl.Sandbox) {
730	n.Lock()
731	n.sbox = sbox
732	n.Unlock()
733}
734
735func (n *network) vxlanID(s *subnet) uint32 {
736	n.Lock()
737	defer n.Unlock()
738
739	return s.vni
740}
741
742func (n *network) setVxlanID(s *subnet, vni uint32) {
743	n.Lock()
744	s.vni = vni
745	n.Unlock()
746}
747
748func (n *network) Key() []string {
749	return []string{"overlay", "network", n.id}
750}
751
752func (n *network) KeyPrefix() []string {
753	return []string{"overlay", "network"}
754}
755
756func (n *network) Value() []byte {
757	m := map[string]interface{}{}
758
759	netJSON := []*subnetJSON{}
760
761	for _, s := range n.subnets {
762		sj := &subnetJSON{
763			SubnetIP: s.subnetIP.String(),
764			GwIP:     s.gwIP.String(),
765			Vni:      s.vni,
766		}
767		netJSON = append(netJSON, sj)
768	}
769
770	b, err := json.Marshal(netJSON)
771	if err != nil {
772		return []byte{}
773	}
774
775	m["secure"] = n.secure
776	m["subnets"] = netJSON
777	m["mtu"] = n.mtu
778	b, err = json.Marshal(m)
779	if err != nil {
780		return []byte{}
781	}
782
783	return b
784}
785
786func (n *network) Index() uint64 {
787	return n.dbIndex
788}
789
790func (n *network) SetIndex(index uint64) {
791	n.dbIndex = index
792	n.dbExists = true
793}
794
795func (n *network) Exists() bool {
796	return n.dbExists
797}
798
799func (n *network) Skip() bool {
800	return false
801}
802
803func (n *network) SetValue(value []byte) error {
804	var (
805		m       map[string]interface{}
806		newNet  bool
807		isMap   = true
808		netJSON = []*subnetJSON{}
809	)
810
811	if err := json.Unmarshal(value, &m); err != nil {
812		err := json.Unmarshal(value, &netJSON)
813		if err != nil {
814			return err
815		}
816		isMap = false
817	}
818
819	if len(n.subnets) == 0 {
820		newNet = true
821	}
822
823	if isMap {
824		if val, ok := m["secure"]; ok {
825			n.secure = val.(bool)
826		}
827		if val, ok := m["mtu"]; ok {
828			n.mtu = int(val.(float64))
829		}
830		bytes, err := json.Marshal(m["subnets"])
831		if err != nil {
832			return err
833		}
834		if err := json.Unmarshal(bytes, &netJSON); err != nil {
835			return err
836		}
837	}
838
839	for _, sj := range netJSON {
840		subnetIPstr := sj.SubnetIP
841		gwIPstr := sj.GwIP
842		vni := sj.Vni
843
844		subnetIP, _ := types.ParseCIDR(subnetIPstr)
845		gwIP, _ := types.ParseCIDR(gwIPstr)
846
847		if newNet {
848			s := &subnet{
849				subnetIP: subnetIP,
850				gwIP:     gwIP,
851				vni:      vni,
852				once:     &sync.Once{},
853			}
854			n.subnets = append(n.subnets, s)
855		} else {
856			sNet := n.getMatchingSubnet(subnetIP)
857			if sNet != nil {
858				sNet.vni = vni
859			}
860		}
861	}
862	return nil
863}
864
865func (n *network) DataScope() string {
866	return datastore.GlobalScope
867}
868
869func (n *network) writeToStore() error {
870	if n.driver.store == nil {
871		return nil
872	}
873
874	return n.driver.store.PutObjectAtomic(n)
875}
876
877func (n *network) releaseVxlanID() ([]uint32, error) {
878	if len(n.subnets) == 0 {
879		return nil, nil
880	}
881
882	if n.driver.store != nil {
883		if err := n.driver.store.DeleteObjectAtomic(n); err != nil {
884			if err == datastore.ErrKeyModified || err == datastore.ErrKeyNotFound {
885				// In both the above cases we can safely assume that the key has been removed by some other
886				// instance and so simply get out of here
887				return nil, nil
888			}
889
890			return nil, fmt.Errorf("failed to delete network to vxlan id map: %v", err)
891		}
892	}
893	var vnis []uint32
894	for _, s := range n.subnets {
895		if n.driver.vxlanIdm != nil {
896			vni := n.vxlanID(s)
897			vnis = append(vnis, vni)
898			n.driver.vxlanIdm.Release(uint64(vni))
899		}
900
901		n.setVxlanID(s, 0)
902	}
903
904	return vnis, nil
905}
906
907func (n *network) obtainVxlanID(s *subnet) error {
908	//return if the subnet already has a vxlan id assigned
909	if s.vni != 0 {
910		return nil
911	}
912
913	if n.driver.store == nil {
914		return fmt.Errorf("no valid vxlan id and no datastore configured, cannot obtain vxlan id")
915	}
916
917	for {
918		if err := n.driver.store.GetObject(datastore.Key(n.Key()...), n); err != nil {
919			return fmt.Errorf("getting network %q from datastore failed %v", n.id, err)
920		}
921
922		if s.vni == 0 {
923			vxlanID, err := n.driver.vxlanIdm.GetID()
924			if err != nil {
925				return fmt.Errorf("failed to allocate vxlan id: %v", err)
926			}
927
928			n.setVxlanID(s, uint32(vxlanID))
929			if err := n.writeToStore(); err != nil {
930				n.driver.vxlanIdm.Release(uint64(n.vxlanID(s)))
931				n.setVxlanID(s, 0)
932				if err == datastore.ErrKeyModified {
933					continue
934				}
935				return fmt.Errorf("network %q failed to update data store: %v", n.id, err)
936			}
937			return nil
938		}
939		return nil
940	}
941}
942
943// contains return true if the passed ip belongs to one the network's
944// subnets
945func (n *network) contains(ip net.IP) bool {
946	for _, s := range n.subnets {
947		if s.subnetIP.Contains(ip) {
948			return true
949		}
950	}
951
952	return false
953}
954
955// getSubnetforIP returns the subnet to which the given IP belongs
956func (n *network) getSubnetforIP(ip *net.IPNet) *subnet {
957	for _, s := range n.subnets {
958		// first check if the mask lengths are the same
959		i, _ := s.subnetIP.Mask.Size()
960		j, _ := ip.Mask.Size()
961		if i != j {
962			continue
963		}
964		if s.subnetIP.Contains(ip.IP) {
965			return s
966		}
967	}
968	return nil
969}
970
971// getMatchingSubnet return the network's subnet that matches the input
972func (n *network) getMatchingSubnet(ip *net.IPNet) *subnet {
973	if ip == nil {
974		return nil
975	}
976	for _, s := range n.subnets {
977		// first check if the mask lengths are the same
978		i, _ := s.subnetIP.Mask.Size()
979		j, _ := ip.Mask.Size()
980		if i != j {
981			continue
982		}
983		if s.subnetIP.IP.Equal(ip.IP) {
984			return s
985		}
986	}
987	return nil
988}
989