1package osl
2
3import (
4	"fmt"
5	"net"
6	"regexp"
7	"sync"
8	"syscall"
9	"time"
10
11	"github.com/docker/libnetwork/ns"
12	"github.com/docker/libnetwork/types"
13	"github.com/sirupsen/logrus"
14	"github.com/vishvananda/netlink"
15	"github.com/vishvananda/netns"
16)
17
18// IfaceOption is a function option type to set interface options
19type IfaceOption func(i *nwIface)
20
21type nwIface struct {
22	srcName     string
23	dstName     string
24	master      string
25	dstMaster   string
26	mac         net.HardwareAddr
27	address     *net.IPNet
28	addressIPv6 *net.IPNet
29	llAddrs     []*net.IPNet
30	routes      []*net.IPNet
31	bridge      bool
32	ns          *networkNamespace
33	sync.Mutex
34}
35
36func (i *nwIface) SrcName() string {
37	i.Lock()
38	defer i.Unlock()
39
40	return i.srcName
41}
42
43func (i *nwIface) DstName() string {
44	i.Lock()
45	defer i.Unlock()
46
47	return i.dstName
48}
49
50func (i *nwIface) DstMaster() string {
51	i.Lock()
52	defer i.Unlock()
53
54	return i.dstMaster
55}
56
57func (i *nwIface) Bridge() bool {
58	i.Lock()
59	defer i.Unlock()
60
61	return i.bridge
62}
63
64func (i *nwIface) Master() string {
65	i.Lock()
66	defer i.Unlock()
67
68	return i.master
69}
70
71func (i *nwIface) MacAddress() net.HardwareAddr {
72	i.Lock()
73	defer i.Unlock()
74
75	return types.GetMacCopy(i.mac)
76}
77
78func (i *nwIface) Address() *net.IPNet {
79	i.Lock()
80	defer i.Unlock()
81
82	return types.GetIPNetCopy(i.address)
83}
84
85func (i *nwIface) AddressIPv6() *net.IPNet {
86	i.Lock()
87	defer i.Unlock()
88
89	return types.GetIPNetCopy(i.addressIPv6)
90}
91
92func (i *nwIface) LinkLocalAddresses() []*net.IPNet {
93	i.Lock()
94	defer i.Unlock()
95
96	return i.llAddrs
97}
98
99func (i *nwIface) Routes() []*net.IPNet {
100	i.Lock()
101	defer i.Unlock()
102
103	routes := make([]*net.IPNet, len(i.routes))
104	for index, route := range i.routes {
105		r := types.GetIPNetCopy(route)
106		routes[index] = r
107	}
108
109	return routes
110}
111
112func (n *networkNamespace) Interfaces() []Interface {
113	n.Lock()
114	defer n.Unlock()
115
116	ifaces := make([]Interface, len(n.iFaces))
117
118	for i, iface := range n.iFaces {
119		ifaces[i] = iface
120	}
121
122	return ifaces
123}
124
125func (i *nwIface) Remove() error {
126	i.Lock()
127	n := i.ns
128	i.Unlock()
129
130	n.Lock()
131	isDefault := n.isDefault
132	nlh := n.nlHandle
133	n.Unlock()
134
135	// Find the network interface identified by the DstName attribute.
136	iface, err := nlh.LinkByName(i.DstName())
137	if err != nil {
138		return err
139	}
140
141	// Down the interface before configuring
142	if err := nlh.LinkSetDown(iface); err != nil {
143		return err
144	}
145
146	err = nlh.LinkSetName(iface, i.SrcName())
147	if err != nil {
148		logrus.Debugf("LinkSetName failed for interface %s: %v", i.SrcName(), err)
149		return err
150	}
151
152	// if it is a bridge just delete it.
153	if i.Bridge() {
154		if err := nlh.LinkDel(iface); err != nil {
155			return fmt.Errorf("failed deleting bridge %q: %v", i.SrcName(), err)
156		}
157	} else if !isDefault {
158		// Move the network interface to caller namespace.
159		if err := nlh.LinkSetNsFd(iface, ns.ParseHandlerInt()); err != nil {
160			logrus.Debugf("LinkSetNsPid failed for interface %s: %v", i.SrcName(), err)
161			return err
162		}
163	}
164
165	n.Lock()
166	for index, intf := range n.iFaces {
167		if intf == i {
168			n.iFaces = append(n.iFaces[:index], n.iFaces[index+1:]...)
169			break
170		}
171	}
172	n.Unlock()
173
174	n.checkLoV6()
175
176	return nil
177}
178
179// Returns the sandbox's side veth interface statistics
180func (i *nwIface) Statistics() (*types.InterfaceStatistics, error) {
181	i.Lock()
182	n := i.ns
183	i.Unlock()
184
185	l, err := n.nlHandle.LinkByName(i.DstName())
186	if err != nil {
187		return nil, fmt.Errorf("failed to retrieve the statistics for %s in netns %s: %v", i.DstName(), n.path, err)
188	}
189
190	stats := l.Attrs().Statistics
191	if stats == nil {
192		return nil, fmt.Errorf("no statistics were returned")
193	}
194
195	return &types.InterfaceStatistics{
196		RxBytes:   uint64(stats.RxBytes),
197		TxBytes:   uint64(stats.TxBytes),
198		RxPackets: uint64(stats.RxPackets),
199		TxPackets: uint64(stats.TxPackets),
200		RxDropped: uint64(stats.RxDropped),
201		TxDropped: uint64(stats.TxDropped),
202	}, nil
203}
204
205func (n *networkNamespace) findDst(srcName string, isBridge bool) string {
206	n.Lock()
207	defer n.Unlock()
208
209	for _, i := range n.iFaces {
210		// The master should match the srcname of the interface and the
211		// master interface should be of type bridge, if searching for a bridge type
212		if i.SrcName() == srcName && (!isBridge || i.Bridge()) {
213			return i.DstName()
214		}
215	}
216
217	return ""
218}
219
220func (n *networkNamespace) AddInterface(srcName, dstPrefix string, options ...IfaceOption) error {
221	i := &nwIface{srcName: srcName, dstName: dstPrefix, ns: n}
222	i.processInterfaceOptions(options...)
223
224	if i.master != "" {
225		i.dstMaster = n.findDst(i.master, true)
226		if i.dstMaster == "" {
227			return fmt.Errorf("could not find an appropriate master %q for %q",
228				i.master, i.srcName)
229		}
230	}
231
232	n.Lock()
233	if n.isDefault {
234		i.dstName = i.srcName
235	} else {
236		i.dstName = fmt.Sprintf("%s%d", dstPrefix, n.nextIfIndex[dstPrefix])
237		n.nextIfIndex[dstPrefix]++
238	}
239
240	path := n.path
241	isDefault := n.isDefault
242	nlh := n.nlHandle
243	nlhHost := ns.NlHandle()
244	n.Unlock()
245
246	// If it is a bridge interface we have to create the bridge inside
247	// the namespace so don't try to lookup the interface using srcName
248	if i.bridge {
249		link := &netlink.Bridge{
250			LinkAttrs: netlink.LinkAttrs{
251				Name: i.srcName,
252			},
253		}
254		if err := nlh.LinkAdd(link); err != nil {
255			return fmt.Errorf("failed to create bridge %q: %v", i.srcName, err)
256		}
257	} else {
258		// Find the network interface identified by the SrcName attribute.
259		iface, err := nlhHost.LinkByName(i.srcName)
260		if err != nil {
261			return fmt.Errorf("failed to get link by name %q: %v", i.srcName, err)
262		}
263
264		// Move the network interface to the destination
265		// namespace only if the namespace is not a default
266		// type
267		if !isDefault {
268			newNs, err := netns.GetFromPath(path)
269			if err != nil {
270				return fmt.Errorf("failed get network namespace %q: %v", path, err)
271			}
272			defer newNs.Close()
273			if err := nlhHost.LinkSetNsFd(iface, int(newNs)); err != nil {
274				return fmt.Errorf("failed to set namespace on link %q: %v", i.srcName, err)
275			}
276		}
277	}
278
279	// Find the network interface identified by the SrcName attribute.
280	iface, err := nlh.LinkByName(i.srcName)
281	if err != nil {
282		return fmt.Errorf("failed to get link by name %q: %v", i.srcName, err)
283	}
284
285	// Down the interface before configuring
286	if err := nlh.LinkSetDown(iface); err != nil {
287		return fmt.Errorf("failed to set link down: %v", err)
288	}
289
290	// Configure the interface now this is moved in the proper namespace.
291	if err := configureInterface(nlh, iface, i); err != nil {
292		// If configuring the device fails move it back to the host namespace
293		// and change the name back to the source name. This allows the caller
294		// to properly cleanup the interface. Its important especially for
295		// interfaces with global attributes, ex: vni id for vxlan interfaces.
296		if nerr := nlh.LinkSetName(iface, i.SrcName()); nerr != nil {
297			logrus.Errorf("renaming interface (%s->%s) failed, %v after config error %v", i.DstName(), i.SrcName(), nerr, err)
298		}
299		if nerr := nlh.LinkSetNsFd(iface, ns.ParseHandlerInt()); nerr != nil {
300			logrus.Errorf("moving inteface %s to host ns failed, %v, after config error %v", i.SrcName(), nerr, err)
301		}
302		return err
303	}
304
305	// Up the interface.
306	cnt := 0
307	for err = nlh.LinkSetUp(iface); err != nil && cnt < 3; cnt++ {
308		logrus.Debugf("retrying link setup because of: %v", err)
309		time.Sleep(10 * time.Millisecond)
310		err = nlh.LinkSetUp(iface)
311	}
312	if err != nil {
313		return fmt.Errorf("failed to set link up: %v", err)
314	}
315
316	// Set the routes on the interface. This can only be done when the interface is up.
317	if err := setInterfaceRoutes(nlh, iface, i); err != nil {
318		return fmt.Errorf("error setting interface %q routes to %q: %v", iface.Attrs().Name, i.Routes(), err)
319	}
320
321	n.Lock()
322	n.iFaces = append(n.iFaces, i)
323	n.Unlock()
324
325	n.checkLoV6()
326
327	return nil
328}
329
330func configureInterface(nlh *netlink.Handle, iface netlink.Link, i *nwIface) error {
331	ifaceName := iface.Attrs().Name
332	ifaceConfigurators := []struct {
333		Fn         func(*netlink.Handle, netlink.Link, *nwIface) error
334		ErrMessage string
335	}{
336		{setInterfaceName, fmt.Sprintf("error renaming interface %q to %q", ifaceName, i.DstName())},
337		{setInterfaceMAC, fmt.Sprintf("error setting interface %q MAC to %q", ifaceName, i.MacAddress())},
338		{setInterfaceIP, fmt.Sprintf("error setting interface %q IP to %v", ifaceName, i.Address())},
339		{setInterfaceIPv6, fmt.Sprintf("error setting interface %q IPv6 to %v", ifaceName, i.AddressIPv6())},
340		{setInterfaceMaster, fmt.Sprintf("error setting interface %q master to %q", ifaceName, i.DstMaster())},
341		{setInterfaceLinkLocalIPs, fmt.Sprintf("error setting interface %q link local IPs to %v", ifaceName, i.LinkLocalAddresses())},
342	}
343
344	for _, config := range ifaceConfigurators {
345		if err := config.Fn(nlh, iface, i); err != nil {
346			return fmt.Errorf("%s: %v", config.ErrMessage, err)
347		}
348	}
349	return nil
350}
351
352func setInterfaceMaster(nlh *netlink.Handle, iface netlink.Link, i *nwIface) error {
353	if i.DstMaster() == "" {
354		return nil
355	}
356
357	return nlh.LinkSetMaster(iface, &netlink.Bridge{
358		LinkAttrs: netlink.LinkAttrs{Name: i.DstMaster()}})
359}
360
361func setInterfaceMAC(nlh *netlink.Handle, iface netlink.Link, i *nwIface) error {
362	if i.MacAddress() == nil {
363		return nil
364	}
365	return nlh.LinkSetHardwareAddr(iface, i.MacAddress())
366}
367
368func setInterfaceIP(nlh *netlink.Handle, iface netlink.Link, i *nwIface) error {
369	if i.Address() == nil {
370		return nil
371	}
372	if err := checkRouteConflict(nlh, i.Address(), netlink.FAMILY_V4); err != nil {
373		return err
374	}
375	ipAddr := &netlink.Addr{IPNet: i.Address(), Label: ""}
376	return nlh.AddrAdd(iface, ipAddr)
377}
378
379func setInterfaceIPv6(nlh *netlink.Handle, iface netlink.Link, i *nwIface) error {
380	if i.AddressIPv6() == nil {
381		return nil
382	}
383	if err := checkRouteConflict(nlh, i.AddressIPv6(), netlink.FAMILY_V6); err != nil {
384		return err
385	}
386	if err := setIPv6(i.ns.path, i.DstName(), true); err != nil {
387		return fmt.Errorf("failed to enable ipv6: %v", err)
388	}
389	ipAddr := &netlink.Addr{IPNet: i.AddressIPv6(), Label: "", Flags: syscall.IFA_F_NODAD}
390	return nlh.AddrAdd(iface, ipAddr)
391}
392
393func setInterfaceLinkLocalIPs(nlh *netlink.Handle, iface netlink.Link, i *nwIface) error {
394	for _, llIP := range i.LinkLocalAddresses() {
395		ipAddr := &netlink.Addr{IPNet: llIP}
396		if err := nlh.AddrAdd(iface, ipAddr); err != nil {
397			return err
398		}
399	}
400	return nil
401}
402
403func setInterfaceName(nlh *netlink.Handle, iface netlink.Link, i *nwIface) error {
404	return nlh.LinkSetName(iface, i.DstName())
405}
406
407func setInterfaceRoutes(nlh *netlink.Handle, iface netlink.Link, i *nwIface) error {
408	for _, route := range i.Routes() {
409		err := nlh.RouteAdd(&netlink.Route{
410			Scope:     netlink.SCOPE_LINK,
411			LinkIndex: iface.Attrs().Index,
412			Dst:       route,
413		})
414		if err != nil {
415			return err
416		}
417	}
418	return nil
419}
420
421// In older kernels (like the one in Centos 6.6 distro) sysctl does not have netns support. Therefore
422// we cannot gather the statistics from /sys/class/net/<dev>/statistics/<counter> files. Per-netns stats
423// are naturally found in /proc/net/dev in kernels which support netns (ifconfig relies on that).
424const (
425	netStatsFile = "/proc/net/dev"
426	base         = "[ ]*%s:([ ]+[0-9]+){16}"
427)
428
429func scanInterfaceStats(data, ifName string, i *types.InterfaceStatistics) error {
430	var (
431		bktStr string
432		bkt    uint64
433	)
434
435	regex := fmt.Sprintf(base, ifName)
436	re := regexp.MustCompile(regex)
437	line := re.FindString(data)
438
439	_, err := fmt.Sscanf(line, "%s %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d",
440		&bktStr, &i.RxBytes, &i.RxPackets, &i.RxErrors, &i.RxDropped, &bkt, &bkt, &bkt,
441		&bkt, &i.TxBytes, &i.TxPackets, &i.TxErrors, &i.TxDropped, &bkt, &bkt, &bkt, &bkt)
442
443	return err
444}
445
446func checkRouteConflict(nlh *netlink.Handle, address *net.IPNet, family int) error {
447	routes, err := nlh.RouteList(nil, family)
448	if err != nil {
449		return err
450	}
451	for _, route := range routes {
452		if route.Dst != nil {
453			if route.Dst.Contains(address.IP) || address.Contains(route.Dst.IP) {
454				return fmt.Errorf("cannot program address %v in sandbox interface because it conflicts with existing route %s",
455					address, route)
456			}
457		}
458	}
459	return nil
460}
461