1// +build !providerless
2
3/*
4Copyright 2016 The Kubernetes Authors.
5
6Licensed under the Apache License, Version 2.0 (the "License");
7you may not use this file except in compliance with the License.
8You may obtain a copy of the License at
9
10    http://www.apache.org/licenses/LICENSE-2.0
11
12Unless required by applicable law or agreed to in writing, software
13distributed under the License is distributed on an "AS IS" BASIS,
14WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15See the License for the specific language governing permissions and
16limitations under the License.
17*/
18
19package openstack
20
21import (
22	"context"
23	"errors"
24	"net"
25
26	"github.com/gophercloud/gophercloud"
27	"github.com/gophercloud/gophercloud/openstack/compute/v2/servers"
28	"github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/layer3/routers"
29	neutronports "github.com/gophercloud/gophercloud/openstack/networking/v2/ports"
30
31	"k8s.io/apimachinery/pkg/types"
32	cloudprovider "k8s.io/cloud-provider"
33	"k8s.io/klog/v2"
34)
35
36var errNoRouterID = errors.New("router-id not set in cloud provider config")
37
38var _ cloudprovider.Routes = (*Routes)(nil)
39
40// Routes implements the cloudprovider.Routes for OpenStack clouds
41type Routes struct {
42	compute *gophercloud.ServiceClient
43	network *gophercloud.ServiceClient
44	opts    RouterOpts
45}
46
47// NewRoutes creates a new instance of Routes
48func NewRoutes(compute *gophercloud.ServiceClient, network *gophercloud.ServiceClient, opts RouterOpts) (cloudprovider.Routes, error) {
49	if opts.RouterID == "" {
50		return nil, errNoRouterID
51	}
52
53	return &Routes{
54		compute: compute,
55		network: network,
56		opts:    opts,
57	}, nil
58}
59
60// ListRoutes lists all managed routes that belong to the specified clusterName
61func (r *Routes) ListRoutes(ctx context.Context, clusterName string) ([]*cloudprovider.Route, error) {
62	klog.V(4).Infof("ListRoutes(%v)", clusterName)
63
64	nodeNamesByAddr := make(map[string]types.NodeName)
65	err := foreachServer(r.compute, servers.ListOpts{}, func(srv *servers.Server) (bool, error) {
66		addrs, err := nodeAddresses(srv)
67		if err != nil {
68			return false, err
69		}
70
71		name := mapServerToNodeName(srv)
72		for _, addr := range addrs {
73			nodeNamesByAddr[addr.Address] = name
74		}
75
76		return true, nil
77	})
78	if err != nil {
79		return nil, err
80	}
81
82	router, err := routers.Get(r.network, r.opts.RouterID).Extract()
83	if err != nil {
84		return nil, err
85	}
86
87	var routes []*cloudprovider.Route
88	for _, item := range router.Routes {
89		nodeName, foundNode := nodeNamesByAddr[item.NextHop]
90		if !foundNode {
91			nodeName = types.NodeName(item.NextHop)
92		}
93		route := cloudprovider.Route{
94			Name:            item.DestinationCIDR,
95			TargetNode:      nodeName, //contains the nexthop address if node was not found
96			Blackhole:       !foundNode,
97			DestinationCIDR: item.DestinationCIDR,
98		}
99		routes = append(routes, &route)
100	}
101
102	return routes, nil
103}
104
105func updateRoutes(network *gophercloud.ServiceClient, router *routers.Router, newRoutes []routers.Route) (func(), error) {
106	origRoutes := router.Routes // shallow copy
107
108	_, err := routers.Update(network, router.ID, routers.UpdateOpts{
109		Routes: newRoutes,
110	}).Extract()
111	if err != nil {
112		return nil, err
113	}
114
115	unwinder := func() {
116		klog.V(4).Infof("Reverting routes change to router %v", router.ID)
117		_, err := routers.Update(network, router.ID, routers.UpdateOpts{
118			Routes: origRoutes,
119		}).Extract()
120		if err != nil {
121			klog.Warningf("Unable to reset routes during error unwind: %v", err)
122		}
123	}
124
125	return unwinder, nil
126}
127
128func updateAllowedAddressPairs(network *gophercloud.ServiceClient, port *neutronports.Port, newPairs []neutronports.AddressPair) (func(), error) {
129	origPairs := port.AllowedAddressPairs // shallow copy
130
131	_, err := neutronports.Update(network, port.ID, neutronports.UpdateOpts{
132		AllowedAddressPairs: &newPairs,
133	}).Extract()
134	if err != nil {
135		return nil, err
136	}
137
138	unwinder := func() {
139		klog.V(4).Infof("Reverting allowed-address-pairs change to port %v", port.ID)
140		_, err := neutronports.Update(network, port.ID, neutronports.UpdateOpts{
141			AllowedAddressPairs: &origPairs,
142		}).Extract()
143		if err != nil {
144			klog.Warningf("Unable to reset allowed-address-pairs during error unwind: %v", err)
145		}
146	}
147
148	return unwinder, nil
149}
150
151// CreateRoute creates the described managed route
152func (r *Routes) CreateRoute(ctx context.Context, clusterName string, nameHint string, route *cloudprovider.Route) error {
153	klog.V(4).Infof("CreateRoute(%v, %v, %v)", clusterName, nameHint, route)
154
155	onFailure := newCaller()
156
157	ip, _, _ := net.ParseCIDR(route.DestinationCIDR)
158	isCIDRv6 := ip.To4() == nil
159	addr, err := getAddressByName(r.compute, route.TargetNode, isCIDRv6)
160
161	if err != nil {
162		return err
163	}
164
165	klog.V(4).Infof("Using nexthop %v for node %v", addr, route.TargetNode)
166
167	router, err := routers.Get(r.network, r.opts.RouterID).Extract()
168	if err != nil {
169		return err
170	}
171
172	routes := router.Routes
173
174	for _, item := range routes {
175		if item.DestinationCIDR == route.DestinationCIDR && item.NextHop == addr {
176			klog.V(4).Infof("Skipping existing route: %v", route)
177			return nil
178		}
179	}
180
181	routes = append(routes, routers.Route{
182		DestinationCIDR: route.DestinationCIDR,
183		NextHop:         addr,
184	})
185
186	unwind, err := updateRoutes(r.network, router, routes)
187	if err != nil {
188		return err
189	}
190	defer onFailure.call(unwind)
191
192	// get the port of addr on target node.
193	portID, err := getPortIDByIP(r.compute, route.TargetNode, addr)
194	if err != nil {
195		return err
196	}
197	port, err := getPortByID(r.network, portID)
198	if err != nil {
199		return err
200	}
201
202	found := false
203	for _, item := range port.AllowedAddressPairs {
204		if item.IPAddress == route.DestinationCIDR {
205			klog.V(4).Infof("Found existing allowed-address-pair: %v", item)
206			found = true
207			break
208		}
209	}
210
211	if !found {
212		newPairs := append(port.AllowedAddressPairs, neutronports.AddressPair{
213			IPAddress: route.DestinationCIDR,
214		})
215		unwind, err := updateAllowedAddressPairs(r.network, port, newPairs)
216		if err != nil {
217			return err
218		}
219		defer onFailure.call(unwind)
220	}
221
222	klog.V(4).Infof("Route created: %v", route)
223	onFailure.disarm()
224	return nil
225}
226
227// DeleteRoute deletes the specified managed route
228func (r *Routes) DeleteRoute(ctx context.Context, clusterName string, route *cloudprovider.Route) error {
229	klog.V(4).Infof("DeleteRoute(%v, %v)", clusterName, route)
230
231	onFailure := newCaller()
232
233	ip, _, _ := net.ParseCIDR(route.DestinationCIDR)
234	isCIDRv6 := ip.To4() == nil
235
236	var addr string
237
238	// Blackhole routes are orphaned and have no counterpart in OpenStack
239	if !route.Blackhole {
240		var err error
241		addr, err = getAddressByName(r.compute, route.TargetNode, isCIDRv6)
242		if err != nil {
243			return err
244		}
245	}
246
247	router, err := routers.Get(r.network, r.opts.RouterID).Extract()
248	if err != nil {
249		return err
250	}
251
252	routes := router.Routes
253	index := -1
254	for i, item := range routes {
255		if item.DestinationCIDR == route.DestinationCIDR && (item.NextHop == addr || route.Blackhole && item.NextHop == string(route.TargetNode)) {
256			index = i
257			break
258		}
259	}
260
261	if index == -1 {
262		klog.V(4).Infof("Skipping non-existent route: %v", route)
263		return nil
264	}
265
266	// Delete element `index`
267	routes[index] = routes[len(routes)-1]
268	routes = routes[:len(routes)-1]
269
270	unwind, err := updateRoutes(r.network, router, routes)
271	// If this was a blackhole route we are done, there are no ports to update
272	if err != nil || route.Blackhole {
273		return err
274	}
275	defer onFailure.call(unwind)
276
277	// get the port of addr on target node.
278	portID, err := getPortIDByIP(r.compute, route.TargetNode, addr)
279	if err != nil {
280		return err
281	}
282	port, err := getPortByID(r.network, portID)
283	if err != nil {
284		return err
285	}
286
287	addrPairs := port.AllowedAddressPairs
288	index = -1
289	for i, item := range addrPairs {
290		if item.IPAddress == route.DestinationCIDR {
291			index = i
292			break
293		}
294	}
295
296	if index != -1 {
297		// Delete element `index`
298		addrPairs[index] = addrPairs[len(addrPairs)-1]
299		addrPairs = addrPairs[:len(addrPairs)-1]
300
301		unwind, err := updateAllowedAddressPairs(r.network, port, addrPairs)
302		if err != nil {
303			return err
304		}
305		defer onFailure.call(unwind)
306	}
307
308	klog.V(4).Infof("Route deleted: %v", route)
309	onFailure.disarm()
310	return nil
311}
312
313func getPortIDByIP(compute *gophercloud.ServiceClient, targetNode types.NodeName, ipAddress string) (string, error) {
314	srv, err := getServerByName(compute, targetNode)
315	if err != nil {
316		return "", err
317	}
318
319	interfaces, err := getAttachedInterfacesByID(compute, srv.ID)
320	if err != nil {
321		return "", err
322	}
323
324	for _, intf := range interfaces {
325		for _, fixedIP := range intf.FixedIPs {
326			if fixedIP.IPAddress == ipAddress {
327				return intf.PortID, nil
328			}
329		}
330	}
331
332	return "", ErrNotFound
333}
334
335func getPortByID(client *gophercloud.ServiceClient, portID string) (*neutronports.Port, error) {
336	targetPort, err := neutronports.Get(client, portID).Extract()
337	if err != nil {
338		return nil, err
339	}
340
341	if targetPort == nil {
342		return nil, ErrNotFound
343	}
344
345	return targetPort, nil
346}
347