1// +build !providerless 2 3/* 4Copyright 2016 The Kubernetes Authors. 5 6Licensed under the Apache License, Version 2.0 (the "License"); 7you may not use this file except in compliance with the License. 8You may obtain a copy of the License at 9 10 http://www.apache.org/licenses/LICENSE-2.0 11 12Unless required by applicable law or agreed to in writing, software 13distributed under the License is distributed on an "AS IS" BASIS, 14WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15See the License for the specific language governing permissions and 16limitations under the License. 17*/ 18 19package openstack 20 21import ( 22 "context" 23 "errors" 24 "net" 25 26 "github.com/gophercloud/gophercloud" 27 "github.com/gophercloud/gophercloud/openstack/compute/v2/servers" 28 "github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/layer3/routers" 29 neutronports "github.com/gophercloud/gophercloud/openstack/networking/v2/ports" 30 31 "k8s.io/apimachinery/pkg/types" 32 cloudprovider "k8s.io/cloud-provider" 33 "k8s.io/klog/v2" 34) 35 36var errNoRouterID = errors.New("router-id not set in cloud provider config") 37 38var _ cloudprovider.Routes = (*Routes)(nil) 39 40// Routes implements the cloudprovider.Routes for OpenStack clouds 41type Routes struct { 42 compute *gophercloud.ServiceClient 43 network *gophercloud.ServiceClient 44 opts RouterOpts 45} 46 47// NewRoutes creates a new instance of Routes 48func NewRoutes(compute *gophercloud.ServiceClient, network *gophercloud.ServiceClient, opts RouterOpts) (cloudprovider.Routes, error) { 49 if opts.RouterID == "" { 50 return nil, errNoRouterID 51 } 52 53 return &Routes{ 54 compute: compute, 55 network: network, 56 opts: opts, 57 }, nil 58} 59 60// ListRoutes lists all managed routes that belong to the specified clusterName 61func (r *Routes) ListRoutes(ctx context.Context, clusterName string) ([]*cloudprovider.Route, error) { 62 klog.V(4).Infof("ListRoutes(%v)", clusterName) 63 64 nodeNamesByAddr := make(map[string]types.NodeName) 65 err := foreachServer(r.compute, servers.ListOpts{}, func(srv *servers.Server) (bool, error) { 66 addrs, err := nodeAddresses(srv) 67 if err != nil { 68 return false, err 69 } 70 71 name := mapServerToNodeName(srv) 72 for _, addr := range addrs { 73 nodeNamesByAddr[addr.Address] = name 74 } 75 76 return true, nil 77 }) 78 if err != nil { 79 return nil, err 80 } 81 82 router, err := routers.Get(r.network, r.opts.RouterID).Extract() 83 if err != nil { 84 return nil, err 85 } 86 87 var routes []*cloudprovider.Route 88 for _, item := range router.Routes { 89 nodeName, foundNode := nodeNamesByAddr[item.NextHop] 90 if !foundNode { 91 nodeName = types.NodeName(item.NextHop) 92 } 93 route := cloudprovider.Route{ 94 Name: item.DestinationCIDR, 95 TargetNode: nodeName, //contains the nexthop address if node was not found 96 Blackhole: !foundNode, 97 DestinationCIDR: item.DestinationCIDR, 98 } 99 routes = append(routes, &route) 100 } 101 102 return routes, nil 103} 104 105func updateRoutes(network *gophercloud.ServiceClient, router *routers.Router, newRoutes []routers.Route) (func(), error) { 106 origRoutes := router.Routes // shallow copy 107 108 _, err := routers.Update(network, router.ID, routers.UpdateOpts{ 109 Routes: newRoutes, 110 }).Extract() 111 if err != nil { 112 return nil, err 113 } 114 115 unwinder := func() { 116 klog.V(4).Infof("Reverting routes change to router %v", router.ID) 117 _, err := routers.Update(network, router.ID, routers.UpdateOpts{ 118 Routes: origRoutes, 119 }).Extract() 120 if err != nil { 121 klog.Warningf("Unable to reset routes during error unwind: %v", err) 122 } 123 } 124 125 return unwinder, nil 126} 127 128func updateAllowedAddressPairs(network *gophercloud.ServiceClient, port *neutronports.Port, newPairs []neutronports.AddressPair) (func(), error) { 129 origPairs := port.AllowedAddressPairs // shallow copy 130 131 _, err := neutronports.Update(network, port.ID, neutronports.UpdateOpts{ 132 AllowedAddressPairs: &newPairs, 133 }).Extract() 134 if err != nil { 135 return nil, err 136 } 137 138 unwinder := func() { 139 klog.V(4).Infof("Reverting allowed-address-pairs change to port %v", port.ID) 140 _, err := neutronports.Update(network, port.ID, neutronports.UpdateOpts{ 141 AllowedAddressPairs: &origPairs, 142 }).Extract() 143 if err != nil { 144 klog.Warningf("Unable to reset allowed-address-pairs during error unwind: %v", err) 145 } 146 } 147 148 return unwinder, nil 149} 150 151// CreateRoute creates the described managed route 152func (r *Routes) CreateRoute(ctx context.Context, clusterName string, nameHint string, route *cloudprovider.Route) error { 153 klog.V(4).Infof("CreateRoute(%v, %v, %v)", clusterName, nameHint, route) 154 155 onFailure := newCaller() 156 157 ip, _, _ := net.ParseCIDR(route.DestinationCIDR) 158 isCIDRv6 := ip.To4() == nil 159 addr, err := getAddressByName(r.compute, route.TargetNode, isCIDRv6) 160 161 if err != nil { 162 return err 163 } 164 165 klog.V(4).Infof("Using nexthop %v for node %v", addr, route.TargetNode) 166 167 router, err := routers.Get(r.network, r.opts.RouterID).Extract() 168 if err != nil { 169 return err 170 } 171 172 routes := router.Routes 173 174 for _, item := range routes { 175 if item.DestinationCIDR == route.DestinationCIDR && item.NextHop == addr { 176 klog.V(4).Infof("Skipping existing route: %v", route) 177 return nil 178 } 179 } 180 181 routes = append(routes, routers.Route{ 182 DestinationCIDR: route.DestinationCIDR, 183 NextHop: addr, 184 }) 185 186 unwind, err := updateRoutes(r.network, router, routes) 187 if err != nil { 188 return err 189 } 190 defer onFailure.call(unwind) 191 192 // get the port of addr on target node. 193 portID, err := getPortIDByIP(r.compute, route.TargetNode, addr) 194 if err != nil { 195 return err 196 } 197 port, err := getPortByID(r.network, portID) 198 if err != nil { 199 return err 200 } 201 202 found := false 203 for _, item := range port.AllowedAddressPairs { 204 if item.IPAddress == route.DestinationCIDR { 205 klog.V(4).Infof("Found existing allowed-address-pair: %v", item) 206 found = true 207 break 208 } 209 } 210 211 if !found { 212 newPairs := append(port.AllowedAddressPairs, neutronports.AddressPair{ 213 IPAddress: route.DestinationCIDR, 214 }) 215 unwind, err := updateAllowedAddressPairs(r.network, port, newPairs) 216 if err != nil { 217 return err 218 } 219 defer onFailure.call(unwind) 220 } 221 222 klog.V(4).Infof("Route created: %v", route) 223 onFailure.disarm() 224 return nil 225} 226 227// DeleteRoute deletes the specified managed route 228func (r *Routes) DeleteRoute(ctx context.Context, clusterName string, route *cloudprovider.Route) error { 229 klog.V(4).Infof("DeleteRoute(%v, %v)", clusterName, route) 230 231 onFailure := newCaller() 232 233 ip, _, _ := net.ParseCIDR(route.DestinationCIDR) 234 isCIDRv6 := ip.To4() == nil 235 236 var addr string 237 238 // Blackhole routes are orphaned and have no counterpart in OpenStack 239 if !route.Blackhole { 240 var err error 241 addr, err = getAddressByName(r.compute, route.TargetNode, isCIDRv6) 242 if err != nil { 243 return err 244 } 245 } 246 247 router, err := routers.Get(r.network, r.opts.RouterID).Extract() 248 if err != nil { 249 return err 250 } 251 252 routes := router.Routes 253 index := -1 254 for i, item := range routes { 255 if item.DestinationCIDR == route.DestinationCIDR && (item.NextHop == addr || route.Blackhole && item.NextHop == string(route.TargetNode)) { 256 index = i 257 break 258 } 259 } 260 261 if index == -1 { 262 klog.V(4).Infof("Skipping non-existent route: %v", route) 263 return nil 264 } 265 266 // Delete element `index` 267 routes[index] = routes[len(routes)-1] 268 routes = routes[:len(routes)-1] 269 270 unwind, err := updateRoutes(r.network, router, routes) 271 // If this was a blackhole route we are done, there are no ports to update 272 if err != nil || route.Blackhole { 273 return err 274 } 275 defer onFailure.call(unwind) 276 277 // get the port of addr on target node. 278 portID, err := getPortIDByIP(r.compute, route.TargetNode, addr) 279 if err != nil { 280 return err 281 } 282 port, err := getPortByID(r.network, portID) 283 if err != nil { 284 return err 285 } 286 287 addrPairs := port.AllowedAddressPairs 288 index = -1 289 for i, item := range addrPairs { 290 if item.IPAddress == route.DestinationCIDR { 291 index = i 292 break 293 } 294 } 295 296 if index != -1 { 297 // Delete element `index` 298 addrPairs[index] = addrPairs[len(addrPairs)-1] 299 addrPairs = addrPairs[:len(addrPairs)-1] 300 301 unwind, err := updateAllowedAddressPairs(r.network, port, addrPairs) 302 if err != nil { 303 return err 304 } 305 defer onFailure.call(unwind) 306 } 307 308 klog.V(4).Infof("Route deleted: %v", route) 309 onFailure.disarm() 310 return nil 311} 312 313func getPortIDByIP(compute *gophercloud.ServiceClient, targetNode types.NodeName, ipAddress string) (string, error) { 314 srv, err := getServerByName(compute, targetNode) 315 if err != nil { 316 return "", err 317 } 318 319 interfaces, err := getAttachedInterfacesByID(compute, srv.ID) 320 if err != nil { 321 return "", err 322 } 323 324 for _, intf := range interfaces { 325 for _, fixedIP := range intf.FixedIPs { 326 if fixedIP.IPAddress == ipAddress { 327 return intf.PortID, nil 328 } 329 } 330 } 331 332 return "", ErrNotFound 333} 334 335func getPortByID(client *gophercloud.ServiceClient, portID string) (*neutronports.Port, error) { 336 targetPort, err := neutronports.Get(client, portID).Extract() 337 if err != nil { 338 return nil, err 339 } 340 341 if targetPort == nil { 342 return nil, ErrNotFound 343 } 344 345 return targetPort, nil 346} 347