1package overlay 2 3import ( 4 "fmt" 5 "net" 6 "sync" 7 "syscall" 8 9 "github.com/sirupsen/logrus" 10) 11 12const ovPeerTable = "overlay_peer_table" 13 14type peerKey struct { 15 peerIP net.IP 16 peerMac net.HardwareAddr 17} 18 19type peerEntry struct { 20 eid string 21 vtep net.IP 22 peerIPMask net.IPMask 23 inSandbox bool 24 isLocal bool 25} 26 27type peerMap struct { 28 mp map[string]peerEntry 29 sync.Mutex 30} 31 32type peerNetworkMap struct { 33 mp map[string]*peerMap 34 sync.Mutex 35} 36 37func (pKey peerKey) String() string { 38 return fmt.Sprintf("%s %s", pKey.peerIP, pKey.peerMac) 39} 40 41func (pKey *peerKey) Scan(state fmt.ScanState, verb rune) error { 42 ipB, err := state.Token(true, nil) 43 if err != nil { 44 return err 45 } 46 47 pKey.peerIP = net.ParseIP(string(ipB)) 48 49 macB, err := state.Token(true, nil) 50 if err != nil { 51 return err 52 } 53 54 pKey.peerMac, err = net.ParseMAC(string(macB)) 55 if err != nil { 56 return err 57 } 58 59 return nil 60} 61 62var peerDbWg sync.WaitGroup 63 64func (d *driver) peerDbWalk(f func(string, *peerKey, *peerEntry) bool) error { 65 d.peerDb.Lock() 66 nids := []string{} 67 for nid := range d.peerDb.mp { 68 nids = append(nids, nid) 69 } 70 d.peerDb.Unlock() 71 72 for _, nid := range nids { 73 d.peerDbNetworkWalk(nid, func(pKey *peerKey, pEntry *peerEntry) bool { 74 return f(nid, pKey, pEntry) 75 }) 76 } 77 return nil 78} 79 80func (d *driver) peerDbNetworkWalk(nid string, f func(*peerKey, *peerEntry) bool) error { 81 d.peerDb.Lock() 82 pMap, ok := d.peerDb.mp[nid] 83 if !ok { 84 d.peerDb.Unlock() 85 return nil 86 } 87 d.peerDb.Unlock() 88 89 pMap.Lock() 90 for pKeyStr, pEntry := range pMap.mp { 91 var pKey peerKey 92 if _, err := fmt.Sscan(pKeyStr, &pKey); err != nil { 93 logrus.Warnf("Peer key scan on network %s failed: %v", nid, err) 94 } 95 96 if f(&pKey, &pEntry) { 97 pMap.Unlock() 98 return nil 99 } 100 } 101 pMap.Unlock() 102 103 return nil 104} 105 106func (d *driver) peerDbSearch(nid string, peerIP net.IP) (net.HardwareAddr, net.IPMask, net.IP, error) { 107 var ( 108 peerMac net.HardwareAddr 109 vtep net.IP 110 peerIPMask net.IPMask 111 found bool 112 ) 113 114 err := d.peerDbNetworkWalk(nid, func(pKey *peerKey, pEntry *peerEntry) bool { 115 if pKey.peerIP.Equal(peerIP) { 116 peerMac = pKey.peerMac 117 peerIPMask = pEntry.peerIPMask 118 vtep = pEntry.vtep 119 found = true 120 return found 121 } 122 123 return found 124 }) 125 126 if err != nil { 127 return nil, nil, nil, fmt.Errorf("peerdb search for peer ip %q failed: %v", peerIP, err) 128 } 129 130 if !found { 131 return nil, nil, nil, fmt.Errorf("peer ip %q not found in peerdb", peerIP) 132 } 133 134 return peerMac, peerIPMask, vtep, nil 135} 136 137func (d *driver) peerDbAdd(nid, eid string, peerIP net.IP, peerIPMask net.IPMask, 138 peerMac net.HardwareAddr, vtep net.IP, isLocal bool) { 139 140 peerDbWg.Wait() 141 142 d.peerDb.Lock() 143 pMap, ok := d.peerDb.mp[nid] 144 if !ok { 145 d.peerDb.mp[nid] = &peerMap{ 146 mp: make(map[string]peerEntry), 147 } 148 149 pMap = d.peerDb.mp[nid] 150 } 151 d.peerDb.Unlock() 152 153 pKey := peerKey{ 154 peerIP: peerIP, 155 peerMac: peerMac, 156 } 157 158 pEntry := peerEntry{ 159 eid: eid, 160 vtep: vtep, 161 peerIPMask: peerIPMask, 162 isLocal: isLocal, 163 } 164 165 pMap.Lock() 166 pMap.mp[pKey.String()] = pEntry 167 pMap.Unlock() 168} 169 170func (d *driver) peerDbDelete(nid, eid string, peerIP net.IP, peerIPMask net.IPMask, 171 peerMac net.HardwareAddr, vtep net.IP) peerEntry { 172 peerDbWg.Wait() 173 174 d.peerDb.Lock() 175 pMap, ok := d.peerDb.mp[nid] 176 if !ok { 177 d.peerDb.Unlock() 178 return peerEntry{} 179 } 180 d.peerDb.Unlock() 181 182 pKey := peerKey{ 183 peerIP: peerIP, 184 peerMac: peerMac, 185 } 186 187 pMap.Lock() 188 189 pEntry, ok := pMap.mp[pKey.String()] 190 if ok { 191 // Mismatched endpoint ID(possibly outdated). Do not 192 // delete peerdb 193 if pEntry.eid != eid { 194 pMap.Unlock() 195 return pEntry 196 } 197 } 198 199 delete(pMap.mp, pKey.String()) 200 pMap.Unlock() 201 202 return pEntry 203} 204 205func (d *driver) peerDbUpdateSandbox(nid string) { 206 d.peerDb.Lock() 207 pMap, ok := d.peerDb.mp[nid] 208 if !ok { 209 d.peerDb.Unlock() 210 return 211 } 212 d.peerDb.Unlock() 213 214 peerDbWg.Add(1) 215 216 var peerOps []func() 217 pMap.Lock() 218 for pKeyStr, pEntry := range pMap.mp { 219 var pKey peerKey 220 if _, err := fmt.Sscan(pKeyStr, &pKey); err != nil { 221 fmt.Printf("peer key scan failed: %v", err) 222 } 223 224 if pEntry.isLocal { 225 continue 226 } 227 228 // Go captures variables by reference. The pEntry could be 229 // pointing to the same memory location for every iteration. Make 230 // a copy of pEntry before capturing it in the following closure. 231 entry := pEntry 232 op := func() { 233 if err := d.peerAdd(nid, entry.eid, pKey.peerIP, entry.peerIPMask, 234 pKey.peerMac, entry.vtep, 235 false); err != nil { 236 fmt.Printf("peerdbupdate in sandbox failed for ip %s and mac %s: %v", 237 pKey.peerIP, pKey.peerMac, err) 238 } 239 } 240 241 peerOps = append(peerOps, op) 242 } 243 pMap.Unlock() 244 245 for _, op := range peerOps { 246 op() 247 } 248 249 peerDbWg.Done() 250} 251 252func (d *driver) peerAdd(nid, eid string, peerIP net.IP, peerIPMask net.IPMask, 253 peerMac net.HardwareAddr, vtep net.IP, updateDb bool) error { 254 255 if err := validateID(nid, eid); err != nil { 256 return err 257 } 258 259 if updateDb { 260 d.peerDbAdd(nid, eid, peerIP, peerIPMask, peerMac, vtep, false) 261 } 262 263 n := d.network(nid) 264 if n == nil { 265 return nil 266 } 267 268 sbox := n.sandbox() 269 if sbox == nil { 270 return nil 271 } 272 273 IP := &net.IPNet{ 274 IP: peerIP, 275 Mask: peerIPMask, 276 } 277 278 s := n.getSubnetforIP(IP) 279 if s == nil { 280 return fmt.Errorf("couldn't find the subnet %q in network %q", IP.String(), n.id) 281 } 282 283 if err := n.obtainVxlanID(s); err != nil { 284 return fmt.Errorf("couldn't get vxlan id for %q: %v", s.subnetIP.String(), err) 285 } 286 287 if err := n.joinSubnetSandbox(s, false); err != nil { 288 return fmt.Errorf("subnet sandbox join failed for %q: %v", s.subnetIP.String(), err) 289 } 290 291 if err := d.checkEncryption(nid, vtep, n.vxlanID(s), false, true); err != nil { 292 logrus.Warn(err) 293 } 294 295 // Add neighbor entry for the peer IP 296 if err := sbox.AddNeighbor(peerIP, peerMac, sbox.NeighborOptions().LinkName(s.vxlanName)); err != nil { 297 return fmt.Errorf("could not add neighbor entry into the sandbox: %v", err) 298 } 299 300 // Add fdb entry to the bridge for the peer mac 301 if err := sbox.AddNeighbor(vtep, peerMac, sbox.NeighborOptions().LinkName(s.vxlanName), 302 sbox.NeighborOptions().Family(syscall.AF_BRIDGE)); err != nil { 303 return fmt.Errorf("could not add fdb entry into the sandbox: %v", err) 304 } 305 306 return nil 307} 308 309func (d *driver) peerDelete(nid, eid string, peerIP net.IP, peerIPMask net.IPMask, 310 peerMac net.HardwareAddr, vtep net.IP, updateDb bool) error { 311 312 if err := validateID(nid, eid); err != nil { 313 return err 314 } 315 316 var pEntry peerEntry 317 if updateDb { 318 pEntry = d.peerDbDelete(nid, eid, peerIP, peerIPMask, peerMac, vtep) 319 } 320 321 n := d.network(nid) 322 if n == nil { 323 return nil 324 } 325 326 sbox := n.sandbox() 327 if sbox == nil { 328 return nil 329 } 330 331 // Delete fdb entry to the bridge for the peer mac only if the 332 // entry existed in local peerdb. If it is a stale delete 333 // request, still call DeleteNeighbor but only to cleanup any 334 // leftover sandbox neighbor cache and not actually delete the 335 // kernel state. 336 if (eid == pEntry.eid && vtep.Equal(pEntry.vtep)) || 337 (eid != pEntry.eid && !vtep.Equal(pEntry.vtep)) { 338 if err := sbox.DeleteNeighbor(vtep, peerMac, 339 eid == pEntry.eid && vtep.Equal(pEntry.vtep)); err != nil { 340 return fmt.Errorf("could not delete fdb entry into the sandbox: %v", err) 341 } 342 } 343 344 // Delete neighbor entry for the peer IP 345 if eid == pEntry.eid { 346 if err := sbox.DeleteNeighbor(peerIP, peerMac, true); err != nil { 347 return fmt.Errorf("could not delete neighbor entry into the sandbox: %v", err) 348 } 349 } 350 351 if err := d.checkEncryption(nid, vtep, 0, false, false); err != nil { 352 logrus.Warn(err) 353 } 354 355 return nil 356} 357 358func (d *driver) pushLocalDb() { 359 d.peerDbWalk(func(nid string, pKey *peerKey, pEntry *peerEntry) bool { 360 if pEntry.isLocal { 361 d.pushLocalEndpointEvent("join", nid, pEntry.eid) 362 } 363 return false 364 }) 365} 366