1 /*
2 * Copyright (c)2013-2020 ZeroTier, Inc.
3 *
4 * Use of this software is governed by the Business Source License included
5 * in the LICENSE.TXT file in the project's root directory.
6 *
7 * Change Date: 2025-01-01
8 *
9 * On the date above, in accordance with the Business Source License, use
10 * of this software will be governed by version 2.0 of the Apache License.
11 */
12 /****/
13
14 #include <stdio.h>
15 #include <stdlib.h>
16
17 #include <algorithm>
18 #include <utility>
19 #include <stdexcept>
20
21 #include "../version.h"
22 #include "../include/ZeroTierOne.h"
23
24 #include "Constants.hpp"
25 #include "RuntimeEnvironment.hpp"
26 #include "Switch.hpp"
27 #include "Node.hpp"
28 #include "InetAddress.hpp"
29 #include "Topology.hpp"
30 #include "Peer.hpp"
31 #include "SelfAwareness.hpp"
32 #include "Packet.hpp"
33 #include "Trace.hpp"
34
35 namespace ZeroTier {
36
Switch(const RuntimeEnvironment * renv)37 Switch::Switch(const RuntimeEnvironment *renv) :
38 RR(renv),
39 _lastBeaconResponse(0),
40 _lastCheckedQueues(0),
41 _lastUniteAttempt(8) // only really used on root servers and upstreams, and it'll grow there just fine
42 {
43 }
44
45 // Returns true if packet appears valid; pos and proto will be set
_ipv6GetPayload(const uint8_t * frameData,unsigned int frameLen,unsigned int & pos,unsigned int & proto)46 static bool _ipv6GetPayload(const uint8_t *frameData,unsigned int frameLen,unsigned int &pos,unsigned int &proto)
47 {
48 if (frameLen < 40)
49 return false;
50 pos = 40;
51 proto = frameData[6];
52 while (pos <= frameLen) {
53 switch(proto) {
54 case 0: // hop-by-hop options
55 case 43: // routing
56 case 60: // destination options
57 case 135: // mobility options
58 if ((pos + 8) > frameLen)
59 return false; // invalid!
60 proto = frameData[pos];
61 pos += ((unsigned int)frameData[pos + 1] * 8) + 8;
62 break;
63
64 //case 44: // fragment -- we currently can't parse these and they are deprecated in IPv6 anyway
65 //case 50:
66 //case 51: // IPSec ESP and AH -- we have to stop here since this is encrypted stuff
67 default:
68 return true;
69 }
70 }
71 return false; // overflow == invalid
72 }
73
onRemotePacket(void * tPtr,const int64_t localSocket,const InetAddress & fromAddr,const void * data,unsigned int len)74 void Switch::onRemotePacket(void *tPtr,const int64_t localSocket,const InetAddress &fromAddr,const void *data,unsigned int len)
75 {
76 int32_t flowId = ZT_QOS_NO_FLOW;
77 try {
78 const int64_t now = RR->node->now();
79
80 const SharedPtr<Path> path(RR->topology->getPath(localSocket,fromAddr));
81 path->received(now);
82
83 if (len == 13) {
84 /* LEGACY: before VERB_PUSH_DIRECT_PATHS, peers used broadcast
85 * announcements on the LAN to solve the 'same network problem.' We
86 * no longer send these, but we'll listen for them for a while to
87 * locate peers with versions <1.0.4. */
88
89 const Address beaconAddr(reinterpret_cast<const char *>(data) + 8,5);
90 if (beaconAddr == RR->identity.address())
91 return;
92 if (!RR->node->shouldUsePathForZeroTierTraffic(tPtr,beaconAddr,localSocket,fromAddr))
93 return;
94 const SharedPtr<Peer> peer(RR->topology->getPeer(tPtr,beaconAddr));
95 if (peer) { // we'll only respond to beacons from known peers
96 if ((now - _lastBeaconResponse) >= 2500) { // limit rate of responses
97 _lastBeaconResponse = now;
98 Packet outp(peer->address(),RR->identity.address(),Packet::VERB_NOP);
99 outp.armor(peer->key(),true,peer->aesKeysIfSupported());
100 path->send(RR,tPtr,outp.data(),outp.size(),now);
101 }
102 }
103
104 } else if (len > ZT_PROTO_MIN_FRAGMENT_LENGTH) { // SECURITY: min length check is important since we do some C-style stuff below!
105 if (reinterpret_cast<const uint8_t *>(data)[ZT_PACKET_FRAGMENT_IDX_FRAGMENT_INDICATOR] == ZT_PACKET_FRAGMENT_INDICATOR) {
106 // Handle fragment ----------------------------------------------------
107
108 Packet::Fragment fragment(data,len);
109 const Address destination(fragment.destination());
110
111 if (destination != RR->identity.address()) {
112 if ( (!RR->topology->amUpstream()) && (!path->trustEstablished(now)) )
113 return;
114
115 if (fragment.hops() < ZT_RELAY_MAX_HOPS) {
116 fragment.incrementHops();
117
118 // Note: we don't bother initiating NAT-t for fragments, since heads will set that off.
119 // It wouldn't hurt anything, just redundant and unnecessary.
120 SharedPtr<Peer> relayTo = RR->topology->getPeer(tPtr,destination);
121 if ((!relayTo)||(!relayTo->sendDirect(tPtr,fragment.data(),fragment.size(),now,false))) {
122 // Don't know peer or no direct path -- so relay via someone upstream
123 relayTo = RR->topology->getUpstreamPeer();
124 if (relayTo)
125 relayTo->sendDirect(tPtr,fragment.data(),fragment.size(),now,true);
126 }
127 }
128 } else {
129 // Fragment looks like ours
130 const uint64_t fragmentPacketId = fragment.packetId();
131 const unsigned int fragmentNumber = fragment.fragmentNumber();
132 const unsigned int totalFragments = fragment.totalFragments();
133
134 if ((totalFragments <= ZT_MAX_PACKET_FRAGMENTS)&&(fragmentNumber < ZT_MAX_PACKET_FRAGMENTS)&&(fragmentNumber > 0)&&(totalFragments > 1)) {
135 // Fragment appears basically sane. Its fragment number must be
136 // 1 or more, since a Packet with fragmented bit set is fragment 0.
137 // Total fragments must be more than 1, otherwise why are we
138 // seeing a Packet::Fragment?
139
140 RXQueueEntry *const rq = _findRXQueueEntry(fragmentPacketId);
141 Mutex::Lock rql(rq->lock);
142 if (rq->packetId != fragmentPacketId) {
143 // No packet found, so we received a fragment without its head.
144
145 rq->flowId = flowId;
146 rq->timestamp = now;
147 rq->packetId = fragmentPacketId;
148 rq->frags[fragmentNumber - 1] = fragment;
149 rq->totalFragments = totalFragments; // total fragment count is known
150 rq->haveFragments = 1 << fragmentNumber; // we have only this fragment
151 rq->complete = false;
152 } else if (!(rq->haveFragments & (1 << fragmentNumber))) {
153 // We have other fragments and maybe the head, so add this one and check
154
155 rq->frags[fragmentNumber - 1] = fragment;
156 rq->totalFragments = totalFragments;
157
158 if (Utils::countBits(rq->haveFragments |= (1 << fragmentNumber)) == totalFragments) {
159 // We have all fragments -- assemble and process full Packet
160
161 for(unsigned int f=1;f<totalFragments;++f)
162 rq->frag0.append(rq->frags[f - 1].payload(),rq->frags[f - 1].payloadLength());
163
164 if (rq->frag0.tryDecode(RR,tPtr,flowId)) {
165 rq->timestamp = 0; // packet decoded, free entry
166 } else {
167 rq->complete = true; // set complete flag but leave entry since it probably needs WHOIS or something
168 }
169 }
170 } // else this is a duplicate fragment, ignore
171 }
172 }
173
174 // --------------------------------------------------------------------
175 } else if (len >= ZT_PROTO_MIN_PACKET_LENGTH) { // min length check is important!
176 // Handle packet head -------------------------------------------------
177
178 const Address destination(reinterpret_cast<const uint8_t *>(data) + 8,ZT_ADDRESS_LENGTH);
179 const Address source(reinterpret_cast<const uint8_t *>(data) + 13,ZT_ADDRESS_LENGTH);
180
181 if (source == RR->identity.address())
182 return;
183
184 if (destination != RR->identity.address()) {
185 if ( (!RR->topology->amUpstream()) && (!path->trustEstablished(now)) && (source != RR->identity.address()) )
186 return;
187
188 Packet packet(data,len);
189
190 if (packet.hops() < ZT_RELAY_MAX_HOPS) {
191 packet.incrementHops();
192 SharedPtr<Peer> relayTo = RR->topology->getPeer(tPtr,destination);
193 if ((relayTo)&&(relayTo->sendDirect(tPtr,packet.data(),packet.size(),now,false))) {
194 if ((source != RR->identity.address())&&(_shouldUnite(now,source,destination))) {
195 const SharedPtr<Peer> sourcePeer(RR->topology->getPeer(tPtr,source));
196 if (sourcePeer)
197 relayTo->introduce(tPtr,now,sourcePeer);
198 }
199 } else {
200 relayTo = RR->topology->getUpstreamPeer();
201 if ((relayTo)&&(relayTo->address() != source)) {
202 if (relayTo->sendDirect(tPtr,packet.data(),packet.size(),now,true)) {
203 const SharedPtr<Peer> sourcePeer(RR->topology->getPeer(tPtr,source));
204 if (sourcePeer)
205 relayTo->introduce(tPtr,now,sourcePeer);
206 }
207 }
208 }
209 }
210 } else if ((reinterpret_cast<const uint8_t *>(data)[ZT_PACKET_IDX_FLAGS] & ZT_PROTO_FLAG_FRAGMENTED) != 0) {
211 // Packet is the head of a fragmented packet series
212
213 const uint64_t packetId = (
214 (((uint64_t)reinterpret_cast<const uint8_t *>(data)[0]) << 56) |
215 (((uint64_t)reinterpret_cast<const uint8_t *>(data)[1]) << 48) |
216 (((uint64_t)reinterpret_cast<const uint8_t *>(data)[2]) << 40) |
217 (((uint64_t)reinterpret_cast<const uint8_t *>(data)[3]) << 32) |
218 (((uint64_t)reinterpret_cast<const uint8_t *>(data)[4]) << 24) |
219 (((uint64_t)reinterpret_cast<const uint8_t *>(data)[5]) << 16) |
220 (((uint64_t)reinterpret_cast<const uint8_t *>(data)[6]) << 8) |
221 ((uint64_t)reinterpret_cast<const uint8_t *>(data)[7])
222 );
223
224 RXQueueEntry *const rq = _findRXQueueEntry(packetId);
225 Mutex::Lock rql(rq->lock);
226 if (rq->packetId != packetId) {
227 // If we have no other fragments yet, create an entry and save the head
228
229 rq->flowId = flowId;
230 rq->timestamp = now;
231 rq->packetId = packetId;
232 rq->frag0.init(data,len,path,now);
233 rq->totalFragments = 0;
234 rq->haveFragments = 1;
235 rq->complete = false;
236 } else if (!(rq->haveFragments & 1)) {
237 // If we have other fragments but no head, see if we are complete with the head
238
239 if ((rq->totalFragments > 1)&&(Utils::countBits(rq->haveFragments |= 1) == rq->totalFragments)) {
240 // We have all fragments -- assemble and process full Packet
241
242 rq->frag0.init(data,len,path,now);
243 for(unsigned int f=1;f<rq->totalFragments;++f)
244 rq->frag0.append(rq->frags[f - 1].payload(),rq->frags[f - 1].payloadLength());
245
246 if (rq->frag0.tryDecode(RR,tPtr,flowId)) {
247 rq->timestamp = 0; // packet decoded, free entry
248 } else {
249 rq->complete = true; // set complete flag but leave entry since it probably needs WHOIS or something
250 }
251 } else {
252 // Still waiting on more fragments, but keep the head
253 rq->frag0.init(data,len,path,now);
254 }
255 } // else this is a duplicate head, ignore
256 } else {
257 // Packet is unfragmented, so just process it
258 IncomingPacket packet(data,len,path,now);
259 if (!packet.tryDecode(RR,tPtr,flowId)) {
260 RXQueueEntry *const rq = _nextRXQueueEntry();
261 Mutex::Lock rql(rq->lock);
262 rq->flowId = flowId;
263 rq->timestamp = now;
264 rq->packetId = packet.packetId();
265 rq->frag0 = packet;
266 rq->totalFragments = 1;
267 rq->haveFragments = 1;
268 rq->complete = true;
269 }
270 }
271
272 // --------------------------------------------------------------------
273 }
274 }
275 } catch ( ... ) {} // sanity check, should be caught elsewhere
276 }
277
onLocalEthernet(void * tPtr,const SharedPtr<Network> & network,const MAC & from,const MAC & to,unsigned int etherType,unsigned int vlanId,const void * data,unsigned int len)278 void Switch::onLocalEthernet(void *tPtr,const SharedPtr<Network> &network,const MAC &from,const MAC &to,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len)
279 {
280 if (!network->hasConfig())
281 return;
282
283 // Check if this packet is from someone other than the tap -- i.e. bridged in
284 bool fromBridged;
285 if ((fromBridged = (from != network->mac()))) {
286 if (!network->config().permitsBridging(RR->identity.address())) {
287 RR->t->outgoingNetworkFrameDropped(tPtr,network,from,to,etherType,vlanId,len,"not a bridge");
288 return;
289 }
290 }
291
292 uint8_t qosBucket = ZT_AQM_DEFAULT_BUCKET;
293
294 /**
295 * A pseudo-unique identifier used by balancing and bonding policies to
296 * categorize individual flows/conversations for assignment to a specific
297 * physical path. This identifier consists of the source port and
298 * destination port of the encapsulated frame.
299 *
300 * A flowId of -1 will indicate that there is no preference for how this
301 * packet shall be sent. An example of this would be an ICMP packet.
302 */
303
304 int32_t flowId = ZT_QOS_NO_FLOW;
305
306 if (etherType == ZT_ETHERTYPE_IPV4 && (len >= 20)) {
307 uint16_t srcPort = 0;
308 uint16_t dstPort = 0;
309 uint8_t proto = (reinterpret_cast<const uint8_t *>(data)[9]);
310 const unsigned int headerLen = 4 * (reinterpret_cast<const uint8_t *>(data)[0] & 0xf);
311 switch(proto) {
312 case 0x01: // ICMP
313 //flowId = 0x01;
314 break;
315 // All these start with 16-bit source and destination port in that order
316 case 0x06: // TCP
317 case 0x11: // UDP
318 case 0x84: // SCTP
319 case 0x88: // UDPLite
320 if (len > (headerLen + 4)) {
321 unsigned int pos = headerLen + 0;
322 srcPort = (reinterpret_cast<const uint8_t *>(data)[pos++]) << 8;
323 srcPort |= (reinterpret_cast<const uint8_t *>(data)[pos]);
324 pos++;
325 dstPort = (reinterpret_cast<const uint8_t *>(data)[pos++]) << 8;
326 dstPort |= (reinterpret_cast<const uint8_t *>(data)[pos]);
327 flowId = dstPort ^ srcPort ^ proto;
328 }
329 break;
330 }
331 }
332
333 if (etherType == ZT_ETHERTYPE_IPV6 && (len >= 40)) {
334 uint16_t srcPort = 0;
335 uint16_t dstPort = 0;
336 unsigned int pos;
337 unsigned int proto;
338 _ipv6GetPayload((const uint8_t *)data, len, pos, proto);
339 switch(proto) {
340 case 0x3A: // ICMPv6
341 //flowId = 0x3A;
342 break;
343 // All these start with 16-bit source and destination port in that order
344 case 0x06: // TCP
345 case 0x11: // UDP
346 case 0x84: // SCTP
347 case 0x88: // UDPLite
348 if (len > (pos + 4)) {
349 srcPort = (reinterpret_cast<const uint8_t *>(data)[pos++]) << 8;
350 srcPort |= (reinterpret_cast<const uint8_t *>(data)[pos]);
351 pos++;
352 dstPort = (reinterpret_cast<const uint8_t *>(data)[pos++]) << 8;
353 dstPort |= (reinterpret_cast<const uint8_t *>(data)[pos]);
354 flowId = dstPort ^ srcPort ^ proto;
355 }
356 break;
357 default:
358 break;
359 }
360 }
361
362 if (to.isMulticast()) {
363 MulticastGroup multicastGroup(to,0);
364
365 if (to.isBroadcast()) {
366 if ( (etherType == ZT_ETHERTYPE_ARP) && (len >= 28) && ((((const uint8_t *)data)[2] == 0x08)&&(((const uint8_t *)data)[3] == 0x00)&&(((const uint8_t *)data)[4] == 6)&&(((const uint8_t *)data)[5] == 4)&&(((const uint8_t *)data)[7] == 0x01)) ) {
367 /* IPv4 ARP is one of the few special cases that we impose upon what is
368 * otherwise a straightforward Ethernet switch emulation. Vanilla ARP
369 * is dumb old broadcast and simply doesn't scale. ZeroTier multicast
370 * groups have an additional field called ADI (additional distinguishing
371 * information) which was added specifically for ARP though it could
372 * be used for other things too. We then take ARP broadcasts and turn
373 * them into multicasts by stuffing the IP address being queried into
374 * the 32-bit ADI field. In practice this uses our multicast pub/sub
375 * system to implement a kind of extended/distributed ARP table. */
376 multicastGroup = MulticastGroup::deriveMulticastGroupForAddressResolution(InetAddress(((const unsigned char *)data) + 24,4,0));
377 } else if (!network->config().enableBroadcast()) {
378 // Don't transmit broadcasts if this network doesn't want them
379 RR->t->outgoingNetworkFrameDropped(tPtr,network,from,to,etherType,vlanId,len,"broadcast disabled");
380 return;
381 }
382 } else if ((etherType == ZT_ETHERTYPE_IPV6)&&(len >= (40 + 8 + 16))) {
383 // IPv6 NDP emulation for certain very special patterns of private IPv6 addresses -- if enabled
384 if ((network->config().ndpEmulation())&&(reinterpret_cast<const uint8_t *>(data)[6] == 0x3a)&&(reinterpret_cast<const uint8_t *>(data)[40] == 0x87)) { // ICMPv6 neighbor solicitation
385 Address v6EmbeddedAddress;
386 const uint8_t *const pkt6 = reinterpret_cast<const uint8_t *>(data) + 40 + 8;
387 const uint8_t *my6 = (const uint8_t *)0;
388
389 // ZT-RFC4193 address: fdNN:NNNN:NNNN:NNNN:NN99:93DD:DDDD:DDDD / 88 (one /128 per actual host)
390
391 // ZT-6PLANE address: fcXX:XXXX:XXDD:DDDD:DDDD:####:####:#### / 40 (one /80 per actual host)
392 // (XX - lower 32 bits of network ID XORed with higher 32 bits)
393
394 // For these to work, we must have a ZT-managed address assigned in one of the
395 // above formats, and the query must match its prefix.
396 for(unsigned int sipk=0;sipk<network->config().staticIpCount;++sipk) {
397 const InetAddress *const sip = &(network->config().staticIps[sipk]);
398 if (sip->ss_family == AF_INET6) {
399 my6 = reinterpret_cast<const uint8_t *>(reinterpret_cast<const struct sockaddr_in6 *>(&(*sip))->sin6_addr.s6_addr);
400 const unsigned int sipNetmaskBits = Utils::ntoh((uint16_t)reinterpret_cast<const struct sockaddr_in6 *>(&(*sip))->sin6_port);
401 if ((sipNetmaskBits == 88)&&(my6[0] == 0xfd)&&(my6[9] == 0x99)&&(my6[10] == 0x93)) { // ZT-RFC4193 /88 ???
402 unsigned int ptr = 0;
403 while (ptr != 11) {
404 if (pkt6[ptr] != my6[ptr])
405 break;
406 ++ptr;
407 }
408 if (ptr == 11) { // prefix match!
409 v6EmbeddedAddress.setTo(pkt6 + ptr,5);
410 break;
411 }
412 } else if (sipNetmaskBits == 40) { // ZT-6PLANE /40 ???
413 const uint32_t nwid32 = (uint32_t)((network->id() ^ (network->id() >> 32)) & 0xffffffff);
414 if ( (my6[0] == 0xfc) && (my6[1] == (uint8_t)((nwid32 >> 24) & 0xff)) && (my6[2] == (uint8_t)((nwid32 >> 16) & 0xff)) && (my6[3] == (uint8_t)((nwid32 >> 8) & 0xff)) && (my6[4] == (uint8_t)(nwid32 & 0xff))) {
415 unsigned int ptr = 0;
416 while (ptr != 5) {
417 if (pkt6[ptr] != my6[ptr])
418 break;
419 ++ptr;
420 }
421 if (ptr == 5) { // prefix match!
422 v6EmbeddedAddress.setTo(pkt6 + ptr,5);
423 break;
424 }
425 }
426 }
427 }
428 }
429
430 if ((v6EmbeddedAddress)&&(v6EmbeddedAddress != RR->identity.address())) {
431 const MAC peerMac(v6EmbeddedAddress,network->id());
432
433 uint8_t adv[72];
434 adv[0] = 0x60; adv[1] = 0x00; adv[2] = 0x00; adv[3] = 0x00;
435 adv[4] = 0x00; adv[5] = 0x20;
436 adv[6] = 0x3a; adv[7] = 0xff;
437 for(int i=0;i<16;++i) adv[8 + i] = pkt6[i];
438 for(int i=0;i<16;++i) adv[24 + i] = my6[i];
439 adv[40] = 0x88; adv[41] = 0x00;
440 adv[42] = 0x00; adv[43] = 0x00; // future home of checksum
441 adv[44] = 0x60; adv[45] = 0x00; adv[46] = 0x00; adv[47] = 0x00;
442 for(int i=0;i<16;++i) adv[48 + i] = pkt6[i];
443 adv[64] = 0x02; adv[65] = 0x01;
444 adv[66] = peerMac[0]; adv[67] = peerMac[1]; adv[68] = peerMac[2]; adv[69] = peerMac[3]; adv[70] = peerMac[4]; adv[71] = peerMac[5];
445
446 uint16_t pseudo_[36];
447 uint8_t *const pseudo = reinterpret_cast<uint8_t *>(pseudo_);
448 for(int i=0;i<32;++i) pseudo[i] = adv[8 + i];
449 pseudo[32] = 0x00; pseudo[33] = 0x00; pseudo[34] = 0x00; pseudo[35] = 0x20;
450 pseudo[36] = 0x00; pseudo[37] = 0x00; pseudo[38] = 0x00; pseudo[39] = 0x3a;
451 for(int i=0;i<32;++i) pseudo[40 + i] = adv[40 + i];
452 uint32_t checksum = 0;
453 for(int i=0;i<36;++i) checksum += Utils::hton(pseudo_[i]);
454 while ((checksum >> 16)) checksum = (checksum & 0xffff) + (checksum >> 16);
455 checksum = ~checksum;
456 adv[42] = (checksum >> 8) & 0xff;
457 adv[43] = checksum & 0xff;
458
459 RR->node->putFrame(tPtr,network->id(),network->userPtr(),peerMac,from,ZT_ETHERTYPE_IPV6,0,adv,72);
460 return; // NDP emulation done. We have forged a "fake" reply, so no need to send actual NDP query.
461 } // else no NDP emulation
462 } // else no NDP emulation
463 }
464
465 // Check this after NDP emulation, since that has to be allowed in exactly this case
466 if (network->config().multicastLimit == 0) {
467 RR->t->outgoingNetworkFrameDropped(tPtr,network,from,to,etherType,vlanId,len,"multicast disabled");
468 return;
469 }
470
471 /* Learn multicast groups for bridged-in hosts.
472 * Note that some OSes, most notably Linux, do this for you by learning
473 * multicast addresses on bridge interfaces and subscribing each slave.
474 * But in that case this does no harm, as the sets are just merged. */
475 if (fromBridged)
476 network->learnBridgedMulticastGroup(tPtr,multicastGroup,RR->node->now());
477
478 // First pass sets noTee to false, but noTee is set to true in OutboundMulticast to prevent duplicates.
479 if (!network->filterOutgoingPacket(tPtr,false,RR->identity.address(),Address(),from,to,(const uint8_t *)data,len,etherType,vlanId,qosBucket)) {
480 RR->t->outgoingNetworkFrameDropped(tPtr,network,from,to,etherType,vlanId,len,"filter blocked");
481 return;
482 }
483
484 RR->mc->send(
485 tPtr,
486 RR->node->now(),
487 network,
488 Address(),
489 multicastGroup,
490 (fromBridged) ? from : MAC(),
491 etherType,
492 data,
493 len);
494 } else if (to == network->mac()) {
495 // Destination is this node, so just reinject it
496 RR->node->putFrame(tPtr,network->id(),network->userPtr(),from,to,etherType,vlanId,data,len);
497 } else if (to[0] == MAC::firstOctetForNetwork(network->id())) {
498 // Destination is another ZeroTier peer on the same network
499
500 Address toZT(to.toAddress(network->id())); // since in-network MACs are derived from addresses and network IDs, we can reverse this
501 SharedPtr<Peer> toPeer(RR->topology->getPeer(tPtr,toZT));
502
503 if (!network->filterOutgoingPacket(tPtr,false,RR->identity.address(),toZT,from,to,(const uint8_t *)data,len,etherType,vlanId,qosBucket)) {
504 RR->t->outgoingNetworkFrameDropped(tPtr,network,from,to,etherType,vlanId,len,"filter blocked");
505 return;
506 }
507
508 network->pushCredentialsIfNeeded(tPtr,toZT,RR->node->now());
509
510 if (!fromBridged) {
511 Packet outp(toZT,RR->identity.address(),Packet::VERB_FRAME);
512 outp.append(network->id());
513 outp.append((uint16_t)etherType);
514 outp.append(data,len);
515 // 1.4.8: disable compression for unicast as it almost never helps
516 //if (!network->config().disableCompression())
517 // outp.compress();
518 aqm_enqueue(tPtr,network,outp,true,qosBucket,flowId);
519 } else {
520 Packet outp(toZT,RR->identity.address(),Packet::VERB_EXT_FRAME);
521 outp.append(network->id());
522 outp.append((unsigned char)0x00);
523 to.appendTo(outp);
524 from.appendTo(outp);
525 outp.append((uint16_t)etherType);
526 outp.append(data,len);
527 // 1.4.8: disable compression for unicast as it almost never helps
528 //if (!network->config().disableCompression())
529 // outp.compress();
530 aqm_enqueue(tPtr,network,outp,true,qosBucket,flowId);
531 }
532 } else {
533 // Destination is bridged behind a remote peer
534
535 // We filter with a NULL destination ZeroTier address first. Filtrations
536 // for each ZT destination are also done below. This is the same rationale
537 // and design as for multicast.
538 if (!network->filterOutgoingPacket(tPtr,false,RR->identity.address(),Address(),from,to,(const uint8_t *)data,len,etherType,vlanId,qosBucket)) {
539 RR->t->outgoingNetworkFrameDropped(tPtr,network,from,to,etherType,vlanId,len,"filter blocked");
540 return;
541 }
542
543 Address bridges[ZT_MAX_BRIDGE_SPAM];
544 unsigned int numBridges = 0;
545
546 /* Create an array of up to ZT_MAX_BRIDGE_SPAM recipients for this bridged frame. */
547 bridges[0] = network->findBridgeTo(to);
548 std::vector<Address> activeBridges(network->config().activeBridges());
549 if ((bridges[0])&&(bridges[0] != RR->identity.address())&&(network->config().permitsBridging(bridges[0]))) {
550 /* We have a known bridge route for this MAC, send it there. */
551 ++numBridges;
552 } else if (!activeBridges.empty()) {
553 /* If there is no known route, spam to up to ZT_MAX_BRIDGE_SPAM active
554 * bridges. If someone responds, we'll learn the route. */
555 std::vector<Address>::const_iterator ab(activeBridges.begin());
556 if (activeBridges.size() <= ZT_MAX_BRIDGE_SPAM) {
557 // If there are <= ZT_MAX_BRIDGE_SPAM active bridges, spam them all
558 while (ab != activeBridges.end()) {
559 bridges[numBridges++] = *ab;
560 ++ab;
561 }
562 } else {
563 // Otherwise pick a random set of them
564 while (numBridges < ZT_MAX_BRIDGE_SPAM) {
565 if (ab == activeBridges.end())
566 ab = activeBridges.begin();
567 if (((unsigned long)RR->node->prng() % (unsigned long)activeBridges.size()) == 0) {
568 bridges[numBridges++] = *ab;
569 ++ab;
570 } else ++ab;
571 }
572 }
573 }
574
575 for(unsigned int b=0;b<numBridges;++b) {
576 if (network->filterOutgoingPacket(tPtr,true,RR->identity.address(),bridges[b],from,to,(const uint8_t *)data,len,etherType,vlanId,qosBucket)) {
577 Packet outp(bridges[b],RR->identity.address(),Packet::VERB_EXT_FRAME);
578 outp.append(network->id());
579 outp.append((uint8_t)0x00);
580 to.appendTo(outp);
581 from.appendTo(outp);
582 outp.append((uint16_t)etherType);
583 outp.append(data,len);
584 // 1.4.8: disable compression for unicast as it almost never helps
585 //if (!network->config().disableCompression())
586 // outp.compress();
587 aqm_enqueue(tPtr,network,outp,true,qosBucket,flowId);
588 } else {
589 RR->t->outgoingNetworkFrameDropped(tPtr,network,from,to,etherType,vlanId,len,"filter blocked (bridge replication)");
590 }
591 }
592 }
593 }
594
aqm_enqueue(void * tPtr,const SharedPtr<Network> & network,Packet & packet,bool encrypt,int qosBucket,int32_t flowId)595 void Switch::aqm_enqueue(void *tPtr, const SharedPtr<Network> &network, Packet &packet,bool encrypt,int qosBucket,int32_t flowId)
596 {
597 if(!network->qosEnabled()) {
598 send(tPtr, packet, encrypt, flowId);
599 return;
600 }
601 NetworkQoSControlBlock *nqcb = _netQueueControlBlock[network->id()];
602 if (!nqcb) {
603 nqcb = new NetworkQoSControlBlock();
604 _netQueueControlBlock[network->id()] = nqcb;
605 // Initialize ZT_QOS_NUM_BUCKETS queues and place them in the INACTIVE list
606 // These queues will be shuffled between the new/old/inactive lists by the enqueue/dequeue algorithm
607 for (int i=0; i<ZT_AQM_NUM_BUCKETS; i++) {
608 nqcb->inactiveQueues.push_back(new ManagedQueue(i));
609 }
610 }
611 // Don't apply QoS scheduling to ZT protocol traffic
612 if (packet.verb() != Packet::VERB_FRAME && packet.verb() != Packet::VERB_EXT_FRAME) {
613 send(tPtr, packet, encrypt, flowId);
614 }
615
616 _aqm_m.lock();
617
618 // Enqueue packet and move queue to appropriate list
619
620 const Address dest(packet.destination());
621 TXQueueEntry *txEntry = new TXQueueEntry(dest,RR->node->now(),packet,encrypt,flowId);
622
623 ManagedQueue *selectedQueue = nullptr;
624 for (size_t i=0; i<ZT_AQM_NUM_BUCKETS; i++) {
625 if (i < nqcb->oldQueues.size()) { // search old queues first (I think this is best since old would imply most recent usage of the queue)
626 if (nqcb->oldQueues[i]->id == qosBucket) {
627 selectedQueue = nqcb->oldQueues[i];
628 }
629 } if (i < nqcb->newQueues.size()) { // search new queues (this would imply not often-used queues)
630 if (nqcb->newQueues[i]->id == qosBucket) {
631 selectedQueue = nqcb->newQueues[i];
632 }
633 } if (i < nqcb->inactiveQueues.size()) { // search inactive queues
634 if (nqcb->inactiveQueues[i]->id == qosBucket) {
635 selectedQueue = nqcb->inactiveQueues[i];
636 // move queue to end of NEW queue list
637 selectedQueue->byteCredit = ZT_AQM_QUANTUM;
638 // DEBUG_INFO("moving q=%p from INACTIVE to NEW list", selectedQueue);
639 nqcb->newQueues.push_back(selectedQueue);
640 nqcb->inactiveQueues.erase(nqcb->inactiveQueues.begin() + i);
641 }
642 }
643 }
644 if (!selectedQueue) {
645 return;
646 }
647
648 selectedQueue->q.push_back(txEntry);
649 selectedQueue->byteLength+=txEntry->packet.payloadLength();
650 nqcb->_currEnqueuedPackets++;
651
652 // DEBUG_INFO("nq=%2lu, oq=%2lu, iq=%2lu, nqcb.size()=%3d, bucket=%2d, q=%p", nqcb->newQueues.size(), nqcb->oldQueues.size(), nqcb->inactiveQueues.size(), nqcb->_currEnqueuedPackets, qosBucket, selectedQueue);
653
654 // Drop a packet if necessary
655 ManagedQueue *selectedQueueToDropFrom = nullptr;
656 if (nqcb->_currEnqueuedPackets > ZT_AQM_MAX_ENQUEUED_PACKETS)
657 {
658 // DEBUG_INFO("too many enqueued packets (%d), finding packet to drop", nqcb->_currEnqueuedPackets);
659 int maxQueueLength = 0;
660 for (size_t i=0; i<ZT_AQM_NUM_BUCKETS; i++) {
661 if (i < nqcb->oldQueues.size()) {
662 if (nqcb->oldQueues[i]->byteLength > maxQueueLength) {
663 maxQueueLength = nqcb->oldQueues[i]->byteLength;
664 selectedQueueToDropFrom = nqcb->oldQueues[i];
665 }
666 } if (i < nqcb->newQueues.size()) {
667 if (nqcb->newQueues[i]->byteLength > maxQueueLength) {
668 maxQueueLength = nqcb->newQueues[i]->byteLength;
669 selectedQueueToDropFrom = nqcb->newQueues[i];
670 }
671 } if (i < nqcb->inactiveQueues.size()) {
672 if (nqcb->inactiveQueues[i]->byteLength > maxQueueLength) {
673 maxQueueLength = nqcb->inactiveQueues[i]->byteLength;
674 selectedQueueToDropFrom = nqcb->inactiveQueues[i];
675 }
676 }
677 }
678 if (selectedQueueToDropFrom) {
679 // DEBUG_INFO("dropping packet from head of largest queue (%d payload bytes)", maxQueueLength);
680 int sizeOfDroppedPacket = selectedQueueToDropFrom->q.front()->packet.payloadLength();
681 delete selectedQueueToDropFrom->q.front();
682 selectedQueueToDropFrom->q.pop_front();
683 selectedQueueToDropFrom->byteLength-=sizeOfDroppedPacket;
684 nqcb->_currEnqueuedPackets--;
685 }
686 }
687 _aqm_m.unlock();
688 aqm_dequeue(tPtr);
689 }
690
control_law(uint64_t t,int count)691 uint64_t Switch::control_law(uint64_t t, int count)
692 {
693 return (uint64_t)(t + ZT_AQM_INTERVAL / sqrt(count));
694 }
695
dodequeue(ManagedQueue * q,uint64_t now)696 Switch::dqr Switch::dodequeue(ManagedQueue *q, uint64_t now)
697 {
698 dqr r;
699 r.ok_to_drop = false;
700 r.p = q->q.front();
701
702 if (r.p == NULL) {
703 q->first_above_time = 0;
704 return r;
705 }
706 uint64_t sojourn_time = now - r.p->creationTime;
707 if (sojourn_time < ZT_AQM_TARGET || q->byteLength <= ZT_DEFAULT_MTU) {
708 // went below - stay below for at least interval
709 q->first_above_time = 0;
710 } else {
711 if (q->first_above_time == 0) {
712 // just went above from below. if still above at
713 // first_above_time, will say it's ok to drop.
714 q->first_above_time = now + ZT_AQM_INTERVAL;
715 } else if (now >= q->first_above_time) {
716 r.ok_to_drop = true;
717 }
718 }
719 return r;
720 }
721
CoDelDequeue(ManagedQueue * q,bool isNew,uint64_t now)722 Switch::TXQueueEntry * Switch::CoDelDequeue(ManagedQueue *q, bool isNew, uint64_t now)
723 {
724 dqr r = dodequeue(q, now);
725
726 if (q->dropping) {
727 if (!r.ok_to_drop) {
728 q->dropping = false;
729 }
730 while (now >= q->drop_next && q->dropping) {
731 q->q.pop_front(); // drop
732 r = dodequeue(q, now);
733 if (!r.ok_to_drop) {
734 // leave dropping state
735 q->dropping = false;
736 } else {
737 ++(q->count);
738 // schedule the next drop.
739 q->drop_next = control_law(q->drop_next, q->count);
740 }
741 }
742 } else if (r.ok_to_drop) {
743 q->q.pop_front(); // drop
744 r = dodequeue(q, now);
745 q->dropping = true;
746 q->count = (q->count > 2 && now - q->drop_next < 8*ZT_AQM_INTERVAL)?
747 q->count - 2 : 1;
748 q->drop_next = control_law(now, q->count);
749 }
750 return r.p;
751 }
752
aqm_dequeue(void * tPtr)753 void Switch::aqm_dequeue(void *tPtr)
754 {
755 // Cycle through network-specific QoS control blocks
756 for(std::map<uint64_t,NetworkQoSControlBlock*>::iterator nqcb(_netQueueControlBlock.begin());nqcb!=_netQueueControlBlock.end();) {
757 if (!(*nqcb).second->_currEnqueuedPackets) {
758 return;
759 }
760
761 uint64_t now = RR->node->now();
762 TXQueueEntry *entryToEmit = nullptr;
763 std::vector<ManagedQueue*> *currQueues = &((*nqcb).second->newQueues);
764 std::vector<ManagedQueue*> *oldQueues = &((*nqcb).second->oldQueues);
765 std::vector<ManagedQueue*> *inactiveQueues = &((*nqcb).second->inactiveQueues);
766
767 _aqm_m.lock();
768
769 // Attempt dequeue from queues in NEW list
770 bool examiningNewQueues = true;
771 while (currQueues->size()) {
772 ManagedQueue *queueAtFrontOfList = currQueues->front();
773 if (queueAtFrontOfList->byteCredit < 0) {
774 queueAtFrontOfList->byteCredit += ZT_AQM_QUANTUM;
775 // Move to list of OLD queues
776 // DEBUG_INFO("moving q=%p from NEW to OLD list", queueAtFrontOfList);
777 oldQueues->push_back(queueAtFrontOfList);
778 currQueues->erase(currQueues->begin());
779 } else {
780 entryToEmit = CoDelDequeue(queueAtFrontOfList, examiningNewQueues, now);
781 if (!entryToEmit) {
782 // Move to end of list of OLD queues
783 // DEBUG_INFO("moving q=%p from NEW to OLD list", queueAtFrontOfList);
784 oldQueues->push_back(queueAtFrontOfList);
785 currQueues->erase(currQueues->begin());
786 }
787 else {
788 int len = entryToEmit->packet.payloadLength();
789 queueAtFrontOfList->byteLength -= len;
790 queueAtFrontOfList->byteCredit -= len;
791 // Send the packet!
792 queueAtFrontOfList->q.pop_front();
793 send(tPtr, entryToEmit->packet, entryToEmit->encrypt, entryToEmit->flowId);
794 (*nqcb).second->_currEnqueuedPackets--;
795 }
796 if (queueAtFrontOfList) {
797 //DEBUG_INFO("dequeuing from q=%p, len=%lu in NEW list (byteCredit=%d)", queueAtFrontOfList, queueAtFrontOfList->q.size(), queueAtFrontOfList->byteCredit);
798 }
799 break;
800 }
801 }
802
803 // Attempt dequeue from queues in OLD list
804 examiningNewQueues = false;
805 currQueues = &((*nqcb).second->oldQueues);
806 while (currQueues->size()) {
807 ManagedQueue *queueAtFrontOfList = currQueues->front();
808 if (queueAtFrontOfList->byteCredit < 0) {
809 queueAtFrontOfList->byteCredit += ZT_AQM_QUANTUM;
810 oldQueues->push_back(queueAtFrontOfList);
811 currQueues->erase(currQueues->begin());
812 } else {
813 entryToEmit = CoDelDequeue(queueAtFrontOfList, examiningNewQueues, now);
814 if (!entryToEmit) {
815 //DEBUG_INFO("moving q=%p from OLD to INACTIVE list", queueAtFrontOfList);
816 // Move to inactive list of queues
817 inactiveQueues->push_back(queueAtFrontOfList);
818 currQueues->erase(currQueues->begin());
819 }
820 else {
821 int len = entryToEmit->packet.payloadLength();
822 queueAtFrontOfList->byteLength -= len;
823 queueAtFrontOfList->byteCredit -= len;
824 queueAtFrontOfList->q.pop_front();
825 send(tPtr, entryToEmit->packet, entryToEmit->encrypt, entryToEmit->flowId);
826 (*nqcb).second->_currEnqueuedPackets--;
827 }
828 if (queueAtFrontOfList) {
829 //DEBUG_INFO("dequeuing from q=%p, len=%lu in OLD list (byteCredit=%d)", queueAtFrontOfList, queueAtFrontOfList->q.size(), queueAtFrontOfList->byteCredit);
830 }
831 break;
832 }
833 }
834 nqcb++;
835 _aqm_m.unlock();
836 }
837 }
838
removeNetworkQoSControlBlock(uint64_t nwid)839 void Switch::removeNetworkQoSControlBlock(uint64_t nwid)
840 {
841 NetworkQoSControlBlock *nq = _netQueueControlBlock[nwid];
842 if (nq) {
843 _netQueueControlBlock.erase(nwid);
844 delete nq;
845 nq = NULL;
846 }
847 }
848
send(void * tPtr,Packet & packet,bool encrypt,int32_t flowId)849 void Switch::send(void *tPtr,Packet &packet,bool encrypt,int32_t flowId)
850 {
851 const Address dest(packet.destination());
852 if (dest == RR->identity.address())
853 return;
854 if (!_trySend(tPtr,packet,encrypt,flowId)) {
855 {
856 Mutex::Lock _l(_txQueue_m);
857 if (_txQueue.size() >= ZT_TX_QUEUE_SIZE) {
858 _txQueue.pop_front();
859 }
860 _txQueue.push_back(TXQueueEntry(dest,RR->node->now(),packet,encrypt,flowId));
861 }
862 if (!RR->topology->getPeer(tPtr,dest))
863 requestWhois(tPtr,RR->node->now(),dest);
864 }
865 }
866
requestWhois(void * tPtr,const int64_t now,const Address & addr)867 void Switch::requestWhois(void *tPtr,const int64_t now,const Address &addr)
868 {
869 if (addr == RR->identity.address())
870 return;
871
872 {
873 Mutex::Lock _l(_lastSentWhoisRequest_m);
874 int64_t &last = _lastSentWhoisRequest[addr];
875 if ((now - last) < ZT_WHOIS_RETRY_DELAY)
876 return;
877 else last = now;
878 }
879
880 const SharedPtr<Peer> upstream(RR->topology->getUpstreamPeer());
881 if (upstream) {
882 int32_t flowId = ZT_QOS_NO_FLOW;
883 Packet outp(upstream->address(),RR->identity.address(),Packet::VERB_WHOIS);
884 addr.appendTo(outp);
885 send(tPtr,outp,true,flowId);
886 }
887 }
888
doAnythingWaitingForPeer(void * tPtr,const SharedPtr<Peer> & peer)889 void Switch::doAnythingWaitingForPeer(void *tPtr,const SharedPtr<Peer> &peer)
890 {
891 {
892 Mutex::Lock _l(_lastSentWhoisRequest_m);
893 _lastSentWhoisRequest.erase(peer->address());
894 }
895
896 const int64_t now = RR->node->now();
897 for(unsigned int ptr=0;ptr<ZT_RX_QUEUE_SIZE;++ptr) {
898 RXQueueEntry *const rq = &(_rxQueue[ptr]);
899 Mutex::Lock rql(rq->lock);
900 if ((rq->timestamp)&&(rq->complete)) {
901 if ((rq->frag0.tryDecode(RR,tPtr,rq->flowId))||((now - rq->timestamp) > ZT_RECEIVE_QUEUE_TIMEOUT))
902 rq->timestamp = 0;
903 }
904 }
905
906 {
907 Mutex::Lock _l(_txQueue_m);
908 for(std::list< TXQueueEntry >::iterator txi(_txQueue.begin());txi!=_txQueue.end();) {
909 if (txi->dest == peer->address()) {
910 if (_trySend(tPtr,txi->packet,txi->encrypt,txi->flowId)) {
911 _txQueue.erase(txi++);
912 } else {
913 ++txi;
914 }
915 } else {
916 ++txi;
917 }
918 }
919 }
920 }
921
doTimerTasks(void * tPtr,int64_t now)922 unsigned long Switch::doTimerTasks(void *tPtr,int64_t now)
923 {
924 const uint64_t timeSinceLastCheck = now - _lastCheckedQueues;
925 if (timeSinceLastCheck < ZT_WHOIS_RETRY_DELAY)
926 return (unsigned long)(ZT_WHOIS_RETRY_DELAY - timeSinceLastCheck);
927 _lastCheckedQueues = now;
928
929 std::vector<Address> needWhois;
930 {
931 Mutex::Lock _l(_txQueue_m);
932
933 for(std::list< TXQueueEntry >::iterator txi(_txQueue.begin());txi!=_txQueue.end();) {
934 if (_trySend(tPtr,txi->packet,txi->encrypt,txi->flowId)) {
935 _txQueue.erase(txi++);
936 } else if ((now - txi->creationTime) > ZT_TRANSMIT_QUEUE_TIMEOUT) {
937 _txQueue.erase(txi++);
938 } else {
939 if (!RR->topology->getPeer(tPtr,txi->dest))
940 needWhois.push_back(txi->dest);
941 ++txi;
942 }
943 }
944 }
945 for(std::vector<Address>::const_iterator i(needWhois.begin());i!=needWhois.end();++i)
946 requestWhois(tPtr,now,*i);
947
948 for(unsigned int ptr=0;ptr<ZT_RX_QUEUE_SIZE;++ptr) {
949 RXQueueEntry *const rq = &(_rxQueue[ptr]);
950 Mutex::Lock rql(rq->lock);
951 if ((rq->timestamp)&&(rq->complete)) {
952 if ((rq->frag0.tryDecode(RR,tPtr,rq->flowId))||((now - rq->timestamp) > ZT_RECEIVE_QUEUE_TIMEOUT)) {
953 rq->timestamp = 0;
954 } else {
955 const Address src(rq->frag0.source());
956 if (!RR->topology->getPeer(tPtr,src))
957 requestWhois(tPtr,now,src);
958 }
959 }
960 }
961
962 {
963 Mutex::Lock _l(_lastUniteAttempt_m);
964 Hashtable< _LastUniteKey,uint64_t >::Iterator i(_lastUniteAttempt);
965 _LastUniteKey *k = (_LastUniteKey *)0;
966 uint64_t *v = (uint64_t *)0;
967 while (i.next(k,v)) {
968 if ((now - *v) >= (ZT_MIN_UNITE_INTERVAL * 8))
969 _lastUniteAttempt.erase(*k);
970 }
971 }
972
973 {
974 Mutex::Lock _l(_lastSentWhoisRequest_m);
975 Hashtable< Address,int64_t >::Iterator i(_lastSentWhoisRequest);
976 Address *a = (Address *)0;
977 int64_t *ts = (int64_t *)0;
978 while (i.next(a,ts)) {
979 if ((now - *ts) > (ZT_WHOIS_RETRY_DELAY * 2))
980 _lastSentWhoisRequest.erase(*a);
981 }
982 }
983
984 return ZT_WHOIS_RETRY_DELAY;
985 }
986
_shouldUnite(const int64_t now,const Address & source,const Address & destination)987 bool Switch::_shouldUnite(const int64_t now,const Address &source,const Address &destination)
988 {
989 Mutex::Lock _l(_lastUniteAttempt_m);
990 uint64_t &ts = _lastUniteAttempt[_LastUniteKey(source,destination)];
991 if ((now - ts) >= ZT_MIN_UNITE_INTERVAL) {
992 ts = now;
993 return true;
994 }
995 return false;
996 }
997
_trySend(void * tPtr,Packet & packet,bool encrypt,int32_t flowId)998 bool Switch::_trySend(void *tPtr,Packet &packet,bool encrypt,int32_t flowId)
999 {
1000 SharedPtr<Path> viaPath;
1001 const int64_t now = RR->node->now();
1002 const Address destination(packet.destination());
1003
1004 const SharedPtr<Peer> peer(RR->topology->getPeer(tPtr,destination));
1005 if (peer) {
1006 if ((peer->bondingPolicy() == ZT_BOND_POLICY_BROADCAST)
1007 && (packet.verb() == Packet::VERB_FRAME || packet.verb() == Packet::VERB_EXT_FRAME)) {
1008 const SharedPtr<Peer> relay(RR->topology->getUpstreamPeer());
1009 Mutex::Lock _l(peer->_paths_m);
1010 for(int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) {
1011 if (peer->_paths[i].p && peer->_paths[i].p->alive(now)) {
1012 _sendViaSpecificPath(tPtr,peer,peer->_paths[i].p,now,packet,encrypt,flowId);
1013 }
1014 }
1015 return true;
1016 }
1017 else {
1018 viaPath = peer->getAppropriatePath(now,false,flowId);
1019 if (!viaPath) {
1020 peer->tryMemorizedPath(tPtr,now); // periodically attempt memorized or statically defined paths, if any are known
1021 const SharedPtr<Peer> relay(RR->topology->getUpstreamPeer());
1022 if ( (!relay) || (!(viaPath = relay->getAppropriatePath(now,false,flowId))) ) {
1023 if (!(viaPath = peer->getAppropriatePath(now,true,flowId)))
1024 return false;
1025 }
1026 }
1027 if (viaPath) {
1028 _sendViaSpecificPath(tPtr,peer,viaPath,now,packet,encrypt,flowId);
1029 return true;
1030 }
1031 }
1032 }
1033 return false;
1034 }
1035
_sendViaSpecificPath(void * tPtr,SharedPtr<Peer> peer,SharedPtr<Path> viaPath,int64_t now,Packet & packet,bool encrypt,int32_t flowId)1036 void Switch::_sendViaSpecificPath(void *tPtr,SharedPtr<Peer> peer,SharedPtr<Path> viaPath,int64_t now,Packet &packet,bool encrypt,int32_t flowId)
1037 {
1038 unsigned int mtu = ZT_DEFAULT_PHYSMTU;
1039 uint64_t trustedPathId = 0;
1040 RR->topology->getOutboundPathInfo(viaPath->address(),mtu,trustedPathId);
1041
1042 unsigned int chunkSize = std::min(packet.size(),mtu);
1043 packet.setFragmented(chunkSize < packet.size());
1044
1045 if (trustedPathId) {
1046 packet.setTrusted(trustedPathId);
1047 } else {
1048 packet.armor(peer->key(),encrypt,peer->aesKeysIfSupported());
1049 RR->node->expectReplyTo(packet.packetId());
1050 }
1051
1052 peer->recordOutgoingPacket(viaPath, packet.packetId(), packet.payloadLength(), packet.verb(), flowId, now);
1053
1054 if (viaPath->send(RR,tPtr,packet.data(),chunkSize,now)) {
1055 if (chunkSize < packet.size()) {
1056 // Too big for one packet, fragment the rest
1057 unsigned int fragStart = chunkSize;
1058 unsigned int remaining = packet.size() - chunkSize;
1059 unsigned int fragsRemaining = (remaining / (mtu - ZT_PROTO_MIN_FRAGMENT_LENGTH));
1060 if ((fragsRemaining * (mtu - ZT_PROTO_MIN_FRAGMENT_LENGTH)) < remaining)
1061 ++fragsRemaining;
1062 const unsigned int totalFragments = fragsRemaining + 1;
1063
1064 for(unsigned int fno=1;fno<totalFragments;++fno) {
1065 chunkSize = std::min(remaining,(unsigned int)(mtu - ZT_PROTO_MIN_FRAGMENT_LENGTH));
1066 Packet::Fragment frag(packet,fragStart,chunkSize,fno,totalFragments);
1067 viaPath->send(RR,tPtr,frag.data(),frag.size(),now);
1068 fragStart += chunkSize;
1069 remaining -= chunkSize;
1070 }
1071 }
1072 }
1073 }
1074
1075 } // namespace ZeroTier
1076