1 // SPDX-License-Identifier: GPL-2.0 2 // Copyright (c) 2017 Facebook 3 #include <stddef.h> 4 #include <stdbool.h> 5 #include <string.h> 6 #include <linux/pkt_cls.h> 7 #include <linux/bpf.h> 8 #include <linux/in.h> 9 #include <linux/if_ether.h> 10 #include <linux/ip.h> 11 #include <linux/ipv6.h> 12 #include <linux/icmp.h> 13 #include <linux/icmpv6.h> 14 #include <linux/tcp.h> 15 #include <linux/udp.h> 16 #include <bpf/bpf_helpers.h> 17 #include <bpf/bpf_endian.h> 18 19 static __always_inline __u32 rol32(__u32 word, unsigned int shift) 20 { 21 return (word << shift) | (word >> ((-shift) & 31)); 22 } 23 24 /* copy paste of jhash from kernel sources to make sure llvm 25 * can compile it into valid sequence of bpf instructions 26 */ 27 #define __jhash_mix(a, b, c) \ 28 { \ 29 a -= c; a ^= rol32(c, 4); c += b; \ 30 b -= a; b ^= rol32(a, 6); a += c; \ 31 c -= b; c ^= rol32(b, 8); b += a; \ 32 a -= c; a ^= rol32(c, 16); c += b; \ 33 b -= a; b ^= rol32(a, 19); a += c; \ 34 c -= b; c ^= rol32(b, 4); b += a; \ 35 } 36 37 #define __jhash_final(a, b, c) \ 38 { \ 39 c ^= b; c -= rol32(b, 14); \ 40 a ^= c; a -= rol32(c, 11); \ 41 b ^= a; b -= rol32(a, 25); \ 42 c ^= b; c -= rol32(b, 16); \ 43 a ^= c; a -= rol32(c, 4); \ 44 b ^= a; b -= rol32(a, 14); \ 45 c ^= b; c -= rol32(b, 24); \ 46 } 47 48 #define JHASH_INITVAL 0xdeadbeef 49 50 typedef unsigned int u32; 51 52 static __noinline 53 u32 jhash(const void *key, u32 length, u32 initval) 54 { 55 u32 a, b, c; 56 const unsigned char *k = key; 57 58 a = b = c = JHASH_INITVAL + length + initval; 59 60 while (length > 12) { 61 a += *(u32 *)(k); 62 b += *(u32 *)(k + 4); 63 c += *(u32 *)(k + 8); 64 __jhash_mix(a, b, c); 65 length -= 12; 66 k += 12; 67 } 68 switch (length) { 69 case 12: c += (u32)k[11]<<24; 70 case 11: c += (u32)k[10]<<16; 71 case 10: c += (u32)k[9]<<8; 72 case 9: c += k[8]; 73 case 8: b += (u32)k[7]<<24; 74 case 7: b += (u32)k[6]<<16; 75 case 6: b += (u32)k[5]<<8; 76 case 5: b += k[4]; 77 case 4: a += (u32)k[3]<<24; 78 case 3: a += (u32)k[2]<<16; 79 case 2: a += (u32)k[1]<<8; 80 case 1: a += k[0]; 81 __jhash_final(a, b, c); 82 case 0: /* Nothing left to add */ 83 break; 84 } 85 86 return c; 87 } 88 89 __noinline 90 u32 __jhash_nwords(u32 a, u32 b, u32 c, u32 initval) 91 { 92 a += initval; 93 b += initval; 94 c += initval; 95 __jhash_final(a, b, c); 96 return c; 97 } 98 99 __noinline 100 u32 jhash_2words(u32 a, u32 b, u32 initval) 101 { 102 return __jhash_nwords(a, b, 0, initval + JHASH_INITVAL + (2 << 2)); 103 } 104 105 struct flow_key { 106 union { 107 __be32 src; 108 __be32 srcv6[4]; 109 }; 110 union { 111 __be32 dst; 112 __be32 dstv6[4]; 113 }; 114 union { 115 __u32 ports; 116 __u16 port16[2]; 117 }; 118 __u8 proto; 119 }; 120 121 struct packet_description { 122 struct flow_key flow; 123 __u8 flags; 124 }; 125 126 struct ctl_value { 127 union { 128 __u64 value; 129 __u32 ifindex; 130 __u8 mac[6]; 131 }; 132 }; 133 134 struct vip_definition { 135 union { 136 __be32 vip; 137 __be32 vipv6[4]; 138 }; 139 __u16 port; 140 __u16 family; 141 __u8 proto; 142 }; 143 144 struct vip_meta { 145 __u32 flags; 146 __u32 vip_num; 147 }; 148 149 struct real_pos_lru { 150 __u32 pos; 151 __u64 atime; 152 }; 153 154 struct real_definition { 155 union { 156 __be32 dst; 157 __be32 dstv6[4]; 158 }; 159 __u8 flags; 160 }; 161 162 struct lb_stats { 163 __u64 v2; 164 __u64 v1; 165 }; 166 167 struct { 168 __uint(type, BPF_MAP_TYPE_HASH); 169 __uint(max_entries, 512); 170 __type(key, struct vip_definition); 171 __type(value, struct vip_meta); 172 } vip_map SEC(".maps"); 173 174 struct { 175 __uint(type, BPF_MAP_TYPE_LRU_HASH); 176 __uint(max_entries, 300); 177 __uint(map_flags, 1U << 1); 178 __type(key, struct flow_key); 179 __type(value, struct real_pos_lru); 180 } lru_cache SEC(".maps"); 181 182 struct { 183 __uint(type, BPF_MAP_TYPE_ARRAY); 184 __uint(max_entries, 12 * 655); 185 __type(key, __u32); 186 __type(value, __u32); 187 } ch_rings SEC(".maps"); 188 189 struct { 190 __uint(type, BPF_MAP_TYPE_ARRAY); 191 __uint(max_entries, 40); 192 __type(key, __u32); 193 __type(value, struct real_definition); 194 } reals SEC(".maps"); 195 196 struct { 197 __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); 198 __uint(max_entries, 515); 199 __type(key, __u32); 200 __type(value, struct lb_stats); 201 } stats SEC(".maps"); 202 203 struct { 204 __uint(type, BPF_MAP_TYPE_ARRAY); 205 __uint(max_entries, 16); 206 __type(key, __u32); 207 __type(value, struct ctl_value); 208 } ctl_array SEC(".maps"); 209 210 struct eth_hdr { 211 unsigned char eth_dest[6]; 212 unsigned char eth_source[6]; 213 unsigned short eth_proto; 214 }; 215 216 static __noinline __u64 calc_offset(bool is_ipv6, bool is_icmp) 217 { 218 __u64 off = sizeof(struct eth_hdr); 219 if (is_ipv6) { 220 off += sizeof(struct ipv6hdr); 221 if (is_icmp) 222 off += sizeof(struct icmp6hdr) + sizeof(struct ipv6hdr); 223 } else { 224 off += sizeof(struct iphdr); 225 if (is_icmp) 226 off += sizeof(struct icmphdr) + sizeof(struct iphdr); 227 } 228 return off; 229 } 230 231 static __attribute__ ((noinline)) 232 bool parse_udp(void *data, void *data_end, 233 bool is_ipv6, struct packet_description *pckt) 234 { 235 236 bool is_icmp = !((pckt->flags & (1 << 0)) == 0); 237 __u64 off = calc_offset(is_ipv6, is_icmp); 238 struct udphdr *udp; 239 udp = data + off; 240 241 if (udp + 1 > data_end) 242 return false; 243 if (!is_icmp) { 244 pckt->flow.port16[0] = udp->source; 245 pckt->flow.port16[1] = udp->dest; 246 } else { 247 pckt->flow.port16[0] = udp->dest; 248 pckt->flow.port16[1] = udp->source; 249 } 250 return true; 251 } 252 253 static __attribute__ ((noinline)) 254 bool parse_tcp(void *data, void *data_end, 255 bool is_ipv6, struct packet_description *pckt) 256 { 257 258 bool is_icmp = !((pckt->flags & (1 << 0)) == 0); 259 __u64 off = calc_offset(is_ipv6, is_icmp); 260 struct tcphdr *tcp; 261 262 tcp = data + off; 263 if (tcp + 1 > data_end) 264 return false; 265 if (tcp->syn) 266 pckt->flags |= (1 << 1); 267 if (!is_icmp) { 268 pckt->flow.port16[0] = tcp->source; 269 pckt->flow.port16[1] = tcp->dest; 270 } else { 271 pckt->flow.port16[0] = tcp->dest; 272 pckt->flow.port16[1] = tcp->source; 273 } 274 return true; 275 } 276 277 static __attribute__ ((noinline)) 278 bool encap_v6(struct xdp_md *xdp, struct ctl_value *cval, 279 struct packet_description *pckt, 280 struct real_definition *dst, __u32 pkt_bytes) 281 { 282 struct eth_hdr *new_eth; 283 struct eth_hdr *old_eth; 284 struct ipv6hdr *ip6h; 285 __u32 ip_suffix; 286 void *data_end; 287 void *data; 288 289 if (bpf_xdp_adjust_head(xdp, 0 - (int)sizeof(struct ipv6hdr))) 290 return false; 291 data = (void *)(long)xdp->data; 292 data_end = (void *)(long)xdp->data_end; 293 new_eth = data; 294 ip6h = data + sizeof(struct eth_hdr); 295 old_eth = data + sizeof(struct ipv6hdr); 296 if (new_eth + 1 > data_end || 297 old_eth + 1 > data_end || ip6h + 1 > data_end) 298 return false; 299 memcpy(new_eth->eth_dest, cval->mac, 6); 300 memcpy(new_eth->eth_source, old_eth->eth_dest, 6); 301 new_eth->eth_proto = 56710; 302 ip6h->version = 6; 303 ip6h->priority = 0; 304 memset(ip6h->flow_lbl, 0, sizeof(ip6h->flow_lbl)); 305 306 ip6h->nexthdr = IPPROTO_IPV6; 307 ip_suffix = pckt->flow.srcv6[3] ^ pckt->flow.port16[0]; 308 ip6h->payload_len = 309 bpf_htons(pkt_bytes + sizeof(struct ipv6hdr)); 310 ip6h->hop_limit = 4; 311 312 ip6h->saddr.in6_u.u6_addr32[0] = 1; 313 ip6h->saddr.in6_u.u6_addr32[1] = 2; 314 ip6h->saddr.in6_u.u6_addr32[2] = 3; 315 ip6h->saddr.in6_u.u6_addr32[3] = ip_suffix; 316 memcpy(ip6h->daddr.in6_u.u6_addr32, dst->dstv6, 16); 317 return true; 318 } 319 320 static __attribute__ ((noinline)) 321 bool encap_v4(struct xdp_md *xdp, struct ctl_value *cval, 322 struct packet_description *pckt, 323 struct real_definition *dst, __u32 pkt_bytes) 324 { 325 326 __u32 ip_suffix = bpf_ntohs(pckt->flow.port16[0]); 327 struct eth_hdr *new_eth; 328 struct eth_hdr *old_eth; 329 __u16 *next_iph_u16; 330 struct iphdr *iph; 331 __u32 csum = 0; 332 void *data_end; 333 void *data; 334 335 ip_suffix <<= 15; 336 ip_suffix ^= pckt->flow.src; 337 if (bpf_xdp_adjust_head(xdp, 0 - (int)sizeof(struct iphdr))) 338 return false; 339 data = (void *)(long)xdp->data; 340 data_end = (void *)(long)xdp->data_end; 341 new_eth = data; 342 iph = data + sizeof(struct eth_hdr); 343 old_eth = data + sizeof(struct iphdr); 344 if (new_eth + 1 > data_end || 345 old_eth + 1 > data_end || iph + 1 > data_end) 346 return false; 347 memcpy(new_eth->eth_dest, cval->mac, 6); 348 memcpy(new_eth->eth_source, old_eth->eth_dest, 6); 349 new_eth->eth_proto = 8; 350 iph->version = 4; 351 iph->ihl = 5; 352 iph->frag_off = 0; 353 iph->protocol = IPPROTO_IPIP; 354 iph->check = 0; 355 iph->tos = 1; 356 iph->tot_len = bpf_htons(pkt_bytes + sizeof(struct iphdr)); 357 /* don't update iph->daddr, since it will overwrite old eth_proto 358 * and multiple iterations of bpf_prog_run() will fail 359 */ 360 361 iph->saddr = ((0xFFFF0000 & ip_suffix) | 4268) ^ dst->dst; 362 iph->ttl = 4; 363 364 next_iph_u16 = (__u16 *) iph; 365 #pragma clang loop unroll(full) 366 for (int i = 0; i < sizeof(struct iphdr) >> 1; i++) 367 csum += *next_iph_u16++; 368 iph->check = ~((csum & 0xffff) + (csum >> 16)); 369 if (bpf_xdp_adjust_head(xdp, (int)sizeof(struct iphdr))) 370 return false; 371 return true; 372 } 373 374 static __attribute__ ((noinline)) 375 int swap_mac_and_send(void *data, void *data_end) 376 { 377 unsigned char tmp_mac[6]; 378 struct eth_hdr *eth; 379 380 eth = data; 381 memcpy(tmp_mac, eth->eth_source, 6); 382 memcpy(eth->eth_source, eth->eth_dest, 6); 383 memcpy(eth->eth_dest, tmp_mac, 6); 384 return XDP_TX; 385 } 386 387 static __attribute__ ((noinline)) 388 int send_icmp_reply(void *data, void *data_end) 389 { 390 struct icmphdr *icmp_hdr; 391 __u16 *next_iph_u16; 392 __u32 tmp_addr = 0; 393 struct iphdr *iph; 394 __u32 csum = 0; 395 __u64 off = 0; 396 397 if (data + sizeof(struct eth_hdr) 398 + sizeof(struct iphdr) + sizeof(struct icmphdr) > data_end) 399 return XDP_DROP; 400 off += sizeof(struct eth_hdr); 401 iph = data + off; 402 off += sizeof(struct iphdr); 403 icmp_hdr = data + off; 404 icmp_hdr->type = 0; 405 icmp_hdr->checksum += 0x0007; 406 iph->ttl = 4; 407 tmp_addr = iph->daddr; 408 iph->daddr = iph->saddr; 409 iph->saddr = tmp_addr; 410 iph->check = 0; 411 next_iph_u16 = (__u16 *) iph; 412 #pragma clang loop unroll(full) 413 for (int i = 0; i < sizeof(struct iphdr) >> 1; i++) 414 csum += *next_iph_u16++; 415 iph->check = ~((csum & 0xffff) + (csum >> 16)); 416 return swap_mac_and_send(data, data_end); 417 } 418 419 static __attribute__ ((noinline)) 420 int send_icmp6_reply(void *data, void *data_end) 421 { 422 struct icmp6hdr *icmp_hdr; 423 struct ipv6hdr *ip6h; 424 __be32 tmp_addr[4]; 425 __u64 off = 0; 426 427 if (data + sizeof(struct eth_hdr) 428 + sizeof(struct ipv6hdr) + sizeof(struct icmp6hdr) > data_end) 429 return XDP_DROP; 430 off += sizeof(struct eth_hdr); 431 ip6h = data + off; 432 off += sizeof(struct ipv6hdr); 433 icmp_hdr = data + off; 434 icmp_hdr->icmp6_type = 129; 435 icmp_hdr->icmp6_cksum -= 0x0001; 436 ip6h->hop_limit = 4; 437 memcpy(tmp_addr, ip6h->saddr.in6_u.u6_addr32, 16); 438 memcpy(ip6h->saddr.in6_u.u6_addr32, ip6h->daddr.in6_u.u6_addr32, 16); 439 memcpy(ip6h->daddr.in6_u.u6_addr32, tmp_addr, 16); 440 return swap_mac_and_send(data, data_end); 441 } 442 443 static __attribute__ ((noinline)) 444 int parse_icmpv6(void *data, void *data_end, __u64 off, 445 struct packet_description *pckt) 446 { 447 struct icmp6hdr *icmp_hdr; 448 struct ipv6hdr *ip6h; 449 450 icmp_hdr = data + off; 451 if (icmp_hdr + 1 > data_end) 452 return XDP_DROP; 453 if (icmp_hdr->icmp6_type == 128) 454 return send_icmp6_reply(data, data_end); 455 if (icmp_hdr->icmp6_type != 3) 456 return XDP_PASS; 457 off += sizeof(struct icmp6hdr); 458 ip6h = data + off; 459 if (ip6h + 1 > data_end) 460 return XDP_DROP; 461 pckt->flow.proto = ip6h->nexthdr; 462 pckt->flags |= (1 << 0); 463 memcpy(pckt->flow.srcv6, ip6h->daddr.in6_u.u6_addr32, 16); 464 memcpy(pckt->flow.dstv6, ip6h->saddr.in6_u.u6_addr32, 16); 465 return -1; 466 } 467 468 static __attribute__ ((noinline)) 469 int parse_icmp(void *data, void *data_end, __u64 off, 470 struct packet_description *pckt) 471 { 472 struct icmphdr *icmp_hdr; 473 struct iphdr *iph; 474 475 icmp_hdr = data + off; 476 if (icmp_hdr + 1 > data_end) 477 return XDP_DROP; 478 if (icmp_hdr->type == 8) 479 return send_icmp_reply(data, data_end); 480 if ((icmp_hdr->type != 3) || (icmp_hdr->code != 4)) 481 return XDP_PASS; 482 off += sizeof(struct icmphdr); 483 iph = data + off; 484 if (iph + 1 > data_end) 485 return XDP_DROP; 486 if (iph->ihl != 5) 487 return XDP_DROP; 488 pckt->flow.proto = iph->protocol; 489 pckt->flags |= (1 << 0); 490 pckt->flow.src = iph->daddr; 491 pckt->flow.dst = iph->saddr; 492 return -1; 493 } 494 495 static __attribute__ ((noinline)) 496 __u32 get_packet_hash(struct packet_description *pckt, 497 bool hash_16bytes) 498 { 499 if (hash_16bytes) 500 return jhash_2words(jhash(pckt->flow.srcv6, 16, 12), 501 pckt->flow.ports, 24); 502 else 503 return jhash_2words(pckt->flow.src, pckt->flow.ports, 504 24); 505 } 506 507 __attribute__ ((noinline)) 508 static bool get_packet_dst(struct real_definition **real, 509 struct packet_description *pckt, 510 struct vip_meta *vip_info, 511 bool is_ipv6, void *lru_map) 512 { 513 struct real_pos_lru new_dst_lru = { }; 514 bool hash_16bytes = is_ipv6; 515 __u32 *real_pos, hash, key; 516 __u64 cur_time; 517 518 if (vip_info->flags & (1 << 2)) 519 hash_16bytes = 1; 520 if (vip_info->flags & (1 << 3)) { 521 pckt->flow.port16[0] = pckt->flow.port16[1]; 522 memset(pckt->flow.srcv6, 0, 16); 523 } 524 hash = get_packet_hash(pckt, hash_16bytes); 525 if (hash != 0x358459b7 /* jhash of ipv4 packet */ && 526 hash != 0x2f4bc6bb /* jhash of ipv6 packet */) 527 return false; 528 key = 2 * vip_info->vip_num + hash % 2; 529 real_pos = bpf_map_lookup_elem(&ch_rings, &key); 530 if (!real_pos) 531 return false; 532 key = *real_pos; 533 *real = bpf_map_lookup_elem(&reals, &key); 534 if (!(*real)) 535 return false; 536 if (!(vip_info->flags & (1 << 1))) { 537 __u32 conn_rate_key = 512 + 2; 538 struct lb_stats *conn_rate_stats = 539 bpf_map_lookup_elem(&stats, &conn_rate_key); 540 541 if (!conn_rate_stats) 542 return true; 543 cur_time = bpf_ktime_get_ns(); 544 if ((cur_time - conn_rate_stats->v2) >> 32 > 0xffFFFF) { 545 conn_rate_stats->v1 = 1; 546 conn_rate_stats->v2 = cur_time; 547 } else { 548 conn_rate_stats->v1 += 1; 549 if (conn_rate_stats->v1 >= 1) 550 return true; 551 } 552 if (pckt->flow.proto == IPPROTO_UDP) 553 new_dst_lru.atime = cur_time; 554 new_dst_lru.pos = key; 555 bpf_map_update_elem(lru_map, &pckt->flow, &new_dst_lru, 0); 556 } 557 return true; 558 } 559 560 __attribute__ ((noinline)) 561 static void connection_table_lookup(struct real_definition **real, 562 struct packet_description *pckt, 563 void *lru_map) 564 { 565 566 struct real_pos_lru *dst_lru; 567 __u64 cur_time; 568 __u32 key; 569 570 dst_lru = bpf_map_lookup_elem(lru_map, &pckt->flow); 571 if (!dst_lru) 572 return; 573 if (pckt->flow.proto == IPPROTO_UDP) { 574 cur_time = bpf_ktime_get_ns(); 575 if (cur_time - dst_lru->atime > 300000) 576 return; 577 dst_lru->atime = cur_time; 578 } 579 key = dst_lru->pos; 580 *real = bpf_map_lookup_elem(&reals, &key); 581 } 582 583 /* don't believe your eyes! 584 * below function has 6 arguments whereas bpf and llvm allow maximum of 5 585 * but since it's _static_ llvm can optimize one argument away 586 */ 587 __attribute__ ((noinline)) 588 static int process_l3_headers_v6(struct packet_description *pckt, 589 __u8 *protocol, __u64 off, 590 __u16 *pkt_bytes, void *data, 591 void *data_end) 592 { 593 struct ipv6hdr *ip6h; 594 __u64 iph_len; 595 int action; 596 597 ip6h = data + off; 598 if (ip6h + 1 > data_end) 599 return XDP_DROP; 600 iph_len = sizeof(struct ipv6hdr); 601 *protocol = ip6h->nexthdr; 602 pckt->flow.proto = *protocol; 603 *pkt_bytes = bpf_ntohs(ip6h->payload_len); 604 off += iph_len; 605 if (*protocol == 45) { 606 return XDP_DROP; 607 } else if (*protocol == 59) { 608 action = parse_icmpv6(data, data_end, off, pckt); 609 if (action >= 0) 610 return action; 611 } else { 612 memcpy(pckt->flow.srcv6, ip6h->saddr.in6_u.u6_addr32, 16); 613 memcpy(pckt->flow.dstv6, ip6h->daddr.in6_u.u6_addr32, 16); 614 } 615 return -1; 616 } 617 618 __attribute__ ((noinline)) 619 static int process_l3_headers_v4(struct packet_description *pckt, 620 __u8 *protocol, __u64 off, 621 __u16 *pkt_bytes, void *data, 622 void *data_end) 623 { 624 struct iphdr *iph; 625 int action; 626 627 iph = data + off; 628 if (iph + 1 > data_end) 629 return XDP_DROP; 630 if (iph->ihl != 5) 631 return XDP_DROP; 632 *protocol = iph->protocol; 633 pckt->flow.proto = *protocol; 634 *pkt_bytes = bpf_ntohs(iph->tot_len); 635 off += 20; 636 if (iph->frag_off & 65343) 637 return XDP_DROP; 638 if (*protocol == IPPROTO_ICMP) { 639 action = parse_icmp(data, data_end, off, pckt); 640 if (action >= 0) 641 return action; 642 } else { 643 pckt->flow.src = iph->saddr; 644 pckt->flow.dst = iph->daddr; 645 } 646 return -1; 647 } 648 649 __attribute__ ((noinline)) 650 static int process_packet(void *data, __u64 off, void *data_end, 651 bool is_ipv6, struct xdp_md *xdp) 652 { 653 654 struct real_definition *dst = NULL; 655 struct packet_description pckt = { }; 656 struct vip_definition vip = { }; 657 struct lb_stats *data_stats; 658 void *lru_map = &lru_cache; 659 struct vip_meta *vip_info; 660 __u32 lru_stats_key = 513; 661 __u32 mac_addr_pos = 0; 662 __u32 stats_key = 512; 663 struct ctl_value *cval; 664 __u16 pkt_bytes; 665 __u8 protocol; 666 __u32 vip_num; 667 int action; 668 669 if (is_ipv6) 670 action = process_l3_headers_v6(&pckt, &protocol, off, 671 &pkt_bytes, data, data_end); 672 else 673 action = process_l3_headers_v4(&pckt, &protocol, off, 674 &pkt_bytes, data, data_end); 675 if (action >= 0) 676 return action; 677 protocol = pckt.flow.proto; 678 if (protocol == IPPROTO_TCP) { 679 if (!parse_tcp(data, data_end, is_ipv6, &pckt)) 680 return XDP_DROP; 681 } else if (protocol == IPPROTO_UDP) { 682 if (!parse_udp(data, data_end, is_ipv6, &pckt)) 683 return XDP_DROP; 684 } else { 685 return XDP_TX; 686 } 687 688 if (is_ipv6) 689 memcpy(vip.vipv6, pckt.flow.dstv6, 16); 690 else 691 vip.vip = pckt.flow.dst; 692 vip.port = pckt.flow.port16[1]; 693 vip.proto = pckt.flow.proto; 694 vip_info = bpf_map_lookup_elem(&vip_map, &vip); 695 if (!vip_info) { 696 vip.port = 0; 697 vip_info = bpf_map_lookup_elem(&vip_map, &vip); 698 if (!vip_info) 699 return XDP_PASS; 700 if (!(vip_info->flags & (1 << 4))) 701 pckt.flow.port16[1] = 0; 702 } 703 if (data_end - data > 1400) 704 return XDP_DROP; 705 data_stats = bpf_map_lookup_elem(&stats, &stats_key); 706 if (!data_stats) 707 return XDP_DROP; 708 data_stats->v1 += 1; 709 if (!dst) { 710 if (vip_info->flags & (1 << 0)) 711 pckt.flow.port16[0] = 0; 712 if (!(pckt.flags & (1 << 1)) && !(vip_info->flags & (1 << 1))) 713 connection_table_lookup(&dst, &pckt, lru_map); 714 if (dst) 715 goto out; 716 if (pckt.flow.proto == IPPROTO_TCP) { 717 struct lb_stats *lru_stats = 718 bpf_map_lookup_elem(&stats, &lru_stats_key); 719 720 if (!lru_stats) 721 return XDP_DROP; 722 if (pckt.flags & (1 << 1)) 723 lru_stats->v1 += 1; 724 else 725 lru_stats->v2 += 1; 726 } 727 if (!get_packet_dst(&dst, &pckt, vip_info, is_ipv6, lru_map)) 728 return XDP_DROP; 729 data_stats->v2 += 1; 730 } 731 out: 732 cval = bpf_map_lookup_elem(&ctl_array, &mac_addr_pos); 733 if (!cval) 734 return XDP_DROP; 735 if (dst->flags & (1 << 0)) { 736 if (!encap_v6(xdp, cval, &pckt, dst, pkt_bytes)) 737 return XDP_DROP; 738 } else { 739 if (!encap_v4(xdp, cval, &pckt, dst, pkt_bytes)) 740 return XDP_DROP; 741 } 742 vip_num = vip_info->vip_num; 743 data_stats = bpf_map_lookup_elem(&stats, &vip_num); 744 if (!data_stats) 745 return XDP_DROP; 746 data_stats->v1 += 1; 747 data_stats->v2 += pkt_bytes; 748 749 data = (void *)(long)xdp->data; 750 data_end = (void *)(long)xdp->data_end; 751 if (data + 4 > data_end) 752 return XDP_DROP; 753 *(u32 *)data = dst->dst; 754 return XDP_DROP; 755 } 756 757 SEC("xdp") 758 int balancer_ingress_v4(struct xdp_md *ctx) 759 { 760 void *data = (void *)(long)ctx->data; 761 void *data_end = (void *)(long)ctx->data_end; 762 struct eth_hdr *eth = data; 763 __u32 eth_proto; 764 __u32 nh_off; 765 766 nh_off = sizeof(struct eth_hdr); 767 if (data + nh_off > data_end) 768 return XDP_DROP; 769 eth_proto = bpf_ntohs(eth->eth_proto); 770 if (eth_proto == ETH_P_IP) 771 return process_packet(data, nh_off, data_end, 0, ctx); 772 else 773 return XDP_DROP; 774 } 775 776 SEC("xdp") 777 int balancer_ingress_v6(struct xdp_md *ctx) 778 { 779 void *data = (void *)(long)ctx->data; 780 void *data_end = (void *)(long)ctx->data_end; 781 struct eth_hdr *eth = data; 782 __u32 eth_proto; 783 __u32 nh_off; 784 785 nh_off = sizeof(struct eth_hdr); 786 if (data + nh_off > data_end) 787 return XDP_DROP; 788 eth_proto = bpf_ntohs(eth->eth_proto); 789 if (eth_proto == ETH_P_IPV6) 790 return process_packet(data, nh_off, data_end, 1, ctx); 791 else 792 return XDP_DROP; 793 } 794 795 char _license[] SEC("license") = "GPL"; 796