1 /* This test is a demo of using get_socket_uid and get_socket_cookie 2 * helper function to do per socket based network traffic monitoring. 3 * It requires iptables version higher then 1.6.1. to load pinned eBPF 4 * program into the xt_bpf match. 5 * 6 * TEST: 7 * ./run_cookie_uid_helper_example.sh -option 8 * option: 9 * -t: do traffic monitoring test, the program will continuously 10 * print out network traffic happens after program started A sample 11 * output is shown below: 12 * 13 * cookie: 877, uid: 0x3e8, Pakcet Count: 20, Bytes Count: 11058 14 * cookie: 132, uid: 0x0, Pakcet Count: 2, Bytes Count: 286 15 * cookie: 812, uid: 0x3e8, Pakcet Count: 3, Bytes Count: 1726 16 * cookie: 802, uid: 0x3e8, Pakcet Count: 2, Bytes Count: 104 17 * cookie: 877, uid: 0x3e8, Pakcet Count: 20, Bytes Count: 11058 18 * cookie: 831, uid: 0x3e8, Pakcet Count: 2, Bytes Count: 104 19 * cookie: 0, uid: 0x0, Pakcet Count: 6, Bytes Count: 712 20 * cookie: 880, uid: 0xfffe, Pakcet Count: 1, Bytes Count: 70 21 * 22 * -s: do getsockopt SO_COOKIE test, the program will set up a pair of 23 * UDP sockets and send packets between them. And read out the traffic data 24 * directly from the ebpf map based on the socket cookie. 25 * 26 * Clean up: if using shell script, the script file will delete the iptables 27 * rule and unmount the bpf program when exit. Else the iptables rule need 28 * to be deleted by hand, see run_cookie_uid_helper_example.sh for detail. 29 */ 30 31 #define _GNU_SOURCE 32 33 #define offsetof(type, member) __builtin_offsetof(type, member) 34 #define ARRAY_SIZE(x) (sizeof(x) / sizeof(*(x))) 35 36 #include <arpa/inet.h> 37 #include <errno.h> 38 #include <error.h> 39 #include <limits.h> 40 #include <linux/bpf.h> 41 #include <linux/if_ether.h> 42 #include <net/if.h> 43 #include <signal.h> 44 #include <stdbool.h> 45 #include <stdint.h> 46 #include <stdio.h> 47 #include <stdlib.h> 48 #include <string.h> 49 #include <sys/socket.h> 50 #include <sys/stat.h> 51 #include <sys/types.h> 52 #include <unistd.h> 53 #include <bpf/bpf.h> 54 #include "bpf_insn.h" 55 56 #define PORT 8888 57 58 struct stats { 59 uint32_t uid; 60 uint64_t packets; 61 uint64_t bytes; 62 }; 63 64 static int map_fd, prog_fd; 65 66 static bool test_finish; 67 68 static void maps_create(void) 69 { 70 map_fd = bpf_create_map(BPF_MAP_TYPE_HASH, sizeof(uint32_t), 71 sizeof(struct stats), 100, 0); 72 if (map_fd < 0) 73 error(1, errno, "map create failed!\n"); 74 } 75 76 static void prog_load(void) 77 { 78 static char log_buf[1 << 16]; 79 80 struct bpf_insn prog[] = { 81 /* 82 * Save sk_buff for future usage. value stored in R6 to R10 will 83 * not be reset after a bpf helper function call. 84 */ 85 BPF_MOV64_REG(BPF_REG_6, BPF_REG_1), 86 /* 87 * pc1: BPF_FUNC_get_socket_cookie takes one parameter, 88 * R1: sk_buff 89 */ 90 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, 91 BPF_FUNC_get_socket_cookie), 92 /* pc2-4: save &socketCookie to r7 for future usage*/ 93 BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_0, -8), 94 BPF_MOV64_REG(BPF_REG_7, BPF_REG_10), 95 BPF_ALU64_IMM(BPF_ADD, BPF_REG_7, -8), 96 /* 97 * pc5-8: set up the registers for BPF_FUNC_map_lookup_elem, 98 * it takes two parameters (R1: map_fd, R2: &socket_cookie) 99 */ 100 BPF_LD_MAP_FD(BPF_REG_1, map_fd), 101 BPF_MOV64_REG(BPF_REG_2, BPF_REG_7), 102 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, 103 BPF_FUNC_map_lookup_elem), 104 /* 105 * pc9. if r0 != 0x0, go to pc+14, since we have the cookie 106 * stored already 107 * Otherwise do pc10-22 to setup a new data entry. 108 */ 109 BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, 14), 110 BPF_MOV64_REG(BPF_REG_1, BPF_REG_6), 111 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, 112 BPF_FUNC_get_socket_uid), 113 /* 114 * Place a struct stats in the R10 stack and sequentially 115 * place the member value into the memory. Packets value 116 * is set by directly place a IMM value 1 into the stack. 117 */ 118 BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_0, 119 -32 + (__s16)offsetof(struct stats, uid)), 120 BPF_ST_MEM(BPF_DW, BPF_REG_10, 121 -32 + (__s16)offsetof(struct stats, packets), 1), 122 /* 123 * __sk_buff is a special struct used for eBPF program to 124 * directly access some sk_buff field. 125 */ 126 BPF_LDX_MEM(BPF_W, BPF_REG_1, BPF_REG_6, 127 offsetof(struct __sk_buff, len)), 128 BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_1, 129 -32 + (__s16)offsetof(struct stats, bytes)), 130 /* 131 * add new map entry using BPF_FUNC_map_update_elem, it takes 132 * 4 parameters (R1: map_fd, R2: &socket_cookie, R3: &stats, 133 * R4: flags) 134 */ 135 BPF_LD_MAP_FD(BPF_REG_1, map_fd), 136 BPF_MOV64_REG(BPF_REG_2, BPF_REG_7), 137 BPF_MOV64_REG(BPF_REG_3, BPF_REG_10), 138 BPF_ALU64_IMM(BPF_ADD, BPF_REG_3, -32), 139 BPF_MOV64_IMM(BPF_REG_4, 0), 140 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, 141 BPF_FUNC_map_update_elem), 142 BPF_JMP_IMM(BPF_JA, 0, 0, 5), 143 /* 144 * pc24-30 update the packet info to a exist data entry, it can 145 * be done by directly write to pointers instead of using 146 * BPF_FUNC_map_update_elem helper function 147 */ 148 BPF_MOV64_REG(BPF_REG_9, BPF_REG_0), 149 BPF_MOV64_IMM(BPF_REG_1, 1), 150 BPF_ATOMIC_OP(BPF_DW, BPF_ADD, BPF_REG_9, BPF_REG_1, 151 offsetof(struct stats, packets)), 152 BPF_LDX_MEM(BPF_W, BPF_REG_1, BPF_REG_6, 153 offsetof(struct __sk_buff, len)), 154 BPF_ATOMIC_OP(BPF_DW, BPF_ADD, BPF_REG_9, BPF_REG_1, 155 offsetof(struct stats, bytes)), 156 BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_6, 157 offsetof(struct __sk_buff, len)), 158 BPF_EXIT_INSN(), 159 }; 160 prog_fd = bpf_load_program(BPF_PROG_TYPE_SOCKET_FILTER, prog, 161 ARRAY_SIZE(prog), "GPL", 0, 162 log_buf, sizeof(log_buf)); 163 if (prog_fd < 0) 164 error(1, errno, "failed to load prog\n%s\n", log_buf); 165 } 166 167 static void prog_attach_iptables(char *file) 168 { 169 int ret; 170 char rules[256]; 171 172 if (bpf_obj_pin(prog_fd, file)) 173 error(1, errno, "bpf_obj_pin"); 174 if (strlen(file) > 50) { 175 printf("file path too long: %s\n", file); 176 exit(1); 177 } 178 ret = snprintf(rules, sizeof(rules), 179 "iptables -A OUTPUT -m bpf --object-pinned %s -j ACCEPT", 180 file); 181 if (ret < 0 || ret >= sizeof(rules)) { 182 printf("error constructing iptables command\n"); 183 exit(1); 184 } 185 ret = system(rules); 186 if (ret < 0) { 187 printf("iptables rule update failed: %d/n", WEXITSTATUS(ret)); 188 exit(1); 189 } 190 } 191 192 static void print_table(void) 193 { 194 struct stats curEntry; 195 uint32_t curN = UINT32_MAX; 196 uint32_t nextN; 197 int res; 198 199 while (bpf_map_get_next_key(map_fd, &curN, &nextN) > -1) { 200 curN = nextN; 201 res = bpf_map_lookup_elem(map_fd, &curN, &curEntry); 202 if (res < 0) { 203 error(1, errno, "fail to get entry value of Key: %u\n", 204 curN); 205 } else { 206 printf("cookie: %u, uid: 0x%x, Packet Count: %lu," 207 " Bytes Count: %lu\n", curN, curEntry.uid, 208 curEntry.packets, curEntry.bytes); 209 } 210 } 211 } 212 213 static void udp_client(void) 214 { 215 struct sockaddr_in si_other = {0}; 216 struct sockaddr_in si_me = {0}; 217 struct stats dataEntry; 218 int s_rcv, s_send, i, recv_len; 219 char message = 'a'; 220 char buf; 221 uint64_t cookie; 222 int res; 223 socklen_t cookie_len = sizeof(cookie); 224 socklen_t slen = sizeof(si_other); 225 226 s_rcv = socket(PF_INET, SOCK_DGRAM, 0); 227 if (s_rcv < 0) 228 error(1, errno, "rcv socket creat failed!\n"); 229 si_other.sin_family = AF_INET; 230 si_other.sin_port = htons(PORT); 231 if (inet_aton("127.0.0.1", &si_other.sin_addr) == 0) 232 error(1, errno, "inet_aton\n"); 233 if (bind(s_rcv, (struct sockaddr *)&si_other, sizeof(si_other)) == -1) 234 error(1, errno, "bind\n"); 235 s_send = socket(PF_INET, SOCK_DGRAM, 0); 236 if (s_send < 0) 237 error(1, errno, "send socket creat failed!\n"); 238 res = getsockopt(s_send, SOL_SOCKET, SO_COOKIE, &cookie, &cookie_len); 239 if (res < 0) 240 printf("get cookie failed: %s\n", strerror(errno)); 241 res = bpf_map_lookup_elem(map_fd, &cookie, &dataEntry); 242 if (res != -1) 243 error(1, errno, "socket stat found while flow not active\n"); 244 for (i = 0; i < 10; i++) { 245 res = sendto(s_send, &message, sizeof(message), 0, 246 (struct sockaddr *)&si_other, slen); 247 if (res == -1) 248 error(1, errno, "send\n"); 249 if (res != sizeof(message)) 250 error(1, 0, "%uB != %luB\n", res, sizeof(message)); 251 recv_len = recvfrom(s_rcv, &buf, sizeof(buf), 0, 252 (struct sockaddr *)&si_me, &slen); 253 if (recv_len < 0) 254 error(1, errno, "receive\n"); 255 res = memcmp(&(si_other.sin_addr), &(si_me.sin_addr), 256 sizeof(si_me.sin_addr)); 257 if (res != 0) 258 error(1, EFAULT, "sender addr error: %d\n", res); 259 printf("Message received: %c\n", buf); 260 res = bpf_map_lookup_elem(map_fd, &cookie, &dataEntry); 261 if (res < 0) 262 error(1, errno, "lookup sk stat failed, cookie: %lu\n", 263 cookie); 264 printf("cookie: %lu, uid: 0x%x, Packet Count: %lu," 265 " Bytes Count: %lu\n\n", cookie, dataEntry.uid, 266 dataEntry.packets, dataEntry.bytes); 267 } 268 close(s_send); 269 close(s_rcv); 270 } 271 272 static int usage(void) 273 { 274 printf("Usage: ./run_cookie_uid_helper_example.sh" 275 " bpfObjName -option\n" 276 " -t traffic monitor test\n" 277 " -s getsockopt cookie test\n"); 278 return 1; 279 } 280 281 static void finish(int ret) 282 { 283 test_finish = true; 284 } 285 286 int main(int argc, char *argv[]) 287 { 288 int opt; 289 bool cfg_test_traffic = false; 290 bool cfg_test_cookie = false; 291 292 if (argc != 3) 293 return usage(); 294 while ((opt = getopt(argc, argv, "ts")) != -1) { 295 switch (opt) { 296 case 't': 297 cfg_test_traffic = true; 298 break; 299 case 's': 300 cfg_test_cookie = true; 301 break; 302 303 default: 304 printf("unknown option %c\n", opt); 305 usage(); 306 return -1; 307 } 308 } 309 maps_create(); 310 prog_load(); 311 prog_attach_iptables(argv[2]); 312 if (cfg_test_traffic) { 313 if (signal(SIGINT, finish) == SIG_ERR) 314 error(1, errno, "register SIGINT handler failed"); 315 if (signal(SIGTERM, finish) == SIG_ERR) 316 error(1, errno, "register SIGTERM handler failed"); 317 while (!test_finish) { 318 print_table(); 319 printf("\n"); 320 sleep(1); 321 } 322 } else if (cfg_test_cookie) { 323 udp_client(); 324 } 325 close(prog_fd); 326 close(map_fd); 327 return 0; 328 } 329