1 #include "common.h"
2
3 extern struct uwsgi_server uwsgi;
4 struct uwsgi_tuntap utt;
5
6 /*
7
8 The tuntap router is a non-blocking highly optimized ip router
9 translating from tuntap device to socket streams.
10
11 It is meant as a replacement for the currently available networking namespaces
12 approaches. Compared to veth or macvlan it is really simple and allows total control
13 over the routing subsystem (in addition to a simple customizable firewalling engine)
14
15 Generally you spawn the tuntap router in the Emperor instance
16
17 [uwsgi]
18 master = true
19 emperor = /etc/uwsgi
20 emperor-use-clone = fs,uts,ipc,pid,net
21 tuntap-router = emperor0 /tmp/tuntap.socket
22 exec-as-root = ifconfig emperor0 192.168.0.1 netmask 255.255.255.0 up
23 exec-as-root = iptables -t nat -F
24 exec-as-root = iptables -t nat -A POSTROUTING -o eth0 -s 192.168.0.0/24 -j MASQUERADE
25 exec-as-root = echo 1 > /proc/sys/net/ipv4/ip_forward
26
27 vassals will run in new namespaces in which they create a tuntap device attached to
28 the tuntap router. UNIX sockets are the only way to connect to the tuntap router
29 after jailing.
30
31 Firewalling is based on 2 chains (in and out), and each rule is formed by 3 parameters: <action> <src> <dst>
32 The firewall is applied to traffic from the clients to the tuntap device (out) and the opposite (in)
33
34
35 The first matching rule stop the chain, if no rule applies, the policy is "allow"
36
37 the following rules allows access from vassals to the internet, but block
38 vassals intercommunication
39
40 --tuntap-router-firewall-out = allow 192.168.0.0/24 192.168.0.1
41 --tuntap-router-firewall-out = deny 192.168.0.0/24 192.168.0.0/24
42 --tuntap-router-firewall-out = allow 192.168.0.0/24 0.0.0.0
43 --tuntap-router-firewall-out = deny
44 --tuntap-router-firewall-in = allow 192.168.0.1 192.168.0.0/24
45 --tuntap-router-firewall-in = deny 192.168.0.0/24 192.168.0.0/24
46 --tuntap-router-firewall-in = allow 0.0.0.0 192.168.0.0/24
47 --tuntap-router-firewall-in = deny
48
49 Author: Roberto De Ioris
50
51 TODO:
52
53 - some form of security to disallow raw access to the tuntap router unix socket
54 - stats server
55 - port to other platforms ?
56
57 */
58
59 static struct uwsgi_option uwsgi_tuntap_options[] = {
60 {"tuntap-router", required_argument, 0, "run the tuntap router (syntax: <device> <socket> [stats] [gateway])", uwsgi_opt_add_string_list, &utt.routers, 0},
61 {"tuntap-device", required_argument, 0, "add a tuntap device to the instance (syntax: <device>[ <socket>])", uwsgi_opt_add_string_list, &utt.devices, 0},
62 {"tuntap-use-credentials", optional_argument, 0, "enable check of SCM_CREDENTIALS for tuntap client/server", uwsgi_opt_set_str, &utt.use_credentials, 0},
63 {"tuntap-router-firewall-in", required_argument, 0, "add a firewall rule to the tuntap router (syntax: <action> <src/mask> <dst/mask>)", uwsgi_tuntap_opt_firewall, &utt.fw_in, 0},
64 {"tuntap-router-firewall-out", required_argument, 0, "add a firewall rule to the tuntap router (syntax: <action> <src/mask> <dst/mask>)", uwsgi_tuntap_opt_firewall, &utt.fw_out, 0},
65 {"tuntap-router-route", required_argument, 0, "add a routing rule to the tuntap router (syntax: <src/mask> <dst/mask> <gateway>)", uwsgi_tuntap_opt_route, &utt.routes, 0},
66 {"tuntap-router-stats", required_argument, 0, "run the tuntap router stats server", uwsgi_opt_set_str, &utt.stats_server, 0},
67 {"tuntap-device-rule", required_argument, 0, "add a tuntap device rule (syntax: <direction> <src/mask> <dst/mask> <action> [target])", uwsgi_opt_add_string_list, &utt.device_rules, 0},
68 {NULL, 0, 0, NULL, NULL, NULL, 0},
69 };
70
uwsgi_tuntap_loop(void * arg)71 static void *uwsgi_tuntap_loop(void *arg) {
72
73 // block signals on this thread
74 sigset_t smask;
75 sigfillset(&smask);
76 #ifndef UWSGI_DEBUG
77 sigdelset(&smask, SIGSEGV);
78 #endif
79 pthread_sigmask(SIG_BLOCK, &smask, NULL);
80
81 struct uwsgi_tuntap_router *uttr = (struct uwsgi_tuntap_router *) arg;
82
83 uwsgi_socket_nb(uttr->fd);
84
85 if (!utt.buffer_size)
86 utt.buffer_size = 8192;
87 uttr->buf = uwsgi_malloc(utt.buffer_size);
88 uttr->write_buf = uwsgi_malloc(utt.buffer_size);
89
90 uttr->queue = event_queue_init();
91
92 if (event_queue_add_fd_read(uttr->queue, uttr->fd)) {
93 exit(1);
94 }
95 if (event_queue_add_fd_read(uttr->queue, uttr->server_fd)) {
96 exit(1);
97 }
98
99 uwsgi_socket_nb(uttr->server_fd);
100
101 struct uwsgi_tuntap_peer *uttp = uwsgi_tuntap_peer_create(uttr, uttr->server_fd, 0);
102
103 for (;;) {
104 int interesting_fd = -1;
105 int ret = event_queue_wait(uttr->queue, -1, &interesting_fd);
106
107 if (ret <= 0)
108 continue;
109
110 if (interesting_fd == uttr->fd) {
111 if (uttr->wait_for_write) {
112 uwsgi_tuntap_enqueue(uttr);
113 continue;
114 }
115 ssize_t rlen = read(uttr->fd, uttr->buf, utt.buffer_size);
116 if (rlen <= 0) {
117 uwsgi_error("uwsgi_tuntap_loop()/read()");
118 exit(1);
119 }
120
121 // check for full write buffer
122 if (uttp->write_buf_pktsize + 4 + rlen > utt.buffer_size) {
123 uttp->dropped++;
124 continue;
125 }
126
127 uint16_t pktsize = rlen;
128 char *ptr = uttp->write_buf + uttp->write_buf_pktsize;
129 memcpy(ptr + 4, uttr->buf, rlen);
130 ptr[0] = 0;
131 ptr[1] = (uint8_t) (pktsize & 0xff);
132 ptr[2] = (uint8_t) ((pktsize >> 8) & 0xff);
133 ptr[3] = 0;
134 uttp->write_buf_pktsize+= pktsize+4;
135 if (uwsgi_tuntap_peer_enqueue(uttr, uttp)) {
136 uwsgi_log_verbose("tuntap server disconnected...\n");
137 exit(1);
138 }
139 continue;
140 }
141
142
143 if (interesting_fd == uttr->server_fd) {
144 // read from the client
145 if (!uttp->wait_for_write) {
146 if (uwsgi_tuntap_peer_dequeue(uttr, uttp, 0)) {
147 uwsgi_log_verbose("tuntap server disconnected...\n");
148 exit(1);
149 }
150 }
151 else {
152 // something is wrong (the tuntap device is blocked)
153 if (uttr->wait_for_write) {
154 continue;
155 }
156
157 // write to the client
158 if (uwsgi_tuntap_peer_enqueue(uttr, uttp)) {
159 uwsgi_log_verbose("tuntap server disconnected...\n");
160 exit(1);
161 }
162 }
163 }
164 }
165
166 return NULL;
167 }
168
uwsgi_tuntap_client()169 static void uwsgi_tuntap_client() {
170
171 if (!utt.devices) return;
172
173 struct uwsgi_string_list *usl;
174 uwsgi_foreach(usl, utt.devices) {
175 char *space = strchr(usl->value, ' ');
176 if (space) {
177 *space = 0;
178 struct uwsgi_tuntap_router *uttr = uwsgi_calloc(sizeof(struct uwsgi_tuntap_router));
179 uttr->fd = uwsgi_tuntap_device(usl->value);
180
181 uttr->server_fd = uwsgi_connect(space+1, 30, 0);
182 if (uttr->server_fd < 0) {
183 uwsgi_error("uwsgi_tuntap_client()/uwsgi_connect()");
184 exit(1);
185 }
186 *space = ' ';
187
188 pthread_t t;
189 pthread_create(&t, NULL, uwsgi_tuntap_loop, uttr);
190 }
191 else {
192 if (uwsgi_tuntap_device(usl->value) < 0) {
193 // foo
194 }
195 }
196 }
197 }
198
199 void tuntaprouter_send_stats(struct uwsgi_tuntap_router *);
200
uwsgi_tuntap_router_loop(int id,void * arg)201 void uwsgi_tuntap_router_loop(int id, void *arg) {
202 int i;
203
204 struct uwsgi_tuntap_router *uttr = (struct uwsgi_tuntap_router *) arg;
205 uttr->buf = uwsgi_malloc(utt.buffer_size);
206 uttr->write_buf = uwsgi_malloc(utt.buffer_size);
207 uttr->queue = event_queue_init();
208
209 uttr->stats_server_fd = -1;
210 uttr->gateway_fd = -1;
211
212 void *events = event_queue_alloc(64);
213 if (event_queue_add_fd_read(uttr->queue, uttr->server_fd))
214 exit(1);
215 if (event_queue_add_fd_read(uttr->queue, uttr->fd))
216 exit(1);
217
218 char *stats_server = utt.stats_server;
219 if (uttr->stats_server) stats_server = uttr->stats_server;
220 if (stats_server) {
221 char *tcp_port = strchr(stats_server, ':');
222 if (tcp_port) {
223 // disable deferred accept for this socket
224 int current_defer_accept = uwsgi.no_defer_accept;
225 uwsgi.no_defer_accept = 1;
226 uttr->stats_server_fd = bind_to_tcp(stats_server, uwsgi.listen_queue, tcp_port);
227 uwsgi.no_defer_accept = current_defer_accept;
228 }
229 else {
230 uttr->stats_server_fd = bind_to_unix(stats_server, uwsgi.listen_queue, uwsgi.chmod_socket, uwsgi.abstract_socket);
231 }
232
233 if (event_queue_add_fd_read(uttr->queue, uttr->stats_server_fd)) exit(1);
234 uwsgi_log("*** tuntap stats server enabled on %s fd: %d ***\n", stats_server, uttr->stats_server_fd);
235 uwsgi_socket_nb(uttr->stats_server_fd);
236 }
237
238 if (uttr->gateway) {
239 uttr->gateway_fd = bind_to_udp(uttr->gateway, 0, 0);
240 if (uttr->gateway_fd < 0) exit(1);
241 if (event_queue_add_fd_read(uttr->queue, uttr->gateway_fd)) exit(1);
242 uwsgi_log("*** tuntap gateway address enabled on %s\n", uttr->gateway);
243 uttr->gateway_buf = uwsgi_malloc(utt.buffer_size);
244 uwsgi_socket_nb(uttr->gateway_fd);
245 }
246
247 for (;;) {
248 int nevents = event_queue_wait_multi(uttr->queue, -1, events, 64);
249 for (i = 0; i < nevents; i++) {
250 int interesting_fd = event_queue_interesting_fd(events, i);
251 if (interesting_fd == uttr->fd) {
252 // if writing, continue enqueuing
253 if (uttr->wait_for_write) {
254 uwsgi_tuntap_enqueue(uttr);
255 continue;
256 }
257 ssize_t rlen = read(uttr->fd, uttr->buf, utt.buffer_size);
258 if (rlen <= 0) {
259 uwsgi_error("uwsgi_tuntap_router_loop()/read()");
260 exit(1);
261 }
262 if (rlen < 20) continue;
263
264 if (uwsgi_tuntap_firewall_check(utt.fw_in, uttr->buf, rlen)) continue;
265
266
267 uint32_t *dst_ip = (uint32_t *) & uttr->buf[16];
268 struct uwsgi_tuntap_peer *uttp = uwsgi_tuntap_peer_get_by_addr(uttr, *dst_ip);
269 if (!uttp)
270 continue;
271
272 if (uwsgi_tuntap_peer_rules_check(uttr, uttp, uttr->buf, rlen, 0)) continue;
273
274 // check for full write buffer
275 if (uttp->write_buf_pktsize + 4 + rlen > utt.buffer_size) {
276 uttp->dropped++;
277 continue;
278 }
279
280 uint16_t pktsize = rlen;
281 char *ptr = uttp->write_buf + uttp->write_buf_pktsize;
282 memcpy(ptr + 4, uttr->buf, rlen);
283 ptr[0] = 0;
284 ptr[1] = (uint8_t) (pktsize & 0xff);
285 ptr[2] = (uint8_t) ((pktsize >> 8) & 0xff);
286 ptr[3] = 0;
287 uttp->write_buf_pktsize+= pktsize+4;
288 if (uwsgi_tuntap_peer_enqueue(uttr, uttp)) {
289 uwsgi_tuntap_peer_destroy(uttr, uttp);
290 }
291 continue;
292 }
293
294 if (interesting_fd == uttr->server_fd) {
295 int client_fd = uwsgi_accept(uttr->server_fd);
296 if (client_fd < 0) {
297 uwsgi_error("uwsgi_tuntap_server_loop()/accept()");
298 continue;
299 }
300 if (utt.use_credentials) {
301 if (uwsgi_socket_passcred(client_fd)) {
302 close(client_fd);
303 continue;
304 }
305 }
306 struct uwsgi_tuntap_peer *uttp = uwsgi_tuntap_peer_create(uttr, client_fd, 1);
307 if (event_queue_add_fd_read(uttr->queue, uttp->fd)) {
308 uwsgi_tuntap_peer_destroy(uttr, uttp);
309 }
310 continue;
311 }
312
313 if (uttr->stats_server_fd > -1 && interesting_fd == uttr->stats_server_fd) {
314 tuntaprouter_send_stats(uttr);
315 continue;
316 }
317
318 if (uttr->gateway_fd > -1 && interesting_fd == uttr->gateway_fd) {
319 ssize_t rlen = read(uttr->gateway_fd, uttr->gateway_buf, utt.buffer_size);
320 if (rlen <= 0) {
321 uwsgi_error("uwsgi_tuntap_router_loop()/read()");
322 continue;
323 }
324 if (rlen < 20) continue;
325 if (uwsgi_tuntap_firewall_check(utt.fw_in, uttr->gateway_buf, rlen)) continue;
326 uint32_t *dst_ip = (uint32_t *) & uttr->gateway_buf[16];
327 struct uwsgi_tuntap_peer *uttp = uwsgi_tuntap_peer_get_by_addr(uttr, *dst_ip);
328 if (!uttp)
329 continue;
330
331 // check for full write buffer
332 if (uttp->write_buf_pktsize + 4 + rlen > utt.buffer_size) {
333 uttp->dropped++;
334 continue;
335 }
336
337 uint16_t pktsize = rlen;
338 char *ptr = uttp->write_buf + uttp->write_buf_pktsize;
339 memcpy(ptr + 4, uttr->gateway_buf, rlen);
340 ptr[0] = 0;
341 ptr[1] = (uint8_t) (pktsize & 0xff);
342 ptr[2] = (uint8_t) ((pktsize >> 8) & 0xff);
343 ptr[3] = 0;
344 uttp->write_buf_pktsize+= pktsize+4;
345 if (uwsgi_tuntap_peer_enqueue(uttr, uttp)) {
346 uwsgi_tuntap_peer_destroy(uttr, uttp);
347 }
348 continue;
349 }
350
351 struct uwsgi_tuntap_peer *uttp = uttr->peers_head;
352 while (uttp) {
353 if (interesting_fd == uttp->fd) {
354 // read from the client
355 if (event_queue_interesting_fd_is_read(events, i)) {
356 if (utt.use_credentials) {
357 if (uttp->addr == 0) {
358 if (!uttp->sent_credentials) {
359 if (uwsgi_recv_cred(uttp->fd, "uwsgi-tuntap", 12, &uttp->pid, &uttp->uid, &uttp->gid)) {
360 uwsgi_tuntap_peer_destroy(uttr, uttp);
361 break;
362 }
363 if (utt.addr_by_credentials) {
364 uttp->addr = utt.addr_by_credentials(uttp->pid, uttp->uid, uttp->gid);
365 if (!uttp->addr) {
366 uwsgi_tuntap_peer_destroy(uttr, uttp);
367 break;
368 }
369 if (uwsgi_tuntap_register_addr(uttr, uttp)) {
370 uwsgi_tuntap_peer_destroy(uttr, uttp);
371 break;
372 }
373 }
374 uttp->sent_credentials = 1;
375 break;
376 }
377 else {
378 // if credentials are sent and a function is available, destroy the peer (if addr is 0)
379 if (utt.addr_by_credentials) {
380 uwsgi_tuntap_peer_destroy(uttr, uttp);
381 break;
382 }
383 }
384 }
385 }
386
387 if (uwsgi_tuntap_peer_dequeue(uttr, uttp, 1)) {
388 uwsgi_tuntap_peer_destroy(uttr, uttp);
389 break;
390 }
391 }
392
393 if (event_queue_interesting_fd_is_write(events, i)) {
394 // something is wrong (the tuntap device is blocked)
395 if (uttr->wait_for_write)
396 break;
397
398 // write to the client
399 if (uwsgi_tuntap_peer_enqueue(uttr, uttp)) {
400 uwsgi_tuntap_peer_destroy(uttr, uttp);
401 break;
402 }
403 }
404 break;
405 }
406 uttp = uttp->next;
407 }
408 }
409 }
410 }
411
uwsgi_tuntap_router()412 static void uwsgi_tuntap_router() {
413
414 if (!utt.routers) return;
415
416 if (!utt.buffer_size)
417 utt.buffer_size = 8192;
418
419 if (utt.use_credentials) {
420 if (utt.use_credentials[0] != 0 && strcmp(utt.use_credentials, "true")) {
421 utt.addr_by_credentials = (uint32_t (*)(pid_t, uid_t, gid_t)) dlsym(RTLD_DEFAULT, utt.use_credentials);
422 if (!utt.addr_by_credentials) {
423 uwsgi_log("[uwsgi-tuntap] unable to find symbol %s\n", utt.use_credentials);
424 exit(1);
425 }
426 }
427 }
428
429 struct uwsgi_string_list *usl;
430 uwsgi_foreach(usl, utt.routers) {
431 size_t rlen = 0;
432 char **args = uwsgi_split_quoted(usl->value, usl->len, " \t", &rlen);
433 if (rlen < 2) {
434 uwsgi_log("invalid tuntap router syntax, must be <device> <socket> [stats] [gateway]\n");
435 exit(1);
436 }
437
438 struct uwsgi_tuntap_router *uttr = uwsgi_calloc(sizeof(struct uwsgi_tuntap_router));
439
440 uttr->server_fd = bind_to_unix(args[1], uwsgi.listen_queue, uwsgi.chmod_socket, uwsgi.abstract_socket);
441
442 uttr->fd = uwsgi_tuntap_device(args[0]);
443
444 if (rlen > 2) uttr->stats_server = args[2];
445 if (rlen > 3) uttr->gateway = args[3];
446
447 if (register_gateway("uWSGI tuntap router", uwsgi_tuntap_router_loop, uttr) == NULL) {
448 uwsgi_log("unable to register the tuntap server gateway\n");
449 exit(1);
450 }
451 }
452 }
453
tuntaprouter_send_stats(struct uwsgi_tuntap_router * uttr)454 void tuntaprouter_send_stats(struct uwsgi_tuntap_router *uttr) {
455
456 struct sockaddr_un client_src;
457 socklen_t client_src_len = 0;
458
459 int client_fd = accept(uttr->stats_server_fd, (struct sockaddr *) &client_src, &client_src_len);
460 if (client_fd < 0) {
461 uwsgi_error("tuntaprouter_send_stats()/accept()");
462 return;
463 }
464
465 if (uwsgi.stats_http) {
466 if (uwsgi_send_http_stats(client_fd)) {
467 close(client_fd);
468 return;
469 }
470 }
471
472 struct uwsgi_stats *us = uwsgi_stats_new(8192);
473
474 if (uwsgi_stats_keyval_comma(us, "version", UWSGI_VERSION)) goto end;
475 if (uwsgi_stats_keylong_comma(us, "pid", (unsigned long long) getpid())) goto end;
476 if (uwsgi_stats_keylong_comma(us, "uid", (unsigned long long) getuid())) goto end;
477 if (uwsgi_stats_keylong_comma(us, "gid", (unsigned long long) getgid())) goto end;
478
479 char *cwd = uwsgi_get_cwd();
480 if (uwsgi_stats_keyval_comma(us, "cwd", cwd)) goto end0;
481
482 if (uwsgi_stats_key(us , "peers")) goto end0;
483 if (uwsgi_stats_list_open(us)) goto end0;
484 struct uwsgi_tuntap_peer *uttp = uttr->peers_head;
485
486 while (uttp) {
487 if (uwsgi_stats_object_open(us)) goto end0;
488 if (uwsgi_stats_keyval_comma(us, "addr", uttp->ip)) goto end0;
489 if (uwsgi_stats_keylong_comma(us, "addr_32", uttp->addr)) goto end0;
490 if (uwsgi_stats_keylong_comma(us, "uid", uttp->uid)) goto end0;
491 if (uwsgi_stats_keylong_comma(us, "gid", uttp->gid)) goto end0;
492 if (uwsgi_stats_keylong_comma(us, "pid", uttp->pid)) goto end0;
493 if (uwsgi_stats_keylong_comma(us, "tx", uttp->tx)) goto end0;
494 if (uwsgi_stats_keylong_comma(us, "rx", uttp->rx)) goto end0;
495 if (uwsgi_stats_keylong(us, "dropped", uttp->dropped)) goto end0;
496 if (uwsgi_stats_object_close(us)) goto end0;
497 uttp = uttp->next;
498 if (uttp) {
499 if (uwsgi_stats_comma(us)) goto end0;
500 }
501 }
502
503 if (uwsgi_stats_list_close(us)) goto end0;
504
505 if (uwsgi_stats_object_close(us)) goto end0;
506
507 size_t remains = us->pos;
508 off_t pos = 0;
509 while(remains > 0) {
510 int ret = uwsgi_waitfd_write(client_fd, uwsgi.socket_timeout);
511 if (ret <= 0) {
512 goto end0;
513 }
514 ssize_t res = write(client_fd, us->base + pos, remains);
515 if (res <= 0) {
516 if (res < 0) {
517 uwsgi_error("tuntaprouter_send_stats()/write()");
518 }
519 goto end0;
520 }
521 pos += res;
522 remains -= res;
523 }
524
525 end0:
526 free(cwd);
527 end:
528 free(us->base);
529 free(us);
530 close(client_fd);
531 }
532
533
534 struct uwsgi_plugin tuntap_plugin = {
535 .name = "tuntap",
536 .options = uwsgi_tuntap_options,
537 .post_jail = uwsgi_tuntap_client,
538 .jail = uwsgi_tuntap_router,
539 };
540