1 /*
2 * ufdbguardd.c - URLfilterDB
3 *
4 * ufdbGuard is copyrighted (C) 2005-2020 by URLfilterDB B.V. with all rights reserved.
5 *
6 * Parts of the ufdbGuard daemon are based on squidGuard.
7 *
8 * Multithreaded ufdbGuard daemon
9 *
10 * $Id: ufdbguardd.c,v 1.197 2020/11/02 13:32:04 root Exp root $
11 */
12
13 #undef UFDB_FREE_MEMORY
14
15 #include "ufdb.h"
16 #include "sg.h"
17 #include "ufdb_globals.h"
18 #include "ufdbdb.h"
19 #include "ufdblocks.h"
20 #include "ufdbchkport.h"
21 #include "httpsQueue.h"
22 #include "httpserver.h"
23 #include "ufdbHostnames.h"
24
25 #include <stdio.h>
26 #include <errno.h>
27 #include <string.h>
28 #include <pthread.h>
29 #include <ctype.h>
30 #include <signal.h>
31 #include <libgen.h>
32 #include <stdlib.h>
33 #include <pwd.h>
34 #include <unistd.h>
35 #include <time.h>
36 #include <dirent.h>
37 #include <fcntl.h>
38 #include <sys/stat.h>
39 #include <sys/types.h>
40 #include <sys/socket.h>
41 #include <sys/select.h>
42 #include <netinet/tcp.h>
43 #include <netinet/in.h>
44 #include <sys/wait.h>
45 #include <sys/time.h>
46 #include <sys/resource.h>
47 #include <sys/utsname.h>
48 #ifdef HAVE_SYS_SYSCALL_H
49 #include <sys/syscall.h>
50 #endif
51 #ifdef __linux__
52 #include <linux/unistd.h>
53 #endif
54 #include <sys/socket.h>
55 #if HAVE_UNIX_SOCKETS
56 #include <sys/un.h>
57 #endif
58 #include <netinet/in.h>
59 #include <netinet/tcp.h>
60 #include <arpa/inet.h>
61
62 #ifdef __linux__
63 #include <execinfo.h>
64 #endif
65
66 #ifdef _POSIX_PRIORITY_SCHEDULING
67 #include <sched.h>
68 #endif
69
70 #include <openssl/ssl.h>
71
72 #ifdef __cplusplus
73 extern "C" {
74 #endif
75
76 #define FULL 1
77 #define EMPTY 0
78
79 UFDB_GCC_ALIGN64
80 extern pthread_rwlock_t TheDynamicCategoriesLock;
81
82 #define _132THREADS 132
83
84 #ifndef UFDB_STACK_SIZE
85 #define UFDB_STACK_SIZE (400 * 1024)
86 #endif
87
88
89 /* enforce one single cache line per mutex/rwlock */
90 UFDB_GCC_ALIGN64
91 static ufdb_mutex sighup_mutex = ufdb_mutex_initializer;
92
93 #if defined(__GNUC__) && __GCC_HAVE_SYNC_COMPARE_AND_SWAP_4 && __SIZEOF_INT__ == 4
94 /* no mutex required */
95 #elif defined(__GNUC__) && __GCC_HAVE_SYNC_COMPARE_AND_SWAP_8 && __SIZEOF_INT__ == 8
96 /* no mutex required */
97 #else
98 UFDB_GCC_ALIGN64
99 static ufdb_mutex incrMutex = ufdb_mutex_initializer;
100 #endif
101
102 UFDB_GCC_ALIGN64
103 static pthread_rwlock_t TheDatabaseLock;
104
105 UFDB_GCC_ALIGN64
106 static volatile int n_active_workers = 0;
107 static int prev_n_workers = 0;
108 #define UFDB_MAX_WORKERS UFDB_MAX_THREADS
109
110 static const char * configFile = DEFAULT_CONFIGFILE;
111
112 static int badSignal = 0;
113 static pthread_t badSignalThread;
114
115 extern int httpsConnectionCacheSize;
116
117 static int portNumCmdLine = -1;
118
119 static pthread_t threadedSignalHandler = (pthread_t) 837;
120 static pthread_t housekeeper = (pthread_t) 837;
121 static pthread_t sockethandler = (pthread_t) 0;
122 static pthread_t dyn_userlist_handler = (pthread_t) 837;
123 static pthread_t dyn_iplist_handler = (pthread_t) 837;
124 static pthread_t dyn_domainlist_handler = (pthread_t) 837;
125 static pthread_t workers[UFDB_MAX_WORKERS+1] = { (pthread_t) 837, (pthread_t) 837 };
126 static pthread_t httpsthr[UFDB_NUM_HTTPS_VERIFIERS] = { (pthread_t) 837, (pthread_t) 837 };
127 static char * workerURLbuffers[UFDB_MAX_WORKERS+1];
128
129
130
131 static time_t lastUpload = 0;
132 static int testMode = 0;
133 static int testConfig = 0;
134 static int allowedToRemovePidFile = 0;
135 UFDB_GCC_ALIGN64
136 volatile unsigned long UFDBlookupCounter = 0; /* hot variable */
137 volatile unsigned long UFDBuploadSeqNo = 0;
138 volatile unsigned long UFDBedufilterCounter = 0;
139 volatile unsigned long UFDBtestBlockCounter = 0;
140 static int parseOnly = 0;
141 static int runAsDaemon = 1;
142 static pid_t httpDaemonPid = 0;
143 UFDB_GCC_ALIGN64
144 volatile unsigned long UFDBsafesearchCounter = 0; /* warm variable */
145 volatile unsigned long UFDBuncategorisedCounter = 0; /* warm variable */
146 volatile unsigned long UFDBblockCounter = 0; /* warm variable */
147 volatile int badSignalHandlerBusy = 0;
148
149 void BadSignalCaught( int signum );
150 static void uploadStatistics( const char * reason );
151 static void * worker_main( void * ptr );
152 static void * housekeeper_main( void * ptr );
153 static void * dynamic_userlist_updater_main( void * ptr );
154 static void * dynamic_iplist_updater_main( void * ptr );
155 static void * dynamic_domainlist_updater_main( void * ptr );
156
157 #define SECSOF8DAYS (60*60*24*8)
158 #define SECSOF2DAYS (60*60*24*2)
159
160 #define UFDB_DEBUG_YY 0
161
162 #if UFDB_DEBUG_YY
163 extern int yydebug;
164 #endif
165
166
usage(char option)167 static void usage( char option )
168 {
169 if (option != '\0')
170 fprintf( stderr, "unknown option '-%c'\n", option );
171
172 fprintf( stderr, "usage: ufdbguardd [-A] [-d] [-v] [-r] [-R] [-T] [-h] [-w %d-%d] [-p port] [-c <configfile>]\n",
173 UFDB_MIN_THREADS, UFDB_MAX_THREADS );
174 fprintf( stderr, "-v print version\n" );
175 fprintf( stderr, "-c FN use configuration file FN\n" );
176 fprintf( stderr, "-T test mode; do not block, only log which URLs might be blocked\n" );
177 fprintf( stderr, "-w N use N worker threads (default: %d)\n", ufdbGV.nWorkers );
178 fprintf( stderr, "-p N use TCP port N to listen on\n" );
179 fprintf( stderr, "-U user run with privileges of \"user\"\n" );
180 fprintf( stderr, "-d extra debug information in log file\n" );
181 fprintf( stderr, "-R debug RE matching\n" );
182 fprintf( stderr, "-r log all redirections\n" );
183 fprintf( stderr, "-N do not analyse uncategorised URLs (not recommended)\n" );
184 fprintf( stderr, "-h print this help text\n" );
185 fprintf( stderr, "-C verify verify a configuration file; must be the last option\n" );
186
187 if (option != '\0')
188 exit( 1 );
189 else
190 exit( 0 );
191 }
192
193
writePidFile(void)194 static int writePidFile( void )
195 {
196 FILE * fp;
197 char * fname;
198 mode_t oldumask;
199 struct stat dir;
200
201 UFDBraisePrivileges( ufdbGV.userName, "write pid file" );
202
203 if (ufdbGV.pidFilename == NULL) // pidfile is not set in the config file, so we use the default pid file
204 {
205 fname = (char *) UFDBGUARDD_PID_FILE;
206 if (stat( DEFAULT_PIDDIR, &dir ) != 0)
207 {
208 ufdbLogMessage( "The directory '%s' where the pid file is stored, does not exist. creating it.",
209 DEFAULT_PIDDIR );
210 if (mkdir( DEFAULT_PIDDIR, S_IRUSR|S_IWUSR|S_IXUSR| S_IRGRP|S_IWGRP|S_IXGRP ) < 0)
211 {
212 ufdbLogError( "Cannot create directory %s: %s", DEFAULT_PIDDIR, strerror(errno) );
213 UFDBdropPrivileges( ufdbGV.userName );
214 return 0;
215 }
216
217 if (ufdbGV.userName[0] != '\0')
218 {
219 struct passwd * user;
220 user = getpwnam( ufdbGV.userName );
221 if (user == NULL)
222 {
223 ufdbLogError( "user \"%s\" does not exist. cannot change ownership of %s *****",
224 ufdbGV.userName, DEFAULT_PIDDIR );
225 UFDBdropPrivileges( ufdbGV.userName );
226 return 0;
227 }
228 else
229 {
230 if (chown( DEFAULT_PIDDIR, user->pw_uid, user->pw_gid ) != 0)
231 {
232 ufdbLogError( "cannot change ownership of %s: %s", DEFAULT_PIDDIR, strerror(errno) );
233 UFDBdropPrivileges( ufdbGV.userName );
234 return 0;
235 }
236 if (chmod( DEFAULT_PIDDIR, S_IRUSR|S_IWUSR|S_IXUSR |S_IRGRP|S_IWGRP|S_IXGRP |S_IROTH|S_IXOTH ) != 0)
237 {
238 ufdbLogError( "cannot change directory permissions (chmod) of %s: %s",
239 DEFAULT_PIDDIR, strerror(errno) );
240 UFDBdropPrivileges( ufdbGV.userName );
241 return 0;
242 }
243 }
244 }
245 }
246 }
247 else
248 fname = ufdbGV.pidFilename;
249
250 if (unlink( fname ) < 0)
251 { ; }
252
253 /* make the pid file world readable */
254 oldumask = umask( S_IXUSR | S_IWGRP|S_IXGRP | S_IWOTH|S_IXOTH );
255 fp = fopen( fname, "w" );
256 umask( oldumask );
257 if (fp == NULL)
258 {
259 ufdbLogFatalError( "cannot write to PID file %s: %s\n"
260 "Check file and directory permission and ownership.",
261 fname, strerror(errno) );
262 UFDBdropPrivileges( ufdbGV.userName );
263 return 0;
264 }
265 else
266 {
267 struct passwd * user;
268 user = getpwnam( ufdbGV.userName );
269 if (user != NULL)
270 fchown( fileno(fp), user->pw_uid, user->pw_gid );
271 fprintf( fp, "%d\n", ufdbGV.pid );
272 fclose( fp );
273 ufdbLogMessage( "PID %d written to %s", ufdbGV.pid, fname );
274 allowedToRemovePidFile = 1;
275 }
276
277 UFDBdropPrivileges( ufdbGV.userName );
278
279 return 1;
280 }
281
282
removePidFile(void)283 static void removePidFile( void )
284 {
285 FILE * fp;
286 #if HAVE_UNIX_SOCKETS
287 char unixsocket[64];
288 #endif
289
290 if (ufdbGV.debug)
291 ufdbLogMessage( "removing pid file allowed=%d", allowedToRemovePidFile );
292
293 /* When the initialisation of the socket failed there is most likely
294 * already an ufdbguardd daemon running and we may not remove
295 * the pid file nor kill any ufdbhttpd daemon.
296 */
297 if (!allowedToRemovePidFile)
298 return;
299
300 #if HAVE_UNIX_SOCKETS
301 sprintf( unixsocket, "/tmp/ufdbguardd-%05d", ufdbGV.portNum );
302 if (unlink( unixsocket ) < 0 && errno != ENOENT)
303 ufdbLogError( "cannot remove socket %s: %s", unixsocket, strerror(errno) );
304 #endif
305
306 if (ufdbGV.pidFilename == NULL)
307 (void) unlink( (char *) UFDBGUARDD_PID_FILE );
308 else
309 (void) unlink( ufdbGV.pidFilename );
310
311 allowedToRemovePidFile = 0;
312
313 if (ufdbGV.httpdPort > 0)
314 {
315 UFDBraisePrivileges( ufdbGV.userName, "kill ufdbhttpd" );
316 fp = fopen( UFDBHTTPD_PID_FILE, "r" );
317 if (fp != NULL)
318 {
319 if (fscanf( fp, "%d", &httpDaemonPid ) == 1)
320 {
321 ufdbLogMessage( "sending TERM signal to the HTTP daemon (pid=%d)", httpDaemonPid );
322 if (kill( httpDaemonPid, SIGTERM ) < 0)
323 ufdbLogError( "cannot send TERM signal to ufdbhttpd with pid %d: %s",
324 httpDaemonPid, strerror(errno) );
325 /* TODO: find out why ufdbhttpd does not act on the TERM signal... */
326 usleep( 210000 );
327 (void) kill( httpDaemonPid, SIGKILL ); /* TODO: we might be killing an other process after a usleep() ... */
328 }
329 fclose( fp );
330 }
331 UFDBdropPrivileges( ufdbGV.userName );
332 }
333 }
334
335
logStatistics(const char * reason)336 static void logStatistics( const char * reason )
337 {
338 struct Acl * acl;
339 struct Category * cat;
340 struct AclCategory * aclcat;
341 struct Source * src;
342 char * logmsg;
343 char * lbuf;
344 char * implicitMsg;
345
346 ufdbLogMessage( "statistics: %s", reason );
347 ufdbLogMessage( "statistics: %lu URL lookups (%lu https). %lu URLs blocked. "
348 "%lu tunnels detected. %lu safe searches. %lu Youtube edufilter. "
349 "%lu uncategorised URLs. %lu nodot URLs. %lu clients. %lu users.",
350 UFDBlookupCounter,
351 UFDB_API_num_https,
352 UFDBblockCounter + UFDBtestBlockCounter,
353 ufdbGV.tunnelCounter,
354 UFDBsafesearchCounter,
355 UFDBedufilterCounter,
356 UFDBuncategorisedCounter,
357 UFDB_API_num_nodot_URL,
358 UFDBgetNumberOfRegisteredIPs(),
359 UFDBgetNumberOfRegisteredUsers() );
360
361 if (ufdbGV.catList == NULL || ufdbGV.sourceList == NULL || ufdbGV.aclList == NULL)
362 return;
363
364 logmsg = (char *) ufdbMalloc( 256 * 1024 );
365 implicitMsg = (char *) ufdbMalloc( 256 * 1024 );
366
367 for (cat = ufdbGV.catList; cat != NULL; cat = cat->next)
368 {
369 ufdbLogMessage( "statistics: category %s was matched %lu times and blocked %lu times",
370 cat->name, cat->nmatches, cat->nblocks );
371 }
372
373 for (src = ufdbGV.sourceList; src != NULL; src = src->next)
374 {
375 ufdbLogMessage( "statistics: source %s was matched %lu times and blocked %lu times",
376 src->name, src->nmatches, src->nblocks );
377 }
378
379 for (acl = ufdbGV.aclList; acl != NULL; acl = acl->next)
380 {
381 if (acl->pass == NULL)
382 continue;
383
384 lbuf = logmsg;
385 if (acl->within == UFDB_ACL_ELSE)
386 lbuf += sprintf( lbuf, "acl %s-else: ", acl->name );
387 else if (acl->within == UFDB_ACL_WITHIN)
388 lbuf += sprintf( lbuf, "acl %s-within-%s: ", acl->name, acl->time->name );
389 else if (acl->within == UFDB_ACL_OUTSIDE)
390 lbuf += sprintf( lbuf, "acl %s-outside-%s: ", acl->name, acl->time->name );
391 else
392 lbuf += sprintf( lbuf, "acl %s: ", acl->name );
393
394 for (aclcat = acl->pass; aclcat != NULL; aclcat = aclcat->next)
395 {
396 cat = aclcat->cat;
397 if (cat != NULL && !cat->active)
398 continue;
399
400 switch (aclcat->type)
401 {
402 case ACL_TYPE_TERMINATOR:
403 if (aclcat->access)
404 lbuf += sprintf( lbuf, " any" );
405 else
406 lbuf += sprintf( lbuf, " none" );
407 break;
408 case ACL_TYPE_DEFAULT:
409 *lbuf++ = ' ';
410 if (!aclcat->access)
411 *lbuf++ = '!';
412 lbuf += sprintf( lbuf, "%s", aclcat->name );
413 break;
414 case ACL_TYPE_INADDR:
415 lbuf += sprintf( lbuf, " in-addr" );
416 break;
417 }
418 lbuf += sprintf( lbuf, ":%lu/%lu", aclcat->nblocks, aclcat->nmatches );
419 }
420
421 implicitMsg[0] = '\0';
422 for (aclcat = acl->implicitPass; aclcat != NULL; aclcat = aclcat->next)
423 {
424 cat = aclcat->cat;
425 if (cat != NULL && !cat->active)
426 continue;
427
428 lbuf += sprintf( lbuf, " %s:%lu/%lu", aclcat->name, aclcat->nblocks, aclcat->nmatches );
429 }
430 if (implicitMsg[0] != '\0')
431 {
432 lbuf += sprintf( logmsg, " -- implicitly allowed: %s", implicitMsg );
433 }
434
435 ufdbLogMessage( "statistics: %s", logmsg );
436 }
437
438 ufdbFree( logmsg );
439 ufdbFree( implicitMsg );
440 }
441
442
startHttpDaemon(void)443 static void startHttpDaemon( void )
444 {
445 int i;
446 const char * argv[24];
447 struct stat statbuf;
448 char portStr[16];
449 char httpdbin[1024];
450
451 if (ufdbGV.debug || ufdbGV.debugHttpd)
452 ufdbLogMessage( "going to start ufdbhttpd" );
453
454 sprintf( httpdbin, "%s/%s", DEFAULT_BINDIR, "ufdbhttpd" );
455 if (stat( httpdbin, &statbuf ) < 0)
456 {
457 ufdbLogError( "cannot find the HTTP daemon executable (%s): %s *****", httpdbin, strerror(errno) );
458 return;
459 }
460
461 /* go back to user root to allow ufdbhttpd to use privileged ports */
462 UFDBraisePrivileges( ufdbGV.userName, "start ufdbhttpd" );
463
464 httpDaemonPid = fork();
465 if (httpDaemonPid == 0) /* child */
466 {
467 ufdbGV.pid = getpid();
468 i = 0;
469 argv[i++] = httpdbin;
470 if (ufdbGV.debug || ufdbGV.debugHttpd)
471 argv[i++] = "-d";
472 argv[i++] = "-p";
473 sprintf( portStr, "%d", ufdbGV.httpdPort );
474 argv[i++] = portStr;
475 argv[i++] = "-l";
476 if (ufdbGV.logDir == NULL)
477 argv[i++] = DEFAULT_LOGDIR;
478 else
479 argv[i++] = ufdbGV.logDir;
480 argv[i++] = "-I";
481 if (ufdbGV.httpdImagesDirectory[0] == '\0')
482 strcpy( ufdbGV.httpdImagesDirectory, DEFAULT_IMAGESDIR );
483 argv[i++] = ufdbGV.httpdImagesDirectory;
484 argv[i++] = "-i";
485 argv[i++] = ufdbGV.httpdInterface;
486 if (ufdbGV.userName[0] != '\0')
487 {
488 argv[i++] = "-U";
489 argv[i++] = ufdbGV.userName;
490 }
491 argv[i] = NULL;
492 ufdbLogMessage( "starting HTTP daemon ..." );
493 /* Note: the child has the logfile still open */
494 execv( httpdbin, (char* const*) argv );
495 ufdbLogError( "failed starting http daemon: execv failed: %s", strerror(errno) );
496 _exit( 2 );
497 }
498
499 UFDBdropPrivileges( ufdbGV.userName );
500
501 if (httpDaemonPid < 0)
502 {
503 ufdbLogError( "cannot start HTTP daemon: fork failed: %s", strerror(errno) );
504 httpDaemonPid = 0;
505 }
506 else
507 {
508 #if 0
509 int status;
510 pid_t pid;
511 #endif
512
513 ufdbLogMessage( "ufdbhttpd is started. pid %ld port=%d interface=%s images=%s",
514 (long) httpDaemonPid, ufdbGV.httpdPort, ufdbGV.httpdInterface, ufdbGV.httpdImagesDirectory );
515 #if 0
516 usleep( 300000 );
517 while ((pid = waitpid( -1, &status, WNOHANG )) > 0)
518 ;
519 #endif
520 }
521 }
522
523
UFDBinstance(void)524 static const char * UFDBinstance( void )
525 {
526 static char instance[8+1+8+1+4];
527 int hash;
528 struct Acl * acl;
529 struct AclCategory * acat;
530
531 hash = 0;
532 for (acl = ufdbGV.aclList; acl != NULL; acl = acl->next)
533 {
534 if (acl->source != NULL && acl->source->name != NULL)
535 hash += acl->source->name[0] * 17 + acl->source->name[1] * 97;
536 for (acat = acl->pass; acat != NULL; acat = acat->next)
537 {
538 if (acat->name != NULL)
539 hash ^= (hash << 3) + acat->name[0] * 7 + acat->name[1] * 13;
540 else
541 hash ^= (hash << 3) + 97;
542 }
543 }
544 sprintf( instance, "%08d-%08x", (int) getpid(), hash );
545 return instance;
546 }
547
548
adjustNumberOfWorkerThreads(void)549 static void adjustNumberOfWorkerThreads( void )
550 {
551 int i;
552 pthread_attr_t attr;
553
554 /* IF the check_https_tunnel_option is aggressive we need more worker threads */
555 /* Note that UFDB_MAX_THREADS currently is 1285 */
556 if (UFDBgetTunnelCheckMethod() == UFDB_API_HTTPS_CHECK_AGGRESSIVE &&
557 ufdbGV.nWorkers < UFDB_MIN_THREADS)
558 {
559 if (ufdbGV.debug)
560 ufdbLogMessage( "adjusting #worker threads: starting %d threads", UFDB_MIN_THREADS - ufdbGV.nWorkers );
561
562 pthread_attr_init( &attr );
563 pthread_attr_setscope( &attr, PTHREAD_SCOPE_SYSTEM );
564 #if HAVE_PTHREAD_ATTR_SETGUARDSIZE && UFDB_DEBUG
565 pthread_attr_setstacksize( &attr, UFDB_STACK_SIZE - (8 * 1024) );
566 pthread_attr_setguardsize( &attr, 8 * 1024 );
567 #else
568 pthread_attr_setstacksize( &attr, UFDB_STACK_SIZE );
569 #endif
570 for (i = ufdbGV.nWorkers; i < UFDB_MIN_THREADS; i++)
571 {
572 pthread_create( &workers[i], &attr, worker_main, (void *) ((long) i) );
573 if (i % 4 == 0)
574 {
575 #ifdef _POSIX_PRIORITY_SCHEDULING
576 sched_yield();
577 #else
578 usleep( 10000 );
579 #endif
580 }
581 }
582 ufdbGV.nWorkers = UFDB_MIN_THREADS;
583 prev_n_workers = ufdbGV.nWorkers;
584 ufdbLogMessage( "#threads is increased to %d because check-proxy-tunnels is set to \"aggressive\"",
585 ufdbGV.nWorkers );
586 }
587
588 if (ufdbGV.nWorkers > prev_n_workers)
589 {
590 if (ufdbGV.debug)
591 ufdbLogMessage( "adjusting #worker threads: starting %d threads", ufdbGV.nWorkers - prev_n_workers );
592
593 pthread_attr_init( &attr );
594 pthread_attr_setscope( &attr, PTHREAD_SCOPE_SYSTEM );
595 #if HAVE_PTHREAD_ATTR_SETGUARDSIZE && UFDB_DEBUG
596 pthread_attr_setstacksize( &attr, UFDB_STACK_SIZE - (8 * 1024) );
597 pthread_attr_setguardsize( &attr, 8 * 1024 );
598 #else
599 pthread_attr_setstacksize( &attr, UFDB_STACK_SIZE );
600 #endif
601 for (i = prev_n_workers; i < ufdbGV.nWorkers; i++)
602 {
603 pthread_create( &workers[i], &attr, worker_main, (void *) ((long) i) );
604 if (i % 4 == 0)
605 {
606 #ifdef _POSIX_PRIORITY_SCHEDULING
607 sched_yield();
608 #else
609 usleep( 10000 );
610 #endif
611 }
612 }
613 prev_n_workers = ufdbGV.nWorkers;
614 ufdbLogMessage( "#threads is increased because num-worker-thread is %d", ufdbGV.nWorkers );
615 }
616 }
617
618
Usr1SignalCaught(int sig)619 static void Usr1SignalCaught( int sig )
620 {
621 FILE * fp;
622 int oldStatus;
623
624 if (sig != 0)
625 { ; }
626
627 if (ufdbGV.terminating)
628 return;
629
630 ufdbLogMessage( "USR1 signal received for logfile rotation" );
631 oldStatus = UFDBchangeStatus( UFDB_API_STATUS_ROLLING_LOGFILE );
632 UFDBrotateLogfile();
633 if (ufdbGV.httpdPort > 0)
634 {
635 fp = fopen( UFDBHTTPD_PID_FILE, "r" );
636 if (fp != NULL)
637 {
638 if (1 != fscanf( fp, "%d", &httpDaemonPid ))
639 httpDaemonPid = -1;
640 fclose( fp );
641 }
642 else
643 httpDaemonPid = -1;
644
645 if (httpDaemonPid > 0)
646 {
647 int retval;
648
649 ufdbLogMessage( "sending USR1 signal to ufdbhttpd (pid=%d)", httpDaemonPid );
650 /* TODO: use setreuid() */
651 retval = kill( httpDaemonPid, SIGUSR1 );
652 if (retval != 0)
653 ufdbLogError( "cannot send USR1 signal to httpd daemon: %s", strerror( errno ) );
654 }
655 }
656 ufdbLogMessage( "USR1 signal received for logfile rotation" );
657 UFDBchangeStatus( oldStatus );
658 }
659
660
Usr2SignalCaught(int sig)661 static void Usr2SignalCaught( int sig )
662 {
663 int oldStatus;
664
665 if (sig != 0)
666 { ; }
667
668 if (ufdbGV.terminating)
669 return;
670
671 ufdbLogMessage( "USR2 signal received to trigger monitoring commands" );
672 oldStatus = UFDBchangeStatus( UFDB_API_STATUS_UPDATE );
673 UFDBchangeStatus( oldStatus );
674 ufdbLogMessage( "status update done" );
675 }
676
677
WinchSignalCaught(int sig)678 static void WinchSignalCaught( int sig )
679 {
680 int ret;
681
682 if (sig != 0)
683 { ; }
684
685 if (ufdbGV.terminating)
686 return;
687
688 ufdbLogMessage( "WinchSignalCaught: WINCH signal received to trigger execiplist refresh" );
689 ret = pthread_kill( dyn_iplist_handler, SIGWINCH );
690 if (ret != 0)
691 ufdbLogError( "WinchSignalCaught: error %d (%s) while sending WINCH signal to dyn_iplist_handler",
692 ret, strerror(ret) );
693 }
694
695
696 /*
697 * OpenBSD is very picky with its implementation of signals and
698 * does not allow violation of the POSIX standard that says that
699 * a signal handler may NOT raise a signal, so calling pthread_kill
700 * is not allowed.
701 * The signal handlers below do nothing. They are installed to
702 * tell the OS to send the signals to the process and the special
703 * signal interception threads will handle the signals asynchroneously.
704 *
705 * These signal handlers can do nothing, not even call
706 * ufdbLogMessage because it uses a mutex.
707 */
catchHUPSignal(int signal)708 static void catchHUPSignal( int signal )
709 {
710 if (signal != 0)
711 { ; }
712 }
713
714
catchUSR1Signal(int signal)715 static void catchUSR1Signal( int signal )
716 {
717 if (signal != 0)
718 { ; }
719 }
720
721
catchUSR2Signal(int signal)722 static void catchUSR2Signal( int signal )
723 {
724 if (signal != 0)
725 { ; }
726 }
727
728
catchWINCHSignal(int signal)729 static void catchWINCHSignal( int signal )
730 {
731 if (signal != 0)
732 { ; }
733 }
734
735
catchChildSignal(int signal)736 static void catchChildSignal( int signal )
737 {
738 /* For some reason, on OpenBSD the SIGCHLD does not get through
739 * to the signal_handler_thread() so we must do a waitpid() here.
740 */
741 if (signal != 0)
742 { ; }
743
744 #if defined(__OpenBSD__) || defined(__NetBSD__) || defined(__FreeBSD__)
745 int status;
746 pid_t pid;
747
748 while ((pid = waitpid( -1, &status, WNOHANG )) > 0)
749 ;
750 #endif
751 }
752
753
754 #if !UFDB_PRODUCE_CORE_DUMPS
catchAbortSignal(int signum)755 static void catchAbortSignal( int signum )
756 {
757 if (signum != 0)
758 { ; }
759
760 badSignal = SIGABRT;
761 badSignalHandlerBusy = 1;
762 ufdbGV.reconfig = UFDB_RECONFIGR_ABORT;
763 ufdbGV.terminating = 1;
764 __asm__ volatile ("" : : : "memory"); // soft barrier for compiler
765 (void) alarm( 0 );
766
767 /* how can it be? we already ignored SIGPIPE but saw the signal raised after a SIGABRT.
768 * So ignore it again...
769 */
770 ufdbSetSignalHandler( SIGPIPE, SIG_IGN );
771
772 ufdbLogMessage( "catchAbortSignal: ABORT signal was sent to ufdbguardd " UFDB_VERSION );
773 BadSignalCaught( SIGABRT );
774
775 removePidFile();
776
777 UFDBchangeStatus( UFDB_API_STATUS_FATAL_ERROR );
778 sleep( 1 );
779 sleep( 1 );
780 UFDBchangeStatus( UFDB_API_STATUS_TERMINATED );
781
782 exit( 3 );
783 }
784 #endif
785
786
catchTermSignal(int signum)787 void catchTermSignal( int signum )
788 {
789 /* do nothing. sigwait() in a thread will deal with the signal */
790 if (signum != 0)
791 { ; }
792 }
793
794
TermSignalCaught(int signum)795 static void TermSignalCaught( int signum )
796 {
797 if (signum != 0)
798 { ; }
799
800 if (ufdbGV.terminating)
801 {
802 (void) alarm( 0 );
803 ufdbLogMessage( "received %s signal but I was already terminating (reconfig=%d) *****",
804 signum == SIGINT ? "INT" : "TERM", ufdbGV.reconfig );
805 ufdbGV.reconfig = UFDB_RECONFIGR_TERM;
806 __asm__ volatile ("" : : : "memory"); // soft barrier for compiler
807 return;
808 }
809
810 ufdbGV.terminating = 1;
811 ufdbGV.reconfig = UFDB_RECONFIGR_TERM;
812 __asm__ volatile ("" : : : "memory"); // soft barrier for compiler
813 (void) alarm( 0 );
814
815 ufdbLogMessage( "received %s signal", signum == SIGINT ? "INT" : "TERM" );
816 uploadStatistics( signum == SIGINT ? "INT" : "TERM" );
817
818 UFDBchangeStatus( UFDB_API_STATUS_TERMINATED );
819
820 usleep( 100000 );
821 if (sockethandler != 0)
822 pthread_cancel( sockethandler ); // main() is waiting with pthread_join()
823
824 /* we are inside the signal_handler_thread and prefer that the main thread does all the cleanup */
825 /* do 8 short sleeps instead of one large sleep since a few of them will be interrupted by signals */
826 sleep( 1 );
827 sleep( 1 );
828 sleep( 1 );
829 sleep( 1 );
830 sleep( 1 );
831 sleep( 1 );
832 sleep( 1 );
833 sleep( 1 );
834
835 /* close all connections with clients */
836 UFDBcloseFilesNoLog();
837
838 removePidFile();
839 ufdbLogMessage( "TermSignalCaught: bye bye" );
840 _exit( 0 );
841 }
842
843
do_upload_crashreport(char * filename)844 static int do_upload_crashreport( char * filename )
845 {
846 int fd;
847 int s;
848 int port;
849 SSL * ssl;
850 int msglen;
851 int nwritten;
852 char * message;
853 char * content;
854 struct stat stbuf;
855 struct utsname sysinfo;
856
857 fd = open( filename, O_RDONLY );
858 if (fd < 0)
859 {
860 ufdbLogError( "cannot open crash report %s", filename );
861 return 0;
862 }
863
864 if (fstat( fd, &stbuf ) != 0)
865 {
866 ufdbLogError( "cannot fstat crash report %s: %s", filename, strerror(errno) );
867 close( fd );
868 return 0;
869 }
870
871 ssl = NULL;
872 port = 443;
873 s = UFDBopenSocket( UFDB_UPLOAD_UNCATEGORISED_URLS_WEBSITE, port );
874 if (s >= 0)
875 {
876 if (UFDBopenssl_connect( UFDB_UPLOAD_UNCATEGORISED_URLS_WEBSITE, port, s, &ssl ) != UFDB_API_OK)
877 {
878 ufdbLogError( "SSL/TLS handshake with %s:%d failed", UFDB_UPLOAD_UNCATEGORISED_URLS_WEBSITE, port );
879 close( s );
880 s = -1;
881 }
882 }
883 else if (ufdbGV.debug)
884 ufdbLogMessage( "do_upload_crashreport: cannot connect to %s:443 trying %s:80",
885 UFDB_UPLOAD_UNCATEGORISED_URLS_WEBSITE, UFDB_UPLOAD_UNCATEGORISED_URLS_WEBSITE );
886
887 if (s < 0)
888 {
889 port = 80;
890 s = UFDBopenSocket( UFDB_UPLOAD_UNCATEGORISED_URLS_WEBSITE, port );
891 if (s < 0)
892 {
893 ufdbLogError( "cannot upload crash report %s; connect to server %s on port 443 and port 80 failed with error: %s",
894 filename, UFDB_UPLOAD_UNCATEGORISED_URLS_WEBSITE, strerror(errno) );
895 close( fd );
896 return 0;
897 }
898 }
899
900 message = (char *) ufdbMalloc( 4096 + stbuf.st_size );
901
902 content = (char *) ufdbMalloc( stbuf.st_size + 1 );
903 if (read( fd, content, stbuf.st_size ) != stbuf.st_size)
904 {
905 ufdbLogError( "cannot read crash report %s : %s", filename, strerror(errno) );
906 ufdbLogMessage( "It is recommended to send the crash report in %s "
907 "by email to support@urlfilterdb.com *****",
908 filename );
909 close( s );
910 close( fd );
911 ufdbFree( message );
912 ufdbFree( content );
913 return 0;
914 }
915 content[stbuf.st_size] = '\0';
916
917 ufdbGetSysInfo( &sysinfo );
918 msglen = sprintf( message,
919 "POST /cgi-bin/crashreport.pl HTTP/1.1\r\n"
920 "Host: " UFDB_UPLOAD_UNCATEGORISED_URLS_WEBSITE "\r\n"
921 "User-Agent: ufdbGuardd-" UFDB_VERSION "\r\n"
922 "Content-Type: text/plain\r\n"
923 "Content-Length: %ld\r\n"
924 "Connection: close\r\n"
925 "X-NodeName: %s\r\n"
926 "X-NumThreads: %d\r\n"
927 "X-FileName: %s\r\n"
928 "\r\n"
929 "%s\r\n",
930 stbuf.st_size,
931 sysinfo.nodename,
932 ufdbGV.nWorkers,
933 filename,
934 content );
935
936 errno = 0;
937 if (port == 443)
938 nwritten = UFDBopenssl_write( message, msglen, ssl );
939 else
940 nwritten = (int) write( s, message, msglen );
941 if (nwritten != msglen)
942 ufdbLogError( "cannot upload crash report %s : nwritten %d, %s", filename, nwritten, strerror(errno) );
943
944 /* TODO: check HTTP reply status */
945
946 ufdbFree( message );
947 ufdbFree( content );
948 if (port == 443)
949 UFDBopenssl_close( ssl );
950 close( s );
951 close( fd );
952
953 return nwritten == msglen;
954 }
955
956
upload_crash_reports(void)957 static void upload_crash_reports( void )
958 {
959 int reports_seen;
960 int previous_status;
961 DIR * dirh;
962 struct dirent * entry;
963
964 reports_seen = 0;
965 previous_status = 0;
966 dirh = opendir( "/tmp" );
967 if (dirh != NULL)
968 {
969 while ((entry = readdir(dirh)) != NULL)
970 {
971 if (strncmp( entry->d_name, "urlfilterdb.crashreport.", 24 ) == 0)
972 {
973 int retval;
974 char fname[1024];
975 char newfname[1024];
976
977 reports_seen = 1;
978 sprintf( fname, "/tmp/%s", entry->d_name );
979 if (ufdbGV.uploadCrashReports)
980 {
981 if (do_upload_crashreport( fname ))
982 {
983 sprintf( newfname, "/tmp/uploaded.%s", entry->d_name );
984 UFDBraisePrivileges( ufdbGV.userName, "rename uploaded crash report" );
985 retval = rename( fname, newfname );
986 UFDBdropPrivileges( ufdbGV.userName );
987 if (retval == 0)
988 ufdbLogMessage( "uploaded and renamed crash report %s to %s", fname, newfname );
989 else
990 ufdbLogError( "could not rename crash report from %s to %s: %s",
991 fname, newfname, strerror(errno) );
992 }
993 else
994 {
995 if (!reports_seen)
996 previous_status = UFDBchangeStatus( UFDB_API_STATUS_CRASH_REPORT_UPLOADED );
997 else
998 (void) UFDBchangeStatus( UFDB_API_STATUS_CRASH_REPORT_UPLOADED );
999 }
1000 }
1001 else
1002 {
1003 ufdbLogMessage( "Not allowed to upload crash report '%s' *****", fname );
1004 ufdbLogMessage( "It is recommended to send the crash report to "
1005 "the support desk of urlfilterdb.com *****" );
1006 ufdbLogMessage( "Set upload-crash-reports to 'on' to upload crash reports. *****" );
1007 if (!reports_seen)
1008 previous_status = UFDBchangeStatus( UFDB_API_STATUS_CRASH_REPORT_NOT_UPLOADED );
1009 else
1010 (void) UFDBchangeStatus( UFDB_API_STATUS_CRASH_REPORT_NOT_UPLOADED );
1011 }
1012 }
1013 }
1014 closedir( dirh );
1015 }
1016
1017 if (reports_seen)
1018 (void) UFDBchangeStatus( previous_status );
1019 }
1020
1021
BadSignalCaught(int signum)1022 void BadSignalCaught( int signum )
1023 {
1024 int i;
1025 int num;
1026 char me[48];
1027
1028 /* We desperately want a crash report but repeating signals prevent this so ignore the bad signal */
1029 ufdbSetSignalHandler( signum, SIG_IGN );
1030 /* pclose() generates a SIGCHLD which we also want to ignore here */
1031 ufdbSetSignalHandler( SIGCHLD, SIG_IGN );
1032
1033 /* find out which thread has a signal */
1034 sprintf( me, "pid %d", (int) getpid() );
1035 if (badSignalThread == threadedSignalHandler)
1036 {
1037 strcpy( me, "thread signal-handler" );
1038 }
1039 else if (badSignalThread == housekeeper)
1040 {
1041 strcpy( me, "thread housekeeper" );
1042 }
1043 else if (badSignalThread == sockethandler)
1044 {
1045 strcpy( me, "thread socket-handler" );
1046 }
1047 else if (badSignalThread == dyn_userlist_handler)
1048 {
1049 strcpy( me, "thread dynamic-userlist-handler" );
1050 }
1051 else if (badSignalThread == dyn_iplist_handler)
1052 {
1053 strcpy( me, "thread dynamic-iplist-handler" );
1054 }
1055 else if (badSignalThread == dyn_domainlist_handler)
1056 {
1057 strcpy( me, "thread dynamic-domainlist-handler" );
1058 }
1059 else
1060 {
1061 num = UFDB_NUM_HTTPS_VERIFIERS;
1062 for (i = 0; i < num; i++)
1063 if (badSignalThread == httpsthr[i])
1064 {
1065 sprintf( me, "thread https-verifier-%02d", i );
1066 break;
1067 }
1068 num = ufdbGV.nWorkers;
1069 for (i = 0; i < num; i++)
1070 if (badSignalThread == workers[i])
1071 {
1072 sprintf( me, "thread worker-%03d", i );
1073 ufdbLogMessage( "worker %03d received signal %d while processing URL '%s'",
1074 i, signum, workerURLbuffers[i] );
1075 break;
1076 }
1077 }
1078 ufdbLogMessage( "%s caught bad signal %d *****", me, signum );
1079
1080 if (strcmp( UFDB_GDB_PATH, "no" ) == 0)
1081 {
1082 ufdbLogMessage( "ufdb-pstack is not called since the gdb package was not installed." );
1083 ufdbLogMessage( "It is highly recommended to install gdb and reinstall ufdbguard "
1084 "to be able to produce a crash report. *****" );
1085 ufdbLogMessage( "Crash reports are essential to be able to resolve bugs quickly." );
1086 for (i = 0; i < ufdbGV.nWorkers; i++)
1087 ufdbLogMessage( "worker %03d is processing URL '%s'", i, workerURLbuffers[i] );
1088 }
1089 else
1090 ufdbExecutePstack( me );
1091
1092 uploadStatistics( "BADSIG" );
1093
1094 UFDBchangeStatus( UFDB_API_STATUS_FATAL_ERROR );
1095 badSignalHandlerBusy = 0;
1096
1097 if (ufdbGV.debugCoreDump)
1098 {
1099 ufdbLogMessage( "sending ABRT signal to myself to generate a core dump" );
1100 ufdbSetSignalHandler( SIGABRT, SIG_DFL );
1101 kill( getpid(), SIGABRT );
1102 }
1103 }
1104
1105
crasher(void)1106 static void crasher( void )
1107 {
1108 if (badSignal == 0)
1109 badSignal = SIGILL;
1110 badSignalHandlerBusy = 1;
1111 ufdbLogFatalError( "crasher: fatal error occurred and this program is about to crash" );
1112 BadSignalCaught( SIGABRT );
1113 sleep( 1 );
1114 exit( 9 );
1115 }
1116
1117
__wrap___stack_chk_fail(void)1118 void __wrap___stack_chk_fail( void )
1119 {
1120 badSignalHandlerBusy = 1;
1121 #if 0
1122 ufdbGV.reconfig = UFDB_RECONFIGR_STACK;
1123 ufdbGV.terminating = 1;
1124 __asm__ volatile ("" : : : "memory"); // soft barrier for compiler
1125 #endif
1126 (void) alarm( 0 );
1127
1128 ufdbLogFatalError( "stack allocation failed. simulating bad signal to call ufdb-pstack." );
1129
1130 badSignalThread = pthread_self();
1131 BadSignalCaught( SIGQUIT );
1132 _exit( 4 );
1133 }
1134
1135
__GI___fortify_fail(const char * msg)1136 void __GI___fortify_fail( const char * msg )
1137 {
1138 badSignalHandlerBusy = 1;
1139 #if 0
1140 ufdbGV.reconfig = UFDB_RECONFIGR_FORTIFY;
1141 ufdbGV.terminating = 1;
1142 __asm__ volatile ("" : : : "memory"); // soft barrier for compiler
1143 #endif
1144 (void) alarm( 0 );
1145
1146 ufdbLogFatalError( "fortified code failed: %s. simulating bad signal to call ufdb-pstack.", msg );
1147
1148 badSignalThread = pthread_self();
1149 BadSignalCaught( SIGQUIT );
1150 _exit( 4 );
1151 }
1152
1153
__wrap___fortify_fail(const char * msg)1154 void __wrap___fortify_fail( const char * msg )
1155 {
1156 badSignalHandlerBusy = 1;
1157 #if 0
1158 ufdbGV.reconfig = UFDB_RECONFIGR_FORTIFY;
1159 ufdbGV.terminating = 1;
1160 __asm__ volatile ("" : : : "memory"); // soft barrier for compiler
1161 #endif
1162 (void) alarm( 0 );
1163
1164 ufdbLogFatalError( "fortified code failed: %s. simulating bad signal to call ufdb-pstack.", msg );
1165
1166 badSignalThread = pthread_self();
1167 BadSignalCaught( SIGQUIT );
1168 _exit( 4 );
1169 }
1170
1171
__wrap___assert_fail(const char * assertion,const char * file,unsigned int line,const char * function)1172 void __wrap___assert_fail( const char * assertion, const char * file, unsigned int line, const char * function )
1173 {
1174 badSignalHandlerBusy = 1;
1175 #if 0
1176 ufdbGV.reconfig = UFDB_RECONFIGR_FORTIFY;
1177 ufdbGV.terminating = 1;
1178 __asm__ volatile ("" : : : "memory"); // soft barrier for compiler
1179 #endif
1180 (void) alarm( 0 );
1181
1182 ufdbLogFatalError( "libc assertion failure: %s line %d: in function %s: %s", file, line, function, assertion );
1183
1184 badSignalThread = pthread_self();
1185 BadSignalCaught( SIGQUIT );
1186 _exit( 4 );
1187 }
1188
1189
1190 #if !UFDB_PRODUCE_CORE_DUMPS
ufdbCatchBadSignal(int signum)1191 static void ufdbCatchBadSignal( int signum )
1192 {
1193 /* Note that we are inside a signal handler and can do almost nothing. */
1194
1195 if (badSignal != signum)
1196 {
1197 badSignalHandlerBusy = 1;
1198 badSignal = signum;
1199 ufdbGV.reconfig = UFDB_RECONFIGR_BADSIG;
1200 ufdbGV.terminating = 1;
1201 __asm__ volatile ("" : : : "memory"); // soft barrier for compiler
1202 (void) alarm( 0 );
1203
1204 badSignalThread = pthread_self();
1205
1206 ufdbLogMessage( "ufdbCatchBadSignal: received signal %d, memory-allocation-errors=%d",
1207 signum, ufdbGV.memoryAllocationErrors );
1208
1209 /* Cancel the sockethandler thread so that the main thread will wake up and
1210 * wind up.
1211 * But sleep first so that ufdb-pstack can do its job in an other thread.
1212 */
1213 sleep( 1 );
1214 if (sockethandler != 0)
1215 pthread_cancel( sockethandler );
1216 #if 0
1217 sleep( 1 );
1218 pthread_kill( sockethandler, SIGTERM );
1219 #endif
1220
1221 removePidFile();
1222
1223 /* If for any reason the termination by main fails, we have to call exit() ourselves. */
1224 sleep( 2 );
1225 sleep( 2 );
1226 sleep( 2 );
1227 sleep( 2 );
1228 sleep( 2 );
1229 ufdbLogError( "ufdbCatchBadSignal: calling exit() after waiting for 10 seconds for main to terminate ufdbguardd" );
1230 exit( 10 );
1231 }
1232 }
1233 #endif
1234
1235
HUPslowRefresh(void)1236 static void HUPslowRefresh( void )
1237 {
1238 int status;
1239 pid_t pid;
1240 int ret;
1241 struct timespec DatabaseLockTimeout;
1242
1243 if (ufdbGV.debug)
1244 ufdbLogMessage( "HUPslowRefresh" );
1245
1246 ufdbGV.reconfig = UFDB_RECONFIGR_HUP;
1247
1248 UFDBchangeStatus( UFDB_API_STATUS_RELOADING );
1249
1250 if (ufdbGV.debug > 1)
1251 {
1252 ufdbLogMessage( "===== current configuration =====" );
1253 UFDBlogConfig( &ufdbGV );
1254 ufdbLogMessage( "===== end =====" );
1255 }
1256
1257 if (ufdbGV.debug)
1258 ufdbLogMessage( "sleeping a little to allow workers to finish current queries" );
1259
1260 sleep( 1 );
1261 if (UFDBgetTunnelCheckMethod() == UFDB_API_HTTPS_CHECK_OFF)
1262 {
1263 usleep( 500 * 1000 ); /* 0.5 seconds */
1264 }
1265 else
1266 {
1267 sleep( 1 );
1268 sleep( 1 );
1269 sleep( 1 );
1270 }
1271
1272 if (UFDBgetTunnelCheckMethod() == UFDB_API_HTTPS_CHECK_AGGRESSIVE)
1273 {
1274 /* with aggressive HTTPS port probing we must wait patiently for workers to release the read lock */
1275 sleep( 1 );
1276 sleep( 1 );
1277 sleep( 1 );
1278 }
1279
1280 /*
1281 * Synchronise with all worker threads.
1282 * ufdbGV.reconfig is set for the last 2-10 seconds so the worker threads have
1283 * most likely no read-locks any more. Try to get a write lock within 30 seconds
1284 * and if this fails, we will sleep 5 seconds more and acquire the rwlock with
1285 * pthread_rwlock_wrlock().
1286 */
1287 ufdbLogMessage( "requesting a write lock for the in-memory database (1st attempt) ..." );
1288
1289 clock_gettime( CLOCK_REALTIME, &DatabaseLockTimeout );
1290 DatabaseLockTimeout.tv_sec += 30;
1291 ret = pthread_rwlock_timedwrlock( &TheDatabaseLock, &DatabaseLockTimeout ); // >==========================
1292 if (ret == 0)
1293 ufdbLogMessage( "lock acquired to reload the URL database" );
1294 else
1295 {
1296 if (ret == EDEADLK)
1297 ufdbLogFatalError( "HUPslowRefresh: deadlock detected trying to acquire database write lock" );
1298 else
1299 if (ret == ETIMEDOUT || ret == EBUSY)
1300 {
1301 ufdbLogMessage( "Could not obtain a lock within 30 seconds to refresh the URL database\n"
1302 "waiting 5 more seconds before attempting to acquire the lock again *****" );
1303 sleep( 1 );
1304 sleep( 1 );
1305 sleep( 1 );
1306 sleep( 1 );
1307 sleep( 1 );
1308 }
1309 else
1310 ufdbLogError( "pthread_rwlock_timedwrlock failed with code %d *****", ret );
1311
1312 ufdbLogMessage( "requesting a write lock for the in-memory database (2nd attempt) ..." );
1313
1314 /*
1315 * Wait many seconds for a write lock ...
1316 * This DOES NOT DISTURB USERS since most workers will be in "reload mode"
1317 * replying requests from Squid.
1318 * This state is not desirable since "reload mode" either does not filter or blocks all access
1319 * so the time to reload a configuration should always be as short as possible.
1320 */
1321 DatabaseLockTimeout.tv_sec += 60;
1322 ret = pthread_rwlock_timedwrlock( &TheDatabaseLock, &DatabaseLockTimeout ); // >======================
1323 if (ret == 0)
1324 ufdbLogMessage( "lock acquired to reload the configuration and URL database" );
1325 else
1326 {
1327 if (ret == EDEADLK)
1328 ufdbLogFatalError( "HUPslowRefresh: deadlock detected trying to acquire database write lock" );
1329 else
1330 if (ret == ETIMEDOUT || ret == EBUSY)
1331 {
1332 ufdbLogError( "HUP signal received but could not acquire a lock on the configuration *****\n"
1333 "Perhaps a dynamic refresh is taking too long or hangs "
1334 "(check all commands used by execuserlist, execiplist and execdomainlist) *****\n"
1335 "Verify that the system load is normal.\n"
1336 "On virtual machines, verify that the virtual machine has sufficient resources.\n"
1337 "Request support from the Support Desk of www.urlfilterdb.com "
1338 "in case there are any doubts." );
1339 ufdbExecutePstack( "debug-cannot-get-rwlock-in-90-seconds" );
1340 }
1341 else
1342 ufdbLogError( "pthread_rwlock_wrlock failed with code %d *****", ret );
1343
1344 /*
1345 * We have two choices here:
1346 * 1) raise SIGABRT since we cannot reload the configuration,
1347 * 2) forget SIGHUP and not reload the configuration.
1348 * Both options are not very desirable, but option 2 is more friendly for the end users.
1349 */
1350 if (ret != ETIMEDOUT && ret != EBUSY)
1351 ufdbExecutePstack( "cannot-get-rwlock-for-database-refresh-after-many-attempts" );
1352 UFDBchangeStatus( UFDB_API_STATUS_FATAL_ERROR );
1353 if (ret == EDEADLK)
1354 {
1355 ufdbLogFatalError( "must terminate because of a fatal deadlock" );
1356 exit( 9 );
1357 }
1358 ufdbGV.reconfig = UFDB_RECONFIGR_NONE;
1359 ufdbHandleAlarmForTimeEvents( UFDB_PARAM_INIT );
1360 return;
1361 }
1362 }
1363
1364 ufdbGV.reconfig = UFDB_RECONFIGR_RELOAD;
1365 __asm__ volatile ("" : : : "memory"); // soft barrier for compiler
1366
1367 /* TODO: reload CA certificates */
1368 /* TODO: clear security cache */
1369
1370 UFDBcopyGV( &ufdbNewGV, &ufdbGV );
1371
1372 ufdbLogMessage( "releasing URL database from memory ..." );
1373 ufdbFreeAllMemory( &ufdbGV );
1374
1375 ufdbLogMessage( "reading configuration file and database ..." );
1376 ufdbNewGV.fatalError = ufdbGV.fatalError = 0;
1377 if (ufdbReadConfig( configFile ) == 0)
1378 {
1379 ufdbGV.reconfig = UFDB_RECONFIGR_FATAL;
1380 __asm__ volatile ("" : : : "memory"); // soft barrier for compiler
1381 ufdbLogFatalError( "exiting due to missing configuration file" );
1382 exit( 3 );
1383 }
1384
1385 ufdbNewGV.terminating = ufdbGV.terminating;
1386 ufdbNewGV.reconfig = ufdbGV.reconfig;
1387 ufdbNewGV.fatalError = ufdbGV.fatalError;
1388 ufdbGV = ufdbNewGV;
1389
1390 /*
1391 * Previous versions released the lock ASAP but that is wrong since during the whole
1392 * time that the configuration is loaded, the internal data structures may not be used
1393 * by any thread, including the threads that (re)load lists of users, domains and IP addresses.
1394 * The lock is released after all processing has been done.
1395 */
1396
1397 if (ufdbGV.terminating)
1398 {
1399 ufdbLogMessage( "HUPslowRefresh: ufdbguardd is terminating *****" );
1400 ufdbLogMessage( "releasing the rwlock" );
1401 (void) pthread_rwlock_unlock( &TheDatabaseLock ); // <==================================
1402 return;
1403 }
1404 if (ufdbGV.fatalError)
1405 {
1406 ufdbLogMessage( "A FATAL ERROR OCCURRED: all requests are %s since url-lookup-result-when-fatal-error "
1407 "is %s (see previous lines with \"FATAL ERROR\" for more details) *****",
1408 ufdbGV.URLlookupResultFatalError == UFDB_ALLOW ? "ALLOWED" : "BLOCKED",
1409 ufdbGV.URLlookupResultFatalError == UFDB_ALLOW ? "allow" : "deny" );
1410 }
1411
1412 /* TODO: only stop/start ufdbhttpd if the parameters have changed */
1413 if (httpDaemonPid > 0)
1414 {
1415 int retval;
1416
1417 UFDBraisePrivileges( ufdbGV.userName, "kill old ufdbhttpd" );
1418
1419 ufdbLogMessage( "killing old ufdbhttpd (pid=%d)", httpDaemonPid );
1420 retval = kill( httpDaemonPid, SIGTERM );
1421 if (retval != 0)
1422 ufdbLogError( "cannot send TERM signal to the HTTP daemon (pid=%d): %s",
1423 httpDaemonPid, strerror( errno ) );
1424
1425 usleep( 400000 );
1426 /* be sure that the daemon is killed. */
1427 if (kill( httpDaemonPid, SIGKILL ) == 0)
1428 usleep( 100000 );
1429
1430 while ((pid = waitpid( -1, &status, WNOHANG )) > 0)
1431 ;
1432
1433 if (kill( httpDaemonPid, 0 ) < 0)
1434 ufdbLogMessage( "HTTP daemon (pid=%d) terminated to be restarted (OK)", httpDaemonPid );
1435 else
1436 ufdbLogError( "HTTP daemon did not terminate: %s", strerror(errno) );
1437
1438 UFDBdropPrivileges( ufdbGV.userName );
1439 }
1440
1441 if (ufdbGV.httpdPort > 0)
1442 startHttpDaemon();
1443
1444 adjustNumberOfWorkerThreads();
1445
1446 UFDBlookupCounter = 0;
1447 UFDB_API_num_https = 0;
1448 UFDB_API_num_bumps = 0;
1449 UFDB_API_num_url_localnet = 0;
1450 UFDBblockCounter = 0;
1451 UFDBtestBlockCounter = 0;
1452 UFDBuncategorisedCounter = 0;
1453 UFDB_API_num_nodot_URL = 0;
1454 ufdbGV.tunnelCounter = 0;
1455 UFDBsafesearchCounter = 0;
1456 UFDBedufilterCounter = 0;
1457 UFDBuploadSeqNo++;
1458
1459 ufdbLogMessage( "releasing the rwlock" );
1460 ufdbGV.reconfig = UFDB_RECONFIGR_NONE;
1461 // __asm__ volatile ("" : : : "memory"); // soft barrier for compiler
1462 (void) pthread_rwlock_unlock( &TheDatabaseLock ); // <==================================
1463
1464 if (ufdbGV.fatalError)
1465 ufdbLogMessage( "A FATAL ERROR OCCURRED: all requests are answered with \"OK\" "
1466 "(see previous lines with \"FATAL ERROR\" for more details) *****" );
1467 else
1468 ufdbLogMessage( "the new configuration and database are loaded for ufdbguardd " UFDB_VERSION );
1469
1470 if (ufdbGV.debug)
1471 {
1472 ufdbLogMessage( "===== new configuration =====" );
1473 UFDBlogConfig( &ufdbGV );
1474 ufdbLogMessage( "===== end new configuration =====" );
1475 }
1476
1477 ufdbHandleAlarmForTimeEvents( UFDB_PARAM_INIT );
1478
1479 if (ufdbGV.fatalError)
1480 UFDBchangeStatus( UFDB_API_STATUS_FATAL_ERROR );
1481 else
1482 UFDBchangeStatus( UFDB_API_STATUS_RELOAD_OK );
1483 }
1484
1485
HUPfastRefresh(void)1486 static void HUPfastRefresh( void )
1487 {
1488 int status;
1489 pid_t pid;
1490 int ret;
1491 struct timespec DatabaseLockTimeout;
1492
1493 // Note that the alarm signal is already disabled
1494
1495 if (ufdbGV.debug)
1496 ufdbLogMessage( "HUPfastRefresh: HUP signal received. "
1497 "Going to load-new-config fast-swap-config unload-old-config ..." );
1498
1499 UFDBchangeStatus( UFDB_API_STATUS_RELOADING );
1500
1501 ufdbGV.fatalError = 0;
1502 UFDBcopyGV( &ufdbNewGV, &ufdbGV );
1503
1504 if (ufdbReadConfig( configFile ) == 0)
1505 {
1506 ufdbGV.reconfig = UFDB_RECONFIGR_FATAL;
1507 __asm__ volatile ("" : : : "memory"); // soft barrier for compiler
1508 ufdbLogFatalError( "received HUP signal to reload but the config file %s is missing", configFile );
1509 UFDBchangeStatus( UFDB_API_STATUS_FATAL_ERROR );
1510 exit( 3 );
1511 }
1512 if (ufdbGV.terminating)
1513 {
1514 ufdbLogMessage( "HUPfastRefresh: ufdbguardd is terminating *****" );
1515 return;
1516 }
1517 if (ufdbGV.fatalError)
1518 {
1519 ufdbLogMessage( "A FATAL ERROR OCCURRED: all requests are %s since url-lookup-result-when-fatal-error "
1520 "is %s (see previous lines with \"FATAL ERROR\" for more details) *****",
1521 ufdbGV.URLlookupResultFatalError == UFDB_ALLOW ? "ALLOWED" : "BLOCKED",
1522 ufdbGV.URLlookupResultFatalError == UFDB_ALLOW ? "allow" : "deny" );
1523 }
1524
1525 adjustNumberOfWorkerThreads();
1526
1527 if (ufdbGV.fatalError || ufdbGV.debug > 1)
1528 {
1529 ufdbLogMessage( "===== old configuration =====" );
1530 UFDBlogConfig( &ufdbGV );
1531 ufdbLogMessage( "===== end old configuration =====" );
1532 }
1533 ufdbLogMessage( "===== new configuration =====" );
1534 UFDBlogConfig( &ufdbNewGV );
1535 ufdbLogMessage( "===== end new configuration =====" );
1536
1537 /*
1538 * Synchronise with all worker threads.
1539 * Try to get a write lock within 3 seconds and if this fails a second attempt to acquire the lock
1540 * will take maximum 87 seconds.
1541 */
1542 ufdbLogMessage( "requesting a write lock for the in-memory database (1st attempt) ..." );
1543 ufdbGV.reconfig = UFDB_RECONFIGR_HUP;
1544 __asm__ volatile ("" : : : "memory"); // soft barrier for compiler
1545
1546 clock_gettime( CLOCK_REALTIME, &DatabaseLockTimeout );
1547 DatabaseLockTimeout.tv_sec += 3;
1548 ret = pthread_rwlock_timedwrlock( &TheDatabaseLock, &DatabaseLockTimeout ); // >==========================
1549 if (ret == 0)
1550 ufdbLogMessage( "lock acquired to reload the URL database" );
1551 else
1552 {
1553 if (ret == EDEADLK)
1554 ufdbLogFatalError( "HUPfastRefresh: deadlock detected trying to acquire database write lock" );
1555 else
1556 if (ret == ETIMEDOUT || ret == EBUSY)
1557 ufdbLogMessage( "Could not obtain a lock within 3 seconds to refresh the URL database\n"
1558 "Attempting to acquire the lock again" );
1559 else
1560 ufdbLogError( "pthread_rwlock_timedwrlock failed with code %d *****", ret );
1561
1562 ufdbLogMessage( "requesting a write lock for the in-memory database (2nd attempt) ..." );
1563
1564 /*
1565 * Wait many seconds for a write lock ...
1566 * This DOES NOT DISTURB USERS since most workers will be in "reload mode"
1567 * replying requests from Squid.
1568 * This state is not desirable since "reload mode" either does not filter or blocks all access
1569 * so the time to reload a configuration should always be as short as possible.
1570 */
1571 DatabaseLockTimeout.tv_sec += 87;
1572 ret = pthread_rwlock_timedwrlock( &TheDatabaseLock, &DatabaseLockTimeout ); // >======================
1573 if (ret == 0)
1574 ufdbLogMessage( "lock acquired to reload the configuration and URL database" );
1575 else
1576 {
1577 if (ret == EDEADLK)
1578 ufdbLogFatalError( "HUPfastRefresh: deadlock detected trying to acquire database write lock" );
1579 else
1580 if (ret == ETIMEDOUT || ret == EBUSY)
1581 {
1582 ufdbLogError( "HUP signal received but could not acquire a lock on the configuration *****\n"
1583 "Perhaps a dynamic refresh is taking too long or hangs "
1584 "(check all commands used by execuserlist, execiplist and execdomainlist) *****\n"
1585 "Verify that the system load is normal.\n"
1586 "On virtual machines, verify that the virtual machine has sufficient resources.\n"
1587 "Request support from the Support Desk of www.urlfilterdb.com "
1588 "in case there are any doubts." );
1589 ufdbExecutePstack( "debug-cannot-get-rwlock-in-90-seconds" );
1590 }
1591 else
1592 ufdbLogError( "pthread_rwlock_wrlock failed with code %d *****", ret );
1593
1594 /*
1595 * We have two choices here:
1596 * 1) raise SIGABRT since we cannot reload the configuration,
1597 * 2) forget SIGHUP and not reload the configuration.
1598 * Both options are not very desirable, but option 2 is more friendly for the end users.
1599 */
1600 if (ret != ETIMEDOUT && ret != EBUSY)
1601 ufdbExecutePstack( "cannot-get-rwlock-for-database-refresh-after-2-attempts" );
1602 UFDBchangeStatus( UFDB_API_STATUS_FATAL_ERROR );
1603 if (ret == EDEADLK)
1604 {
1605 ufdbLogFatalError( "must terminate because of a fatal deadlock" );
1606 exit( 9 );
1607 }
1608 // abort the refresh
1609 ufdbGV.reconfig = UFDB_RECONFIGR_NONE;
1610 ufdbHandleAlarmForTimeEvents( UFDB_PARAM_INIT );
1611 sleep( 30 );
1612 ufdbFreeAllMemory( &ufdbNewGV );
1613 return;
1614 }
1615 }
1616
1617 ufdbGV.reconfig = UFDB_RECONFIGR_RELOAD;
1618 ufdbNewGV.reconfig = UFDB_RECONFIGR_RELOAD;
1619 ufdbNewGV.terminating = ufdbGV.terminating;
1620 ufdbNewGV.fatalError = ufdbGV.fatalError;
1621
1622 ufdbOldGV = ufdbGV;
1623 ufdbGV = ufdbNewGV;
1624 __asm__ volatile ("" : : : "memory"); // soft barrier for compiler
1625
1626 UFDBlookupCounter = 0;
1627 UFDB_API_num_https = 0;
1628 UFDB_API_num_bumps = 0;
1629 UFDB_API_num_url_localnet = 0;
1630 UFDBblockCounter = 0;
1631 UFDBtestBlockCounter = 0;
1632 UFDBuncategorisedCounter = 0;
1633 UFDB_API_num_nodot_URL = 0;
1634 ufdbGV.tunnelCounter = 0;
1635 UFDBsafesearchCounter = 0;
1636 UFDBedufilterCounter = 0;
1637 UFDBuploadSeqNo++;
1638
1639 /* TODO: reload CA certificates */
1640 /* TODO: clear security cache */
1641
1642 ufdbGV.reconfig = UFDB_RECONFIGR_NONE;
1643 (void) pthread_rwlock_unlock( &TheDatabaseLock ); // <==================================
1644
1645 ufdbLogMessage( "new configuration was activated. The lock has just been released" );
1646
1647 if (ufdbGV.fatalError)
1648 ufdbLogMessage( "A FATAL ERROR OCCURRED: all requests are answered with \"OK\" "
1649 "(see previous lines with \"FATAL ERROR\" for more details) *****" );
1650
1651 /* TODO: only stop/start ufdbhttpd if the parameters have changed */
1652 if (httpDaemonPid > 0)
1653 {
1654 int retval;
1655
1656 UFDBraisePrivileges( ufdbGV.userName, "kill old ufdbhttpd" );
1657
1658 ufdbLogMessage( "killing old ufdbhttpd (pid=%d)", httpDaemonPid );
1659 retval = kill( httpDaemonPid, SIGTERM );
1660 if (retval != 0)
1661 ufdbLogError( "cannot send TERM signal to the HTTP daemon (pid=%d): %s",
1662 httpDaemonPid, strerror( errno ) );
1663
1664 usleep( 50000 );
1665 /* be sure that the daemon is killed. */
1666 if (kill( httpDaemonPid, SIGKILL ) == 0)
1667 usleep( 50000 );
1668
1669 while ((pid = waitpid( -1, &status, WNOHANG )) > 0)
1670 ;
1671
1672 if (kill( httpDaemonPid, 0 ) < 0)
1673 ufdbLogMessage( "HTTP daemon (pid=%d) terminated to be restarted (OK)", httpDaemonPid );
1674 else
1675 ufdbLogError( "HTTP daemon did not terminate: %s", strerror(errno) );
1676
1677 UFDBdropPrivileges( ufdbGV.userName );
1678 }
1679 if (ufdbGV.httpdPort > 0)
1680 startHttpDaemon();
1681
1682 ufdbHandleAlarmForTimeEvents( UFDB_PARAM_INIT );
1683
1684 sleep( 1 );
1685
1686 ufdbLogMessage( "releasing old config and URL database from memory ..." );
1687 ufdbFreeAllMemory( &ufdbOldGV );
1688
1689 if (ufdbGV.fatalError)
1690 UFDBchangeStatus( UFDB_API_STATUS_FATAL_ERROR );
1691 else
1692 UFDBchangeStatus( UFDB_API_STATUS_RELOAD_OK );
1693 }
1694
1695
HupSignalCaught(int sig)1696 static void HupSignalCaught( int sig )
1697 {
1698 time_t t;
1699 struct tm thetime;
1700 FILE * fp;
1701
1702 if (sig != 0)
1703 { ; }
1704
1705 if (ufdbGV.terminating)
1706 return;
1707
1708 if (ufdb_mutex_trylock( &sighup_mutex ) != 0) // >>===============================
1709 {
1710 ufdbLogMessage( "HUP signal is already being processed" );
1711 return;
1712 }
1713 ufdbLogMessage( "===== HUP signal received to reload the configuration and database =====" );
1714
1715 /* going to reload the configuration but first wait for others to finish */
1716 while (ufdbGV.reconfig)
1717 usleep( 100000 );
1718
1719 // stop processing of time-dependent events
1720 alarm( 0 );
1721
1722 uploadStatistics( "HUP" );
1723
1724 /* To prevent counting 'old' clients:
1725 * if today is Wednesday, we reset the IP counters.
1726 * If reset was more than 8 days ago, we also do a reset now.
1727 */
1728 t = UFDBtime();
1729 if (lastUpload != 0 && t - lastUpload > SECSOF8DAYS)
1730 {
1731 UFDBinitializeIPcounters();
1732 UFDBinitializeUserCounters();
1733 }
1734 else
1735 {
1736 localtime_r( &t, &thetime );
1737 if (thetime.tm_wday == 3)
1738 {
1739 UFDBinitializeIPcounters();
1740 UFDBinitializeUserCounters();
1741 }
1742 }
1743
1744 /* Get the pid of the HTTP daemon. It is used later to kill it. */
1745 httpDaemonPid = -1;
1746 if (ufdbGV.httpdPort > 0)
1747 {
1748 fp = fopen( UFDBHTTPD_PID_FILE, "r" );
1749 if (fp != NULL)
1750 {
1751 if (1 != fscanf( fp, "%d", &httpDaemonPid ))
1752 {
1753 ufdbLogError( "file %s does not have a process id", UFDBHTTPD_PID_FILE );
1754 httpDaemonPid = -1;
1755 }
1756 fclose( fp );
1757 }
1758 else
1759 {
1760 ufdbLogError( "cannot open file %s: %s", UFDBHTTPD_PID_FILE, strerror(errno) );
1761 }
1762 }
1763
1764 if (ufdbGV.fastRefresh)
1765 HUPfastRefresh();
1766 else
1767 HUPslowRefresh();
1768
1769 (void) ufdb_mutex_unlock( &sighup_mutex ); // <<===============================
1770 }
1771
1772
incr_active_workers(void)1773 static void incr_active_workers( void )
1774 {
1775 #if defined(__GNUC__) && __GCC_HAVE_SYNC_COMPARE_AND_SWAP_4 && __SIZEOF_INT__ == 4
1776 (void) __sync_add_and_fetch( &n_active_workers, 1 );
1777 #elif defined(__GNUC__) && __GCC_HAVE_SYNC_COMPARE_AND_SWAP_8 && __SIZEOF_INT__ == 8
1778 (void) __sync_add_and_fetch( &n_active_workers, 1 );
1779 #else
1780 ufdb_mutex_lock( &incrMutex );
1781 n_active_workers++;
1782 ufdb_mutex_unlock( &incrMutex );
1783 #endif
1784 }
1785
1786
decr_active_workers(void)1787 static void decr_active_workers( void )
1788 {
1789 #if defined(__GNUC__) && __GCC_HAVE_SYNC_COMPARE_AND_SWAP_4 && __SIZEOF_LONG__ == 4
1790 (void) __sync_sub_and_fetch( &n_active_workers, 1 );
1791 #elif defined(__GNUC__) && __GCC_HAVE_SYNC_COMPARE_AND_SWAP_8 && __SIZEOF_LONG__ == 8
1792 (void) __sync_sub_and_fetch( &n_active_workers, 1 );
1793 #else
1794 ufdb_mutex_lock( &incrMutex );
1795 n_active_workers--;
1796 ufdb_mutex_unlock( &incrMutex );
1797 #endif
1798 }
1799
1800
1801 /*
1802 * Communication between the worker and other threads is done via
1803 * RW-locks and data is passed via a queue.
1804 *
1805 * Since this queue is a FIFO we could use a tail queue. But we don't
1806 * like the malloc() overhead, so therefore we use a static array.
1807 */
1808
1809 #define UFDB_MAX_FD_QUEUE UFDB_MAX_THREADS
1810
1811
1812 static struct {
1813 int fd;
1814 } q[UFDB_MAX_FD_QUEUE];
1815
1816 UFDB_GCC_ALIGN64
1817 static pthread_mutex_t fdq_lock = PTHREAD_MUTEX_INITIALIZER;
1818
1819 UFDB_GCC_ALIGN64
1820 static pthread_cond_t empty = PTHREAD_COND_INITIALIZER;
1821
1822 UFDB_GCC_ALIGN64
1823 static volatile int n_queued = 0;
1824 static volatile int ihead = 0; /* array index for the head */
1825 static volatile int itail = 0; /* array index for the tail */
1826
1827
1828 /*
1829 * enqueue a new fd.
1830 */
putFdQueue(int fd)1831 static void putFdQueue( int fd )
1832 {
1833 int ret;
1834
1835 ret = pthread_mutex_lock( &fdq_lock ); /* ======================================= */
1836 if (ret != 0)
1837 #ifdef UFDB_DEBUG
1838 ufdbLogError( "putFdQueue: mutex_lock failed with code %d *****", ret );
1839 #else
1840 { ; }
1841 #endif
1842
1843 if (n_queued < UFDB_MAX_FD_QUEUE)
1844 {
1845 /*
1846 * insert it at the tail
1847 */
1848 q[itail].fd = fd;
1849
1850 itail = (itail + 1) % UFDB_MAX_FD_QUEUE;
1851 n_queued++;
1852
1853 /*
1854 * leave the critical section
1855 */
1856 ret = pthread_mutex_unlock( &fdq_lock ); /* ======================================= */
1857 if (ret != 0)
1858 #ifdef UFDB_DEBUG
1859 ufdbLogError( "putFdQueue: mutex_unlock failed with code %d *****", ret );
1860 #else
1861 { ; }
1862 #endif
1863
1864 /*
1865 * signal anyone who is waiting
1866 */
1867 if (n_queued == 1)
1868 pthread_cond_signal( &empty );
1869 else
1870 pthread_cond_broadcast( &empty );
1871 }
1872 else
1873 {
1874 ret = pthread_mutex_unlock( &fdq_lock ); /* ======================================= */
1875 ufdbLogFatalError( "connection queue is full. *****\n"
1876 "There are too many ufdbgclient processes !!\n"
1877 "the maximum is %d when the -w option is used\n"
1878 "closing fd %d",
1879 UFDB_MAX_FD_QUEUE, fd );
1880 /* there is no other option than to close the connection */
1881 /* TO-DO: maybe: close all queued fd's ? */
1882 close( fd );
1883 }
1884
1885 if (n_queued + n_active_workers > ufdbGV.nWorkers)
1886 {
1887 usleep( 100000 );
1888 if (n_queued + n_active_workers > ufdbGV.nWorkers)
1889 {
1890 ufdbLogError( "NOT ENOUGH WORKER THREADS *****\n"
1891 "There are curently %d active worker threads and\n"
1892 "the configured maximum number of worker threads is %d and\n"
1893 "there are %d connections pending to be serviced.\n"
1894 "Count the number of ufdbgclient processes which should be less than %d.\n"
1895 "You may also request help at support@urlfilterdb.com",
1896 n_active_workers,
1897 ufdbGV.nWorkers,
1898 n_queued,
1899 ufdbGV.nWorkers );
1900 }
1901 }
1902 }
1903
1904
1905 /*
1906 * get a fd where there is new content.
1907 */
getFdQueue(int * fd)1908 static void getFdQueue( int * fd )
1909 {
1910 int ret;
1911
1912 allover:
1913 ret = pthread_mutex_lock( &fdq_lock );
1914 if (ret != 0)
1915 #ifdef UFDB_DEBUG
1916 ufdbLogError( "getFdQueue: mutex_lock failed with code %d *****", ret );
1917 #else
1918 { ; }
1919 #endif
1920
1921 while (1)
1922 {
1923 /*
1924 * if there are jobs in the queue
1925 */
1926 if (n_queued > 0)
1927 {
1928 n_queued--;
1929 /*
1930 * get the one at the head
1931 */
1932 *fd = q[ihead].fd;
1933 ihead = (ihead + 1) % UFDB_MAX_FD_QUEUE;
1934
1935 ret = pthread_mutex_unlock( &fdq_lock );
1936 if (ret != 0)
1937 #ifdef UFDB_DEBUG
1938 ufdbLogError( "getFdQueue: mutex_unlock failed with code %d *****", ret );
1939 #else
1940 { ; }
1941 #endif
1942
1943 return;
1944 }
1945 else /* otherwise wait until there are fds available */
1946 {
1947 pthread_cond_wait( &empty, &fdq_lock );
1948 /*
1949 * when I'm here, I've been signaled because there
1950 * are jobs in the queue. Go try and get one.
1951 */
1952 ret = pthread_mutex_unlock( &fdq_lock );
1953 if (ret != 0)
1954 #ifdef UFDB_DEBUG
1955 ufdbLogError( "getFdQueue: mutex_unlock failed with code %d *****", ret );
1956 #else
1957 { ; }
1958 #endif
1959 usleep( ((unsigned long) pthread_self()) % 821 );
1960 goto allover;
1961 }
1962 }
1963 }
1964
1965
daemon_accept_connections(int s,int protocol)1966 static void daemon_accept_connections(
1967 int s,
1968 int protocol )
1969 {
1970 int n;
1971 int newfd;
1972 fd_set fds;
1973 struct timeval tv;
1974
1975 /*
1976 * Allow that this thread can be cancelled without delays at any time.
1977 */
1978 pthread_setcancelstate( PTHREAD_CANCEL_ENABLE, NULL );
1979 pthread_setcanceltype( PTHREAD_CANCEL_ASYNCHRONOUS, NULL );
1980
1981 #if 0
1982 if (fcntl( s, F_SETFD, FD_CLOEXEC ))
1983 { ; }
1984 #endif
1985
1986 while (!ufdbGV.terminating)
1987 {
1988 FD_ZERO( &fds );
1989 FD_SET( s, &fds );
1990 errno = 0;
1991 n = select( s+1, &fds, (fd_set *) NULL, (fd_set *) NULL, (struct timeval *) NULL );
1992
1993 if (ufdbGV.terminating)
1994 break;
1995
1996 if (n > 0)
1997 {
1998 newfd = accept( s, NULL, NULL );
1999 if (newfd < 0)
2000 {
2001 if (errno == EINTR)
2002 {
2003 continue;
2004 }
2005 ufdbLogError( "SRV: accept on socket failed: %s", strerror(errno) );
2006 return;
2007 }
2008
2009 #if 0
2010 if (fcntl( newfd, F_SETFD, FD_CLOEXEC ))
2011 { ; }
2012 #endif
2013
2014 if (protocol == AF_INET)
2015 {
2016 int sock_parm;
2017
2018 sock_parm = 1;
2019 setsockopt( newfd, IPPROTO_TCP, TCP_NODELAY, (void *) &sock_parm, sizeof(sock_parm) );
2020 sock_parm = 1;
2021 setsockopt( newfd, SOL_SOCKET, SO_KEEPALIVE, (void *) &sock_parm, sizeof(sock_parm) );
2022 sock_parm = 64 * 1024;
2023 setsockopt( newfd, SOL_SOCKET, SO_SNDBUF, (void *) &sock_parm, sizeof(sock_parm) );
2024 sock_parm = 64 * 1024;
2025 setsockopt( newfd, SOL_SOCKET, SO_RCVBUF, (void *) &sock_parm, sizeof(sock_parm) );
2026 }
2027
2028 /*
2029 * In case of troubles, the timeout prevents that a thread will block for ever.
2030 */
2031 #if 0
2032 tv.tv_sec = 16 * 60; /* we need unlimited read timeout since we simply wait for input */
2033 tv.tv_usec = 0;
2034 setsockopt( newfd, SOL_SOCKET, SO_RCVTIMEO, (void *) &tv, sizeof(tv) );
2035 #endif
2036 tv.tv_sec = 5; /* 5 second send timeout */
2037 tv.tv_usec = 0;
2038 setsockopt( newfd, SOL_SOCKET, SO_SNDTIMEO, (void *) &tv, sizeof(tv) );
2039
2040 if (ufdbGV.debug > 2)
2041 ufdbLogMessage( "SRV: accept new fd %2d", newfd );
2042
2043 putFdQueue( newfd );
2044 }
2045 }
2046
2047 ufdbLogMessage( "connection handler: terminating=%d badsig=%d memory-allocation-errors=%d",
2048 ufdbGV.terminating, badSignal, ufdbGV.memoryAllocationErrors );
2049 if (badSignal)
2050 {
2051 ufdbLogMessage( "connection handler: sleeping to let ufdb-pstack make a crash report" );
2052 sleep( 1 );
2053 sleep( 1 );
2054 sleep( 1 );
2055 }
2056 }
2057
2058
allowed_to_remove_unix_socket(void)2059 static int allowed_to_remove_unix_socket( void )
2060 {
2061 int retval;
2062 pid_t pid;
2063 FILE * fp;
2064 const char * pidfile;
2065
2066 if (ufdbGV.pidFilename == NULL)
2067 pidfile = UFDBGUARDD_PID_FILE;
2068 else
2069 pidfile = ufdbGV.pidFilename;
2070
2071 /* read the pid file, if no file return 1 */
2072 pid = -1;
2073 fp = fopen( pidfile, "r" );
2074 if (fp != NULL)
2075 {
2076 if (1 != fscanf( fp, "%d", &pid ))
2077 {
2078 ufdbLogError( "allowed_to_remove_unix_socket: pid file \"%s\" does not have a process id\n", pidfile );
2079 fclose( fp );
2080 return 1;
2081 }
2082 else if (ufdbGV.debug)
2083 ufdbLogMessage( "allowed_to_remove_unix_socket: read pid %d from file \"%s\"", pid, pidfile );
2084 fclose( fp );
2085 if (pid < 0)
2086 return 1;
2087 }
2088 else
2089 {
2090 if (ufdbGV.debug)
2091 ufdbLogMessage( "allowed_to_remove_unix_socket: cannot open file \"%s\": %s",
2092 pidfile, strerror(errno) );
2093 return 1;
2094 }
2095
2096 /* check if the pid exists, if not return 1 */
2097 UFDBraisePrivileges( ufdbGV.userName, "check if an other ufdbguardd process exists" );
2098 retval = kill( pid, 0 );
2099 UFDBdropPrivileges( ufdbGV.userName );
2100 if (retval != 0)
2101 {
2102 if (ufdbGV.debug)
2103 ufdbLogMessage( "allowed_to_remove_unix_socket: sending signal 0 to pid %d failed: %s",
2104 pid, strerror(errno) );
2105 return 1;
2106 }
2107
2108 ufdbLogFatalError( "another instance of ufdbguardd with pid %d is running.", pid );
2109 return 0;
2110 }
2111
2112
socket_handler_thread(void * ptr)2113 static void * socket_handler_thread( void * ptr )
2114 {
2115 int protocol;
2116 sigset_t sigset;
2117 int s;
2118 int sock_parm;
2119 int retval;
2120 long num_cpus;
2121 struct utsname sysinfo;
2122 struct sockaddr_in addr_in;
2123 #if HAVE_UNIX_SOCKETS
2124 struct sockaddr_un addr_un;
2125 #endif
2126
2127 if (ptr != NULL)
2128 { ; }
2129
2130 /* Most signals must be blocked.
2131 * This is a requirement to use sigwait() in a thread.
2132 */
2133 sigemptyset( &sigset );
2134 sigaddset( &sigset, SIGHUP );
2135 sigaddset( &sigset, SIGUSR1 );
2136 sigaddset( &sigset, SIGUSR2 );
2137 sigaddset( &sigset, SIGWINCH );
2138 sigaddset( &sigset, SIGCHLD );
2139 sigaddset( &sigset, SIGTERM );
2140 sigaddset( &sigset, SIGINT );
2141 sigaddset( &sigset, SIGALRM );
2142 pthread_sigmask( SIG_BLOCK, &sigset, NULL );
2143
2144 /*
2145 * Create the daemon socket that the daemon accepts connections on.
2146 * if available, first try a UNIX socket and then an IP socket.
2147 */
2148 #if HAVE_UNIX_SOCKETS
2149 protocol = AF_UNIX;
2150 s = socket( protocol, SOCK_STREAM, 0 );
2151 if (s < 0)
2152 {
2153 protocol = AF_INET;
2154 s = socket( protocol, SOCK_STREAM, 0 );
2155 }
2156 #else
2157 protocol = AF_INET;
2158 s = socket( protocol, SOCK_STREAM, 0 );
2159 #endif
2160
2161 if (s < 0)
2162 {
2163 ufdbLogFatalError( "cannot create daemon socket: %s", strerror(errno) );
2164 UFDBchangeStatus( UFDB_API_STATUS_FATAL_ERROR );
2165 pthread_exit( (void *) 0 );
2166 }
2167
2168 /*
2169 * The port number can be specified in the config file (ufdbGV.portNum is set by the parser)
2170 * or a command line parameter (portNumCmdLine is set in main).
2171 * The command line setting has preference.
2172 */
2173 if (portNumCmdLine >= 0)
2174 ufdbGV.portNum = portNumCmdLine;
2175
2176 errno = 0;
2177 #if HAVE_UNIX_SOCKETS
2178 if (protocol == AF_UNIX)
2179 {
2180 sprintf( addr_un.sun_path, "/tmp/ufdbguardd-%05d", ufdbGV.portNum );
2181 #if 0
2182 if (unlink( addr_un.sun_path ) < 0 && errno != ENOENT)
2183 ufdbLogFatalError( "cannot remove socket %s: %s", addr_un.sun_path, strerror(errno) );
2184 #endif
2185 addr_un.sun_family = AF_UNIX;
2186 /* with anti-aliasing warnings ON, connect/bind cause compiler warnings which we may ignore */
2187 retval = bind( s, (struct sockaddr *) &addr_un, sizeof(addr_un) );
2188 /* allow anybody to connect to the socket */
2189 if (retval == 0)
2190 {
2191 (void) chmod( addr_un.sun_path, S_IRUSR|S_IWUSR| S_IRGRP|S_IWGRP| S_IROTH|S_IWOTH );
2192 ufdbLogMessage( "UNIX socket \"%s\" successfully created", addr_un.sun_path );
2193 }
2194 else
2195 {
2196 if (allowed_to_remove_unix_socket())
2197 {
2198 ufdbLogMessage( "socket_handler: first attempt: cannot bind daemon socket: %s (protocol=UNIX)",
2199 strerror(errno) );
2200 errno = 0;
2201 retval = unlink( addr_un.sun_path );
2202 if (retval != 0 && errno != ENOENT)
2203 ufdbLogFatalError( "socket_handler: could not remove '%s': %s",
2204 addr_un.sun_path, strerror(errno) );
2205 else
2206 {
2207 retval = bind( s, (struct sockaddr *) &addr_un, sizeof(addr_un) );
2208 if (retval != 0)
2209 {
2210 ufdbLogFatalError( "socket_handler: second attempt: failed to bind socket '%s': %s",
2211 addr_un.sun_path, strerror(errno) );
2212 UFDBchangeStatus( UFDB_API_STATUS_FATAL_ERROR );
2213 pthread_exit( (void *) 0 );
2214 }
2215 else
2216 {
2217 (void) chmod( addr_un.sun_path, S_IRUSR|S_IWUSR| S_IRGRP|S_IWGRP| S_IROTH|S_IWOTH );
2218 ufdbLogMessage( "UNIX socket \"%s\" successfully created with the second attempt",
2219 addr_un.sun_path );
2220 }
2221 }
2222 }
2223 }
2224 }
2225 else
2226 #endif
2227 {
2228 /*
2229 * Allow server-side addresses to be reused (don't have to wait for timeout).
2230 */
2231 sock_parm = 1;
2232 setsockopt( s, SOL_SOCKET, SO_REUSEADDR, (void *) &sock_parm, sizeof(sock_parm) );
2233
2234 addr_in.sin_family = AF_INET;
2235 addr_in.sin_port = htons( ufdbGV.portNum );
2236 if (strcmp( ufdbGV.interface, "all" ) == 0)
2237 addr_in.sin_addr.s_addr = htonl( INADDR_ANY );
2238 else
2239 {
2240 struct in_addr iaddr;
2241 if (inet_pton( AF_INET, ufdbGV.interface, &iaddr ) == 0)
2242 {
2243 addr_in.sin_addr.s_addr = htonl( INADDR_ANY );
2244 ufdbLogError( "interface parameter '%s' is invalid. I will listen on port %d on ALL interfaces.",
2245 ufdbGV.interface, ufdbGV.portNum );
2246 }
2247 else
2248 addr_in.sin_addr.s_addr = iaddr.s_addr;
2249 }
2250 /* with anti-aliasing warnings ON, connect/bind cause compiler warnings which we may ignore */
2251 retval = bind( s, (struct sockaddr *) &addr_in, sizeof(addr_in) );
2252 if (retval >= 0)
2253 ufdbLogMessage( "IP socket port %d on interface \"%s\" successfully created",
2254 ufdbGV.portNum, ufdbGV.interface );
2255 }
2256
2257 if (retval < 0)
2258 {
2259 ufdbLogFatalError( "cannot bind daemon socket: %s (protocol=%s)", strerror(errno),
2260 protocol==AF_INET ? "IP" : "UNIX" );
2261 fprintf( stderr, "cannot bind daemon socket: %s (protocol=%s) *****\n", strerror(errno),
2262 protocol==AF_INET ? "IP" : "UNIX" );
2263 ufdbLogMessage( "check for and kill old daemon processes" );
2264 fprintf( stderr, "check for and kill old daemon processes\n" );
2265 #if HAVE_UNIX_SOCKETS
2266 ufdbLogMessage( "and remove UNIX socket file \"%s\"", addr_un.sun_path );
2267 fprintf( stderr, "and remove UNIX socket file \"%s\"\n", addr_un.sun_path );
2268 #endif
2269 close( s );
2270 UFDBchangeStatus( UFDB_API_STATUS_FATAL_ERROR );
2271 pthread_exit( (void *) 0 );
2272 }
2273
2274 /*
2275 * According to a comment in the Apache httpd source code, these socket
2276 * options should only be set after a successful bind....
2277 */
2278 sock_parm = 1;
2279 setsockopt( s, SOL_SOCKET, SO_KEEPALIVE, (void *) &sock_parm, sizeof(sock_parm) );
2280
2281 if (protocol == AF_INET)
2282 {
2283 sock_parm = 1;
2284 setsockopt( s, IPPROTO_TCP, TCP_NODELAY, (void *) &sock_parm, sizeof(sock_parm) );
2285 }
2286
2287 if (listen( s, UFDB_MAX_THREADS ) < 0)
2288 {
2289 ufdbLogFatalError( "cannot listen on daemon socket: %s", strerror(errno) );
2290 close( s );
2291 UFDBchangeStatus( UFDB_API_STATUS_FATAL_ERROR );
2292 pthread_exit( (void *) 0 );
2293 }
2294
2295 if (protocol == AF_INET)
2296 ufdbLogMessage( "listening on interface %s port %d", ufdbGV.interface, ufdbGV.portNum );
2297 #if HAVE_UNIX_SOCKETS
2298 else
2299 ufdbLogMessage( "listening on UNIX socket \"%s\"", addr_un.sun_path );
2300 #endif
2301
2302 #if HAVE_PTHREAD_RWLOCK_PREFER_WRITER_NP || __FreeBSD__ || __OpenBSD__ || __NetBSD__ || __bsdi__ || __DragonFly__ || (__APPLE__ && __MACH__)
2303 ufdbLogMessage( "using rwlock for database locking with preference for \"writer\"" );
2304 #else
2305 ufdbLogMessage( "using rwlock for database locking but could not set preferred locking for \"writer\"" );
2306 #endif
2307
2308 #ifdef _POSIX_PRIORITY_SCHEDULING
2309 if (ufdbGV.debug)
2310 ufdbLogMessage( "processor yielding is enabled" );
2311 #endif
2312
2313 ufdbGetSysInfo( &sysinfo );
2314 num_cpus = ufdbGetNumCPUs();
2315 ufdbLogMessage( "system: %s %s %s %s on %ld CPU%s", sysinfo.machine, sysinfo.sysname, sysinfo.release,
2316 sysinfo.nodename, num_cpus, num_cpus>1?"s":"" );
2317 ufdbLogMessage( "ufdbguardd " UFDB_VERSION " started with %d URL verification threads and "
2318 "%d TLS/SSL verification threads",
2319 ufdbGV.nWorkers, UFDB_NUM_HTTPS_VERIFIERS );
2320 if (ufdbGV.httpdPort > 0)
2321 startHttpDaemon();
2322
2323 /* let the other threads finish their initialisation before we accept connections */
2324 usleep( 200000 );
2325 ufdbGV.reconfig = UFDB_RECONFIGR_NONE;
2326 __asm__ volatile ("" : : : "memory"); // soft barrier for compiler
2327
2328 daemon_accept_connections( s, protocol );
2329
2330 #if 0
2331 removePidFile();
2332 #endif
2333 pthread_exit( NULL );
2334
2335 return NULL;
2336 }
2337
2338
2339 /*
2340 * This thread waits and handles the following signals:
2341 * HUP, CHLD, USR1, USR2, WINCH, TERM, INT and ALRM.
2342 */
2343 UFDB_GCC_HOT
signal_handler_thread(void * ptr)2344 void * signal_handler_thread( void * ptr )
2345 {
2346 sigset_t sigset;
2347 int sig;
2348
2349 if (ptr != NULL)
2350 { ; }
2351
2352 ufdbSetThreadCPUaffinity( 0 );
2353
2354 /* The HUP and other signals must be blocked.
2355 * This is a requirement to use sigwait() in a thread.
2356 */
2357 sigemptyset( &sigset );
2358 sigaddset( &sigset, SIGHUP );
2359 sigaddset( &sigset, SIGUSR1 );
2360 sigaddset( &sigset, SIGUSR2 );
2361 sigaddset( &sigset, SIGWINCH );
2362 sigaddset( &sigset, SIGCHLD );
2363 sigaddset( &sigset, SIGTERM );
2364 sigaddset( &sigset, SIGINT );
2365 sigaddset( &sigset, SIGALRM );
2366 pthread_sigmask( SIG_BLOCK, &sigset, NULL );
2367
2368 sig = 0;
2369 while (1)
2370 {
2371 int status;
2372 pid_t pid;
2373
2374 /* do not disturb pclose() and let it do its waitpid() */
2375 if (sig == SIGCHLD)
2376 usleep( 100000 );
2377
2378 while ((pid = waitpid( -1, &status, WNOHANG )) > 0)
2379 {
2380 if (ufdbGV.debug > 1 && WIFEXITED(status))
2381 ufdbLogMessage( "signal handler: child process %d terminated with status %d",
2382 pid, WEXITSTATUS(status) );
2383 }
2384
2385 sig = 0;
2386 if (0 != sigwait( &sigset, &sig ))
2387 {
2388 ufdbLogError( "signal_handler_thread: sigwait() failed with %s", strerror(errno) );
2389 continue;
2390 }
2391
2392 switch (sig)
2393 {
2394 case SIGCHLD:
2395 break;
2396 case SIGTERM:
2397 case SIGINT:
2398 TermSignalCaught( sig );
2399 break;
2400 case SIGHUP:
2401 HupSignalCaught( sig );
2402 break;
2403 case SIGUSR1:
2404 Usr1SignalCaught( sig );
2405 break;
2406 case SIGUSR2:
2407 Usr2SignalCaught( sig );
2408 break;
2409 case SIGWINCH:
2410 WinchSignalCaught( sig );
2411 break;
2412 case SIGALRM:
2413 if (ufdbGV.terminating)
2414 break;
2415 if (ufdbGV.reconfig)
2416 alarm( 15 );
2417 else
2418 {
2419 (void) pthread_rwlock_rdlock( &TheDatabaseLock ); /* >======================= */
2420 ufdbHandleAlarmForTimeEvents( UFDB_PARAM_ALARM );
2421 (void) pthread_rwlock_unlock( &TheDatabaseLock ); /* <======================= */
2422 }
2423 break;
2424 default:
2425 ufdbLogError( "signal handler: unexpected signal %d received *****", sig );
2426 }
2427 }
2428
2429 return NULL;
2430 }
2431
2432
2433 /*
2434 * A pseudo-random sample of URLs is checked against all known categories
2435 * and if it appears to be uncategorised, it is buffered to be sent
2436 * to URLfilterDB at a later date (when SIGHUP is received).
2437 *
2438 * return 0 iff the URL is uncategorised, 1 otherwise.
2439 */
ufdbVerifyURLallCats(UFDBrevURL * revURL,char * URL)2440 int ufdbVerifyURLallCats(
2441 UFDBrevURL * revURL,
2442 char * URL )
2443 {
2444 struct Category * cat;
2445 struct UFDBmemTable * mt;
2446
2447 if (ufdbGV.reconfig)
2448 return 1;
2449
2450 if (strncmp( URL, "function%", 9 ) == 0)
2451 return 1;
2452
2453 if (ufdbGV.debug > 1)
2454 ufdbLogMessage( " ufdbVerifyURLallCats: %s", URL );
2455
2456 if (ufdbGV.checkedDB.table.nNextLevels > 0)
2457 {
2458 if (UFDBlookupRevUrl( &(ufdbGV.checkedDB.table.nextLevel[0]), revURL ))
2459 return 1;
2460 }
2461 if (ufdbGV.checkedExpressions != NULL)
2462 {
2463 if (ufdbRegExpMatch( ufdbGV.checkedExpressions, URL ) == UFDB_API_MATCH)
2464 return 1;
2465 }
2466
2467 for (cat = ufdbGV.catList; cat != NULL; cat = cat->next)
2468 {
2469 if (ufdbGV.reconfig)
2470 return 1;
2471 if (cat->domainlistDb != NULL)
2472 {
2473 mt = (struct UFDBmemTable *) cat->domainlistDb->dbcp;
2474 if (UFDBlookupRevUrl( &(mt->table.nextLevel[0]), revURL ))
2475 return 1;
2476 }
2477 /* RE matching is not used in ufdbVerifyURLallCats() since we want those URLs to be marked as uncategorised to get in the database tables */
2478 }
2479
2480 return 0;
2481 }
2482
2483
_blockReason(int reason)2484 static const char * _blockReason(
2485 int reason )
2486 {
2487 switch (reason)
2488 {
2489 case UFDB_API_BLOCKR_PASS: return "PASS";
2490 case UFDB_API_BLOCKR_ACL: return "BLOCK";
2491 case UFDB_API_BLOCKR_ACL_NONE: return "BLOCK";
2492 case UFDB_API_BLOCKR_HTTPS_OPTION: return "BLOCK";
2493 case UFDB_API_BLOCKR_SKYPE: return "BLOCK";
2494 case UFDB_API_BLOCKR_SAFESEARCH: return "REDIR";
2495 case UFDB_API_BLOCKR_LOADING_DB: return "BLOCK-LD";
2496 case UFDB_API_BLOCKR_FATAL_ERROR: return "BLOCK-FATAL";
2497 case UFDB_API_BLOCKR_CHECKED: return "PASS";
2498 case UFDB_API_BLOCKR_YOUTUBE_EDUFILTER: return "REDIR";
2499 }
2500 return "BLOCK";
2501 }
2502
2503
2504 /*
2505 * housekeeper thread
2506 */
2507 UFDB_GCC_HOT
housekeeper_main(void * ptr)2508 static void * housekeeper_main( void * ptr )
2509 {
2510 sigset_t sigset;
2511 time_t t;
2512
2513 if (ptr != NULL)
2514 { ; }
2515
2516 /* The HUP signal must be blocked.
2517 * This is a requirement to use sigwait() in a thread.
2518 */
2519 sigemptyset( &sigset );
2520 sigaddset( &sigset, SIGHUP );
2521 sigaddset( &sigset, SIGUSR1 );
2522 sigaddset( &sigset, SIGUSR2 );
2523 sigaddset( &sigset, SIGWINCH );
2524 sigaddset( &sigset, SIGCHLD );
2525 sigaddset( &sigset, SIGTERM );
2526 sigaddset( &sigset, SIGINT );
2527 sigaddset( &sigset, SIGALRM );
2528 pthread_sigmask( SIG_BLOCK, &sigset, NULL );
2529
2530 ufdbSetThreadCPUaffinity( 0 );
2531
2532 while (1)
2533 {
2534 if (ufdbGV.terminating)
2535 break;
2536
2537 pthread_testcancel();
2538
2539 sleep( 120 ); /* wakeup every 2 minutes */
2540 t = time( NULL );
2541 if (lastUpload == 0)
2542 lastUpload = t;
2543
2544 if (ufdbGV.terminating)
2545 break;
2546
2547 if (!ufdbGV.reconfig)
2548 {
2549 if (t - lastUpload >= SECSOF2DAYS)
2550 {
2551 ufdbLogMessage( "No new URL database was loaded in the past 48 hours. "
2552 "It is highly recommended to refresh the URL database. *****" );
2553 if (ufdbGV.debug)
2554 ufdbLogMessage( "housekeeper: generating statistics since it was not done in the past 48 hours" );
2555 uploadStatistics( "2DAYS" );
2556 UFDBlookupCounter = 0;
2557 UFDB_API_num_https = 0;
2558 UFDB_API_num_bumps = 0;
2559 UFDB_API_num_url_localnet = 0;
2560 UFDBblockCounter = 0;
2561 UFDBtestBlockCounter = 0;
2562 UFDBuncategorisedCounter = 0;
2563 UFDB_API_num_nodot_URL = 0;
2564 ufdbGV.tunnelCounter = 0;
2565 UFDBsafesearchCounter = 0;
2566 UFDBedufilterCounter = 0;
2567 UFDBuploadSeqNo++;
2568 if (ufdbGV.lastIPcounterResetDate != 0 && t - ufdbGV.lastIPcounterResetDate >= SECSOF8DAYS)
2569 {
2570 if (ufdbGV.debug)
2571 ufdbLogMessage( "housekeeper: resetting user counters since there was no reset in 8 days" );
2572 UFDBinitializeIPcounters();
2573 UFDBinitializeUserCounters();
2574 }
2575 if (ufdbGV.checkedDB.mem != NULL && ufdbGV.checkedDB.flags[1] == 'P')
2576 {
2577 if (ufdbGV.debug)
2578 ufdbLogMessage( "housekeeper: ufdbUpdate did not run... going to reload the configuration" );
2579 pthread_kill( threadedSignalHandler, SIGHUP );
2580 }
2581 }
2582 else
2583 if (ufdbGV.checkedDB.mem != NULL && ufdbGV.checkedDB.age <= 24 && t - lastUpload >= 8*60*60)
2584 {
2585 struct tm tm;
2586 (void) localtime_r( &t, &tm );
2587 if (tm.tm_hour == 22 && (tm.tm_min>=2 && tm.tm_min<=5))
2588 {
2589 uploadStatistics( "EOB" ); /* end of business - upload uncategorised URLs */
2590 UFDBlookupCounter = 0;
2591 UFDB_API_num_https = 0;
2592 UFDB_API_num_bumps = 0;
2593 UFDB_API_num_url_localnet = 0;
2594 UFDBblockCounter = 0;
2595 UFDBtestBlockCounter = 0;
2596 UFDBuncategorisedCounter = 0;
2597 UFDB_API_num_nodot_URL = 0;
2598 ufdbGV.tunnelCounter = 0;
2599 UFDBsafesearchCounter = 0;
2600 UFDBedufilterCounter = 0;
2601 UFDBuploadSeqNo++;
2602 }
2603 }
2604 }
2605 }
2606
2607 pthread_exit( NULL );
2608 }
2609
2610
2611 UFDB_GCC_INLINE UFDB_GCC_HOT
write_answer_ok(int fd,char * replybuf,struct SquidInfo * si)2612 static int write_answer_ok(
2613 int fd,
2614 char * replybuf,
2615 struct SquidInfo * si )
2616 {
2617 char * r;
2618 char * channelid;
2619
2620 /* NOTE: the optional channel ID is dealt with by us since 1.32.5 */
2621 r = replybuf;
2622 *r = '\0';
2623 channelid = si->channelid;
2624 if (*channelid != '\0')
2625 {
2626 while (*channelid != '\0')
2627 *r++ = *channelid++;
2628 *r++ = ' ';
2629 }
2630
2631 if (ufdbGV.SquidHelperProtocol == UFDB_SQUID_HELPER_PROTOCOL3)
2632 {
2633 /* strcpy( r, "OK\n" ); */
2634 *r++ = 'O';
2635 *r++ = 'K';
2636 *r++ = '\n';
2637 *r = '\0';
2638 /* See also http://wiki.squid-cache.org/Features/AddonHelpers and
2639 * http://www.squid-cache.org/Versions/v3/3.4/cfgman/url_rewrite_program.html
2640 */
2641 }
2642 else
2643 {
2644 *r++ = '\n';
2645 *r = '\0';
2646 }
2647
2648 if (ufdbGV.debug > 1)
2649 ufdbLogMessage( "W%03d: sending OK with protocol %d: '%s'", si->worker, ufdbGV.SquidHelperProtocol, replybuf );
2650
2651 return (int) write( fd, replybuf, strlen(replybuf) );
2652 }
2653
2654
2655 UFDB_GCC_INLINE UFDB_GCC_HOT
write_answer_rewrite(int fd,char * replyBuf,char * URL,struct SquidInfo * si)2656 static int write_answer_rewrite(
2657 int fd,
2658 char * replyBuf,
2659 char * URL,
2660 struct SquidInfo * si )
2661 {
2662 int replyLen;
2663
2664 /* NOTE: the optional channel ID is dealt with by ufdbgclient */
2665
2666 if (ufdbGV.SquidHelperProtocol == UFDB_SQUID_HELPER_PROTOCOL3)
2667 {
2668 if (si->channelid[0] == '\0')
2669 replyLen = sprintf( replyBuf, "OK rewrite-url=\"%s\"\n", URL );
2670 else
2671 replyLen = sprintf( replyBuf, "%s OK rewrite-url=\"%s\"\n", si->channelid, URL );
2672 }
2673 else if (ufdbGV.SquidHelperProtocol == UFDB_SQUID_HELPER_PROTOCOL2)
2674 {
2675 if (si->channelid[0] == '\0')
2676 replyLen = sprintf( replyBuf, "%s\n", URL );
2677 else
2678 replyLen = sprintf( replyBuf, "%s %s\n", si->channelid, URL );
2679 }
2680 else
2681 {
2682 if (si->channelid[0] == '\0')
2683 replyLen = sprintf( replyBuf, "%s %s/%s %s %s\n",
2684 URL,
2685 si->srcIP[0] ? si->srcIP : "-",
2686 si->srcDomain[0] ? si->srcDomain : "-",
2687 si->ident[0] ? si->ident : "-",
2688 si->method );
2689 else
2690 replyLen = sprintf( replyBuf, "%s %s %s/%s %s %s\n",
2691 si->channelid,
2692 URL,
2693 si->srcIP[0] ? si->srcIP : "-",
2694 si->srcDomain[0] ? si->srcDomain : "-",
2695 si->ident[0] ? si->ident : "-",
2696 si->method );
2697 }
2698
2699 if (ufdbGV.debug > 1)
2700 ufdbLogMessage( "W%03d: sending rewrite with protocol %d: '%s'",
2701 si->worker, ufdbGV.SquidHelperProtocol, replyBuf );
2702
2703 return (int) write( fd, replyBuf, replyLen );
2704 }
2705
2706
2707 UFDB_GCC_INLINE UFDB_GCC_HOT
write_answer_redir(int fd,char * replyBuf,char * URL,struct SquidInfo * si)2708 static int write_answer_redir(
2709 int fd,
2710 char * replyBuf,
2711 char * URL,
2712 struct SquidInfo * si )
2713 {
2714 int replyLen;
2715
2716 /* NOTE: the optional channel ID is dealt with by ufdbgclient */
2717
2718 if (ufdbGV.SquidHelperProtocol == UFDB_SQUID_HELPER_PROTOCOL3)
2719 {
2720 if (si->blockReason == UFDB_API_BLOCKR_SAFESEARCH ||
2721 si->blockReason == UFDB_API_BLOCKR_YOUTUBE_EDUFILTER)
2722 {
2723 return write_answer_rewrite( fd, replyBuf, URL, si );
2724 }
2725
2726 /* the redirection URL may have a "30N:" prefix */
2727 if (*URL == '3' && *(URL+1) == '0' && (*(URL+2) >= '0' && *(URL+2) <= '9') && *(URL+3) == ':')
2728 {
2729 if (si->channelid[0] == '\0')
2730 replyLen = sprintf( replyBuf, "OK status=%3.3s url=\"%s\"\n", URL, URL+4 );
2731 else
2732 replyLen = sprintf( replyBuf, "%s OK status=%3.3s url=\"%s\"\n", si->channelid, URL, URL+4 );
2733 }
2734 else
2735 {
2736 if (strcmp( si->method, "CONNECT" ) == 0)
2737 {
2738 /* must force rewrite for HTTPS to make Squid reconnect */
2739 if (si->channelid[0] == '\0')
2740 replyLen = sprintf( replyBuf, "OK rewrite-url=\"%s\"\n", URL );
2741 else
2742 replyLen = sprintf( replyBuf, "%s OK rewrite-url=\"%s\"\n", si->channelid, URL );
2743 }
2744 else
2745 {
2746 if (si->channelid[0] == '\0')
2747 replyLen = sprintf( replyBuf, "OK status=" UFDB_DEFAULT_HTTP_RETURN_CODE " url=\"%s\"\n",
2748 URL );
2749 else
2750 replyLen = sprintf( replyBuf, "%s OK status=" UFDB_DEFAULT_HTTP_RETURN_CODE " url=\"%s\"\n",
2751 si->channelid, URL );
2752 }
2753 }
2754 }
2755 else if (ufdbGV.SquidHelperProtocol == UFDB_SQUID_HELPER_PROTOCOL2)
2756 {
2757 if (si->channelid[0] == '\0')
2758 replyLen = sprintf( replyBuf, "%s\n", URL );
2759 else
2760 replyLen = sprintf( replyBuf, "%s %s\n", si->channelid, URL );
2761 }
2762 else
2763 {
2764 if (si->channelid[0] == '\0')
2765 replyLen = sprintf( replyBuf, "%s %s/%s %s %s\n",
2766 URL,
2767 si->srcIP[0] ? si->srcIP : "-",
2768 si->srcDomain[0] ? si->srcDomain : "-",
2769 si->ident[0] ? si->ident : "-",
2770 si->method );
2771 else
2772 replyLen = sprintf( replyBuf, "%s %s %s/%s %s %s\n",
2773 si->channelid,
2774 URL,
2775 si->srcIP[0] ? si->srcIP : "-",
2776 si->srcDomain[0] ? si->srcDomain : "-",
2777 si->ident[0] ? si->ident : "-",
2778 si->method );
2779 }
2780
2781 if (ufdbGV.debug > 1)
2782 ufdbLogMessage( "W%03d: sending redirect with protocol %d: '%s'",
2783 si->worker, ufdbGV.SquidHelperProtocol, replyBuf );
2784
2785 return (int) write( fd, replyBuf, replyLen );
2786 }
2787
2788
2789 /*
2790 * main of the worker threads.
2791 */
2792 UFDB_GCC_HOT
worker_main(void * ptr)2793 static void * worker_main( void * ptr )
2794 {
2795 sigset_t sigset;
2796 UFDBthreadAdmin * admin;
2797 int tnum;
2798 int fd;
2799 int nbytes;
2800 int isconnect;
2801 unsigned long num_reqs;
2802 struct Category * cat;
2803 struct Source * src;
2804 struct Acl * acl;
2805 char * redirect;
2806 char * line;
2807 char * answerBuf;
2808 char * tmp;
2809 int reconfiguring = -1;
2810 int ret;
2811 int lock_used = 0;
2812 char cn[1024];
2813 struct SquidInfo squidInfo;
2814
2815 tnum = (int) ((long) ptr);
2816
2817 /* The HUP signal must be blocked.
2818 * This is a requirement to use sigwait() in a thread.
2819 */
2820 sigemptyset( &sigset );
2821 sigaddset( &sigset, SIGHUP );
2822 sigaddset( &sigset, SIGUSR1 );
2823 sigaddset( &sigset, SIGUSR2 );
2824 sigaddset( &sigset, SIGWINCH );
2825 sigaddset( &sigset, SIGCHLD );
2826 sigaddset( &sigset, SIGTERM );
2827 sigaddset( &sigset, SIGINT );
2828 sigaddset( &sigset, SIGALRM );
2829 pthread_sigmask( SIG_BLOCK, &sigset, NULL );
2830
2831 ufdbSetThreadCPUaffinity( tnum );
2832
2833 admin = UFDBallocThreadAdmin();
2834 line = (char *) ufdbMallocAligned( UFDB_CACHELINE_SIZE, UFDB_HTTP_1_1_MAX_URI_LENGTH + 512 );
2835 *line = '\0';
2836 workerURLbuffers[tnum] = line;
2837 tmp = (char *) ufdbMallocAligned( UFDB_CACHELINE_SIZE, UFDB_HTTP_1_1_MAX_URI_LENGTH + 8192 );
2838 answerBuf = (char *) ufdbMallocAligned( UFDB_CACHELINE_SIZE, UFDB_MAX_URL_LENGTH + 8192 );
2839 fd = -1;
2840
2841 /* many threads are created, so give some CPU time to others now */
2842 sched_yield();
2843
2844 while (1)
2845 {
2846 if (ufdbGV.terminating)
2847 break;
2848
2849 *line = '\0';
2850 *tmp = '\0';
2851 *answerBuf = '\0';
2852
2853 squidInfo.channelid[0] = '\0';
2854 squidInfo.protocol[0] = '\0';
2855 squidInfo.method[0] = '\0';
2856 squidInfo.revUrl = NULL;
2857 squidInfo.aclpass = NULL;
2858 squidInfo.referer[0] = '\0';
2859 squidInfo.ident[0] = '\0';
2860 squidInfo.worker = tnum;
2861
2862 pthread_testcancel();
2863 getFdQueue( &fd );
2864 incr_active_workers();
2865 num_reqs = 0UL;
2866 if (ufdbGV.debug > 2)
2867 ufdbLogMessage( "W%03d: open fd %2d", tnum, fd );
2868
2869 while ((nbytes = read( fd, line, UFDB_HTTP_1_1_MAX_URI_LENGTH-2 )) > 0)
2870 {
2871 int decision;
2872
2873 cat = NULL;
2874 lock_used = 0;
2875 isconnect = 0;
2876 line[nbytes] = '\0';
2877 if (nbytes > UFDB_HTTP_1_1_MAX_URI_LENGTH-2-2)
2878 {
2879 ufdbLogMessage( "W%03d: input line has more than %d bytes and is truncated here",
2880 tnum, UFDB_HTTP_1_1_MAX_URI_LENGTH-2-2 );
2881 line[UFDB_MAX_URL_LENGTH-2-2] = '\0';
2882 }
2883 else if (line[nbytes-1] == '\n')
2884 {
2885 nbytes--;
2886 line[nbytes] = '\0'; /* remove the trailing \n */
2887 }
2888
2889 pthread_testcancel();
2890
2891 num_reqs++;
2892 /* no mutex nor atomic: faster and we do not care about the counter being off by a small number */
2893 UFDBlookupCounter++;
2894
2895 if (ufdbGV.debug > 1)
2896 ufdbLogMessage( "W%03d: request line=%s", tnum, line );
2897
2898 if (strncmp( line, "http://qdaemonstatus.urlfilterdb.com", 36 ) == 0)
2899 {
2900 sprintf( tmp, "http://cgibin.urlfilterdb.com/cgi-bin/URLblocked.cgi?reconfig=%d", reconfiguring );
2901 if (write_answer_redir( fd, answerBuf, tmp, &squidInfo ) < 0)
2902 {
2903 ufdbLogError( "W%03d: write failed: fd=%d %s", tnum, fd, strerror(errno) );
2904 goto write_error;
2905 }
2906 continue;
2907 }
2908
2909 if (parseLine(admin, line, &squidInfo) != 1) /* break down input line into struct squidInfo */
2910 {
2911 ufdbLogError( "W%03d: error parsing input line: %s", tnum, line );
2912 if (write_answer_ok( fd, answerBuf, &squidInfo ) < 0) /* TODO: use write_answer_error() */
2913 {
2914 ufdbLogError( "W%03d: write failed: fd=%d %s", tnum, fd, strerror(errno) );
2915 UFDBfreeRevURL( admin, squidInfo.revUrl );
2916 goto write_error;
2917 }
2918 continue;
2919 }
2920
2921 /* The strstr() is to prevent loops with redirect 302:... ads and other URLs
2922 * that are matched by regular expressions.
2923 * NEVER block these URLs:
2924 */
2925 if (strstr( squidInfo.orig, "/URLblocked.cgi" ) != NULL ||
2926 strncmp( squidInfo.orig, "blockedhttps.urlfilterdb.com", 28 ) == 0 ||
2927 strncmp( squidInfo.orig, "https://blockedhttps.urlfilterdb.com", 36 ) == 0 ||
2928 strncmp( squidInfo.orig, "http://cgibin.urlfilterdb.com/", 30 ) == 0)
2929 {
2930 if (write_answer_ok( fd, answerBuf, &squidInfo ) < 0)
2931 {
2932 ufdbLogError( "W%03d: write failed: fd=%d %s", tnum, fd, strerror(errno) );
2933 UFDBfreeRevURL( admin, squidInfo.revUrl );
2934 goto write_error;
2935 }
2936 continue;
2937 }
2938
2939 if (squidInfo.srcDomain[0] == '\0')
2940 {
2941 squidInfo.srcDomain[0] = '-';
2942 squidInfo.srcDomain[1] = '\0';
2943 }
2944
2945 if (squidInfo.ident[0] == '\0')
2946 {
2947 squidInfo.ident[0] = '-';
2948 squidInfo.ident[1] = '\0';
2949 }
2950
2951 #if UFDB_TEST_CRASH || 0
2952 if (strcmp( squidInfo.url, "debug.urlfilterdb.com/debug-thread-crash" ) == 0 &&
2953 strcmp( squidInfo.ident, "eviltest" ) == 0)
2954 {
2955 struct Source * s0;
2956 /* test/debug crash handler and ufdb-pstack */
2957 s0 = NULL;
2958 ufdbLogMessage( "W%03d: before forced SEGV", tnum );
2959 s0->name = (char *) "crash-here-with SEGV-signal";
2960 ufdbLogError( "W%03d: after forced SEGV", tnum );
2961 }
2962 #endif
2963
2964 if (reconfiguring == UFDB_RECONFIGR_NONE)
2965 {
2966 UFDBregisterCountedIP( squidInfo.srcIP );
2967 UFDBregisterCountedUser( squidInfo.ident );
2968 }
2969
2970 if (ufdbGV.reconfig) // 2ms pause to give the owner of the lock opportunity to change the
2971 usleep( 2000 ); // configuration and reset ufdbGV.reconfig
2972 reconfiguring = ufdbGV.reconfig;
2973 if (ufdbGV.terminating)
2974 break;
2975
2976 src = NULL;
2977 if (reconfiguring)
2978 {
2979 lock_used = 0;
2980 src = NULL;
2981 }
2982 else
2983 {
2984 lock_used = 1;
2985 ret = pthread_rwlock_rdlock( &TheDatabaseLock ); // >=================================
2986 if (ret != 0)
2987 {
2988 char errstr[128];
2989
2990 ufdbLogFatalError( "W%03d: pthread_rwlock_rdlock failed with code %d", tnum, ret );
2991 ufdbGV.reconfig = UFDB_RECONFIGR_FATAL;
2992 ufdbGV.terminating = 1;
2993 __asm__ volatile ("" : : : "memory"); // soft barrier for compiler
2994 (void) alarm( 0 );
2995 badSignal = SIGABRT;
2996 badSignalThread = pthread_self();
2997 sprintf( errstr, "pthread_rwlock_rdlock-failed-thread-%d-with-retval-%d", tnum, ret );
2998 ufdbExecutePstack( errstr );
2999 exit( 9 );
3000 }
3001 if (ufdbGV.reconfig)
3002 {
3003 reconfiguring = ufdbGV.reconfig;
3004 __asm__ volatile ("" : : : "memory"); // soft barrier for compiler
3005 (void) pthread_rwlock_unlock( &TheDatabaseLock ); // <======
3006 lock_used = 0;
3007 src = NULL;
3008 }
3009 else
3010 src = ufdbGV.sourceList;
3011 }
3012
3013 for ( ; ; )
3014 {
3015 const char * categoryIdent;
3016 const char * ACLident;
3017
3018 src = UFDBfindSource( src, &squidInfo );
3019 do_next_src:
3020 if (src != NULL)
3021 src->nmatches++; // no atomic increment: we may loose a few, no problem
3022 acl = UFDBfindACLbySource( src, &squidInfo ); // returns acl of src or ufdbGV.defaultAcl
3023
3024 if (parseOnly)
3025 {
3026 if (write_answer_ok( fd, answerBuf, &squidInfo ) < 0)
3027 {
3028 ufdbLogError( "W%03d: write failed: fd=%d %s", tnum, fd, strerror(errno) );
3029 UFDBfreeRevURL( admin, squidInfo.revUrl );
3030 goto write_error;
3031 }
3032 break;
3033 }
3034
3035 tmp[0] = '\0';
3036 isconnect = (strcmp( squidInfo.method, "CONNECT" ) == 0);
3037 redirect = tmp;
3038 /* >95% is decided here */
3039 decision = UFDBdecideAccessBasedOnURL( reconfiguring, acl, &squidInfo, redirect );
3040 /* squidInfo.matchedAny may be set here */
3041
3042 /* url_has_ipvN ? */
3043 if (decision == UFDB_ACL_ACCESS_DUNNO && (squidInfo.url_has_ipv4 || squidInfo.url_has_ipv6))
3044 {
3045 if (squidInfo.islocalnet)
3046 {
3047 decision = UFDB_ACL_ACCESS_ALLOW;
3048 if (ufdbGV.debug || ufdbGV.logPass || ufdbGV.logAllRequests)
3049 {
3050 ufdbLogMessage( "W%03d: IP %s is in localnet and is allowed",
3051 squidInfo.worker, squidInfo.domain );
3052 strcpy( squidInfo.matchedBy, "local-ip" );
3053 }
3054 }
3055 else
3056 {
3057 char host[1024];
3058 if (ufdbGV.lookupReverseIP &&
3059 UFDBfindCorrectHostNameForIP( squidInfo.worker, squidInfo.orig_domain, host ))
3060 {
3061 decision = UFDBdecideAccessBasedOnHostname( reconfiguring, acl, &squidInfo, admin,
3062 host, redirect );
3063 if (decision == UFDB_ACL_ACCESS_BLOCK)
3064 {
3065 /* Reset matchedAny since most likely it was set in UFDBdecideAccessBasedOnURL()
3066 * and since we block now based on reverse-ip it is not correct any more.
3067 */
3068 squidInfo.matchedAny = 0;
3069 if (ufdbGV.debug || ufdbGV.logBlock || ufdbGV.logAllRequests)
3070 {
3071 ufdbLogMessage( "W%03d: reverse IP of '%s' is '%s' and will be blocked",
3072 squidInfo.worker, squidInfo.domain, host );
3073 strcpy( squidInfo.matchedBy, "reverse-ip" );
3074 }
3075 }
3076 }
3077 }
3078 }
3079
3080 if (decision == UFDB_ACL_ACCESS_DUNNO && /* HTTPS probing */
3081 !squidInfo.BlockedBumpedConnectAllowed &&
3082 (squidInfo.port == 443 || isconnect))
3083 {
3084 if (ufdbGV.httpsWithHostname && (squidInfo.url_has_ipv4 || squidInfo.url_has_ipv6) &&
3085 !ufdbGV.SquidUsesActiveBumping)
3086 /* TODO: fix this when intercepted CONNECT has FQDN */
3087 {
3088 decision = UFDB_ACL_ACCESS_BLOCK;
3089 squidInfo.blockReason = UFDB_API_BLOCKR_HTTPS_OPTION;
3090 if (ufdbGV.debug)
3091 ufdbLogMessage( "%s:%d blocked because enforce-https-with-hostname is on",
3092 squidInfo.domain, squidInfo.port );
3093 }
3094 else
3095 {
3096 if (reconfiguring == UFDB_RECONFIGR_NONE && ufdbGV.reconfig == UFDB_RECONFIGR_NONE &&
3097 isconnect && ufdbGV.tunnelCheckMethod != UFDB_API_HTTPS_CHECK_OFF)
3098 {
3099 char * content;
3100 int certErrors;
3101 int peekStatus;
3102
3103 cn[0] = '\0';
3104 content = NULL;
3105 certErrors = 0;
3106
3107 /* NOTE: we only peek at URLs with CONNECT, so other ports are not peeked */
3108 peekStatus = UFDBsslPeekServer( squidInfo.orig_domain, squidInfo.port, cn, &certErrors,
3109 &content, squidInfo.worker );
3110 if (peekStatus == UFDB_API_OK)
3111 {
3112 if (certErrors && ufdbGV.httpsOfficialCertificate)
3113 {
3114 decision = UFDB_ACL_ACCESS_BLOCK;
3115 squidInfo.blockReason = UFDB_API_BLOCKR_HTTPS_OPTION;
3116 if (ufdbGV.debug)
3117 ufdbLogMessage( "%s:%d blocked because enforce-https-official-certificate is on",
3118 squidInfo.orig_domain, squidInfo.port );
3119 }
3120 /* make decision based on certificate CN iff URL has IP address */
3121 else if (!certErrors && (squidInfo.url_has_ipv4 || squidInfo.url_has_ipv6) &&
3122 cn[0] != '\0')
3123 {
3124 decision = UFDBdecideAccessBasedOnHostname( reconfiguring, acl, &squidInfo, admin, cn,
3125 redirect );
3126 if (decision != UFDB_ACL_ACCESS_DUNNO && ufdbGV.debug)
3127 ufdbLogMessage( "%s:%d allowed/blocked because the CN of the TLS/SSL certificate "
3128 "is %s and matched category %s",
3129 squidInfo.orig_domain, squidInfo.port, cn,
3130 squidInfo.aclpass->name );
3131 }
3132
3133 #ifdef UFDB_DO_HTTPS_CONTENT_SCANNING
3134 if (decision == UFDB_ACL_ACCESS_DUNNO && content != NULL)
3135 {
3136 /* Note that the result of UFDBfindContentCategory() cannot be cached since a URL
3137 * can be in more than one category and depending on the ACL it may evaluate to
3138 * one or the other.
3139 */
3140 if (UFDBfindContentCategory( acl, content, &category )) // TODO
3141 decision = UFDBdecideAccessBasedOnCategory( acl, &squidInfo, category, redirect );
3142 }
3143 #endif
3144
3145 if (content != NULL)
3146 ufdbFree( content );
3147 }
3148 else /* peekStatus not OK */
3149 {
3150 decision = UFDBdecideAccessByPeekStatus( peekStatus, acl, &squidInfo, redirect );
3151 if (decision == UFDB_ACL_ACCESS_BLOCK && ufdbGV.debug)
3152 ufdbLogMessage( "W%03d: %s:%d is blocked because it was probed and status is %s",
3153 squidInfo.worker, squidInfo.orig_domain, squidInfo.port,
3154 ufdbAPIstatusString(peekStatus) );
3155
3156 if (decision == UFDB_ACL_ACCESS_DUNNO && cn[0] != '\0')
3157 {
3158 /* the certificate has an error AND we retrieved a CN */
3159 /* for some CN's we make a decision here */
3160 if (strcmp( cn, "*.teamviewer.com" ) == 0)
3161 {
3162 decision = ufdbGV.TeamviewerOverHttps ? UFDB_ACL_ACCESS_ALLOW : UFDB_ACL_ACCESS_BLOCK;
3163 if (decision == UFDB_ACL_ACCESS_BLOCK)
3164 {
3165 squidInfo.blockReason = UFDB_API_BLOCKR_HTTPS_OPTION;
3166 ufdbLogMessage( "W%03d: %s:%d blocked because the CN of the TLS/SSL certificate "
3167 "is %s (teamviewer)",
3168 squidInfo.worker, squidInfo.orig_domain, squidInfo.port, cn );
3169 }
3170 }
3171 }
3172 }
3173 }
3174 }
3175 }
3176
3177 if (decision == UFDB_ACL_ACCESS_DUNNO && !squidInfo.matchedAny &&
3178 src != NULL && src->cont_search && src->next != NULL)
3179 {
3180 struct Source * nextsrc;
3181 nextsrc = UFDBfindSource( src->next, &squidInfo );
3182 if (nextsrc != NULL)
3183 {
3184 if (ufdbGV.debug > 1)
3185 ufdbLogMessage( "W%03d: source %s has no match and 'continue' so skipping to source %s",
3186 tnum, src->name, nextsrc->name );
3187 src = nextsrc;
3188 goto do_next_src;
3189 }
3190 }
3191
3192 if (decision == UFDB_ACL_ACCESS_DUNNO && squidInfo.port != 80 &&
3193 !squidInfo.BlockedBumpedConnectAllowed)
3194 {
3195 /* Scanning of implicitpass was skipped to favor calling UFDBsslPeekServer().
3196 * Lets evaluate implicitpass now.
3197 */
3198 decision = UFDBdecideImplicitAccessBasedOnURL( reconfiguring, acl, &squidInfo, redirect );
3199 if (decision != UFDB_ACL_ACCESS_DUNNO && ufdbGV.debug > 1)
3200 ufdbLogMessage( "W%03d: %s:%d is allowed because TLS peek was not decisive and the URL is "
3201 "in implicit category \"%s\"",
3202 tnum, squidInfo.orig_domain, squidInfo.port, squidInfo.aclpass->name );
3203 }
3204
3205 if (ufdbGV.debugRedirect)
3206 {
3207 ufdbLogMessage( "W%03d: REDIRECT %s %s %s %s", tnum,
3208 redirect,
3209 squidInfo.matchedBy[0] == 'h' && squidInfo.srcDomain[0] != '-' &&
3210 squidInfo.srcDomain[1] != '\0' ?
3211 squidInfo.srcDomain : squidInfo.srcIP,
3212 squidInfo.ident,
3213 squidInfo.method );
3214 }
3215
3216 if (squidInfo.blockReason == UFDB_API_BLOCKR_FATAL_ERROR)
3217 {
3218 ACLident = "config";
3219 categoryIdent = "fatal-error";
3220 }
3221 else if (squidInfo.blockReason == UFDB_API_BLOCKR_LOADING_DB)
3222 {
3223 ACLident = "config";
3224 categoryIdent = "loading-database";
3225 }
3226 else if (squidInfo.blockReason == UFDB_API_BLOCKR_HTTPS_OPTION)
3227 {
3228 ACLident = "config";
3229 categoryIdent = "https-option";
3230 }
3231 else if (squidInfo.blockReason == UFDB_API_BLOCKR_CHECKED)
3232 {
3233 if (acl == NULL)
3234 ACLident = "default";
3235 else
3236 ACLident = acl->name;
3237 categoryIdent = "checked";
3238 }
3239 else if (squidInfo.blockReason == UFDB_API_BLOCKR_PASS)
3240 {
3241 if (acl == NULL)
3242 ACLident = "default";
3243 else
3244 ACLident = acl->name;
3245 if (squidInfo.aclpass != NULL)
3246 {
3247 categoryIdent = squidInfo.aclpass->name;
3248 }
3249 else
3250 categoryIdent = "Implicit";
3251 }
3252 else
3253 {
3254 if (acl == NULL)
3255 ACLident = "acl-null";
3256 else
3257 ACLident = acl->name;
3258
3259 if (ufdbGV.fatalError)
3260 categoryIdent = "fatal-error";
3261 else if (reconfiguring)
3262 categoryIdent = "loading-database";
3263 else if (squidInfo.matchedNone)
3264 categoryIdent = "none";
3265 else if (squidInfo.matchedAny || squidInfo.aclpass == NULL)
3266 categoryIdent = "any";
3267 else if (squidInfo.aclpass->name == NULL)
3268 categoryIdent = "any";
3269 else
3270 categoryIdent = squidInfo.aclpass->name;
3271 }
3272 if (categoryIdent == NULL)
3273 {
3274 categoryIdent = "Null";
3275 ufdbLogError( "categoryIdent is NULL. url %s %s", squidInfo.url2log, squidInfo.matchedBy );
3276 }
3277
3278 if (decision == UFDB_ACL_ACCESS_ALLOW && squidInfo.blockReason == UFDB_API_BLOCKR_SAFESEARCH)
3279 {
3280 if (ufdbGV.logAllRequests || ufdbGV.logPass || ufdbGV.debug > 1 ||
3281 ufdbGV.debugRedirect)
3282 {
3283 ufdbLogMessage( "REDIR %-16s %-15s %-10s %-13s %s %s %s %s",
3284 squidInfo.ident,
3285 squidInfo.matchedBy[0] == 'h' && squidInfo.srcDomain[0] != '-' &&
3286 squidInfo.srcDomain[1] != '\0' ? squidInfo.srcDomain : squidInfo.srcIP,
3287 ACLident,
3288 "SAFESEARCH",
3289 squidInfo.url2log,
3290 squidInfo.method,
3291 isconnect ? squidInfo.sni : "",
3292 squidInfo.matchedBy );
3293 }
3294
3295 if (write_answer_rewrite( fd, answerBuf, squidInfo.orig, &squidInfo ) < 0)
3296 {
3297 ufdbLogError( "W%03d: write failed for safesearch redirect: fd=%d %s",
3298 tnum, fd, strerror(errno) );
3299 UFDBfreeRevURL( admin, squidInfo.revUrl );
3300 goto write_error;
3301 }
3302 break;
3303 }
3304 else if (decision == UFDB_ACL_ACCESS_DUNNO || decision == UFDB_ACL_ACCESS_ALLOW)
3305 {
3306 /* global SafeSearch */
3307 if (ufdbGV.safeSearch &&
3308 UFDBaddSafeSearch( squidInfo.domain, squidInfo.surl, squidInfo.orig )
3309 == UFDB_API_MODIFIED_FOR_SAFESEARCH)
3310 {
3311 UFDBsafesearchCounter++;
3312 if (ufdbGV.logAllRequests || ufdbGV.logPass || ufdbGV.debugRedirect)
3313 {
3314 ufdbLogMessage( "REDIR %-16s %-15s %-10s %-13s %s %s %s %s",
3315 squidInfo.ident,
3316 squidInfo.matchedBy[0] == 'h' && squidInfo.srcDomain[0] != '-' &&
3317 squidInfo.srcDomain[1] != '\0' ?
3318 squidInfo.srcDomain : squidInfo.srcIP,
3319 ACLident,
3320 "SAFESEARCH",
3321 squidInfo.url2log,
3322 squidInfo.method,
3323 isconnect ? squidInfo.sni : "",
3324 squidInfo.matchedBy );
3325 }
3326
3327 /* send a REDIRECT to Squid */
3328 if (write_answer_rewrite( fd, answerBuf, squidInfo.orig, &squidInfo ) < 0)
3329 {
3330 ufdbLogError( "W%03d: write failed: fd=%d %s", tnum, fd, strerror(errno) );
3331 UFDBfreeRevURL( admin, squidInfo.revUrl );
3332 goto write_error;
3333 }
3334 break;
3335 }
3336
3337 if (write_answer_ok( fd, answerBuf, &squidInfo ) < 0)
3338 {
3339 ufdbLogError( "W%03d: write failed: fd=%d %s", tnum, fd, strerror(errno) );
3340 UFDBfreeRevURL( admin, squidInfo.revUrl );
3341 goto write_error;
3342 }
3343
3344 if (ufdbGV.logAllRequests || ufdbGV.logPass)
3345 {
3346 if (squidInfo.BlockedBumpedConnectAllowed)
3347 {
3348 ufdbLogMessage( "BPASS %-16s %-15s %-10s %-13s %s %s %s %s",
3349 squidInfo.ident,
3350 squidInfo.srcIP,
3351 ACLident,
3352 squidInfo.islocalnet && squidInfo.matchedAny ? "localnet" : categoryIdent,
3353 squidInfo.url2log,
3354 squidInfo.method,
3355 isconnect ? squidInfo.sni : "",
3356 squidInfo.matchedBy );
3357 }
3358
3359 ufdbLogMessage( "PASS %-16s %-15s %-10s %-13s %s %s %s %s",
3360 squidInfo.ident,
3361 squidInfo.matchedBy[0] == 'h' && squidInfo.srcDomain[0] != '-' &&
3362 squidInfo.srcDomain[1] != '\0' ? squidInfo.srcDomain : squidInfo.srcIP,
3363 ACLident,
3364 squidInfo.islocalnet && squidInfo.matchedAny ? "localnet" : categoryIdent,
3365 squidInfo.url2log,
3366 squidInfo.method,
3367 isconnect ? squidInfo.sni : "",
3368 squidInfo.matchedBy );
3369 }
3370
3371 if (!reconfiguring && squidInfo.aclpass != NULL)
3372 {
3373 squidInfo.aclpass->nmatches++;
3374 cat = ufdbCategoryFindByName( &ufdbGV, squidInfo.aclpass->name );
3375 if (cat != NULL)
3376 cat->nmatches++;
3377 }
3378
3379 /* keep track of uncategorised URLs */
3380 if (!reconfiguring && !squidInfo.islocalnet &&
3381 (decision == UFDB_ACL_ACCESS_DUNNO || squidInfo.matchedAny || squidInfo.matchedNone))
3382 {
3383 if (!ufdbVerifyURLallCats( squidInfo.revUrl, squidInfo.url2log ))
3384 {
3385 ufdbRegisterUnknownURL( squidInfo.domain, squidInfo.port, squidInfo.referer );
3386 if (strchr( squidInfo.domain, '.' ) == NULL)
3387 UFDB_API_num_nodot_URL++;
3388 else
3389 UFDBuncategorisedCounter++;
3390 }
3391 }
3392
3393 break;
3394 }
3395 else /* decision == UFDB_ACL_ACCESS_BLOCK */
3396 {
3397 if (testMode)
3398 {
3399 ufdbLogMessage( "TEST-%-5s %s %s %s %s %s %s %s %s",
3400 _blockReason(squidInfo.blockReason),
3401 squidInfo.ident,
3402 squidInfo.matchedBy[0] == 'h' && squidInfo.srcDomain[0] != '-' &&
3403 squidInfo.srcDomain[1] != '\0' ? squidInfo.srcDomain : squidInfo.srcIP,
3404 ACLident,
3405 squidInfo.islocalnet && strcmp(categoryIdent,"any")==0 ?
3406 "localnet" : categoryIdent,
3407 squidInfo.url2log,
3408 squidInfo.method,
3409 isconnect ? squidInfo.sni : "",
3410 squidInfo.matchedBy );
3411 if (write_answer_ok( fd, answerBuf, &squidInfo ) < 0) /* send back an "OK" message */
3412 {
3413 ufdbLogError( "W%03d: write failed: fd=%d %s", tnum, fd, strerror(errno) );
3414 UFDBfreeRevURL( admin, squidInfo.revUrl );
3415 goto write_error;
3416 }
3417 UFDBtestBlockCounter++;
3418 }
3419 else
3420 {
3421 if (ufdbGV.logAllRequests || ufdbGV.logBlock)
3422 {
3423 ufdbLogMessage( "%-5s %-16s %-15s %-10s %-13s %s %s %s %s",
3424 _blockReason(squidInfo.blockReason),
3425 squidInfo.ident,
3426 squidInfo.matchedBy[0] == 'h' && squidInfo.srcDomain[0] != '-' &&
3427 squidInfo.srcDomain[1] != '\0' ? squidInfo.srcDomain : squidInfo.srcIP,
3428 ACLident,
3429 squidInfo.islocalnet && squidInfo.matchedAny ? "localnet" : categoryIdent,
3430 squidInfo.url2log,
3431 squidInfo.method,
3432 isconnect ? squidInfo.sni : "",
3433 squidInfo.matchedBy );
3434 }
3435
3436 if (redirect[0] == '\0')
3437 {
3438 if (isconnect)
3439 strcpy( redirect, ufdbGV.redirectHttps );
3440 else if (squidInfo.port == 443)
3441 strcpy( redirect, ufdbGV.redirectBumpedHttps );
3442 else
3443 strcpy( redirect, "http://cgibin.urlfilterdb.com/cgi-bin/URLblocked.cgi?"
3444 "category=%t&url=%u" );
3445 }
3446
3447 if (write_answer_redir( fd, answerBuf, redirect, &squidInfo ) < 0)
3448 {
3449 ufdbLogError( "W%03d: write failed: fd=%d %s", tnum, fd, strerror(errno) );
3450 UFDBfreeRevURL( admin, squidInfo.revUrl );
3451 goto write_error;
3452 }
3453 if (ufdbGV.debug > 2)
3454 ufdbLogMessage( " %s is redirected to %s", squidInfo.url2log, redirect );
3455 UFDBblockCounter++;
3456 if (src != NULL)
3457 src->nblocks++;
3458 if (!reconfiguring && squidInfo.aclpass != NULL)
3459 {
3460 squidInfo.aclpass->nblocks++;
3461 squidInfo.aclpass->nmatches++;
3462 cat = ufdbCategoryFindByName( &ufdbGV, squidInfo.aclpass->name );
3463 if (cat != NULL)
3464 {
3465 cat->nblocks++;
3466 cat->nmatches++;
3467 }
3468 }
3469 }
3470 break;
3471 }
3472 }
3473
3474 if (isconnect &&
3475 (ufdbGV.SquidUsesActiveBumping ||
3476 (cat != NULL && cat->activeBumping == UFDB_ACTIVE_BUMPING_ON)))
3477 UFDB_API_num_bumps++;
3478
3479 if (squidInfo.islocalnet)
3480 UFDB_API_num_url_localnet++;
3481
3482 if (lock_used)
3483 {
3484 ret = pthread_rwlock_unlock( &TheDatabaseLock ); // <=========================
3485 if (ret != 0)
3486 {
3487 char errstr[128];
3488
3489 ufdbLogFatalError( "W%03d: error releasing lock: %d", tnum, ret );
3490 ufdbGV.reconfig = UFDB_RECONFIGR_FATAL;
3491 ufdbGV.terminating = 1;
3492 __asm__ volatile ("" : : : "memory"); // soft barrier for compiler
3493 (void) alarm( 0 );
3494 badSignal = SIGABRT;
3495 badSignalThread = pthread_self();
3496 sprintf( errstr, "pthread_rwlock_unlock-failed-thread-%d-retval-%d", tnum, ret );
3497 ufdbExecutePstack( errstr );
3498 exit( 9 );
3499 }
3500 lock_used = 0;
3501 }
3502
3503 UFDBfreeRevURL( admin, squidInfo.revUrl );
3504 *line = '\0';
3505 *tmp = '\0';
3506 *answerBuf = '\0';
3507 squidInfo.protocol[0] = '\0';
3508 squidInfo.method[0] = '\0';
3509 squidInfo.revUrl = NULL;
3510 squidInfo.aclpass = NULL;
3511 squidInfo.referer[0] = '\0';
3512 squidInfo.ident[0] = '\0';
3513 squidInfo.url2log[0] = '\0';
3514 squidInfo.orig[0] = '\0';
3515 squidInfo.url[0] = '\0';
3516 squidInfo.surl[0] = '\0';
3517
3518 /* If configured, all traffic is slowed down artificially by introducing a 0.05 second delay.
3519 * The effect of this delay is that
3520 * 1) the database reload thread gets the CPU more often.
3521 * 2) during the database reload only a few URLs are verified (slowly) which means
3522 * that the users experience a slower internet connection and much less URLs
3523 * are passed through unfiltered.
3524 */
3525 if (reconfiguring)
3526 {
3527 if (ufdbGV.URLlookupDelayDBreload)
3528 usleep( 50000 );
3529 else
3530 {
3531 /* the sighup thread must have high priority to obtain the database lock and to load quickly */
3532 sched_yield();
3533 }
3534 }
3535
3536 if (UFDBhttpsVerificationQueued() > 3)
3537 {
3538 static volatile int ny = 0;
3539 if (++ny > 30)
3540 {
3541 ny = 0;
3542 sched_yield();
3543 }
3544 }
3545 if (ufdbGV.sig_other)
3546 break;
3547 } /* end of while-read loop */
3548
3549 if (nbytes < 0 && !ufdbGV.terminating)
3550 {
3551 ufdbLogError( "W%03d: read failed: fd=%d %s", tnum, fd, strerror(errno) );
3552 }
3553 write_error:
3554 *line = '\0';
3555 *tmp = '\0';
3556 *answerBuf = '\0';
3557
3558 decr_active_workers();
3559
3560 if (lock_used)
3561 {
3562 ret = pthread_rwlock_unlock( &TheDatabaseLock ); /* <======================== */
3563 if (ret != 0)
3564 {
3565 char errstr[128];
3566
3567 ufdbLogFatalError( "W%03d: pthread_rwlock_unlock after write error failed with code %d", tnum, ret );
3568 ufdbGV.reconfig = UFDB_RECONFIGR_FATAL;
3569 ufdbGV.terminating = 1;
3570 __asm__ volatile ("" : : : "memory"); // soft barrier for compiler
3571 (void) alarm( 0 );
3572 badSignal = SIGABRT;
3573 badSignalThread = pthread_self();
3574 sprintf( errstr, "pthread_rwlock_unlock-after-write-error-failed-thread-%d-retval-%d", tnum, ret );
3575 ufdbExecutePstack( errstr );
3576 exit( 9 );
3577 }
3578 lock_used = 0;
3579 }
3580
3581 if (num_reqs > 1UL)
3582 ufdbLogMessage( "W%03d: %lu URL verifications", tnum, num_reqs );
3583
3584 if (ufdbGV.debug > 2)
3585 ufdbLogMessage( "W%03d: close fd %2d", tnum, fd );
3586 close( fd );
3587 fd = -1;
3588 }
3589
3590 ufdbFree( line );
3591 ufdbFree( tmp );
3592 ufdbFree( answerBuf );
3593 if (fd >= 0)
3594 close( fd );
3595
3596 pthread_exit( NULL );
3597 }
3598
3599
3600 /*
3601 * main of thread that deals with dynamic lists of usernames defined by execuserlist.
3602 */
dynamic_userlist_updater_main(void * ptr)3603 static void * dynamic_userlist_updater_main( void * ptr )
3604 {
3605 sigset_t sigset;
3606 struct Source * src;
3607 int i, n;
3608 int ret;
3609 char * commands[UFDB_MAX_USERLISTS+1];
3610
3611 if (ptr != NULL)
3612 { ; }
3613
3614 sigemptyset( &sigset );
3615 sigaddset( &sigset, SIGHUP );
3616 sigaddset( &sigset, SIGUSR1 );
3617 sigaddset( &sigset, SIGUSR2 );
3618 sigaddset( &sigset, SIGWINCH );
3619 sigaddset( &sigset, SIGTERM );
3620 sigaddset( &sigset, SIGINT );
3621 sigaddset( &sigset, SIGALRM );
3622 sigaddset( &sigset, SIGCHLD );
3623 pthread_sigmask( SIG_BLOCK, &sigset, NULL );
3624
3625 ufdbSetThreadCPUaffinity( 0 );
3626
3627 sleep( 15 );
3628 goto first_refresh_short_delay;
3629
3630 while (1)
3631 {
3632 struct timespec tv;
3633
3634 pthread_testcancel();
3635
3636 if (ufdbGV.debug)
3637 ufdbLogMessage( "dynamic_userlist_updater_main: sleeping for %d minutes",
3638 ufdbGV.refreshUserlistInterval );
3639
3640 while (ufdbGV.reconfig)
3641 sleep( 1 );
3642
3643 errno = 0;
3644 tv.tv_sec = ufdbGV.refreshUserlistInterval * 60;
3645 tv.tv_nsec = 0;
3646 if (nanosleep( &tv, NULL ) < 0 && errno == EINTR)
3647 {
3648 if (tv.tv_sec > ufdbGV.refreshUserlistInterval * 30)
3649 (void) nanosleep( &tv, NULL );
3650 }
3651
3652 first_refresh_short_delay:
3653 if (ufdbGV.terminating)
3654 break;
3655 if (ufdbGV.reconfig)
3656 continue;
3657
3658 /* prepare a list of commands to call UFDBrefreshExecUserlist */
3659 ret = pthread_rwlock_rdlock( &TheDatabaseLock ); /* >============================== */
3660 if (ret != 0)
3661 {
3662 ufdbLogError( "dynamic_userlist_updater_main: database lock unlock failed with code %d *****", ret );
3663 continue;
3664 }
3665
3666 if (ufdbGV.reconfig) /* check again */
3667 {
3668 pthread_rwlock_unlock( &TheDatabaseLock ); /* <==== */
3669 continue;
3670 }
3671 if (ufdbGV.terminating)
3672 {
3673 pthread_rwlock_unlock( &TheDatabaseLock ); /* <==== */
3674 break;
3675 }
3676
3677 if (ufdbGV.debug || ufdbGV.debugExternalScripts)
3678 ufdbLogMessage( "dynamic_userlist_updater_main: going to refresh execuserlists" );
3679
3680 n = 0;
3681 src = ufdbGV.sourceList;
3682 while (src != NULL)
3683 {
3684 if (src->active && src->userDb != NULL && src->userDb->type == SGDBTYPE_EXECUSERLIST)
3685 {
3686 commands[n] = ufdbStrdup( src->sarg0 );
3687 ++n;
3688 }
3689 src = src->next;
3690 }
3691 commands[n] = NULL;
3692
3693 ret = pthread_rwlock_unlock( &TheDatabaseLock ); /* <============================== */
3694 if (ret != 0)
3695 ufdbLogError( "dynamic_userlist_updater_main: error releasing database lock: %d", ret );
3696
3697 /* call UFDBrefreshExecUserlist for each command */
3698 for (i = 0; i < n; ++i)
3699 {
3700 if (ufdbGV.terminating)
3701 break;
3702 if (ufdbGV.reconfig)
3703 {
3704 ufdbLogMessage( "dynamic_userlist_updater_main: NOT refreshing execuserlist for \"%s\" since "
3705 "the whole configuration is being reloaded",
3706 commands[i] );
3707 continue; /* continue here to print warnings for all commands */
3708 }
3709
3710 if (ufdbGV.debug || ufdbGV.debugExternalScripts)
3711 ufdbLogMessage( "dynamic_userlist_updater_main: refreshing execuserlist for \"%s\"", commands[i] );
3712
3713 /* NOTE: this may take a long time in some environments */
3714 UFDBrefreshExecUserlist( &ufdbGV, commands[i] );
3715
3716 if (ufdbGV.debug || ufdbGV.debugExternalScripts)
3717 ufdbLogMessage( "dynamic_userlist_updater_main: refresh completed for \"%s\"", commands[i] );
3718 ufdbFree( commands[i] );
3719 }
3720
3721 if (ufdbGV.debug || ufdbGV.debugExternalScripts)
3722 ufdbLogMessage( "dynamic_userlist_updater_main: refresh completed for all execuserlists "
3723 "(%d userlists)", n );
3724 }
3725
3726 /*NOTREACHED*/
3727 pthread_exit( NULL );
3728 }
3729
3730
3731 /*
3732 * main of thread that deals with dynamic IP address lists defined by execiplist.
3733 */
dynamic_iplist_updater_main(void * ptr)3734 static void * dynamic_iplist_updater_main( void * ptr )
3735 {
3736 sigset_t sigset;
3737 struct Source * src;
3738 int ret;
3739 int redoRefresh;
3740 struct timespec tv;
3741
3742 if (ptr != NULL)
3743 { ; }
3744
3745 sigemptyset( &sigset );
3746 sigaddset( &sigset, SIGWINCH );
3747 pthread_sigmask( SIG_UNBLOCK, &sigset, NULL );
3748
3749 sigemptyset( &sigset );
3750 sigaddset( &sigset, SIGHUP );
3751 sigaddset( &sigset, SIGUSR1 );
3752 sigaddset( &sigset, SIGUSR2 );
3753 sigaddset( &sigset, SIGTERM );
3754 sigaddset( &sigset, SIGINT );
3755 sigaddset( &sigset, SIGALRM );
3756 sigaddset( &sigset, SIGCHLD );
3757 pthread_sigmask( SIG_BLOCK, &sigset, NULL );
3758
3759 ufdbSetThreadCPUaffinity( 0 );
3760
3761 while (1)
3762 {
3763 pthread_testcancel();
3764
3765 while (ufdbGV.reconfig != UFDB_RECONFIGR_NONE)
3766 sleep( 1 );
3767
3768 if (ufdbGV.debug)
3769 ufdbLogMessage( "dynamic_iplist_updater_main: going to sleep for %d minutes",
3770 ufdbGV.refreshIPlistInterval );
3771
3772 errno = 0;
3773 tv.tv_sec = ufdbGV.refreshIPlistInterval * 60;
3774 tv.tv_nsec = 0;
3775
3776 errno = 0;
3777 (void) nanosleep( &tv, NULL ); // may be interrupted by WINCH signal to force immediate refresh
3778 if (ufdbGV.debug && errno == EINTR)
3779 {
3780 ufdbLogMessage( "dynamic_iplist_updater_main: nanosleep was interrupted to trigger immediate refresh" );
3781 // XXX TODO: we might receive a bunch of WINCH signals and want to eat them all here
3782 // TODO: use sigpending+sigwait and sigprocmask+sigsuspend after obtaining the database lock
3783 }
3784
3785 redo:
3786 if (ufdbGV.terminating)
3787 {
3788 ufdbLogMessage( "dynamic_iplist_updater_main: aborting refresh since ufdbguardd is terminating" );
3789 break;
3790 }
3791
3792 if (ufdbGV.debug || ufdbGV.debugExternalScripts)
3793 ufdbLogMessage( "dynamic_iplist_updater_main: going to refresh execiplists" );
3794
3795 ret = pthread_rwlock_rdlock( &TheDatabaseLock ); // >==================================
3796 if (ret != 0)
3797 ufdbLogError( "dynamic_iplist_updater_main: pthread_rwlock_rdlock database rdlock failed with code %d",
3798 ret );
3799
3800 #if 0
3801 if (ufdbGV.reconfig) /* check again */
3802 {
3803 if (ufdbGV.debug)
3804 ufdbLogMessage( "dynamic_iplist_updater_main: postponing refresh since reconfigure is happening now" );
3805 pthread_rwlock_unlock( &TheDatabaseLock ); // <=======
3806 while (ufdbGV.reconfig)
3807 sleep( 1 );
3808 goto redo;
3809 }
3810 #endif
3811
3812 if (ufdbGV.terminating)
3813 {
3814 ufdbLogMessage( "dynamic_iplist_updater_main: aborting refresh since ufdbguardd is terminating" );
3815 pthread_rwlock_unlock( &TheDatabaseLock ); // <=======
3816 break;
3817 }
3818
3819 // sometimes we may get a storm of WINCH signals so remove them all from the queue
3820 tv.tv_sec = 0;
3821 tv.tv_nsec = 0;
3822 sigemptyset( &sigset );
3823 sigaddset( &sigset, SIGWINCH );
3824 while (sigtimedwait( &sigset, NULL, &tv ) == SIGWINCH)
3825 if (ufdbGV.debug)
3826 ufdbLogMessage( "dynamic_iplist_updater_main: removed extra WINCH signal from the signals queue" );
3827
3828 // call UFDBrefreshExecIPlist for each execiplist
3829 redoRefresh = 0;
3830 src = ufdbGV.sourceList;
3831 while (src != NULL)
3832 {
3833 if (src->active && src->execiplistCommand != NULL)
3834 {
3835 if (ufdbGV.terminating)
3836 break;
3837 if (ufdbGV.reconfig)
3838 {
3839 ufdbLogMessage( "dynamic_iplist_updater_main: NOT refreshing execiplist for source \"%s\" since"
3840 " the configuration is going to be reloaded",
3841 src->name );
3842 redoRefresh = 1;
3843 src = src->next;
3844 continue; /* continue here to print message for all sources */
3845 }
3846 if (ufdbGV.debug || ufdbGV.debugExternalScripts)
3847 ufdbLogMessage( "dynamic_iplist_updater_main: refreshing execiplist for source \"%s\": %s",
3848 src->name, src->execiplistCommand );
3849
3850 /* NOTE: this may take a long time in some environments */
3851 UFDBrefreshExecIPlist( &ufdbGV, src->execiplistCommand, &src->ipv4hosts, &src->ipv4,
3852 &src->ipv6hosts, &src->ipv6 );
3853
3854 if (ufdbGV.debug || ufdbGV.debugExternalScripts)
3855 ufdbLogMessage( "dynamic_iplist_updater_main: refresh completed for source \"%s\"", src->name );
3856 }
3857 src = src->next;
3858 }
3859
3860 ret = pthread_rwlock_unlock( &TheDatabaseLock ); // <=================================
3861 if (ret != 0)
3862 ufdbLogError( "dynamic_iplist_updater_main: error releasing database rdlock: %d", ret );
3863
3864 if (ufdbGV.terminating)
3865 {
3866 ufdbLogMessage( "dynamic_iplist_updater_main: exiting thread since we are terminating" );
3867 break;
3868 }
3869
3870 if (redoRefresh)
3871 {
3872 while (ufdbGV.reconfig != UFDB_RECONFIGR_NONE)
3873 sleep( 1 );
3874 if (ufdbGV.debug)
3875 ufdbLogMessage( "dynamic_iplist_updater_main: going to redo the refresh since "
3876 "ufdbguardd was reconfigured" );
3877 goto redo;
3878 }
3879 }
3880
3881 pthread_exit( NULL );
3882 }
3883
3884
3885 /*
3886 * main of thread that deals with dynamic categories
3887 * defined by execdomainlist.
3888 */
dynamic_domainlist_updater_main(void * ptr)3889 static void * dynamic_domainlist_updater_main( void * ptr )
3890 {
3891 sigset_t sigset;
3892 struct Category * clist;
3893 int ret;
3894 int n;
3895
3896 if (ptr != NULL)
3897 { ; }
3898
3899 sigemptyset( &sigset );
3900 sigaddset( &sigset, SIGHUP );
3901 sigaddset( &sigset, SIGUSR1 );
3902 sigaddset( &sigset, SIGUSR2 );
3903 sigaddset( &sigset, SIGWINCH );
3904 sigaddset( &sigset, SIGTERM );
3905 sigaddset( &sigset, SIGINT );
3906 sigaddset( &sigset, SIGALRM );
3907 sigaddset( &sigset, SIGCHLD );
3908 pthread_sigmask( SIG_BLOCK, &sigset, NULL );
3909
3910 ufdbSetThreadCPUaffinity( 0 );
3911
3912 sleep( 60 );
3913 goto first_refresh_no_delay;
3914
3915 while (1)
3916 {
3917 struct timespec tv;
3918
3919 pthread_testcancel();
3920
3921 if (ufdbGV.debug)
3922 ufdbLogMessage( "dynamic_domainlist_updater_main: sleeping for %d minutes",
3923 ufdbGV.refreshDomainlistInterval );
3924
3925 while (ufdbGV.reconfig)
3926 sleep( 1 );
3927
3928 tv.tv_sec = ufdbGV.refreshDomainlistInterval * 60;
3929 tv.tv_nsec = 0;
3930 (void) nanosleep( &tv, NULL );
3931
3932 first_refresh_no_delay:
3933 if (ufdbGV.terminating)
3934 break;
3935 if (ufdbGV.reconfig)
3936 continue;
3937
3938 if (ufdbGV.debug || ufdbGV.debugExternalScripts)
3939 ufdbLogMessage( "dynamic_domainlist_updater_main: going to refresh execdomainlists" );
3940
3941 ret = pthread_rwlock_rdlock( &TheDatabaseLock ); /* >============================== */
3942 if (ret != 0)
3943 {
3944 ufdbLogError( "dynamic_domainlist_updater_main: pthread_rwlock_rdlock database rdlock failed "
3945 "with code %d *****", ret );
3946 continue;
3947 }
3948
3949 n = 0;
3950 for (clist = ufdbGV.catList; clist != NULL; clist = clist->next)
3951 {
3952 if (clist->active && clist->execdomainlist != NULL)
3953 {
3954 struct sgDb * oldddb;
3955 struct sgDb * newddb;
3956
3957 if (ufdbGV.debug || ufdbGV.debugExternalScripts)
3958 ufdbLogMessage( "dynamic_domainlist_updater_main: refreshing execdomainlist for %s (%s)",
3959 clist->name, clist->execdomainlist );
3960
3961 newddb = UFDBretrieveExecDomainlist( clist );
3962 if (newddb == NULL)
3963 {
3964 if (ufdbGV.debug || ufdbGV.debugExternalScripts)
3965 ufdbLogMessage( "refresh execdomainlist for %s presented no new domainlist", clist->name );
3966 continue;
3967 }
3968
3969 ret = pthread_rwlock_wrlock( &TheDynamicCategoriesLock ); // >-------
3970 if (ret != 0)
3971 ufdbLogError( "dynamic_domainlist_updater_main: pthread_rwlock_wrlock TheDynamicCategoriesLock "
3972 "failed with code %d", ret );
3973
3974 oldddb = clist->domainlistDb;
3975 clist->domainlistDb = newddb;
3976
3977 ret = pthread_rwlock_unlock( &TheDynamicCategoriesLock ); // <-------
3978 if (ret != 0)
3979 ufdbLogError( "dynamic_domainlist_updater_main: pthread_rwlock_unlock TheDynamicCategoriesLock "
3980 "failed with code %d", ret );
3981
3982 n++;
3983 ufdbFreeDomainDb( oldddb );
3984
3985 if (ufdbGV.debug > 1 || ufdbGV.debugExternalScripts)
3986 ufdbLogMessage( "dynamic_domainlist_updater_main: refresh execdomainlist for \"%s\" completed",
3987 clist->name );
3988 }
3989 if (ufdbGV.terminating)
3990 break;
3991 }
3992
3993 ret = pthread_rwlock_unlock( &TheDatabaseLock ); /* <============================== */
3994 if (ret != 0)
3995 ufdbLogError( "dynamic_domainlist_updater_main: error releasing database rdlock: %d *****", ret );
3996
3997 if (ufdbGV.debug || ufdbGV.debugExternalScripts)
3998 ufdbLogMessage( "dynamic_domainlist_updater_main: refresh completed for all execdomainlists "
3999 "(%d domainlists)", n );
4000
4001 if (ufdbGV.terminating)
4002 break;
4003 }
4004
4005 pthread_exit( NULL );
4006 }
4007
4008
doCommand(char * command)4009 void doCommand( char * command )
4010 {
4011 /* There is a serious problem sending signals to ufdbguardd and ufdbhttpd.
4012 * 1) ufdbguardd cannot send SIGTERM, but can send SIGKILL to ufdbhttpd.
4013 * 2) ufdbUpdate cannot send SIGHUP to ufdbguardd.
4014 */
4015
4016 if (strcmp( command, "verify" ) == 0)
4017 {
4018 testConfig = 1;
4019 runAsDaemon = 0;
4020 ufdbGV.uploadStats = 0;
4021 ufdbGlobalLogfile = stderr;
4022 }
4023 }
4024
4025
4026 extern char ** environ;
4027
main(int argc,char ** argv)4028 int main(
4029 int argc,
4030 char ** argv )
4031 {
4032 int i;
4033 pid_t pid;
4034 sigset_t sigset;
4035 pthread_attr_t attr;
4036 void * dummy_ptr;
4037 struct rlimit openfiles;
4038
4039 char ** envp = environ;
4040 pthread_rwlockattr_t mylock_attr;
4041
4042 UFDBinitializeGV( &ufdbNewGV );
4043 ufdbGV.maxLogfileSize = 200 * 1024 * 1024;
4044 UFDBresetGV( &ufdbNewGV );
4045 strcpy( ufdbNewGV.progname, "ufdbguardd" );
4046 ufdbNewGV.reconfig = UFDB_RECONFIGR_INIT;
4047 // the threads use ufdbGV and need a set of variables already set so just copy the (mostly zeroed)
4048 // ufdbNewGV to ufdbGV:
4049 ufdbGV = ufdbNewGV;
4050 __asm__ volatile ("" : : : "memory"); // soft barrier for compiler
4051
4052 /* TO-DO: unset LC variables; regex cannot deal with multibyte characters */
4053
4054 while ((i = getopt(argc, argv, "AC:TNdDRPSh?rt:c:L:p:U:vw:")) != EOF)
4055 {
4056 switch (i) {
4057 case 'A':
4058 fprintf( stderr, "-A option is obsolete\n");
4059 break;
4060 case 'C':
4061 doCommand( optarg );
4062 break;
4063 case 'N':
4064 fprintf( stderr, "-N option is obsolete.\n" );
4065 break;
4066 case 'D':
4067 runAsDaemon = 0; /* undocumented -D option to prevent running as daemon */
4068 break;
4069 case 'd':
4070 ufdbNewGV.debug++;
4071 #if UFDB_DEBUG_YY
4072 yydebug = 1;
4073 #endif
4074 break;
4075 case 'c':
4076 configFile = optarg;
4077 break;
4078 case 'p':
4079 portNumCmdLine = atoi( optarg );
4080 if (portNumCmdLine <= 0) {
4081 fprintf( stderr, "port number must be > 0\n" );
4082 exit( 1 );
4083 }
4084 break;
4085 case 'P': /* undocumented -P option for development purposes */
4086 parseOnly = 1;
4087 break;
4088 case 'R':
4089 ufdbNewGV.debugRegexp = 1;
4090 break;
4091 case 'S':
4092 ufdbNewGV.uploadStats = 0;
4093 break;
4094 case 'r':
4095 ufdbNewGV.debugRedirect = 1;
4096 break;
4097 case 't': /* undocumented -t option for time-related testing */
4098 ufdbLogError( "ufdbguardd does no longer support the -t option" );
4099 break;
4100 case 'T':
4101 testMode = 1;
4102 break;
4103 case 'U':
4104 if (strlen(optarg) <= 31)
4105 {
4106 strcpy( ufdbGV.userName, optarg );
4107 strcpy( ufdbNewGV.userName, optarg );
4108 }
4109 else
4110 ufdbLogFatalError( "username supplied with -U option is too long" );
4111 break;
4112 case 'L': /* TODO: remove undocumented -L option for alternate PID file */
4113 ufdbNewGV.pidFilename = ufdbStrdup( optarg );
4114 break;
4115 case 'v':
4116 fprintf( stderr, "ufdbguardd %s\n", UFDB_VERSION );
4117 fprintf( stderr, "The ufdbGuard software suite is Free and Open Source Software.\n" );
4118 fprintf( stderr, "Copyright (C) 2005-2020 by URLfilterDB B.V. and others.\n" );
4119 exit( 0 );
4120 break;
4121 case 'w':
4122 ufdbNewGV.nWorkers = ufdbGV.nWorkers = atoi( optarg );
4123 if (ufdbNewGV.nWorkers < UFDB_MIN_THREADS)
4124 {
4125 fprintf( stderr, "-w option is too low. The number of threads is set to %d\n",
4126 UFDB_MIN_THREADS );
4127 ufdbNewGV.nWorkers = ufdbGV.nWorkers = UFDB_MIN_THREADS;
4128 }
4129 else if (ufdbNewGV.nWorkers > UFDB_MAX_WORKERS)
4130 {
4131 fprintf( stderr, "-w option exceeds the maximum. The number of threads is set to %d\n",
4132 UFDB_MAX_WORKERS );
4133 ufdbNewGV.nWorkers = ufdbGV.nWorkers = UFDB_MAX_WORKERS;
4134 }
4135 break;
4136 case '?':
4137 case 'h':
4138 usage( '\0' );
4139 break;
4140 default:
4141 usage( i );
4142 }
4143 }
4144
4145 if (runAsDaemon)
4146 {
4147 if ((pid = fork()) != 0) /* the parent exits */
4148 {
4149 /* systemd likes to see the pid file when we exit but since the child process
4150 * creates the pid file (rather late) we will busy-wait for the creation of the pid file
4151 * with a maximum of 10 seconds.
4152 * Note that the pid file may have an alternative name and the busy-wait waits too long...
4153 */
4154 for (i = 0; i < 10; i++)
4155 {
4156 struct stat stbuf;
4157 sleep( 1 );
4158 errno = 0;
4159 /* TODO: use ufdbGV.PidFile if not NULL */
4160 if (stat( UFDBGUARDD_PID_FILE, &stbuf ) == 0)
4161 break;
4162 }
4163 exit( 0 );
4164 }
4165 #ifndef UFDB_DEBUG
4166 close( 0 );
4167 close( 1 );
4168 #endif
4169 setsid(); /* TODO: is setsid necessary when called from systemd ? */
4170 }
4171 ufdbGV.pid = ufdbNewGV.pid = getpid();
4172
4173 ufdbSetGlobalErrorLogFile( NULL, NULL, 0 );
4174
4175 /* currently UFDB_MAX_THREADS is 1285 */
4176 i = getrlimit( RLIMIT_NOFILE, &openfiles );
4177 if (i == 0)
4178 {
4179 if (openfiles.rlim_cur < UFDB_MAX_THREADS * 2 + 1)
4180 {
4181 if (UFDB_MAX_THREADS * 2 + 1 <= openfiles.rlim_max)
4182 openfiles.rlim_cur = UFDB_MAX_THREADS * 2 + 1;
4183 else
4184 openfiles.rlim_cur = openfiles.rlim_max;
4185 }
4186 /* do not touch openfiles.rlim_max */
4187 }
4188 else
4189 {
4190 openfiles.rlim_cur = UFDB_MAX_THREADS * 2 + 1;
4191 openfiles.rlim_max = UFDB_MAX_THREADS * 2 + 1;
4192 }
4193 i = setrlimit( RLIMIT_NOFILE, &openfiles );
4194 if (i != 0)
4195 ufdbLogError( "cannot set maximum number of open files to %ld: %s *****",
4196 (long) openfiles.rlim_cur, strerror(errno) );
4197
4198 UFDBdropPrivileges( ufdbGV.userName );
4199
4200 UFDBinitializeIPcounters();
4201 UFDBinitializeUserCounters();
4202
4203 /*
4204 * Initialise signal handlers.
4205 * The HUP signal is dealt with on a per thread basis.
4206 */
4207 #if !UFDB_PRODUCE_CORE_DUMPS
4208 ufdbSetSignalHandler( SIGILL, ufdbCatchBadSignal );
4209 ufdbSetSignalHandler( SIGBUS, ufdbCatchBadSignal );
4210 ufdbSetSignalHandler( SIGSEGV, ufdbCatchBadSignal );
4211 ufdbSetSignalHandler( SIGQUIT, ufdbCatchBadSignal );
4212 ufdbSetSignalHandler( SIGABRT, catchAbortSignal );
4213 #endif
4214
4215 ufdbSetSignalHandler( SIGPIPE, SIG_IGN );
4216
4217 ufdbSetSignalHandler( SIGTERM, catchTermSignal );
4218 ufdbSetSignalHandler( SIGINT, catchTermSignal );
4219
4220 ufdbSetSignalHandler( SIGHUP, catchHUPSignal );
4221 ufdbSetSignalHandler( SIGUSR1, catchUSR1Signal );
4222 ufdbSetSignalHandler( SIGUSR2, catchUSR2Signal );
4223 ufdbSetSignalHandler( SIGWINCH, catchWINCHSignal );
4224
4225 ufdbSetSignalHandler( SIGCHLD, catchChildSignal );
4226
4227 /* All signals must be blocked.
4228 * This is a requirement to use sigwait() in a thread.
4229 */
4230 sigemptyset( &sigset );
4231 sigaddset( &sigset, SIGHUP );
4232 sigaddset( &sigset, SIGUSR1 );
4233 sigaddset( &sigset, SIGUSR2 );
4234 sigaddset( &sigset, SIGWINCH );
4235 sigaddset( &sigset, SIGCHLD );
4236 sigaddset( &sigset, SIGTERM );
4237 sigaddset( &sigset, SIGINT );
4238 sigaddset( &sigset, SIGALRM );
4239 pthread_sigmask( SIG_BLOCK, &sigset, NULL );
4240
4241 ufdbNewGV.Argv = argv;
4242 ufdbNewGV.Envp = envp;
4243
4244 if (testConfig)
4245 {
4246 ufdbNewGV.fatalError = 0;
4247 ufdbNewGV.httpdPort = 0;
4248 if (ufdbReadConfig( configFile ) == 0)
4249 {
4250 ufdbLogFatalError( "missing configuration file" );
4251 exit( 3 );
4252 }
4253 if (ufdbNewGV.fatalError)
4254 {
4255 ufdbLogMessage( "A FATAL ERROR OCCURRED: (see *previous* lines with \"FATAL ERROR\" "
4256 "for more details) *****" );
4257 exit( 1 );
4258 }
4259 exit( 0 );
4260 }
4261
4262 char * ev;
4263 ev = getenv( "LC_CTYPE" );
4264 if (ev == NULL)
4265 ufdbLogMessage( "LC_CTYPE is not set" );
4266 else
4267 ufdbLogMessage( "LC_CTYPE is '%s'", ev );
4268
4269 ev = getenv( "LANG" );
4270 if (ev == NULL)
4271 ufdbLogMessage( "LANG is not set" );
4272 else
4273 ufdbLogMessage( "LANG is '%s'", ev );
4274
4275 pthread_rwlockattr_init( &mylock_attr );
4276 #if HAVE_PTHREAD_RWLOCK_PREFER_WRITER_NP
4277 pthread_rwlockattr_setkind_np( &mylock_attr, PTHREAD_RWLOCK_PREFER_WRITER_NP );
4278 #else
4279 ufdbLogMessage( "WARNING: could not set writer lock preference for the database lock. *****\n"
4280 "This may delay processing updates and loading a new configuration. *****" );
4281 #endif
4282 pthread_rwlock_init( &TheDatabaseLock, &mylock_attr );
4283
4284 UFDBaclEngineInit();
4285
4286 /*
4287 * create the threads.
4288 */
4289
4290 pthread_attr_init( &attr );
4291 pthread_attr_setscope( &attr, PTHREAD_SCOPE_SYSTEM );
4292 #if HAVE_PTHREAD_ATTR_SETGUARDSIZE && UFDB_DEBUG
4293 pthread_attr_setstacksize( &attr, UFDB_STACK_SIZE - (8 * 1024) );
4294 pthread_attr_setguardsize( &attr, 8 * 1024 );
4295 #else
4296 pthread_attr_setstacksize( &attr, UFDB_STACK_SIZE );
4297 #endif
4298
4299 pthread_create( &threadedSignalHandler, &attr, signal_handler_thread, (void *) 0 );
4300 /* sleep a bit since we really want that signal_handler_thread does its initialisation before going ahead */
4301 usleep( 50000 );
4302
4303 UFDBinitHTTPSchecker();
4304
4305 ufdbGV.nWorkers = ufdbNewGV.nWorkers;
4306 for (i = 0; i < ufdbGV.nWorkers; i++)
4307 {
4308 pthread_create( &workers[i], &attr, worker_main, (void *) ((long) i) );
4309 if (i % 4 == 0)
4310 sched_yield();
4311 }
4312 ufdbLogMessage( "%d worker threads created.", ufdbGV.nWorkers );
4313
4314 for (i = 0; i < UFDB_NUM_HTTPS_VERIFIERS; i++)
4315 {
4316 pthread_create( &httpsthr[i], &attr, UFDBhttpsTunnelVerifier, (void *) ((long) i) );
4317 if (i % 4 == 0)
4318 sched_yield();
4319 }
4320 ufdbLogMessage( "%d HTTPS verification threads created.", UFDB_NUM_HTTPS_VERIFIERS );
4321
4322 pthread_create( &dyn_userlist_handler, &attr, dynamic_userlist_updater_main, (void *) 0 );
4323 pthread_create( &dyn_iplist_handler, &attr, dynamic_iplist_updater_main, (void *) 0 );
4324 pthread_create( &dyn_domainlist_handler, &attr, dynamic_domainlist_updater_main, (void *) 0 );
4325 pthread_create( &housekeeper, &attr, housekeeper_main, (void *) 0 );
4326
4327 /* sleep a bit to give the system the opportunity to finish creating all threads */
4328 usleep( 50000 );
4329
4330 ufdbResetUnknownURLs();
4331
4332 prev_n_workers = ufdbGV.nWorkers;
4333 ufdbNewGV.fatalError = 0;
4334 if (ufdbReadConfig( configFile ) == 0)
4335 {
4336 ufdbLogFatalError( "exiting due to missing configuration file" );
4337 UFDBchangeStatus( UFDB_API_STATUS_FATAL_ERROR );
4338 exit( 3 );
4339 }
4340 if (ufdbNewGV.debug)
4341 ufdbLogMessage( "ufdbReadConfig() finished. fatal=%d", ufdbGV.fatalError );
4342
4343 /* Now is the right moment to write the PID to the pid file.
4344 * Note that systemd monitors the creation of the PID file.
4345 */
4346 if (writePidFile() == 0)
4347 {
4348 ufdbLogError( "make sure that %s exists and has the correct permissions", DEFAULT_PIDDIR );
4349 ufdbLogFatalError( "cannot write my PID to the pidfile (see previous lines)" );
4350 }
4351 atexit( removePidFile );
4352
4353 UFDBlogConfig( &ufdbNewGV );
4354 ufdbNewGV.terminating = ufdbGV.terminating;
4355 ufdbNewGV.reconfig = ufdbGV.reconfig;
4356 ufdbNewGV.fatalError = ufdbGV.fatalError;
4357 ufdbGV = ufdbNewGV;
4358
4359 ufdbRegisterFatalErrorCrashfun( crasher );
4360
4361 adjustNumberOfWorkerThreads(); /* create more worker threads if necessary */
4362
4363 #ifdef _POSIX_PRIORITY_SCHEDULING
4364 sched_yield();
4365 #else
4366 usleep( 50000 );
4367 #endif
4368
4369 if (testMode)
4370 ufdbLogMessage( "-T option is used. In test mode no URLs are ever blocked." );
4371
4372 if (ufdbNewGV.fatalError)
4373 {
4374 ufdbLogError( "A FATAL ERROR OCCURRED: ALL REQUESTS ARE ANSWERED WITH \"OK\" "
4375 "(see previous lines with \"FATAL ERROR\" for more information) *****" );
4376 UFDBchangeStatus( UFDB_API_STATUS_FATAL_ERROR );
4377 }
4378 else
4379 {
4380 ufdbHandleAlarmForTimeEvents( UFDB_PARAM_INIT );
4381 UFDBchangeStatus( UFDB_API_STATUS_STARTED_OK );
4382 }
4383
4384 /* The last thread to create is socket_handler_thread() which listens on the socket
4385 * where clients can connect.
4386 */
4387 pthread_create( &sockethandler, &attr, socket_handler_thread, (void *) 0 );
4388
4389 /*
4390 * and finally we have some time to upload crash reports.
4391 */
4392 sleep( 1 );
4393 upload_crash_reports();
4394
4395 /*
4396 * prevent an immediate exit.
4397 */
4398 sleep( 2 );
4399
4400 /*
4401 * Exit when the sockethandler thread terminates, but
4402 * skip the call to pthread_join() if we already received a bad signal.
4403 */
4404 if (!badSignal)
4405 {
4406 pthread_join( sockethandler, &dummy_ptr );
4407 ufdbLogMessage( "Socket handler stopped. ufdbguardd is going to stop." );
4408 }
4409 else
4410 pthread_kill( sockethandler, SIGTERM );
4411
4412 /*
4413 * Note: sockethandler terminated because of a signal.
4414 */
4415 ufdbGV.reconfig = UFDB_RECONFIGR_FINISH;
4416 ufdbGV.terminating = 1;
4417 __asm__ volatile ("" : : : "memory"); // soft barrier for compiler
4418 alarm( 0 );
4419
4420 if (badSignal)
4421 {
4422 ufdbLogMessage( "main: bad signal %d was caught", badSignal );
4423 removePidFile();
4424 badSignalHandlerBusy = 1;
4425 BadSignalCaught( badSignal );
4426 while (badSignalHandlerBusy)
4427 usleep( 150000 );
4428 }
4429
4430 ufdbLogMessage( "terminating connections with clients" );
4431 UFDBcloseFilesNoLog();
4432
4433 if (ufdbGV.debug)
4434 {
4435 usleep( 150000 );
4436 ufdbLogMessage( "cancelling threads ..." );
4437 pthread_cancel( housekeeper );
4438 pthread_cancel( dyn_userlist_handler );
4439 pthread_cancel( dyn_iplist_handler );
4440 pthread_cancel( dyn_domainlist_handler );
4441 for (i = 0; i < UFDB_NUM_HTTPS_VERIFIERS; i++)
4442 pthread_cancel( httpsthr[i] );
4443 for (i = 0; i < ufdbGV.nWorkers; i++)
4444 pthread_cancel( workers[i] );
4445 usleep( 500000 );
4446 }
4447
4448 removePidFile();
4449 UFDBchangeStatus( UFDB_API_STATUS_TERMINATED );
4450 ufdbLogMessage( "ufdbGuard daemon stopped" );
4451
4452 if (!badSignal && (ufdbGV.debug > 1 || getenv( "UFDB_FREE_MEMORY_ON_EXIT" ) != NULL))
4453 {
4454 usleep( 250000 );
4455 ufdbLogMessage( "freeing all memory ..." );
4456 ufdbFreeAllMemory( &ufdbGV );
4457 ufdbFreeLastBits( &ufdbGV );
4458 UFDBdeleteUserlistCache();
4459 /* TODO: also free IPcache, timeElements, hashTable, HTTPScache */
4460 ufdbLogMessage( "done freeing memory." );
4461 }
4462
4463 _exit( 0 );
4464
4465 return 0;
4466 }
4467
4468
isaclient(void)4469 static int isaclient( void )
4470 {
4471 return ufdbGV.checkedDB.mem != NULL && ufdbGV.checkedDB.flags[1] == 'P';
4472 }
4473
4474
makeBlockedCategoriesString(void)4475 static char * makeBlockedCategoriesString( void )
4476 {
4477 struct Category * cat;
4478 int sz;
4479 static char blocked[48001]; /* only one thread will call us so we may return a static array */
4480
4481 sz = 0;
4482 blocked[0] = '\0';
4483 for (cat = ufdbGV.catList; cat != NULL; cat = cat->next)
4484 {
4485 sz += sprintf( &blocked[sz], "%s:%lu/%lu ", cat->name, cat->nmatches, cat->nblocks );
4486 if (sz > 47000)
4487 {
4488 strcat( &blocked[sz], " ..." );
4489 break;
4490 }
4491 }
4492
4493 return blocked;
4494 }
4495
4496
setHeaders(char headers[30000],char ** message,size_t * length)4497 static void setHeaders( char headers[30000], char ** message, size_t * length )
4498 {
4499 int hi = 0;
4500 char * msg;
4501
4502 msg = *message;
4503
4504 // scan message for header that start with "X-"
4505
4506 while (*msg != '\0' && *msg != '\r')
4507 {
4508 char * r = strchr( msg, '\r' );
4509 int hdrlen = (int) (r - msg);
4510 if (*msg == 'X' && *(msg+1) == '-')
4511 {
4512 hi += sprintf( &headers[hi], " --header '%*.*s'", hdrlen, hdrlen, msg );
4513 }
4514 msg = r + 2;
4515 }
4516 if (*msg == '\r' && *(msg+1) == '\n')
4517 msg += 2;
4518
4519 *message = msg;
4520 *length = strlen( msg );
4521 }
4522
4523
uploadStatisticsWithProxy(char * message,size_t length)4524 static void uploadStatisticsWithProxy( char * message, size_t length )
4525 {
4526 int fd;
4527 ssize_t nwritten;
4528 int retval;
4529 char * proxy;
4530 char * httpuser;
4531 char tmpfilename[64];
4532 char credentials[64];
4533 char headers[30000];
4534 char command[32000];
4535
4536 setHeaders( headers, &message, &length );
4537
4538 strcpy( tmpfilename, "/tmp/ufdbguardd.upload.XXXXXX" );
4539 fd = mkstemp( tmpfilename );
4540 if (fd < 0)
4541 {
4542 ufdbLogError( "cannot upload statistics and uncategorised URLs since cannot create temporary file %s *****",
4543 tmpfilename );
4544 return;
4545 }
4546 nwritten = write( fd, message, length );
4547 if (nwritten < 0)
4548 {
4549 ufdbLogError( "cannot upload statistics and uncategorised URLs since "
4550 "cannot write temporary file %s, error: %s *****",
4551 tmpfilename, strerror(errno) );
4552 close( fd );
4553 if (unlink( tmpfilename ))
4554 { ; }
4555 return;
4556 }
4557 close( fd );
4558
4559 credentials[0] = '\0';
4560 httpuser = getenv( "PROXY_USER" );
4561 if (httpuser != NULL && *httpuser != '\0')
4562 {
4563 char * httppasswd;
4564 httppasswd = getenv( "PROXY_PASSWORD" );
4565 if (httppasswd == NULL || *httppasswd == '\0')
4566 {
4567 ufdbLogError( "cannot upload statistics and uncategorised URLs since envvar PROXY_USER is %s but "
4568 "envvar PROXY_PASSWORD is not set *****", httpuser );
4569 return;
4570 }
4571 else
4572 sprintf( credentials, " --proxy-user '%s:%s'", httpuser, httppasswd );
4573 }
4574 else if (ufdbGV.debug)
4575 ufdbLogMessage( "uploadStatisticsWithProxy: PROXY_USER is not set" );
4576
4577 // now upload the file with curl
4578 proxy = getenv( "https_proxy" );
4579 sprintf( command, UFDB_CURL_PATH
4580 " --proxy %s"
4581 " --proxy-insecure"
4582 " --insecure"
4583 " --user-agent ufdbGuardd-" UFDB_VERSION
4584 " --header 'X-Sent-By: curl'"
4585 "%s"
4586 " --upload-file %s"
4587 " --request POST"
4588 " --silent"
4589 " --output /dev/null"
4590 "%s"
4591 " https://" UFDB_UPLOAD_UNCATEGORISED_URLS_WEBSITE "/cgi-bin/uncat.pl",
4592 proxy,
4593 headers,
4594 tmpfilename,
4595 credentials );
4596 if (ufdbGV.debug)
4597 ufdbLogMessage( "uploadStatisticsWithProxy: executing command:\n%s", command );
4598 retval = system( command );
4599 if (retval < 0)
4600 {
4601 ufdbLogError( "cannot upload statistics and uncategorised URLs since "
4602 "system(3) failed with command \"%s\" *****",
4603 command );
4604 }
4605 else
4606 {
4607 retval = WEXITSTATUS( retval );
4608 if (retval > 0)
4609 {
4610 ufdbLogError( "cannot upload statistics and uncategorised URLs since "
4611 "curl failed; command \"%s\" has exit code %d *****",
4612 command, retval );
4613 ufdbLogMessage( "make sure that https_proxy, PROXY_USER and PROXY_PASSWORD "
4614 "are set and exported in " UFDB_SYSTEM_CONFIG );
4615 }
4616 }
4617 #if !UFDB_TEST_PROXY
4618 if (unlink( tmpfilename ) < 0)
4619 { ; }
4620 #endif
4621 }
4622
4623
uploadStatistics(const char * reason)4624 static void uploadStatistics( const char * reason )
4625 {
4626 char * URLs;
4627 char * message;
4628 int length;
4629 int s;
4630 int port;
4631 SSL * ssl;
4632 int sslStatus;
4633 int nbytes;
4634 int written;
4635 int dummy;
4636 int status;
4637 int onehour;
4638 time_t now;
4639 long num_cpus;
4640 unsigned long num_clients;
4641 unsigned long num_users;
4642 struct utsname sysinfo;
4643
4644 now = time( NULL );
4645 /* do not upload stats more than once per 11 minutes, except when debugging */
4646 if (ufdbGV.debug == 0 && lastUpload != 0 && (now - lastUpload) < 11*60)
4647 {
4648 if (strcmp( reason, "HUP" ) == 0 && !isaclient())
4649 return;
4650 }
4651
4652 onehour = (strcmp( reason, "1HR" ) == 0);
4653 if (onehour)
4654 {
4655 if (ufdbGV.checkedDB.mem == NULL || !ufdbGV.uploadStats)
4656 return;
4657 if (ufdbGV.reconfig || ufdbGV.terminating)
4658 return;
4659 }
4660
4661 if (!onehour)
4662 logStatistics( reason );
4663
4664 lastUpload = now;
4665
4666 num_clients = UFDBgetNumberOfRegisteredIPs();
4667 num_users = UFDBgetNumberOfRegisteredUsers();
4668 if (ufdbGV.checkedDB.mem == NULL)
4669 {
4670 // do not upload insignificant data
4671 if (UFDBlookupCounter < 200)
4672 return;
4673 if (num_clients == 0)
4674 return;
4675 }
4676
4677 if (ufdbGV.debug)
4678 ufdbLogMessage( "uploadStatistics: upload uncategorised URLs: %s, upload statistics: %s",
4679 ufdbGV.analyseUncategorisedURLs ? "yes" : "no",
4680 ufdbGV.uploadStats ? "yes" : "no" );
4681
4682 if (ufdbGV.analyseUncategorisedURLs)
4683 {
4684 URLs = ufdbGetUnknownURLs();
4685 if (URLs == NULL)
4686 {
4687 if (onehour)
4688 return;
4689 URLs = (char *) "example.com|";
4690 }
4691 }
4692 else
4693 {
4694 if (onehour)
4695 return;
4696 URLs = (char *) "example.com|";
4697 }
4698
4699 if (ufdbGV.logUncategorisedURLs)
4700 {
4701 if (!ufdbGV.analyseUncategorisedURLs)
4702 ufdbLogError( "log-uncategorised-urls is ON but analyse-uncategorised-urls is OFF and therefore "
4703 "there are no uncategorised URLs to be logged" );
4704 else if (UFDBuncategorisedCounter == 0)
4705 {
4706 ufdbLogMessage( "there are no uncategorised URLs" );
4707 }
4708 else
4709 {
4710 char * u;
4711 u = URLs;
4712 if (*u == '|' && *(u+1) == 'N' && *(u+2) <= '9' && *(u+3) == '|')
4713 u += 4;
4714 if (*u == '|')
4715 u++;
4716 while (*u != '\0')
4717 {
4718 char * sep;
4719 char * hash;
4720 int len;
4721 sep = strchr( u, '|' );
4722 if (sep == NULL)
4723 break;
4724 hash = strchr( u, '#' );
4725 if (hash == NULL)
4726 len = (int) (sep - u);
4727 else
4728 len = (int) (hash - u);
4729 ufdbLogMessage( "uncategorised URL: %*.*s", len, len, u );
4730 u = sep + 1;
4731 }
4732 }
4733 }
4734
4735 length = strlen( URLs );
4736 if (length <= 1)
4737 {
4738 if (onehour)
4739 return;
4740 URLs = (char *) "example.com|";
4741 length = strlen( URLs );
4742 }
4743
4744 message = (char *) ufdbMalloc( 1500 + length + 48000 );
4745 if (message == NULL)
4746 {
4747 ufdbLogError( "could not malloc %d bytes for upload message. Aborting upload of uncategorised URLs",
4748 1500 + length + 48000 );
4749 ufdbResetUnknownURLs();
4750 return;
4751 }
4752
4753 ufdbGetSysInfo( &sysinfo );
4754 num_cpus = ufdbGetNumCPUs();
4755
4756 if (ufdbGV.uploadStats)
4757 {
4758 if (onehour)
4759 sprintf( message,
4760 "POST /cgi-bin/uncat.pl HTTP/1.1\r\n"
4761 "Host: " UFDB_UPLOAD_UNCATEGORISED_URLS_WEBSITE "\r\n"
4762 "User-Agent: ufdbGuardd-" UFDB_VERSION "\r\n"
4763 "Content-Type: text/plain\r\n"
4764 "Content-Length: %d\r\n"
4765 "Connection: close\r\n"
4766 "X-HasURLfilterDB: %s\r\n"
4767 "X-DatabaseDate: %s\r\n"
4768 "X-SiteInfo: %s %ld %s %s\r\n"
4769 "X-NodeName: %s\r\n"
4770 "X-Instance: %s\r\n"
4771 "X-SquidVersion: %s\r\n"
4772 "X-SquidUsesBumping: %s\r\n"
4773 "X-NumClients: %lu\r\n"
4774 "X-NumUsers: %lu\r\n"
4775 "X-NumThreads: %d\r\n"
4776 "X-UploadSeqNo: %lu\r\n"
4777 "X-UploadReason: %s\r\n"
4778 "\r\n"
4779 "%s\r\n",
4780 length,
4781 "yes",
4782 ufdbGV.checkedDB.date,
4783 sysinfo.machine, num_cpus, sysinfo.sysname, sysinfo.release,
4784 sysinfo.nodename,
4785 UFDBinstance(),
4786 ufdbGV.SquidVersion,
4787 ufdbGV.SquidUsesActiveBumping ? "yes" : "no",
4788 num_clients,
4789 num_users,
4790 ufdbGV.nWorkers,
4791 UFDBuploadSeqNo,
4792 reason,
4793 URLs );
4794 else
4795 sprintf( message,
4796 "POST /cgi-bin/uncat.pl HTTP/1.1\r\n"
4797 "Host: " UFDB_UPLOAD_UNCATEGORISED_URLS_WEBSITE "\r\n"
4798 "User-Agent: ufdbGuardd-" UFDB_VERSION "\r\n"
4799 "Content-Type: text/plain\r\n"
4800 "Content-Length: %d\r\n"
4801 "Connection: close\r\n"
4802 "X-HasURLfilterDB: %s\r\n"
4803 "X-DatabaseDate: %s\r\n"
4804 "X-SiteInfo: %s %ld %s %s\r\n"
4805 "X-NodeName: %s\r\n"
4806 "X-Instance: %s\r\n"
4807 "X-SquidVersion: %s\r\n"
4808 "X-SquidUsesBumping: %s\r\n"
4809 "X-NumClients: %lu\r\n"
4810 "X-NumUsers: %lu\r\n"
4811 "X-NumThreads: %d\r\n"
4812 "X-NumLookups: %lu\r\n"
4813 "X-NumHttpsLookups: %lu\r\n"
4814 "X-NumBumps: %lu\r\n"
4815 "X-Localnet: %lu\r\n"
4816 "X-NumTunnelsDetected: %lu\r\n"
4817 "X-NumTestBlock: %lu\r\n"
4818 "X-NumBlock: %lu\r\n"
4819 "X-NumSafeSearch: %lu\r\n"
4820 "X-NumEduFilter: %lu\r\n"
4821 "X-Uncategorised: %lu\r\n"
4822 "X-NumNodot: %lu\r\n"
4823 "X-BlockedCategories: %s\r\n"
4824 "X-UploadSeqNo: %lu\r\n"
4825 "X-UploadReason: %s\r\n"
4826 "\r\n"
4827 "%s\r\n",
4828 length,
4829 (ufdbGV.checkedDB.mem != NULL && ufdbGV.checkedDB.age <= 24 &&
4830 ufdbGV.checkedDB.flags[1] == 'P' ? "yes" : "no" ),
4831 (ufdbGV.checkedDB.mem == NULL ? "void" : ufdbGV.checkedDB.date),
4832 sysinfo.machine, num_cpus, sysinfo.sysname, sysinfo.release,
4833 sysinfo.nodename,
4834 UFDBinstance(),
4835 ufdbGV.SquidVersion,
4836 ufdbGV.SquidUsesActiveBumping ? "yes" : "no",
4837 num_clients,
4838 num_users,
4839 ufdbGV.nWorkers,
4840 UFDBlookupCounter,
4841 UFDB_API_num_https,
4842 UFDB_API_num_bumps,
4843 UFDB_API_num_url_localnet,
4844 ufdbGV.tunnelCounter,
4845 UFDBtestBlockCounter,
4846 UFDBblockCounter,
4847 UFDBsafesearchCounter,
4848 UFDBedufilterCounter,
4849 UFDBuncategorisedCounter,
4850 UFDB_API_num_nodot_URL,
4851 makeBlockedCategoriesString(),
4852 UFDBuploadSeqNo,
4853 reason,
4854 URLs );
4855 }
4856 else
4857 {
4858 sprintf( message,
4859 "POST /cgi-bin/uncat.pl HTTP/1.1\r\n"
4860 "Host: " UFDB_UPLOAD_UNCATEGORISED_URLS_WEBSITE "\r\n"
4861 "User-Agent: ufdbGuardd-" UFDB_VERSION "\r\n"
4862 "Content-Type: text/plain\r\n"
4863 "Content-Length: %d\r\n"
4864 "X-HasURLfilterDB: %s\r\n"
4865 "X-DatabaseDate: %s\r\n"
4866 "X-SiteInfo: %s %ld %s %s\r\n"
4867 "X-NodeName: %s\r\n"
4868 "X-Instance: %s\r\n"
4869 "X-SquidVersion: %s\r\n"
4870 "X-NumClients: %lu\r\n"
4871 "X-NumUsers: %lu\r\n"
4872 "X-NumThreads: %d\r\n"
4873 "X-NumLookups: %lu\r\n"
4874 "X-UploadSeqNo: %lu\r\n"
4875 "X-UploadReason: %s\r\n"
4876 "\r\n",
4877 length,
4878 (ufdbGV.checkedDB.mem != NULL && ufdbGV.checkedDB.age <= 24 ? "yes" : "no"),
4879 (ufdbGV.checkedDB.mem == NULL ? "void" : ufdbGV.checkedDB.date),
4880 sysinfo.machine, num_cpus, sysinfo.sysname, sysinfo.release,
4881 sysinfo.nodename,
4882 UFDBinstance(),
4883 ufdbGV.SquidVersion,
4884 num_clients,
4885 num_users,
4886 ufdbGV.nWorkers,
4887 UFDBlookupCounter,
4888 UFDBuploadSeqNo,
4889 reason );
4890 }
4891 length = strlen( message );
4892
4893 /* This function migth be called N times when a bad signal is received.
4894 * Let's try to execute the upload only once by resetting the unknown URLs quickly.
4895 */
4896 ufdbResetUnknownURLs();
4897
4898 ssl = NULL;
4899 port = 443;
4900 #if UFDB_TEST_PROXY
4901 s = -1;
4902 #else
4903 s = UFDBopenSocket( UFDB_UPLOAD_UNCATEGORISED_URLS_WEBSITE, port );
4904 #endif
4905 if (s >= 0)
4906 {
4907 sslStatus = UFDBopenssl_connect( UFDB_UPLOAD_UNCATEGORISED_URLS_WEBSITE, port, s, &ssl );
4908 if (sslStatus != UFDB_API_OK)
4909 {
4910 ufdbLogError( "SSL/TLS handshake with %s:443 failed", UFDB_UPLOAD_UNCATEGORISED_URLS_WEBSITE );
4911 close( s );
4912 s = -1;
4913 }
4914 }
4915 else if (ufdbGV.debug)
4916 ufdbLogMessage( "cannot connect to %s:443 trying %s:80",
4917 UFDB_UPLOAD_UNCATEGORISED_URLS_WEBSITE, UFDB_UPLOAD_UNCATEGORISED_URLS_WEBSITE );
4918
4919 if (s < 0)
4920 {
4921 port = 80;
4922 #if UFDB_TEST_PROXY
4923 s = -1;
4924 #else
4925 s = UFDBopenSocket( UFDB_UPLOAD_UNCATEGORISED_URLS_WEBSITE, port );
4926 #endif
4927 if (s < 0)
4928 {
4929 char * proxy = getenv( "https_proxy" );
4930 if (proxy != NULL)
4931 {
4932 if (ufdbGV.debug)
4933 ufdbLogMessage( "ufdbguardd cannot connect to %s:80; "
4934 "trying to use proxy %s ...", UFDB_UPLOAD_UNCATEGORISED_URLS_WEBSITE, proxy );
4935 uploadStatisticsWithProxy( message, length );
4936 }
4937 else
4938 {
4939 ufdbLogError( "cannot upload statistics and uncategorised URLs\n"
4940 "cannot open a communication socket to %s:443 nor %s:80 (%s)\n"
4941 "Make sure that the firewall allows access to %s OR "
4942 "set https_proxy to upload statistics and uncategorised URLs via a proxy *****",
4943 UFDB_UPLOAD_UNCATEGORISED_URLS_WEBSITE, UFDB_UPLOAD_UNCATEGORISED_URLS_WEBSITE,
4944 strerror(errno),
4945 UFDB_UPLOAD_UNCATEGORISED_URLS_WEBSITE );
4946 }
4947 ufdbFree( message );
4948 return;
4949 }
4950 }
4951
4952 written = 0;
4953 while (length > 0)
4954 {
4955 if (port == 443)
4956 nbytes = UFDBopenssl_write( message+written, length, ssl );
4957 else
4958 nbytes = write( s, message+written, length );
4959 if (nbytes == 0)
4960 break;
4961 if (nbytes < 0)
4962 {
4963 if (errno != EINTR)
4964 break;
4965 }
4966 else
4967 {
4968 written += nbytes;
4969 length -= nbytes;
4970 }
4971 }
4972
4973 if (ufdbGV.debug > 1)
4974 ufdbLogMessage( "upload statistics with HTTP POST to %s:%d : %s",
4975 UFDB_UPLOAD_UNCATEGORISED_URLS_WEBSITE, port, message );
4976
4977 errno = 0;
4978 status = 0;
4979 if (port == 443)
4980 nbytes = UFDBopenssl_read( message, 700, ssl );
4981 else
4982 nbytes = read( s, message, 700 );
4983 if (nbytes < 0 || nbytes < 9)
4984 {
4985 ufdbLogError( "did not receive a HTTP response header. URLs are not uploaded: %s *****",
4986 strerror(errno) );
4987 written = 0;
4988 }
4989 else if (sscanf( message, "HTTP/%d.%d %d ", &dummy, &dummy, &status ) != 3)
4990 {
4991 ufdbLogError( "received a malformed HTTP response header: URLs are not uploaded *****" );
4992 written = 0;
4993 }
4994 else if (status != 200)
4995 {
4996 ufdbLogError( "received an unexpected HTTP response status: code %d. URLs are not uploaded *****",
4997 status );
4998 written = 0;
4999 }
5000
5001 if (ufdbGV.debug)
5002 {
5003 if (nbytes > 0)
5004 {
5005 message[nbytes] = '\0';
5006 ufdbLogMessage( "URL upload received this HTTP reply:\n%s", message );
5007 ufdbLogMessage( "HTTP reply status is %d (status message has %d bytes)", status, nbytes );
5008 }
5009 }
5010
5011 if (port == 443)
5012 UFDBopenssl_close( ssl );
5013 close( s );
5014 ufdbFree( message );
5015
5016 if (written > 0 && ufdbGV.analyseUncategorisedURLs)
5017 ufdbLogMessage( "uncategorised URLs have been uploaded" );
5018 }
5019
5020
5021 #ifdef __cplusplus
5022 }
5023 #endif
5024