1 //: http://e2guardian.org/
2 // Released under the GPL v2, with the OpenSSL exception described in the README file.
3
4 // INCLUDES
5 #ifdef HAVE_CONFIG_H
6 #include "dgconfig.h"
7 #endif
8 //#include "NaughtyFilter.hpp"
9 //#include "StoryBoard.hpp"
10 #include "ConnectionHandler.hpp"
11 #include "DataBuffer.hpp"
12 #include "UDSocket.hpp"
13 //#include "Auth.hpp"
14 #include "FDTunnel.hpp"
15 #include "BackedStore.hpp"
16 #include "Queue.hpp"
17 #include "ImageContainer.hpp"
18 #include "FDFuncs.hpp"
19 #include <signal.h>
20 #include <arpa/nameser.h>
21 #include <resolv.h>
22
23 #ifdef __SSLMITM
24 #include "CertificateAuthority.hpp"
25 #endif //__SSLMITM
26
27 #include <syslog.h>
28 #include <cerrno>
29 #include <cstdio>
30 #include <ctime>
31 #include <algorithm>
32 #include <netdb.h>
33 #include <cstdlib>
34 #include <unistd.h>
35 #include <sys/time.h>
36 #include <strings.h>
37 #include <fcntl.h>
38 #include <string.h>
39 #include <sys/stat.h>
40 #include <sys/types.h>
41 #include <sys/socket.h>
42 #include <istream>
43 #include <sstream>
44 #include <memory>
45
46 #ifdef ENABLE_ORIG_IP
47 #include <linux/types.h>
48 #include <linux/netfilter_ipv4.h>
49 #endif
50
51 #ifdef __SSLMITM
52 #include "openssl/ssl.h"
53 #include "openssl/x509v3.h"
54 #include "String.hpp"
55 #endif
56
57 // GLOBALS
58 extern OptionContainer o;
59 extern bool is_daemonised;
60 extern std::atomic<bool> ttg;
61 extern thread_local std::string thread_id;
62
63
64 // IMPLEMENTATION
65
ConnectionHandler()66 ConnectionHandler::ConnectionHandler()
67 : clienthost(NULL) {
68 //ch_isiphost.comp(",*[a-z|A-Z].*");
69
70 // initialise SBauth structure
71 SBauth.filter_group = 0;
72 SBauth.is_authed = false;
73 SBauth.user_name = "";
74 }
75
76
77 // Custom exception class for POST filtering errors
78 class postfilter_exception : public std::runtime_error {
79 public:
postfilter_exception(const char * const & msg)80 postfilter_exception(const char *const &msg)
81 : std::runtime_error(msg) {};
82 };
83
84 //
85 // URL cache funcs
86 //
87
88 // check the URL cache to see if we've already flagged an address as clean
wasClean(HTTPHeader & header,String & url,const int fg)89 bool wasClean(HTTPHeader &header, String &url, const int fg) {
90 return false; // this function needs rewriting always return false
91 }
92
93 // add a known clean URL to the cache
addToClean(String & url,const int fg)94 void addToClean(String &url, const int fg) {
95 return; // this function needs rewriting
96 }
97
98 //
99 // ConnectionHandler class
100 //
101
peerDiag(const char * message,Socket & peersock)102 void ConnectionHandler::peerDiag(const char *message, Socket &peersock) {
103 if (o.logconerror) {
104 //int peerport = peersock.getPeerSourcePort();
105 std::string peer_ip = peersock.getPeerIP();
106 int err = peersock.getErrno();
107
108 if (peersock.isTimedout())
109 syslog(LOG_INFO, "%s %s Client at %s Connection timedout - errno: %d", thread_id.c_str(), message,
110 peer_ip.c_str(), err);
111 else if (peersock.isHup())
112 syslog(LOG_INFO, "%s %s Client at %s has disconnected - errno: %d", thread_id.c_str(), message,
113 peer_ip.c_str(), err);
114 else if (peersock.sockError())
115 syslog(LOG_INFO, "%s %s Client at %s Connection socket error - errno: %d", thread_id.c_str(), message,
116 peer_ip.c_str(), err);
117 else if (peersock.isNoRead())
118 syslog(LOG_INFO, "%s %s cant read Client Connection at %s - errno: %d ", thread_id.c_str(), message,
119 peer_ip.c_str(), err);
120 else if (peersock.isNoWrite())
121 syslog(LOG_INFO, "%s %s cant write Client Connection at %s - errno: %d ", thread_id.c_str(), message,
122 peer_ip.c_str(), err);
123 else if (peersock.isNoOpp())
124 syslog(LOG_INFO, "%s %s Client Connection is no-op - errno: %d", thread_id.c_str(), message, err);
125 else
126 syslog(LOG_INFO, "%s %s Client Connection at %s problem - errno: %d", thread_id.c_str(), message,
127 peer_ip.c_str(), err);
128 }
129 }
130
upstreamDiag(const char * message,Socket & proxysock)131 void ConnectionHandler::upstreamDiag(const char *message, Socket &proxysock) {
132 if (o.logconerror) {
133
134 int err = proxysock.getErrno();
135 if (proxysock.isTimedout())
136 syslog(LOG_INFO, "%s %s upstream timedout - errno: %d:", thread_id.c_str(), message, err);
137 else if (proxysock.isHup())
138 syslog(LOG_INFO, "%s %s upstream has disconnected - errno: %d", thread_id.c_str(), message, err);
139 else if (proxysock.sockError())
140 syslog(LOG_INFO, "%s %s upstream socket error - errno: %d", thread_id.c_str(), message, err);
141 else if (proxysock.isNoRead())
142 syslog(LOG_INFO, "%s %s cant read upstream Connection - errno: %d ", thread_id.c_str(), message, err);
143 else if (proxysock.isNoWrite())
144 syslog(LOG_INFO, "%s %s cant write upstream Connection - errno: %d", thread_id.c_str(), message, err);
145 else if (proxysock.isNoOpp())
146 syslog(LOG_INFO, "%s %s upstream Connection is no-op - errno: %d", thread_id.c_str(), message, err);
147 else
148 syslog(LOG_INFO, "%s %s upstream Connection problem - errno: %d", thread_id.c_str(), message, err);
149 }
150 if (proxysock.isNoOpp())
151 proxysock.close();
152 }
153
154
155 // perform URL encoding on a string
miniURLEncode(const char * s)156 std::string ConnectionHandler::miniURLEncode(const char *s) {
157 std::string encoded;
158 char *buf = new char[3];
159 unsigned char c;
160 for (int i = 0; i < (signed) strlen(s); i++) {
161 c = s[i];
162 // allowed characters in a url that have non special meaning
163 if ((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9')) {
164 encoded += c;
165 continue;
166 }
167 // all other characters get encoded
168 sprintf(buf, "%02x", c);
169 encoded += "%";
170 encoded += buf;
171 }
172 delete[] buf;
173 return encoded;
174 }
175
176 // create a temporary bypass URL for the banned page
hashedURL(String * url,int filtergroup,std::string * clientip,bool infectionbypass,std::string * user)177 String ConnectionHandler::hashedURL(String *url, int filtergroup, std::string *clientip,
178 bool infectionbypass, std::string *user) {
179 // filter/virus bypass hashes last for a certain time only
180 //String timecode(time(NULL) + (infectionbypass ? (*ldl->fg[filtergroup]).infection_bypass_mode : (*ldl->fg[filtergroup]).bypass_mode));
181 String timecode(time(NULL) + (infectionbypass ? (*ldl->fg[filtergroup]).infection_bypass_mode
182 : (*ldl->fg[filtergroup]).bypass_mode));
183 // use the standard key in normal bypass mode, and the infection key in infection bypass mode
184 String magic(infectionbypass ? ldl->fg[filtergroup]->imagic.c_str() : ldl->fg[filtergroup]->magic.c_str());
185 magic += clientip->c_str();
186 if(ldl->fg[filtergroup]->bypass_v2)
187 magic += user->c_str();
188 magic += timecode;
189 String res(infectionbypass ? "GIBYPASS=" : "GBYPASS=");
190 if (!url->after("://").contains("/")) {
191 String newurl((*url));
192 newurl += "/";
193 res += newurl.md5(magic.toCharArray());
194 } else {
195 res += url->md5(magic.toCharArray());
196 }
197 res += timecode;
198 #ifdef DGDEBUG
199 std::cerr << thread_id << " -generate Bypass hashedurl data " << clientip->c_str() << " " << *url << " " << clientuser << " " << timecode << " result " << res << std::endl;
200 #endif
201 return res;
202 }
203
204 // create temporary bypass cookie
hashedCookie(String * url,const char * magic,std::string * clientip,int bypasstimestamp)205 String ConnectionHandler::hashedCookie(String *url, const char *magic, std::string *clientip, int bypasstimestamp) {
206 String timecode(bypasstimestamp);
207 String data(magic);
208 data += clientip->c_str();
209 if(ldl->fg[filtergroup]->bypass_v2)
210 data += clientuser;
211 data += timecode;
212 #ifdef DGDEBUG
213 std::cerr << thread_id << " -generate Bypass hashedCookie data " << clientip->c_str() << " " << *url << " " << clientuser << " " << timecode << std::endl;
214 #endif
215 String res(url->md5(data.toCharArray()));
216 res += timecode;
217
218 #ifdef DGDEBUG
219 std::cerr << thread_id << " -Bypass hashedCookie=" << res << std::endl;
220 #endif
221 return res;
222 }
223
224
225 // is this a temporary filter bypass URL?
isBypassURL(String url,const char * magic,const char * clientip,bool * isvirusbypass,std::string & user)226 int ConnectionHandler::isBypassURL(String url, const char *magic, const char *clientip, bool *isvirusbypass, std::string &user)
227 {
228 if ((url).length() <= 45)
229 return false; // Too short, can't be a bypass
230
231 // check to see if this is a bypass URL, and which type it is
232 bool filterbypass = false;
233 bool virusbypass = false;
234 if ((isvirusbypass == NULL) && ((url).contains("GBYPASS="))) {
235 filterbypass = true;
236 } else if ((isvirusbypass != NULL) && (url).contains("GIBYPASS=")) {
237 virusbypass = true;
238 }
239 if (!(filterbypass || virusbypass))
240 return 0;
241
242 #ifdef DGDEBUG
243 std::cerr << thread_id << "URL " << (filterbypass ? "GBYPASS" : "GIBYPASS") << " found checking..." << " Line: " << __LINE__ << " Function: " << __func__ << std::endl;
244 #endif
245
246 String url_left((url).before(filterbypass ? "GBYPASS=" : "GIBYPASS="));
247 url_left.chop(); // remove the ? or &
248 String url_right((url).after(filterbypass ? "GBYPASS=" : "GIBYPASS="));
249
250 String url_hash(url_right.subString(0, 32));
251 String url_time(url_right.after(url_hash.toCharArray()));
252 #ifdef DGDEBUG
253 std::cerr << thread_id << "URL: " << url_left << ", HASH: " << url_hash << ", TIME: " << url_time << " Line: " << __LINE__ << " Function: " << __func__ << std::endl;
254 #endif
255
256 String mymagic(magic);
257 mymagic += clientip;
258 if(ldl->fg[filtergroup]->bypass_v2)
259 mymagic += user;
260 mymagic += url_time;
261 String hashed(url_left.md5(mymagic.toCharArray()));
262
263 if(ldl->fg[filtergroup]->cgi_bypass_v2) {
264 mymagic = hashed;
265 hashed = mymagic.md5(ldl->fg[filtergroup]->cgi_magic.c_str());
266 }
267
268 if (hashed != url_hash) {
269 #ifdef DGDEBUG
270 std::cerr << thread_id << "URL " << (filterbypass ? "GBYPASS" : "GIBYPASS") << " hash mismatch" << " Line: " << __LINE__ << " Function: " << __func__ << std::endl;
271 #endif
272 return 0;
273 }
274
275 time_t timen = time(NULL);
276 time_t timeu = url_time.toLong();
277
278 if (timeu < 1) {
279 #ifdef DGDEBUG
280 std::cerr << thread_id << "URL " << (filterbypass ? "GBYPASS" : "GIBYPASS") << " bad time value" << " Line: " << __LINE__ << " Function: " << __func__ << std::endl;
281 #endif
282 return 1; // bad time value
283 }
284 if (timeu < timen) { // expired key
285 #ifdef DGDEBUG
286 std::cerr << thread_id << "URL " << (filterbypass ? "GBYPASS" : "GIBYPASS") << " expired" << " Line: " << __LINE__ << " Function: " << __func__ << std::endl;
287 #endif
288 return 1; // denotes expired but there
289 }
290 #ifdef DGDEBUG
291 std::cerr << thread_id << "URL " << (filterbypass ? "GBYPASS" : "GIBYPASS") << " not expired" << " Line: " << __LINE__ << " Function: " << __func__ << std::endl;
292 #endif
293 if (virusbypass)
294 (*isvirusbypass) = true;
295 return (int)timeu;
296 }
297
298 // is this a scan bypass URL? i.e. a "magic" URL for retrieving a previously scanned file
isScanBypassURL(String url,const char * magic,const char * clientip)299 bool ConnectionHandler::isScanBypassURL(String url, const char *magic, const char *clientip)
300 {
301 if ((url).length() <= 45)
302 return false; // Too short, can't be a bypass
303
304 if (!(url).contains("GSBYPASS=")) { // If this is not a bypass url
305 return false;
306 }
307 #ifdef DGDEBUG
308 std::cerr << thread_id << "URL GSBYPASS found checking..." << " Line: " << __LINE__ << " Function: " << __func__ << std::endl;
309 #endif
310
311 String url_left((url).before("GSBYPASS="));
312 url_left.chop(); // remove the ? or &
313 String url_right((url).after("GSBYPASS="));
314
315 String url_hash(url_right.subString(0, 32));
316 #ifdef DGDEBUG
317 std::cerr << thread_id << "URL: " << url_left << ", HASH: " << url_hash << " Line: " << __LINE__ << " Function: " << __func__ << std::endl;
318 #endif
319
320 // format is:
321 // GSBYPASS=hash(ip+url+tempfilename+mime+disposition+secret)
322 // &N=tempfilename&M=mimetype&D=dispos
323
324 String tempfilename(url_right.after("&N="));
325 String tempfilemime(tempfilename.after("&M="));
326 String tempfiledis(tempfilemime.after("&D="));
327 tempfilemime = tempfilemime.before("&D=");
328 tempfilename = tempfilename.before("&M=");
329
330 String tohash(clientip + url_left + tempfilename + tempfilemime + tempfiledis + magic);
331 String hashed(tohash.md5());
332
333 if(ldl->fg[filtergroup]->cgi_bypass_v2) {
334 tohash = hashed;
335 hashed = tohash.md5(ldl->fg[filtergroup]->cgi_magic.c_str());
336 }
337
338 #ifdef DGDEBUG
339 std::cerr << thread_id << "checking hash: " << clientip << " " << url_left << " " << tempfilename << " "
340 << " " << tempfilemime << " " << tempfiledis << " " << magic << " " << hashed << " Line: " << __LINE__ << " Function: " << __func__ << std::endl;
341 #endif
342
343 if (hashed == url_hash) {
344 return true;
345 }
346 #ifdef DGDEBUG
347 std::cerr << thread_id << "URL GSBYPASS HASH mismatch" << " Line: " << __LINE__ << " Function: " << __func__ << std::endl;
348 #endif
349
350 return false;
351 }
352
353 // send a file to the client - used during bypass of blocked downloads
354 off_t
sendFile(Socket * peerconn,NaughtyFilter & cm,String & url,bool is_icap,ICAPHeader * icap_head)355 ConnectionHandler::sendFile(Socket *peerconn, NaughtyFilter &cm, String &url, bool is_icap, ICAPHeader *icap_head) {
356 String filedis = cm.tempfiledis;
357 int fd = open(cm.tempfilename.toCharArray(), O_RDONLY);
358 if (fd < 0) { // file access error
359 syslog(LOG_ERR, "%sError reading file to send", thread_id.c_str());
360 #ifdef DGDEBUG
361 std::cerr << thread_id << " -Error reading file to send:" << cm.tempfilename << std::endl;
362 #endif
363 String fnf(o.language_list.getTranslation(1230));
364 String head("HTTP/1.1 404 " + fnf + "\r\nContent-Type: text/html\r\n\r\n");
365 String body("<HTML><HEAD><TITLE>" + fnf + "</TITLE></HEAD><BODY><H1>" + fnf + "</H1></BODY></HTML>\r\n");
366
367 if (is_icap) {
368 icap_head->out_res_header = head;
369 icap_head->out_res_body = body;
370 icap_head->out_res_hdr_flag = true;
371 icap_head->out_res_body_flag = true;
372 icap_head->respond(*peerconn);
373 } else {
374 peerconn->writeString(head.toCharArray());
375 peerconn->writeString(body.toCharArray());
376 }
377
378 return 0;
379 }
380
381 off_t filesize = lseek(fd, 0, SEEK_END);
382 lseek(fd, 0, SEEK_SET);
383 String head("HTTP/1.1 200 OK\r\nContent-Type: " + cm.tempfilemime + "\r\nContent-Length: " + String(filesize));
384 if (filedis.length() == 0) {
385 filedis = url.before("?");
386 while (filedis.contains("/"))
387 filedis = filedis.after("/");
388 }
389 head += "\r\nContent-disposition: attachment; filename=" + filedis;
390 head += "\r\n\r\n";
391
392 if (is_icap) {
393 icap_head->out_res_header = head;
394 icap_head->out_res_hdr_flag = true;
395 icap_head->out_res_body_flag = true;
396 icap_head->respond(*peerconn);
397 } else {
398 if (!peerconn->writeString(head.toCharArray())) {
399 close(fd);
400 return 0;
401 }
402 }
403
404 // perform the actual sending
405 off_t sent = 0;
406 int rc;
407 //char *buffer = new char[250000];
408 char *buffer = new char[64000];
409 while (sent < filesize) {
410 rc = readEINTR(fd, buffer, 64000);
411 #ifdef DGDEBUG
412 std::cerr << thread_id << " -reading send file rc:" << rc << std::endl;
413 #endif
414 if (rc < 0) {
415 #ifdef DGDEBUG
416 std::cerr << thread_id << " -error reading send file so aborting" << std::endl;
417 #endif
418 delete[] buffer;
419 // throw std::exception/();
420 //cleanThrow("error reading send file", *peerconn);
421 return 0;
422 }
423 if (rc == 0) {
424 #ifdef DGDEBUG
425 std::cerr << thread_id << " -got zero bytes reading send file" << std::endl;
426 #endif
427 break; // should never happen
428 }
429 if (is_icap) {
430 if (!peerconn->writeChunk(buffer, rc, 100000)) {
431 delete[] buffer;
432 peerDiag("Error sending file to client", *peerconn);
433 return 0;
434 }
435 } else {
436 // as it's cached to disk the buffer must be reasonably big
437 if (!peerconn->writeToSocket(buffer, rc, 0, 100000)) {
438 delete[] buffer;
439 peerDiag("Error sending file to client", *peerconn);
440 return 0;
441 // throw std::exception();
442 }
443 }
444 sent += rc;
445 #ifdef DGDEBUG
446 std::cerr << thread_id << " -total sent from temp:" << sent << std::endl;
447 #endif
448 }
449 if (is_icap) {
450 String n;
451 peerconn->writeChunkTrailer(n);
452 }
453 delete[] buffer;
454 close(fd);
455 return sent;
456 }
457
458 int
connectUpstream(Socket & sock,NaughtyFilter & cm,int port=0)459 ConnectionHandler::connectUpstream(Socket &sock, NaughtyFilter &cm, int port = 0) // connects to to proxy or directly
460 {
461 if (port == 0)
462 port = cm.request_header->port;
463 String sport(port);
464 int lerr_mess = 0;
465 int retry = -1;
466 bool may_be_loop = false;
467 for (auto it = o.filter_ports.begin(); it != o.filter_ports.end(); it++) {
468 if (*it == sport) {
469 may_be_loop = true;
470 break;
471 }
472 }
473 #ifdef DGDEBUG
474 std::cerr << thread_id << "May_be_loop = " << may_be_loop << " " << " port " << port << std::endl;
475 #endif
476
477 while (++retry < o.connect_retries) {
478 lerr_mess = 0;
479 if (retry > 0) {
480 if (o.logconerror)
481 syslog(LOG_INFO, "%s retry %d to connect to %s", thread_id.c_str(), retry, cm.urldomain.c_str());
482 if (!sock.isTimedout())
483 usleep(1000); // don't hammer upstream
484 }
485 cm.upfailure = false;
486 if (cm.isdirect) {
487 String des_ip;
488 if (cm.isiphost) {
489 des_ip = cm.urldomain;
490 if (may_be_loop) { // check check_ip list
491 bool do_break = false;
492 if (o.check_ip.size() > 0) {
493 for (auto it = o.check_ip.begin(); it != o.check_ip.end(); it++) {
494 if (*it == des_ip) {
495 do_break = true;
496 lerr_mess = 212;
497 break;
498 }
499 }
500 }
501 if (do_break) break;
502 may_be_loop = false;
503 }
504
505 sock.setTimeout(o.connect_timeout);
506 #ifdef DGDEBUG
507 std::cerr << thread_id << "Connecting to IPHost " << des_ip << " port " << port << std::endl;
508 #endif
509 int rc = sock.connect(des_ip, port);
510 if (rc < 0) {
511 lerr_mess = 203;
512 continue;
513 }
514 return rc;
515 } else {
516 //dns lookup
517 struct addrinfo hints, *infoptr;
518 memset(&hints, 0, sizeof(addrinfo));
519 hints.ai_family = AF_INET;
520 hints.ai_socktype = SOCK_STREAM;
521 hints.ai_flags = 0;
522 hints.ai_protocol = 0;
523 hints.ai_canonname = NULL;
524 hints.ai_addr = NULL;
525 hints.ai_next = NULL;
526 int rc = getaddrinfo(cm.connect_site.toCharArray(), NULL, &hints, &infoptr);
527 if (rc) // problem
528 {
529 #ifdef DGDEBUG
530 std::cerr << thread_id << "connectUpstream: getaddrinfo returned " << rc << " for " << cm.connect_site << " " << gai_strerror(rc) << std::endl;
531 #endif
532 bool rt = false;
533 switch (rc) {
534 case EAI_NONAME:
535 lerr_mess = 207;
536 break;
537 #ifdef EAI_NODATA
538 case EAI_NODATA:
539 lerr_mess = 208;
540 break;
541 #endif
542 case EAI_AGAIN:
543 lerr_mess = 209;
544 rt = true;
545 break;
546 case EAI_FAIL:
547 lerr_mess = 210;
548 break;
549 default:
550 lerr_mess = 210; //TODO this should have it's own message??
551 break;
552 }
553 sock.close();
554 if (rt) continue;
555 else break;
556 }
557 char t[256];
558 struct addrinfo *p;
559 for (p = infoptr; p != NULL; p = p->ai_next) {
560 getnameinfo(p->ai_addr, p->ai_addrlen, t, sizeof(t), NULL, 0, NI_NUMERICHOST);
561 if (may_be_loop) { // check check_ip list
562 bool do_break = false;
563 if (o.check_ip.size() > 0) {
564 for (auto it = o.check_ip.begin(); it != o.check_ip.end(); it++) {
565 if (*it == t) {
566 do_break = true;
567 lerr_mess = 212;
568 break;
569 }
570 }
571 }
572 if (do_break) break;
573 may_be_loop = false;
574 }
575 #ifdef DGDEBUG
576 std::cerr << thread_id << "Connecting to IP " << t << " port " << port << std::endl;
577 #endif
578 int rc = sock.connect(t, port);
579 if (rc == 0) {
580 freeaddrinfo(infoptr);
581 #ifdef DGDEBUG
582 std::cerr << thread_id << "Got connection upfailure is " << cm.upfailure << std::endl;
583 #endif
584 return 0;
585 }
586 }
587 freeaddrinfo(infoptr);
588 if (may_be_loop) break;
589 lerr_mess = 203;
590 continue;
591 }
592 } else { //is via proxy
593 sock.setTimeout(o.proxy_timeout);
594 int rc = sock.connect(o.proxy_ip, o.proxy_port);
595 if (rc < 0) {
596 if (sock.isTimedout())
597 lerr_mess = 201;
598 else
599 lerr_mess = 202;
600 continue;
601 }
602 return rc;
603 }
604 }
605
606 // only get here if failed
607 cm.upfailure = true;
608 cm.message_no = lerr_mess;
609 cm.whatIsNaughty = "";
610 cm.whatIsNaughtyLog = "";
611 cm.isItNaughty = true;
612 cm.blocktype = 3;
613 cm.isexception = false;
614 cm.isbypass = false;
615 return -1;
616 }
617
618 // pass data between proxy and client, filtering as we go.
619 // this is the only public function of ConnectionHandler
handlePeer(Socket & peerconn,String & ip,stat_rec * & dystat,unsigned int lc_type)620 int ConnectionHandler::handlePeer(Socket &peerconn, String &ip, stat_rec *&dystat, unsigned int lc_type) {
621 persistent_authed = false;
622 is_real_user = false;
623 int rc = 0;
624 //#ifdef DGDEBUG
625 // for debug info only - TCP peer port
626 //thread_id = peerconn.getPeerSourcePort();
627 //#endif
628 Socket proxysock; // also used for direct connection
629
630 switch (lc_type) {
631 case CT_PROXY:
632 rc = handleConnection(peerconn, ip, false, proxysock, dystat);
633 break;
634
635 #ifdef __SSLMITM
636 case CT_THTTPS:
637 rc = handleTHTTPSConnection(peerconn, ip, proxysock, dystat);
638 break;
639 #endif
640
641 case CT_ICAP:
642 rc = handleICAPConnection(peerconn, ip, proxysock, dystat);
643 break;
644
645 }
646 //if ( ldl->reload_id != load_id)
647 // rc = -1;
648 return rc;
649 }
650
651
handleConnection(Socket & peerconn,String & ip,bool ismitm,Socket & proxysock,stat_rec * & dystat)652 int ConnectionHandler::handleConnection(Socket &peerconn, String &ip, bool ismitm, Socket &proxysock,
653 stat_rec *&dystat) {
654 struct timeval thestart;
655 gettimeofday(&thestart, NULL);
656
657 //peerconn.setTimeout(o.proxy_timeout);
658 peerconn.setTimeout(o.pcon_timeout);
659
660 // ldl = o.currentLists();
661
662 HTTPHeader docheader(__HEADER_RESPONSE); // to hold the returned page header from proxy
663 HTTPHeader header(__HEADER_REQUEST); // to hold the incoming client request headeri(ldl)
664
665 // set a timeout as we don't want blocking 4 eva
666 // this also sets how long a peerconn will wait for other requests
667 header.setTimeout(o.pcon_timeout);
668 docheader.setTimeout(o.exchange_timeout);
669
670
671 //int bypasstimestamp = 0;
672
673
674 // Content scanning plugins to use for request (POST) & response data
675 std::deque<CSPlugin *> requestscanners;
676 std::deque<CSPlugin *> responsescanners;
677
678 std::string clientip(ip.toCharArray()); // hold the clients ip
679 header.setClientIP(ip);
680
681 if (clienthost) delete clienthost;
682
683 clienthost = NULL; // and the hostname, if available
684 matchedip = false;
685
686 // clear list of parameters extracted from URL
687 urlparams.clear();
688
689 // clear out info about POST data
690 postparts.clear();
691
692 #ifdef DGDEBUG // debug stuff surprisingly enough
693 std::cerr << thread_id << " -got peer connection" << std::endl;
694 std::cerr << thread_id << clientip << std::endl;
695 #endif
696
697 try {
698 //int rc;
699
700 #ifdef DGDEBUG
701 int pcount = 0;
702 #endif
703
704 // assume all requests over the one persistent connection are from
705 // the same user. means we only need to query the auth plugin until
706 // we get credentials, then assume they are valid for all reqs. on
707 // the persistent connection.
708 std::string oldclientuser;
709 std::string room;
710
711 //int oldfg = 0;
712 bool authed = false;
713 //bool isbanneduser = false;
714 //bool isscanbypass = false;
715 //bool isbypass = false;
716 //bool isvirusbypass = false;
717 //int bypasstimestamp = 0;
718 //bool iscookiebypass = false;
719
720 AuthPlugin *auth_plugin = NULL;
721
722 // RFC states that connections are persistent
723 bool persistOutgoing = true;
724 bool persistPeer = true;
725 bool persistProxy = true;
726 String last_domain_port;
727 bool last_isdirect = false;
728
729 bool firsttime = true;
730 if (!header.in(&peerconn, true)) { // get header from client, allowing persistency
731 if (o.logconerror) {
732 if (peerconn.getFD() > -1) {
733
734 int err = peerconn.getErrno();
735 //int pport = peerconn.getPeerSourcePort();
736 std::string peerIP = peerconn.getPeerIP();
737
738 syslog(LOG_INFO, "%s No header recd from client at %s - errno: %d", thread_id.c_str(),
739 peerIP.c_str(), err);
740 #ifdef DGDEBUG
741 std::cerr << thread_id << " No header recd from client - errno: " << err << std::endl;
742 #endif
743 } else {
744 syslog(LOG_INFO, "%s Client connection closed early - no request header received",
745 thread_id.c_str());
746 }
747 }
748 firsttime = false;
749 persistPeer = false;
750 } else {
751 ++dystat->reqs;
752 }
753 //
754 // End of set-up section
755 //
756 // Start of main loop
757 //
758
759 // maintain a persistent connection
760 while ((firsttime || persistPeer) && !ttg)
761 // while ((firsttime || persistPeer) && !reloadconfig)
762 {
763 #ifdef DGDEBUG
764 std::cerr << thread_id << " firsttime =" << firsttime << "ismitm =" << ismitm << " clientuser =" << clientuser << " group = " << filtergroup << std::endl;
765 #endif
766 ldl = o.currentLists();
767 NaughtyFilter checkme(header, docheader);
768 DataBuffer docbody;
769 docbody.setTimeout(o.exchange_timeout);
770 FDTunnel fdt;
771
772 if (firsttime) {
773 // reset flags & objects next time round the loop
774 firsttime = false;
775 gettimeofday(&thestart, NULL);
776 checkme.thestart = thestart;
777
778 // quick trick for the very first connection :-)
779 if (!ismitm)
780 persistProxy = false;
781 } else {
782 // another round...
783 #ifdef DGDEBUG
784 std::cerr << thread_id << " -persisting (count " << ++pcount << ")" << std::endl;
785 // syslog(LOG_ERR, "Served %d requests on this connection so far - ismitm=%d", pcount, ismitm);
786 std::cerr << thread_id << " - " << clientip << std::endl;
787 #endif
788 header.reset();
789 if (!header.in(&peerconn, true)) {
790 #ifdef DGDEBUG
791 std::cerr << thread_id << " -Persistent connection closed" << std::endl;
792 #endif
793 break;
794 }
795 ++dystat->reqs;
796
797 // we will actually need to do *lots* of resetting of flags etc. here for pconns to work
798 gettimeofday(&thestart, NULL);
799 checkme.thestart = thestart;
800
801 checkme.bypasstimestamp = 0;
802
803 authed = false;
804
805 requestscanners.clear();
806 responsescanners.clear();
807
808 matchedip = false;
809 urlparams.clear();
810 postparts.clear();
811 checkme.mimetype = "-";
812 room = ""; // CHECK THIS - surely room is persistant?????
813
814 // reset docheader & docbody
815 // headers *should* take care of themselves on the next in()
816 // actually not entirely true for docheader - we may read
817 // certain properties of it (in denyAccess) before we've
818 // actually performed the next in(), so make sure we do a full
819 // reset now.
820 docheader.reset();
821 docbody.reset();
822 peerconn.resetChunk();
823 proxysock.resetChunk();
824
825 }
826 //
827 // do this normalisation etc just the once at the start.
828 checkme.setURL(ismitm);
829
830 if(o.log_requests) {
831 std::string fnt;
832 if(ismitm)
833 fnt = "MITM";
834 else if(header.isProxyRequest) {
835 fnt = "PROXY";
836 } else fnt = "TRANS";
837 doRQLog(clientuser, clientip, checkme, fnt);
838 }
839
840 //If proxy connection is not persistent..// do this later after checking if direct or via proxy
841
842 #ifdef DGDEBUG
843 std::cerr << thread_id << getpid() << "Start URL " << checkme.url.c_str() << "is_ssl=" << checkme.is_ssl << "ismitm=" << ismitm << std::endl;
844 #endif
845
846 // checks for bad URLs to prevent security holes/domain obfuscation.
847 if (header.malformedURL(checkme.url)) {
848 // The requested URL is malformed.
849 writeback_error(checkme, peerconn, 200, 0, "400 Bad Request");
850 proxysock.close(); // close connection to proxy
851 break;
852 }
853
854 // TODO this needs moving is proxy operation is still to be tested
855 if (checkme.urldomain == "internal.test.e2guardian.org") {
856 peerconn.writeString(
857 "HTTP/1.1 200 \nContent-Type: text/html\n\n<HTML><HEAD><TITLE>e2guardian internal test</TITLE></HEAD><BODY><H1>e2guardian internal test OK</H1> ");
858 peerconn.writeString("</BODY></HTML>\n");
859 proxysock.close(); // close connection to proxy
860 break;
861 }
862
863 // total block list checking now done in pre-auth story
864
865 // don't let the client connection persist if the client doesn't want it to.
866 persistOutgoing = header.isPersistent();
867 // now check if in input proxy mode and direct upstream if upstream needs closing
868 if (persistProxy && last_isdirect &&
869 ((last_domain_port != checkme.urldomainport)|| !o.no_proxy)) {
870 proxysock.close();
871 persistProxy = false;
872 }
873 last_domain_port = checkme.urldomainport;
874 last_isdirect = checkme.isdirect;
875
876
877 //
878 //
879 // Now check if machine is banned and room-based checking
880 //
881 //
882
883 // is this user banned?
884 //isbanneduser = false;
885 if (o.use_xforwardedfor) {
886 bool use_xforwardedfor;
887 if (o.xforwardedfor_filter_ip.size() > 0) {
888 use_xforwardedfor = false;
889 for (unsigned int i = 0; i < o.xforwardedfor_filter_ip.size(); i++) {
890 if (strcmp(clientip.c_str(), o.xforwardedfor_filter_ip[i].c_str()) == 0) {
891 use_xforwardedfor = true;
892 break;
893 }
894 }
895 } else {
896 use_xforwardedfor = true;
897 }
898 if (use_xforwardedfor == 1) {
899 std::string xforwardip(header.getXForwardedForIP());
900 if (xforwardip.length() > 6) {
901 clientip = xforwardip;
902 }
903 #ifdef DGDEBUG
904 std::cerr << thread_id << " -using x-forwardedfor:" << clientip << std::endl;
905 #endif
906 }
907 }
908 checkme.clientip = clientip;
909
910 // Look up reverse DNS name of client if needed
911 if (o.reverse_client_ip_lookups) {
912 getClientFromIP(clientip.c_str(),checkme.clienthost);
913 // std::unique_ptr<std::deque<String> > hostnames;
914 // hostnames.reset(ipToHostname(clientip.c_str()));
915 // checkme.clienthost = std::string(hostnames->front().toCharArray());
916 }
917
918 //CALL SB pre-authcheck
919 ldl->StoryA.runFunctEntry(ENT_STORYA_PRE_AUTH, checkme);
920 #ifdef DGDEBUG
921 std::cerr << "After StoryA pre-authcheck" << checkme.isexception << " mess_no "
922 << checkme.message_no << std::endl;
923 #endif
924 checkme.isItNaughty = checkme.isBlocked;
925 bool isbannedip = checkme.isBlocked;
926 bool part_banned;
927 if (isbannedip) {
928 // matchedip = clienthost == NULL;
929 } else {
930 if (ldl->inRoom(clientip, room, &(checkme.clienthost), &isbannedip, &part_banned, &checkme.isexception,
931 checkme.urld)) {
932 #ifdef DGDEBUG
933 std::cerr << " isbannedip = " << isbannedip << "ispart_banned = " << part_banned << " isexception = " << checkme.isexception << std::endl;
934 #endif
935 if (isbannedip) {
936 // matchedip = clienthost == NULL;
937 checkme.isBlocked = checkme.isItNaughty = true;
938 }
939 if (checkme.isexception) {
940 // do reason codes etc
941 checkme.exceptionreason = o.language_list.getTranslation(630);
942 checkme.exceptionreason.append(room);
943 checkme.exceptionreason.append(o.language_list.getTranslation(631));
944 checkme.message_no = 632;
945 }
946 }
947 }
948
949
950 #ifdef ENABLE_ORIG_IP
951 // if working in transparent mode and grabbing of original IP addresses is
952 // enabled, does the original IP address match one of those that the host
953 // we are going to resolves to?
954 // Resolves http://www.kb.cert.org/vuls/id/435052
955 if (o.get_orig_ip) {
956 // XXX This will currently only work on Linux/Netfilter.
957 sockaddr_in origaddr;
958 socklen_t origaddrlen(sizeof(sockaddr_in));
959 // Note: we assume that for non-redirected connections, this getsockopt call will
960 // return the proxy server's IP, and not -1. Hence, comparing the result with
961 // the return value of Socket::getLocalIP() should tell us that the client didn't
962 // connect transparently, and we can assume they aren't vulnerable.
963 if (getsockopt(peerconn.getFD(), SOL_IP, SO_ORIGINAL_DST, &origaddr, &origaddrlen) < 0) {
964 syslog(LOG_ERR, "Failed to get client's original destination IP: %s", strerror(errno));
965 break;
966 }
967
968 char res[INET_ADDRSTRLEN];
969 std::string orig_dest_ip(inet_ntop(AF_INET,&origaddr.sin_addr, res, sizeof(res)));
970 if (orig_dest_ip == peerconn.getLocalIP()) {
971 // The destination IP before redirection is the same as the IP the
972 // client has actually been connected to - they aren't connecting transparently.
973 #ifdef DGDEBUG
974 std::cerr << thread_id << " -SO_ORIGINAL_DST and getLocalIP are equal; client not connected transparently" << std::endl;
975 #endif
976 } else {
977 // Look up domain from request URL, and check orig IP against resolved IPs
978 addrinfo hints;
979 memset(&hints, 0, sizeof(hints));
980 hints.ai_family = AF_INET;
981 hints.ai_socktype = SOCK_STREAM;
982 hints.ai_protocol = IPPROTO_TCP;
983 addrinfo *results;
984 int result = getaddrinfo(checkme.urldomain.c_str(), NULL, &hints, &results);
985 if (result) {
986 freeaddrinfo(results);
987 syslog(LOG_ERR, "Cannot resolve hostname for host header checks: %s", gai_strerror(errno));
988 break;
989 }
990 addrinfo *current = results;
991 bool matched = false;
992 while (current != NULL) {
993 char res[INET_ADDRSTRLEN];
994 if (orig_dest_ip == inet_ntop(AF_INET,&((sockaddr_in *)(current->ai_addr))->sin_addr, res, sizeof(res))) {
995 #ifdef DGDEBUG
996 std::cerr << thread_id << checkme.urldomain << " matched to original destination of " << orig_dest_ip << std::endl;
997 #endif
998 matched = true;
999 break;
1000 }
1001 current = current->ai_next;
1002 }
1003 freeaddrinfo(results);
1004 if (!matched) {
1005 // Host header/URL said one thing, but the original destination IP said another.
1006 // This is exactly the vulnerability we want to prevent.
1007 #ifdef DGDEBUG
1008 std::cerr << thread_id << checkme.urldomain << " DID NOT MATCH original destination of " << orig_dest_ip << std::endl;
1009 #endif
1010 syslog(LOG_ERR, "Destination host of %s did not match the original destination IP of %s", checkme.urldomain.c_str(), orig_dest_ip.c_str());
1011 writeback_error(checkme, peerconn, 200, 0, "400 Bad Request");
1012 break;
1013 }
1014 }
1015 }
1016 #endif
1017
1018
1019 //
1020 //
1021 // Start of Authentication Checks
1022 //
1023 //
1024 // don't have credentials for this connection yet? get some!
1025 overide_persist = false;
1026 if (!persistent_authed) {
1027 bool only_ip_auth;
1028 if (header.isProxyRequest) {
1029 filtergroup = o.default_fg;
1030 only_ip_auth = false;
1031 } else {
1032 filtergroup = o.default_trans_fg;
1033 only_ip_auth = true;
1034 }
1035 #ifdef DGDEBUG
1036 std::cerr << thread_id << "isProxyRequest is " << header.isProxyRequest << " only_ip_auth is " << only_ip_auth << " needs proxy for auth plugin is " << o.auth_needs_proxy_in_plugin << std::endl;
1037 #endif
1038
1039 if (!persistProxy && o.auth_needs_proxy_in_plugin && header.isProxyRequest) // open upstream connection early if required for ntml auth
1040 {
1041 if (connectUpstream(proxysock, checkme, header.port) < 0) {
1042 if (checkme.isconnect && ldl->fg[filtergroup]->ssl_mitm && ldl->fg[filtergroup]->automitm &&
1043 checkme.upfailure)
1044 {
1045 checkme.gomitm = true; // so that we can deliver a status message to user over half MITM
1046 } else {
1047 checkme.gomitm = false; // if not automitm
1048 }
1049 } else {
1050 persistProxy = true;
1051 }
1052 }
1053
1054 if (!doAuth(checkme.auth_result, authed, filtergroup, auth_plugin, peerconn, proxysock, header,
1055 only_ip_auth,
1056 checkme.isconnect)) {
1057 if ((checkme.auth_result == DGAUTH_REDIRECT) && checkme.isconnect &&
1058 ldl->fg[filtergroup]->ssl_mitm) {
1059 checkme.gomitm = true;
1060 checkme.isdone = true;
1061 } else {
1062 break;
1063 }
1064 }
1065 //checkme.filtergroup = filtergroup;
1066 } else {
1067 #ifdef DGDEBUG
1068 std::cerr << thread_id << " -Already got credentials for this connection - not querying auth plugins" << std::endl;
1069 #endif
1070 authed = true;
1071 }
1072 checkme.filtergroup = filtergroup;
1073
1074 #ifdef DGDEBUG
1075 std::cerr << thread_id << " -username: " << clientuser << std::endl;
1076 std::cerr << thread_id << " -filtergroup: " << filtergroup << std::endl;
1077 #endif
1078 //
1079 //
1080 // End of Authentication Checking
1081 //
1082 //
1083
1084 #ifdef __SSLMITM
1085 // Set if candidate for MITM
1086 // (Exceptions will not go MITM)
1087 checkme.ismitmcandidate = checkme.isconnect && ldl->fg[filtergroup]->ssl_mitm && (header.port == 443);
1088 #endif
1089
1090 //
1091 // Start of by pass
1092 //
1093 if (!checkme.isdone && checkByPass(checkme, ldl, header, proxysock, peerconn, clientip)
1094 && sendScanFile(peerconn, checkme)) {
1095 persistProxy = false;
1096 break;
1097 }
1098
1099 //
1100 // End by pass
1101 //
1102
1103 // virus checking candidate?
1104 // checkme.noviruscheck defaults to true
1105 if (!(checkme.isdone || checkme.isconnect || checkme.ishead) // can't scan connect or head or not yet authed
1106 && !(checkme.isBlocked) // or already blocked
1107 && authed // and is authed
1108 && (o.csplugins.size() > 0) // and we have scan plugins
1109 && !ldl->fg[filtergroup]->disable_content_scan // and is not disabled
1110 && !(checkme.isexception && !ldl->fg[filtergroup]->content_scan_exceptions)
1111 // and not exception unless scan exceptions enabled
1112 )
1113 checkme.noviruscheck = false; // note this may be reset by Storyboard to enable exceptions
1114
1115 //
1116 // Start of Storyboard checking
1117 //
1118 // if (!(checkme.isBlocked || checkme.isbypass))
1119 if (!(checkme.isBlocked || checkme.isdone) && authed) {
1120 // Main checking is now done in Storyboard function(s)
1121 // String funct = "checkrequest";
1122 // ldl->fg[filtergroup]->StoryB.runFunct(funct, checkme);
1123 ldl->fg[filtergroup]->StoryB.runFunctEntry(ENT_STORYB_PROXY_REQUEST, checkme);
1124 #ifdef DGDEBUG
1125 std::cerr << thread_id << "After StoryB checkrequest isexception " << checkme.isexception << " gomitm "
1126 << checkme.gomitm << " mess_no "
1127 << checkme.message_no << std::endl;
1128 #endif
1129 if (ldl->fg[filtergroup]->reporting_level != -1){
1130 checkme.isItNaughty = checkme.isBlocked;
1131 } else {
1132 checkme.isItNaughty = false;
1133 checkme.isBlocked = false;
1134 }
1135 }
1136
1137 if (checkme.isdirect) {
1138 header.setDirect();
1139 last_isdirect = true;
1140 if(!o.no_proxy) { // we are in mixed mode proxy and direct
1141 if(persistProxy) { // if upstream socket is open close it
1142 proxysock.close();
1143 persistProxy = false;
1144 }
1145 }
1146 }
1147
1148 if (checkme.isbypass && !(checkme.iscookiebypass || checkme.isvirusbypass)) {
1149 #ifdef DGDEBUG
1150 std::cout << thread_id << "Setting GBYPASS cookie; bypasstimestamp = " << checkme.bypasstimestamp << __func__ << std::endl;
1151 #endif
1152 String ud(checkme.urldomain);
1153 if (ud.startsWith("www.")) {
1154 ud = ud.after("www.");
1155 }
1156 // redirect user to URL with GBYPASS parameter no longer appended
1157 String outhead = "HTTP/1.1 302 Redirect\r\n";
1158 outhead += "Set-Cookie: GBYPASS=";
1159 outhead += hashedCookie(&ud, ldl->fg[filtergroup]->cookie_magic.c_str(), &clientip,
1160 checkme.bypasstimestamp).toCharArray();
1161 outhead += "; path=/; domain=.";
1162 outhead += ud;
1163 outhead += "\r\n";
1164 outhead += "Location: ";
1165 outhead += header.getUrl(true);
1166 outhead += "\r\n";
1167 outhead += "\r\n";
1168 peerconn.writeString(outhead.c_str());
1169 return 0;
1170 }
1171
1172
1173 //check for redirect
1174 // URL regexp search and edirect
1175 if (checkme.urlredirect) {
1176 checkme.url = header.redirecturl();
1177 proxysock.close();
1178 String writestring("HTTP/1.1 302 Redirect\nLocation: ");
1179 writestring += checkme.url;
1180 writestring += "\n\n";
1181 peerconn.writeString(writestring.toCharArray());
1182 break;
1183 }
1184
1185 //if is a search - content check search terms
1186 if (!checkme.isdone) {
1187 if (checkme.isGrey && checkme.isSearch)
1188 check_search_terms(checkme);
1189 } // will set isItNaughty if needed
1190
1191
1192 // TODO V5 call POST scanning code New NaughtyFilter function????
1193
1194 // don't run willScanRequest if content scanning is disabled, or on exceptions if contentscanexceptions is off,
1195 // or on SSL (CONNECT) requests, or on HEAD requests, or if in AV bypass mode
1196
1197 //if not naughty now send upstream and get response
1198 if (checkme.isItNaughty) {
1199 if (checkme.isconnect && ldl->fg[filtergroup]->ssl_mitm && ldl->fg[filtergroup]->automitm) {
1200 checkme.gomitm = true; // so that we can deliver a status message to user over half MITM
1201 if (checkme.isdirect) { // send connection estabilished to client
1202 std::string msg = "HTTP/1.1 200 Connection established\r\n\r\n";
1203 if (!peerconn.writeString(msg.c_str())) {
1204 peerDiag("Unable to send 200 connection established to client ", peerconn);
1205 break;
1206 }
1207 }
1208 } else {
1209 checkme.gomitm = false; // if not automitm
1210 }
1211 } else {
1212 if (!persistProxy) // open upstream connection
1213 {
1214 if (connectUpstream(proxysock, checkme, header.port) < 0) {
1215 if (checkme.isconnect && ldl->fg[filtergroup]->ssl_mitm && ldl->fg[filtergroup]->automitm &&
1216 checkme.upfailure)
1217 {
1218
1219 checkme.gomitm = true; // so that we can deliver a status message to user over half MITM
1220 // give error - depending on answer
1221 // timeout -etc
1222 } else {
1223 checkme.gomitm = false; // if not automitm
1224 }
1225 }
1226 }
1227 if (!checkme.upfailure) {
1228 if (!proxysock.breadyForOutput(o.proxy_timeout)) {
1229 upstreamDiag("Unable to write upstream", proxysock);
1230 break;
1231 }
1232 #ifdef DGDEBUG
1233 std::cerr << thread_id << " got past line 990 rfo " << std::endl;
1234 #endif
1235 }
1236 if (checkme.isdirect && checkme.isconnect) { // send connection estabilished to client
1237 std::string msg = "HTTP/1.1 200 Connection established\r\n\r\n";
1238 if (!peerconn.writeString(msg.c_str())) {
1239 peerDiag("Unable to send 200 connection established to client ", peerconn);
1240 break;
1241 }
1242 } else if (!checkme.upfailure) // in all other cases send header upstream and get response
1243 {
1244 if (!(header.out(&peerconn, &proxysock, __DGHEADER_SENDALL, true) // send proxy the request
1245 && (docheader.in_handle_100(&proxysock, persistOutgoing, header.expects_100)))) {
1246 if (proxysock.isTimedout()) {
1247 // writeback_error(checkme, peerconn, 203, 204, "408 Request Time-out");
1248 writeback_error(checkme, peerconn, 0, 0, "408 Request Time-out");
1249 } else {
1250 if(!ismitm) {
1251 writeback_error(checkme, peerconn, 0, 0, "408 Request Time-out");
1252 //writeback_error(checkme, peerconn, 205, 206, "502 Gateway Error");
1253 }
1254 }
1255 persistPeer = false;
1256 persistProxy = false;
1257 break;
1258 }
1259 persistProxy = docheader.isPersistent();
1260 persistPeer = persistOutgoing && docheader.wasPersistent();
1261 #ifdef DGDEBUG
1262 std::cerr << thread_id << " -persistPeer: " << persistPeer << std::endl;
1263 #endif
1264
1265 //check response code
1266 if ((!checkme.isItNaughty) && (!checkme.upfailure)) {
1267 int rcode = docheader.returnCode();
1268 if (rcode == 407) { // proxy auth required
1269 // tunnel thru - may be content
1270 checkme.tunnel_rest = true;
1271 checkme.tunnel_2way = false;
1272 // treat connect like normal get
1273 checkme.isconnect = false;
1274 checkme.isexception = true;
1275 } else if (!authed && checkme.request_header->isProxyRequest
1276 && o.auth_needs_proxy_in_plugin && !checkme.isexception) {
1277 checkme.isItNaughty = true;
1278 checkme.message_no = 110;
1279 }
1280 if (checkme.isconnect) {
1281 if (rcode == 200) {
1282 persistProxy = false;
1283 persistPeer = false;
1284 } else { // some sort of problem or needs proxy auth - pass back to client
1285 checkme.ismitmcandidate = false; // only applies to connect
1286 checkme.tunnel_rest = true;
1287 checkme.tunnel_2way = false;
1288 }
1289 }
1290
1291 if (docheader.contentLength() == 0) // no content
1292 checkme.tunnel_rest = true;
1293
1294 }
1295 }
1296
1297 if (checkme.isconnect && checkme.isGrey) { // allow legacy SSL behavour when mitm not enabled
1298 checkme.tunnel_2way = true;
1299 checkme.tunnel_rest = false;
1300 }
1301 }
1302 if (checkme.isexception && !checkme.upfailure) {
1303 if (checkme.isconnect) {
1304 checkme.tunnel_2way = true;
1305 checkme.tunnel_rest = false;
1306 persistPeer = false;
1307 persistProxy = false;
1308 } else {
1309 if (!checkme.noviruscheck && !ldl->fg[filtergroup]->content_scan_exceptions)
1310 checkme.noviruscheck = true;
1311 if (checkme.noviruscheck) {
1312 checkme.tunnel_2way = false;
1313 checkme.tunnel_rest = true;
1314 }
1315 }
1316 }
1317
1318 #ifdef __SSLMITM
1319 //if ismitm - GO MITM
1320 // ssl_grey is covered in storyboard
1321 if (!checkme.tunnel_rest && checkme.isconnect && checkme.gomitm)
1322 {
1323 #ifdef DGDEBUG
1324 std::cerr << "Going MITM ...." << std::endl;
1325 #endif
1326 if(!ldl->fg[filtergroup]->mitm_check_cert)
1327 checkme.nocheckcert = true;
1328 goMITM(checkme, proxysock, peerconn, persistProxy, authed, persistent_authed, ip, dystat, clientip,checkme.isdirect);
1329 persistPeer = false;
1330 persistProxy = false;
1331 //if (!checkme.isItNaughty) // surely we should just break here whatever? - No we need to log error
1332 break;
1333 }
1334 #endif
1335
1336 //CALL SB checkresponse
1337 if ((!checkme.isItNaughty) && (!checkme.upfailure) && (!checkme.isconnect) && !checkme.tunnel_rest) {
1338 ldl->fg[filtergroup]->StoryB.runFunctEntry(ENT_STORYB_PROXY_RESPONSE, checkme);
1339 #ifdef DGDEBUG
1340 std::cerr << thread_id << "After StoryB checkresponse " << checkme.isexception << " mess_no "
1341 << checkme.message_no << std::endl;
1342 #endif
1343 if (ldl->fg[filtergroup]->reporting_level != -1){
1344 checkme.isItNaughty = checkme.isBlocked;
1345 } else {
1346 checkme.isItNaughty = false;
1347 checkme.isBlocked = false;
1348 checkme.isGrey = true;
1349 }
1350
1351 if (checkme.ishead || (docheader.contentLength() == 0 && !docheader.chunked))
1352 checkme.tunnel_rest = true;
1353
1354 if (checkme.isexception && !ldl->fg[filtergroup]->content_scan_exceptions) {
1355 checkme.tunnel_rest = true;
1356 checkme.noviruscheck = true;
1357 }
1358
1359 if (!checkme.noviruscheck) {
1360 for (std::deque<Plugin *>::iterator i = o.csplugins_begin; i != o.csplugins_end; ++i) {
1361 int csrc = ((CSPlugin *) (*i))->willScanRequest(header.getUrl(), clientuser.c_str(),
1362 ldl->fg[filtergroup], clientip.c_str(), false,
1363 false, checkme.isexception, checkme.isbypass);
1364 if (csrc > 0)
1365 responsescanners.push_back((CSPlugin *) (*i));
1366 else if (csrc < 0)
1367 syslog(LOG_ERR, "%swillScanRequest returned error: %d", thread_id.c_str(), csrc);
1368 }
1369 #ifdef DGDEBUG
1370 std::cerr << thread_id << " -Content scanners interested in response data: " << responsescanners.size() << " Line: " << __LINE__ << " Function: " << __func__ << std::endl;
1371 #endif
1372 }
1373
1374 //- if grey content check
1375 // can't do content filtering on HEAD or redirections (no content)
1376 // actually, redirections CAN have content
1377 if ((checkme.isGrey || !checkme.noviruscheck) && !checkme.tunnel_rest) {
1378 check_content(checkme, docbody, proxysock, peerconn, responsescanners);
1379 }
1380 }
1381
1382 //send response header to client
1383 if ((!checkme.isItNaughty) && (!checkme.upfailure) && !(checkme.isconnect && checkme.isdirect)) {
1384 if (!docheader.out(NULL, &peerconn, __DGHEADER_SENDALL, false)) {
1385 peerDiag("Unable to send return header to client", peerconn);
1386 break;
1387 }
1388
1389 if ((!checkme.isItNaughty) && checkme.waschecked) {
1390 if (!docbody.out(&peerconn))
1391 checkme.pausedtoobig = false;
1392 if (checkme.pausedtoobig)
1393 checkme.tunnel_rest = true;
1394 }
1395 }
1396
1397 //if not grey tunnel response
1398 if (!checkme.isItNaughty) {
1399 if (checkme.tunnel_rest) {
1400 bool chunked = docheader.transferEncoding().contains("chunked");
1401 #ifdef DGDEBUG
1402 std::cerr << thread_id << " -Tunnelling to client" << std::endl;
1403 std::cerr << thread_id << " - Content-Length:" << docheader.contentLength() << "cm.docsize:" << checkme.docsize << std::endl;
1404 #endif
1405
1406 if (!fdt.tunnel(proxysock, peerconn, checkme.isconnect, docheader.contentLength() - checkme.docsize,
1407 true, chunked))
1408 persistProxy = false;
1409 checkme.docsize += fdt.throughput;
1410 } else if (checkme.tunnel_2way) {
1411 if (!fdt.tunnel(proxysock, peerconn, true))
1412 persistProxy = false;
1413 checkme.docsize = fdt.throughput;
1414 }
1415 }
1416
1417
1418 #ifdef DGDEBUG
1419 std::cerr << thread_id << " -Forwarding body to client : Upfailure is " << checkme.upfailure << std::endl;
1420 #endif
1421 if (checkme.upfailure || checkme.isItNaughty) {
1422 if (denyAccess(&peerconn, &proxysock, &header, &docheader, &checkme.url, &checkme, &clientuser,
1423 &clientip,
1424 filtergroup, checkme.ispostblock, checkme.headersent, checkme.wasinfected,
1425 checkme.scanerror))
1426 persistPeer = false;
1427 }
1428
1429 //Log
1430 if (!checkme.isourwebserver) { // don't log requests to the web server
1431 doLog(clientuser, clientip, checkme);
1432 }
1433
1434
1435 if (!persistProxy)
1436 proxysock.close(); // close connection to proxy
1437
1438 if (persistPeer) {
1439 continue;
1440 }
1441
1442 break;
1443 }
1444 } catch (std::exception &e) {
1445 #ifdef DGDEBUG
1446 std::cerr << thread_id << " -connection handler caught an exception: " << e.what() << " Line: " << __LINE__ << " Function: " << __func__ << std::endl;
1447 #endif
1448 if (o.logconerror)
1449 syslog(LOG_ERR, "%s-connection handler caught an exception %s", thread_id.c_str(), e.what());
1450
1451 // close connection to proxy
1452 proxysock.close();
1453 return -1;
1454 }
1455 if (!ismitm)
1456 try {
1457 #ifdef DGDEBUG
1458 std::cerr << thread_id << " -Attempting graceful connection close" << " Line: " << __LINE__ << " Function: " << __func__ << std::endl;
1459 #endif
1460 //syslog(LOG_INFO, " -Attempting graceful connection close" );
1461 int fd = peerconn.getFD();
1462 if (fd > -1) {
1463 if (shutdown(fd, SHUT_WR) == 0) {
1464 char buff[2];
1465 peerconn.readFromSocket(buff, 2, 0, 5000);
1466 };
1467 };
1468
1469 // close connection to the client
1470 peerconn.close();
1471 proxysock.close();
1472 } catch (std::exception &e) {
1473 #ifdef DGDEBUG
1474 std::cerr << thread_id << " -connection handler caught an exception on connection closedown: " << e.what() << " Line: " << __LINE__ << " Function: " << __func__ << std::endl;
1475 #endif
1476 // close connection to the client
1477 peerconn.close();
1478 proxysock.close();
1479 }
1480
1481 return 0;
1482 }
1483
1484
doLog(std::string & who,std::string & from,NaughtyFilter & cm)1485 void ConnectionHandler::doLog(std::string &who, std::string &from, NaughtyFilter &cm) {
1486 String rtype = cm.request_header->requestType();
1487 String where = cm.logurl;
1488 unsigned int port = cm.request_header->port;
1489 std::string what;
1490 if(o.log_requests) {
1491 what = thread_id;
1492 }
1493 what += cm.whatIsNaughtyLog;
1494 String how = rtype;
1495 off_t size = cm.docsize;
1496 std::string *cat = &cm.whatIsNaughtyCategories;
1497 bool isnaughty = cm.isItNaughty;
1498 int naughtytype = cm.blocktype;
1499 bool isexception = cm.isexception;
1500 bool istext = cm.is_text;
1501 struct timeval *thestart = &cm.thestart;
1502 bool cachehit = false;
1503 //int code = (cm.wasrequested ? cm.response_header->returnCode() : 200); //cm.wasrequested is never set anywhere!!
1504 int code = (cm.response_header->returnCode());
1505 if (isnaughty) code = 403;
1506 std::string mimetype = cm.mimetype;
1507 bool wasinfected = cm.wasinfected;
1508 bool wasscanned = cm.wasscanned;
1509 int naughtiness = cm.naughtiness;
1510 int filtergroup = cm.filtergroup;
1511 HTTPHeader *reqheader = cm.request_header;
1512 int message_no = cm.message_no;
1513 bool contentmodified = cm.contentmodified;
1514 bool urlmodified = cm.urlmodified;
1515 bool headermodified = cm.headermodified;
1516 bool headeradded = cm.headeradded;
1517
1518 // don't log if logging disabled entirely, or if it's an ad block and ad logging is disabled,
1519 // or if it's an exception and exception logging is disabled
1520 if (
1521 (o.ll == 0) || ((cat != NULL) && !o.log_ad_blocks && (strstr(cat->c_str(), "ADs") != NULL)) ||
1522 ((o.log_exception_hits == 0) && isexception)) {
1523 #ifdef DGDEBUG
1524 if (o.ll != 0) {
1525 if (isexception)
1526 std::cerr << thread_id << " -Not logging exceptions" << std::endl;
1527 else
1528 std::cerr << thread_id << " -Not logging 'ADs' blocks" << std::endl;
1529 }
1530 #endif
1531 return;
1532 }
1533
1534 std::string data, cr("\n");
1535
1536 if ((isexception && (o.log_exception_hits == 2))
1537 || isnaughty || o.ll == 3 || (o.ll == 2 && istext)) {
1538 // put client hostname in log if enabled.
1539 // for banned & exception IP/hostname matches, we want to output exactly what was matched against,
1540 // be it hostname or IP - therefore only do lookups here when we don't already have a cached hostname,
1541 // and we don't have a straight IP match agaisnt the banned or exception IP lists.
1542 if (o.log_client_hostnames && (cm.clienthost == "") && !matchedip && !cm.anon_log) {
1543 #ifdef DGDEBUG
1544 std::cerr << "logclienthostnames enabled but reverseclientiplookups disabled; lookup forced." << std::endl;
1545 #endif
1546 getClientFromIP(from.c_str(),cm.clienthost);
1547 //std::deque<String> *names = ipToHostname(from.c_str());
1548 //if (names->size() > 0) {
1549 //clienthost = new std::string(names->front().toCharArray());
1550 //cm.clienthost = *clienthost;
1551 //}
1552 //delete names;
1553 }
1554
1555 // Build up string describing POST data parts, if any
1556 std::ostringstream postdata;
1557 for (std::list<postinfo>::iterator i = postparts.begin(); i != postparts.end(); ++i) {
1558 // Replace characters which would break log format with underscores
1559 std::string::size_type loc = 0;
1560 while ((loc = i->filename.find_first_of(",;\t ", loc)) != std::string::npos)
1561 i->filename[loc] = '_';
1562 // Build up contents of log column
1563 postdata << i->mimetype << "," << i->filename << "," << i->size
1564 << "," << i->blocked << "," << i->storedname << "," << i->bodyoffset << ";";
1565 }
1566 postdata << std::flush;
1567
1568 // Formatting code moved into log_listener in FatController.cpp
1569 // Original patch by J. Gauthier
1570
1571 // Item length limit put back to avoid log listener
1572 // overload with very long urls Philip Pearce Jan 2014
1573 if ((cat != NULL) && (cat->length() > o.max_logitem_length))
1574 cat->resize(o.max_logitem_length);
1575 if (what.length() > o.max_logitem_length)
1576 what.resize(o.max_logitem_length);
1577 if (where.length() > o.max_logitem_length)
1578 where.limitLength(o.max_logitem_length);
1579 if (o.dns_user_logging && !is_real_user) {
1580 String user;
1581 if (getdnstxt(from, user)) {
1582 who = who + ":" + user;
1583 };
1584 is_real_user = true; // avoid looping on persistent connections
1585 };
1586 std::string l_who = who;
1587 std::string l_from = from;
1588 std::string l_clienthost;
1589 l_clienthost = cm.clienthost;
1590
1591 if (cm.anon_log) {
1592 l_who = "";
1593 l_from = "0.0.0.0";
1594 l_clienthost = "";
1595 }
1596
1597 #ifdef DGDEBUG
1598 std::cerr << thread_id << " -Building raw log data string... ";
1599 #endif
1600
1601 data = String(isexception) + cr;
1602 data += (cat ? (*cat) + cr : cr);
1603 data += String(isnaughty) + cr;
1604 data += String(naughtytype) + cr;
1605 data += String(naughtiness) + cr;
1606 data += where + cr;
1607 data += what + cr;
1608 data += how + cr;
1609 data += l_who + cr;
1610 data += l_from + cr;
1611 data += String(port) + cr;
1612 data += String(wasscanned) + cr;
1613 data += String(wasinfected) + cr;
1614 data += String(contentmodified) + cr;
1615 data += String(urlmodified) + cr;
1616 data += String(headermodified) + cr;
1617 data += String(size) + cr;
1618 data += String(filtergroup) + cr;
1619 data += String(code) + cr;
1620 data += String(cachehit) + cr;
1621 data += String(mimetype) + cr;
1622 data += String((*thestart).tv_sec) + cr;
1623 data += String((*thestart).tv_usec) + cr;
1624 data += l_clienthost + cr;
1625 if (o.log_user_agent)
1626 data += (reqheader ? reqheader->userAgent() + cr : cr);
1627 else
1628 data += cr;
1629 data += urlparams + cr;
1630 data += postdata.str().c_str() + cr;
1631 data += String(message_no) + cr;
1632 data += String(headeradded) + cr;
1633
1634 #ifdef DGDEBUG
1635 std::cerr << thread_id << " -...built" << std::endl;
1636 #endif
1637
1638 //delete newcat;
1639 // push on log queue
1640 o.log_Q->push(data);
1641 // connect to dedicated logging proc
1642 }
1643 }
1644
doRQLog(std::string & who,std::string & from,NaughtyFilter & cm,std::string & funct)1645 void ConnectionHandler::doRQLog(std::string &who, std::string &from, NaughtyFilter &cm, std::string &funct) {
1646 String rtype = cm.request_header->requestType();
1647 String where = cm.logurl;
1648 unsigned int port = cm.request_header->port;
1649 std::string what = thread_id;
1650 what += funct;
1651 if (cm.isTLS)
1652 what += "_TLS";
1653 if (cm.hasSNI)
1654 what += "_SNI";
1655 if (cm.ismitm)
1656 what += "_MITM";
1657 String how = rtype;
1658 off_t size = cm.docsize;
1659 std::string *cat = &cm.whatIsNaughtyCategories;
1660 bool isnaughty = cm.isItNaughty;
1661 int naughtytype = cm.blocktype;
1662 bool isexception = cm.isexception;
1663 bool istext = cm.is_text;
1664 struct timeval *thestart = &cm.thestart;
1665 bool cachehit = false;
1666 int code = 0;
1667 std::string mimetype = cm.mimetype;
1668 bool wasinfected = cm.wasinfected;
1669 bool wasscanned = cm.wasscanned;
1670 int naughtiness = cm.naughtiness;
1671 int filtergroup = cm.filtergroup;
1672 HTTPHeader *reqheader = cm.request_header;
1673 int message_no = cm.message_no;
1674 bool contentmodified = cm.contentmodified;
1675 bool urlmodified = cm.urlmodified;
1676 bool headermodified = cm.headermodified;
1677 bool headeradded = cm.headeradded;
1678
1679 std::string data, cr("\n");
1680
1681 // if ((isexception && (o.log_exception_hits == 2))
1682 // || isnaughty || o.ll == 3 || (o.ll == 2 && istext))
1683 if(true) {
1684
1685 // Item length limit put back to avoid log listener
1686 // overload with very long urls Philip Pearce Jan 2014
1687 if ((cat != NULL) && (cat->length() > o.max_logitem_length))
1688 cat->resize(o.max_logitem_length);
1689 if (what.length() > o.max_logitem_length)
1690 what.resize(o.max_logitem_length);
1691 if (where.length() > o.max_logitem_length)
1692 where.limitLength(o.max_logitem_length);
1693 if (o.dns_user_logging && !is_real_user) {
1694 String user;
1695 if (getdnstxt(from, user)) {
1696 who = who + ":" + user;
1697 };
1698 is_real_user = true; // avoid looping on persistent connections
1699 };
1700 std::string l_who = who;
1701 std::string l_from = from;
1702 std::string l_clienthost;
1703 l_clienthost = cm.clienthost;
1704
1705
1706 #ifdef DGDEBUG
1707 std::cerr << thread_id << " -Building raw log data string... ";
1708 #endif
1709
1710 data = String(isexception) + cr;
1711 data += (cat ? (*cat) + cr : cr);
1712 data += String(isnaughty) + cr;
1713 data += String(naughtytype) + cr;
1714 data += String(naughtiness) + cr;
1715 data += where + cr;
1716 data += what + cr;
1717 data += how + cr;
1718 data += l_who + cr;
1719 data += l_from + cr;
1720 data += String(port) + cr;
1721 data += String(wasscanned) + cr;
1722 data += String(wasinfected) + cr;
1723 data += String(contentmodified) + cr;
1724 data += String(urlmodified) + cr;
1725 data += String(headermodified) + cr;
1726 data += String(size) + cr;
1727 data += String(filtergroup) + cr;
1728 data += String(code) + cr;
1729 data += String(cachehit) + cr;
1730 data += String(mimetype) + cr;
1731 data += String((*thestart).tv_sec) + cr;
1732 data += String((*thestart).tv_usec) + cr;
1733 data += l_clienthost + cr;
1734 if (o.log_user_agent)
1735 data += (reqheader ? reqheader->userAgent() + cr : cr);
1736 else
1737 data += cr;
1738 data += urlparams + cr;
1739 data += cr;
1740 data += String(message_no) + cr;
1741 data += String(headeradded) + cr;
1742
1743 #ifdef DGDEBUG
1744 std::cerr << thread_id << " -...built" << std::endl;
1745 #endif
1746
1747 //delete newcat;
1748 // push on log queue
1749 o.RQlog_Q->push(data);
1750 // connect to dedicated logging proc
1751 }
1752 }
1753
1754
1755 // TODO - V5
1756 #ifdef NOTDEF
1757 // check if embeded url trusted referer
embededRefererChecks(HTTPHeader * header,String * urld,String * url,int filtergroup)1758 bool ConnectionHandler::embededRefererChecks(HTTPHeader *header, String *urld, String *url,
1759 int filtergroup)
1760 {
1761
1762 char *i;
1763 int j;
1764 String temp;
1765 temp = (*urld);
1766 temp.hexDecode();
1767
1768 if (ldl->fg[filtergroup]->inRefererExceptionLists(header->getReferer())) {
1769 return true;
1770 }
1771 #ifdef DGDEBUG
1772 std::cerr << thread_id << " -checking for embed url in " << temp << std::endl;
1773 #endif
1774
1775 if (ldl->fg[filtergroup]->inEmbededRefererLists(temp)) {
1776
1777 // look for referer URLs within URLs
1778 #ifdef DGDEBUG
1779 std::cerr << thread_id << " -starting embeded referer deep analysis" << std::endl;
1780 #endif
1781 String deepurl(temp.after("p://"));
1782 deepurl = header->decode(deepurl, true);
1783 while (deepurl.contains(":")) {
1784 deepurl = deepurl.after(":");
1785 while (deepurl.startsWith(":") || deepurl.startsWith("/")) {
1786 deepurl.lop();
1787 }
1788
1789 if (ldl->fg[filtergroup]->inRefererExceptionLists(deepurl)) {
1790 #ifdef DGDEBUG
1791 std::cerr << "deep site found in trusted referer list; " << std::endl;
1792 #endif
1793 return true;
1794 }
1795 }
1796 #ifdef DGDEBUG
1797 std::cerr << thread_id << " -done embdeded referer deep analysis" << std::endl;
1798 #endif
1799 }
1800 return false;
1801 }
1802 #endif
1803
1804 // based on patch by Aecio F. Neto (afn@harvest.com.br) - Harvest Consultoria (http://www.harvest.com.br)
1805 // show the relevant banned page/image/CGI based on report level setting, request type etc.
genDenyAccess(Socket & peerconn,String & eheader,String & ebody,HTTPHeader * header,HTTPHeader * docheader,String * url,NaughtyFilter * checkme,std::string * clientuser,std::string * clientip,int filtergroup,bool ispostblock,int headersent,bool wasinfected,bool scanerror,bool forceshow)1806 bool ConnectionHandler::genDenyAccess(Socket &peerconn, String &eheader, String &ebody, HTTPHeader *header,
1807 HTTPHeader *docheader,
1808 String *url, NaughtyFilter *checkme, std::string *clientuser,
1809 std::string *clientip, int filtergroup,
1810 bool ispostblock, int headersent, bool wasinfected, bool scanerror,
1811 bool forceshow) {
1812 int reporting_level = ldl->fg[filtergroup]->reporting_level;
1813 #ifdef DGDEBUG
1814
1815 std::cerr << thread_id << " -reporting level is " << reporting_level << std::endl;
1816
1817 #endif
1818 if (checkme->whatIsNaughty == "" && checkme->message_no > 0) {
1819 checkme->whatIsNaughty = o.language_list.getTranslation(checkme->message_no);
1820 }
1821 if (checkme->whatIsNaughtyLog == "") {
1822 if (checkme->log_message_no > 0) {
1823 checkme->whatIsNaughtyLog = o.language_list.getTranslation(checkme->log_message_no);
1824 } else {
1825 checkme->whatIsNaughtyLog = checkme->whatIsNaughty;
1826 }
1827 }
1828
1829 try { // writestring throws exception on error/timeout
1830
1831 // flags to enable filter/infection / hash generation
1832 bool filterhash = false;
1833 bool virushash = false;
1834 // flag to enable internal generation of hashes (i.e. obey the "-1" setting; to allow the modes but disable hash generation)
1835 // (if disabled, just output '1' or '2' to show that the CGI should generate a filter/virus bypass hash;
1836 // otherwise, real hashes get put into substitution variables/appended to the ban CGI redirect URL)
1837 bool dohash = false;
1838 if (reporting_level > 0) {
1839 // generate a filter bypass hash
1840 if (!wasinfected && (checkme->isbypassallowed) && !ispostblock) {
1841 #ifdef DGDEBUG
1842 std::cerr << thread_id << " -Enabling filter bypass hash generation" << std::endl;
1843 #endif
1844 filterhash = true;
1845 if (ldl->fg[filtergroup]->bypass_mode > 0 )
1846 dohash = true;
1847 }
1848 // generate an infection bypass hash
1849 else if (wasinfected && checkme->isinfectionbypassallowed) {
1850 // only generate if scanerror (if option to only bypass scan errors is enabled)
1851 if ((*ldl->fg[filtergroup]).infection_bypass_errors_only ? scanerror : true) {
1852 #ifdef DGDEBUG
1853 std::cerr << thread_id << " -Enabling infection bypass hash generation" << std::endl;
1854 #endif
1855 virushash = true;
1856 if (ldl->fg[filtergroup]->infection_bypass_mode > 0)
1857 dohash = true;
1858 }
1859 }
1860 }
1861 #ifdef DGDEBUG
1862 std::cerr << thread_id << " - filter bypass hash generation" << " virushah " << virushash << " dohash " << dohash << " filterhash " << filterhash <<std::endl;
1863 #endif
1864 // the user is using the full whack of custom banned images and/or HTML templates
1865 #ifdef __SSLMITM
1866 if (reporting_level == 3 || (headersent > 0 && reporting_level > 0) || forceshow || (*header).requestType().startsWith("CONNECT"))
1867 #else
1868 if (reporting_level == 3 || (headersent > 0 && reporting_level > 0) || (*header).requestType().startsWith("CONNECT"))
1869 #endif
1870 {
1871 // if reporting_level = 1 or 2 and headersent then we can't
1872 // send a redirect so we have to display the template instead
1873
1874
1875 #ifdef __SSLMITM
1876 if ((*header).requestType().startsWith("CONNECT") && !(peerconn).isSsl())
1877 #else
1878 if ((*header).requestType().startsWith("CONNECT"))
1879 #endif
1880 {
1881 // Block ssl website
1882 // Buggy with FF < 65 https://bugzilla.mozilla.org/show_bug.cgi?id=1522093
1883 // Connections still opened after a refresh
1884 // 403 requests made ICAP error with high load
1885 eheader = "HTTP/1.1 302 Redirect";
1886 eheader += "\r\nLocation: http://internal.test.e2guardian.org";
1887 eheader += "\r\nServer: e2guardian";
1888 eheader += "\r\nConnection: close";
1889 eheader += "\r\n\r\n";
1890 } else {
1891 // we're dealing with a non-SSL'ed request, and have the option of using the custom banned image/page directly
1892 bool replaceimage = false;
1893 bool replaceflash = false;
1894 if (o.use_custom_banned_image) {
1895
1896 // It would be much nicer to do a mime comparison
1897 // and see if the type is image/* but the header
1898 // never (almost) gets back from squid because
1899 // it gets denied before then.
1900 // This method is prone to over image replacement
1901 // but will work most of the time.
1902
1903 String lurl((*url));
1904 lurl.toLower();
1905 if (lurl.endsWith(".gif") || lurl.endsWith(".jpg") || lurl.endsWith(".jpeg") ||
1906 lurl.endsWith(".jpe")
1907 || lurl.endsWith(".png") || lurl.endsWith(".bmp") ||
1908 (*docheader).isContentType("image/", ldl->fg[filtergroup])) {
1909 replaceimage = true;
1910 }
1911 }
1912
1913 if (o.use_custom_banned_flash) {
1914 String lurl((*url));
1915 lurl.toLower();
1916 if (lurl.endsWith(".swf") ||
1917 (*docheader).isContentType("application/x-shockwave-flash", ldl->fg[filtergroup])) {
1918 replaceflash = true;
1919 }
1920 }
1921
1922 // if we're denying an image request, show the image; otherwise, show the HTML page.
1923 // (or advanced ad block page, or HTML page with bypass URLs)
1924 if (replaceimage) {
1925 if (headersent == 0) {
1926 eheader = "HTTP/1.1 200 OK\r\n";
1927 }
1928 o.banned_image.display_hb(eheader, ebody);
1929 } else if (replaceflash) {
1930 if (headersent == 0) {
1931 eheader = "HTTP/1.1 200 OK\r\n";
1932 }
1933 o.banned_flash.display_hb(eheader, ebody);
1934 } else {
1935 // advanced ad blocking - if category contains ADs, wrap ad up in an "ad blocked" message,
1936 // which provides a link to the original URL if you really want it. primarily
1937 // for IFRAMEs, which will end up containing this link instead of the ad (standard non-IFRAMEd
1938 // ad images still get image-replaced.)
1939 if (strstr(checkme->whatIsNaughtyCategories.c_str(), "ADs") != NULL) {
1940 eheader = "HTTP/1.1 200 \r\n";
1941 eheader += o.language_list.getTranslation(1101); // advert blocked
1942 eheader += "\r\nContent-Type: text/html\r\n";
1943 ebody = "<HTML><HEAD><TITLE>E2guardian - ";
1944 ebody += o.language_list.getTranslation(1101); // advert blocked
1945 ebody += "</TITLE></HEAD><BODY><CENTER><FONT SIZE=\"-1\"><A HREF=\"";
1946 ebody += (*url);
1947 ebody += "\" TARGET=\"_BLANK\">";
1948 ebody += o.language_list.getTranslation(1101); // advert blocked
1949 ebody += "</A></FONT></CENTER></BODY></HTML>\r\n";
1950 eheader += "Content-Length: ";
1951 eheader += std::to_string(ebody.size());
1952 eheader += "\r\n\r\n";
1953 }
1954
1955 // Mod by Ernest W Lessenger Mon 2nd February 2004
1956 // Other bypass code mostly written by Ernest also
1957 // create temporary bypass URL to show on denied page
1958 else {
1959 String hashed;
1960 // generate valid hash locally if enabled
1961 if (dohash) {
1962 hashed = hashedURL(url, filtergroup, clientip, virushash, clientuser);
1963 }
1964 // otherwise, just generate flags showing what to generate
1965 else if (filterhash) {
1966 hashed = "HASH=1";
1967 } else if (virushash) {
1968 hashed = "HASH=2";
1969 }
1970
1971 if(ldl->fg[filtergroup]->cgi_bypass_v2) {
1972 if (filterhash) {
1973 hashed += "&HASH=1";
1974 } else if (virushash) {
1975 hashed += "&HASH=2";
1976 }
1977 }
1978
1979 if (headersent == 0) {
1980 eheader = "HTTP/1.1 200 \r\n";
1981 }
1982 if (headersent < 2) {
1983 eheader += "Content-type: text/html\r\n\r\n";
1984 }
1985 // if the header has been sent then likely displaying the
1986 // template will break the download, however as this is
1987 // only going to be happening if the unsafe trickle
1988 // buffer method is used and we want the download to be
1989 // broken we don't mind too much
1990 //String fullurl = header->getLogUrl(true);
1991 String fullurl = checkme->logurl;
1992 String localip = peerconn.getLocalIP();
1993 ldl->fg[filtergroup]->getHTMLTemplate(checkme->upfailure)->display_hb(ebody,
1994 &fullurl,
1995 (*checkme).whatIsNaughty,
1996 (*checkme).whatIsNaughtyLog,
1997 // grab either the full category list or the thresholded list
1998 (checkme->usedisplaycats
1999 ? checkme->whatIsNaughtyDisplayCategories
2000 : checkme->whatIsNaughtyCategories),
2001 clientuser, clientip,
2002 &(checkme->clienthost), filtergroup,
2003 ldl->fg[filtergroup]->name,
2004 hashed, localip);
2005 }
2006 }
2007 }
2008 }
2009 // the user is using the CGI rather than the HTML template - so issue a redirect with parameters filled in on GET string
2010 else if (reporting_level > 0) {
2011 // grab either the full category list or the thresholded list
2012 std::string cats;
2013 cats = checkme->usedisplaycats ? checkme->whatIsNaughtyDisplayCategories : checkme->whatIsNaughtyCategories;
2014
2015 String hashed;
2016 // generate valid hash locally if enabled
2017 if (dohash) {
2018 hashed = hashedURL(url, filtergroup, clientip, virushash, clientuser);
2019 }
2020 // otherwise, just generate flags showing what to generate
2021 else if (filterhash) {
2022 hashed = "1";
2023 } else if (virushash) {
2024 hashed = "2";
2025 }
2026
2027 if ((*checkme).whatIsNaughty.length() > 2048) {
2028 (*checkme).whatIsNaughty = String((*checkme).whatIsNaughty.c_str()).subString(0, 2048).toCharArray();
2029 }
2030 if ((*checkme).whatIsNaughtyLog.length() > 2048) {
2031 (*checkme).whatIsNaughtyLog = String((*checkme).whatIsNaughtyLog.c_str()).subString(0,
2032 2048).toCharArray();
2033 }
2034
2035 #ifdef __SSLMITM
2036 if ((*header).requestType().startsWith("CONNECT") && !(peerconn).isSsl())
2037 #else
2038 if ((*header).requestType().startsWith("CONNECT"))
2039 #endif
2040 {
2041 // Block ssl website
2042 // Buggy with FF < 65 https://bugzilla.mozilla.org/show_bug.cgi?id=1522093
2043 // Connections still opened after a refresh
2044 // 403 requests made ICAP error with high load
2045 eheader = "HTTP/1.1 302 Redirect";
2046 eheader += "\r\nLocation: http://internal.test.e2guardian.org";
2047 eheader += "\r\nServer: e2guardian";
2048 eheader += "\r\nConnection: close";
2049 eheader += "\r\n\r\n";
2050 // we're dealing with a non-SSL'ed request, and have the option of using the custom banned image/page directly
2051 } else {
2052 eheader = "HTTP/1.1 302 Redirect\r\n";
2053 eheader += "Location: ";
2054 eheader += ldl->fg[filtergroup]->access_denied_address;
2055 }
2056 if (ldl->fg[filtergroup]->non_standard_delimiter) {
2057 eheader += "?DENIEDURL==";
2058 eheader += miniURLEncode((*url).toCharArray()).c_str();
2059 eheader += "::IP==";
2060 eheader += (*clientip).c_str();
2061 eheader += "::USER==";
2062 eheader += (*clientuser).c_str();
2063 eheader += "::FILTERGROUP==";
2064 eheader += ldl->fg[filtergroup]->name;
2065 if (checkme->clienthost != "") {
2066 eheader += "::HOST==";
2067 eheader += checkme->clienthost.c_str();
2068 }
2069 eheader += "::CATEGORIES==";
2070 eheader += miniURLEncode(cats.c_str()).c_str();
2071 if (virushash || filterhash) {
2072 // output either a genuine hash, or just flags
2073 if (dohash) {
2074 eheader += "::";
2075 eheader += hashed.before("=").toCharArray();
2076 eheader += "==";
2077 eheader += hashed.after("=").toCharArray();
2078 } else {
2079 eheader += "::HASH==";
2080 eheader += hashed.toCharArray();
2081 }
2082 if(ldl->fg[filtergroup]->cgi_bypass_v2) {
2083 String data = *clientip;
2084 data += *clientuser;
2085 data += ldl->fg[filtergroup]->cgi_magic;
2086 String checkh(url->md5(data.c_str()));
2087 eheader += "::CHECK==";
2088 eheader += checkh.toCharArray();
2089 }
2090 }
2091 eheader += "::REASON==";
2092 } else {
2093 eheader += "?DENIEDURL=";
2094 eheader += miniURLEncode((*url).toCharArray()).c_str();
2095 eheader += "&IP=";
2096 eheader += (*clientip).c_str();
2097 eheader += "&USER=";
2098 eheader += (*clientuser).c_str();
2099 eheader += "&FILTERGROUP=";
2100 eheader += ldl->fg[filtergroup]->name;
2101 if (checkme->clienthost != "") {
2102 eheader += "&HOST=";
2103 eheader += checkme->clienthost.c_str();
2104 }
2105 eheader += "&CATEGORIES=";
2106 eheader += miniURLEncode(cats.c_str()).c_str();
2107 if (virushash || filterhash) {
2108 // output either a genuine hash, or just flags
2109 if (dohash) {
2110 eheader += "&";
2111 eheader += hashed.toCharArray();
2112 } else {
2113 eheader += "&HASH=";
2114 eheader += hashed.toCharArray();
2115 }
2116 if(ldl->fg[filtergroup]->cgi_bypass_v2) {
2117 String data = *clientip;
2118 data += *clientuser;
2119 data += ldl->fg[filtergroup]->cgi_magic;
2120 String checkh(url->md5(data.c_str()));
2121 eheader += "&CHECK=";
2122 eheader += checkh.toCharArray();
2123 }
2124 }
2125 eheader += "&REASON=";
2126 }
2127 if (reporting_level == 1) {
2128 eheader += miniURLEncode((*checkme).whatIsNaughty.c_str()).c_str();
2129 } else {
2130 eheader += miniURLEncode((*checkme).whatIsNaughtyLog.c_str()).c_str();
2131 }
2132 eheader += "\r\n\r\n";
2133 }
2134
2135 // the user is using the barebones banned page
2136 else if (reporting_level == 0) {
2137 eheader = "HTTP/1.1 200 OK\r\n";
2138 eheader += "Content-type: text/html\r\n";
2139 ebody = "<HTML><HEAD><TITLE>e2guardian - ";
2140 ebody += o.language_list.getTranslation(1); // access denied
2141 ebody += "</TITLE></HEAD><BODY><CENTER><H1>e2guardian - ";
2142 ebody += o.language_list.getTranslation(1); // access denied
2143 ebody += "</H1></CENTER></BODY></HTML>\r\n";
2144 eheader += "Content-Length: ";
2145 eheader += std::to_string(ebody.size());
2146 eheader += "\r\n\r\n";
2147 }
2148
2149 // stealth mode
2150 else if (reporting_level == -1) {
2151 (*checkme).isItNaughty = false; // dont block
2152 return false;
2153 }
2154 } catch (std::exception &e) {
2155 }
2156 return true;
2157 } // end of deny request loop
2158
2159 // show the relevant banned page/image/CGI based on report level setting, request type etc.
denyAccess(Socket * peerconn,Socket * proxysock,HTTPHeader * header,HTTPHeader * docheader,String * url,NaughtyFilter * checkme,std::string * clientuser,std::string * clientip,int filtergroup,bool ispostblock,int headersent,bool wasinfected,bool scanerror,bool forceshow)2160 bool ConnectionHandler::denyAccess(Socket *peerconn, Socket *proxysock, HTTPHeader *header, HTTPHeader *docheader,
2161 String *url, NaughtyFilter *checkme, std::string *clientuser, std::string *clientip,
2162 int filtergroup,
2163 bool ispostblock, int headersent, bool wasinfected, bool scanerror, bool forceshow) {
2164 String eheader, ebody;
2165 if (genDenyAccess(*peerconn, eheader, ebody, header, docheader, &checkme->logurl, checkme, clientuser, clientip, filtergroup,
2166 ispostblock, headersent, wasinfected, scanerror, forceshow)) {
2167 peerconn->writeString(eheader.toCharArray());
2168 if (ebody.length() > 0)
2169 peerconn->writeString(ebody.toCharArray());
2170 };
2171
2172 // we blocked the request, so flush the client connection & close the proxy connection.
2173 if ((*checkme).isItNaughty) {
2174 (*peerconn).breadyForOutput(o.proxy_timeout); //as best a flush as I can
2175 (*proxysock).close(); // close connection to proxy
2176 // we said no to the request, so return true, indicating exit the connhandler
2177 return true;
2178 }
2179 ldl.reset();
2180 return false;
2181 } // end of deny request loop
2182
2183 // do content scanning (AV filtering) and naughty filtering
contentFilter(HTTPHeader * docheader,HTTPHeader * header,DataBuffer * docbody,Socket * proxysock,Socket * peerconn,int * headersent,bool * pausedtoobig,off_t * docsize,NaughtyFilter * checkme,bool wasclean,int filtergroup,std::deque<CSPlugin * > & responsescanners,std::string * clientuser,std::string * clientip,bool * wasinfected,bool * wasscanned,bool isbypass,String & url,String & domain,bool * scanerror,bool & contentmodified,String * csmessage)2184 void ConnectionHandler::contentFilter(HTTPHeader *docheader, HTTPHeader *header, DataBuffer *docbody,
2185 Socket *proxysock, Socket *peerconn, int *headersent, bool *pausedtoobig,
2186 off_t *docsize, NaughtyFilter *checkme,
2187 bool wasclean, int filtergroup, std::deque<CSPlugin *> &responsescanners,
2188 std::string *clientuser, std::string *clientip, bool *wasinfected,
2189 bool *wasscanned, bool isbypass,
2190 String &url, String &domain, bool *scanerror, bool &contentmodified,
2191 String *csmessage) {
2192 //int rc = 0;
2193
2194 //proxysock->bcheckForInput(120000);
2195 bool compressed = docheader->isCompressed();
2196 if (compressed) {
2197 #ifdef DGDEBUG
2198 std::cerr << thread_id << " -Decompressing as we go....." << std::endl;
2199 #endif
2200 docbody->setDecompress(docheader->contentEncoding());
2201 }
2202 #ifdef DGDEBUG
2203 std::cerr << thread_id << docheader->contentEncoding() << std::endl;
2204 std::cerr << thread_id << " -about to get body from proxy" << std::endl;
2205 #endif
2206 (*pausedtoobig) = docbody->in(proxysock, peerconn, header, docheader, !responsescanners.empty(),
2207 headersent); // get body from proxy
2208 // checkme: surely if pausedtoobig is true, we just want to break here?
2209 // the content is larger than max_content_filecache_scan_size if it was downloaded for scanning,
2210 // and larger than max_content_filter_size if not.
2211 // in fact, why don't we check the content length (when it's not -1) before even triggering the download managers?
2212 #ifdef DGDEBUG
2213 if ((*pausedtoobig)) {
2214 std::cerr << thread_id << " -got PARTIAL body " << std::endl;
2215 } else {
2216 std::cerr << thread_id << " -got body" << std::endl;
2217 }
2218 #endif
2219 off_t dblen;
2220 bool isfile = false;
2221 if (docbody->tempfilesize > 0) {
2222 dblen = docbody->tempfilesize;
2223 isfile = true;
2224 } else {
2225 dblen = docbody->buffer_length;
2226 }
2227 // don't scan zero-length buffers (waste of AV resources, especially with external scanners (ICAP)).
2228 // these were encountered browsing opengroup.org, caused by a stats script. (PRA 21/09/2005)
2229 // if we wanted to honour a hypothetical min_content_scan_size, we'd do it here.
2230 if (((*docsize) = dblen) == 0) {
2231 #ifdef DGDEBUG
2232 std::cerr << thread_id << " -Not scanning zero-length body" << std::endl;
2233 #endif
2234 // it's not inconceivable that we received zlib or gzip encoded content
2235 // that is, after decompression, zero length. we need to cater for this.
2236 // seen on SW's internal MediaWiki.
2237 docbody->swapbacktocompressed();
2238 return;
2239 }
2240
2241 if (!wasclean) { // was not clean or no urlcache
2242
2243 // fixed to obey maxcontentramcachescansize
2244 if (!responsescanners.empty() &&
2245 (isfile ? dblen <= o.max_content_filecache_scan_size : dblen <= o.max_content_ramcache_scan_size)) {
2246 int csrc = 0;
2247 #ifdef DGDEBUG
2248 int k = 0;
2249 #endif
2250 for (std::deque<CSPlugin *>::iterator i = responsescanners.begin(); i != responsescanners.end(); i++) {
2251 (*wasscanned) = true;
2252 if (isfile) {
2253 #ifdef DGDEBUG
2254 std::cerr << thread_id << " -Running scanFile" << std::endl;
2255 #endif
2256 csrc = (*i)->scanFile(header, docheader, clientuser->c_str(), ldl->fg[filtergroup],
2257 clientip->c_str(), docbody->tempfilepath.toCharArray(), checkme);
2258 if ((csrc != DGCS_CLEAN) && (csrc != DGCS_WARNING)) {
2259 unlink(docbody->tempfilepath.toCharArray());
2260 // delete infected (or unscanned due to error) file straight away
2261 }
2262 } else {
2263 #ifdef DGDEBUG
2264 std::cerr << thread_id << " -Running scanMemory" << std::endl;
2265 #endif
2266 csrc = (*i)->scanMemory(header, docheader, clientuser->c_str(), ldl->fg[filtergroup],
2267 clientip->c_str(), docbody->data, docbody->buffer_length, checkme);
2268 }
2269 #ifdef DGDEBUG
2270 std::cerr << thread_id << " -AV scan " << k << " returned: " << csrc << std::endl;
2271 #endif
2272 if (csrc == DGCS_WARNING) {
2273 // Scanner returned a warning. File wasn't infected, but wasn't scanned properly, either.
2274 (*wasscanned) = false;
2275 (*scanerror) = false;
2276 #ifdef DGDEBUG
2277 std::cerr << thread_id << (*i)->getLastMessage() << std::endl;
2278 #endif
2279 (*csmessage) = (*i)->getLastMessage();
2280 } else if (csrc == DGCS_BLOCKED) {
2281 (*wasscanned) = true;
2282 (*scanerror) = false;
2283 break;
2284 } else if (csrc == DGCS_INFECTED) {
2285 (*wasinfected) = true;
2286 (*scanerror) = false;
2287 break;
2288 }
2289 //if its not clean / we errored then treat it as infected
2290 else if (csrc != DGCS_CLEAN) {
2291 if (csrc < 0) {
2292 syslog(LOG_ERR, "Unknown return code from content scanner: %d", csrc);
2293 if (ldl->fg[filtergroup]->disable_content_scan_error) {
2294 syslog(LOG_ERR, "disablecontentscanerror is on : bypass actived USER: %s URL: %s ",
2295 clientip->c_str(), url.c_str());
2296 (*wasscanned) = false;
2297 (*wasinfected) = false;
2298 break;
2299 }
2300 } else {
2301 syslog(LOG_ERR, "scanFile/Memory returned error: %d", csrc);
2302 }
2303 //TODO: have proper error checking/reporting here?
2304 //at the very least, integrate with the translation system.
2305 //checkme->whatIsNaughty = "WARNING: Could not perform content scan!";
2306 checkme->message_no = 1203;
2307 checkme->whatIsNaughty = o.language_list.getTranslation(1203);
2308 checkme->whatIsNaughtyLog = (*i)->getLastMessage().toCharArray();
2309 //checkme->whatIsNaughtyCategories = "Content scanning";
2310 checkme->whatIsNaughtyCategories = o.language_list.getTranslation(72);
2311 checkme->isItNaughty = true;
2312 checkme->isException = false;
2313 (*scanerror) = true;
2314 break;
2315 }
2316 #ifdef DGDEBUG
2317 k++;
2318 #endif
2319 }
2320
2321 #ifdef DGDEBUG
2322 std::cerr << thread_id << " -finished running AV" << std::endl;
2323 // rc = system("date");
2324 #endif
2325 }
2326 #ifdef DGDEBUG
2327 else if (!responsescanners.empty()) {
2328 std::cerr << thread_id << " -content length large so skipping content scanning (virus) filtering" << std::endl;
2329 }
2330 // rc = system("date");
2331 #endif
2332 if (!checkme->isItNaughty && !checkme->isException && !isbypass && (dblen <= o.max_content_filter_size)
2333 && !docheader->authRequired() && (docheader->isContentType("text", ldl->fg[filtergroup]) ||
2334 docheader->isContentType("-", ldl->fg[filtergroup]))) {
2335 #ifdef DGDEBUG
2336 std::cerr << thread_id << " -Start content filtering: ";
2337 #endif
2338 checkme->checkme(docbody->data, docbody->buffer_length, &url, &domain,
2339 ldl->fg[filtergroup], ldl->fg[filtergroup]->banned_phrase_list,
2340 ldl->fg[filtergroup]->naughtyness_limit);
2341 #ifdef DGDEBUG
2342 std::cerr << thread_id << " -Done content filtering: ";
2343 #endif
2344 }
2345 #ifdef DGDEBUG
2346 else {
2347 std::cerr << thread_id << " -Skipping content filtering: ";
2348 if (dblen > o.max_content_filter_size)
2349 std::cerr << thread_id << " -Content too large";
2350 else if (checkme->isException)
2351 std::cerr << thread_id << " -Is flagged as an exception";
2352 else if (checkme->isItNaughty)
2353 std::cerr << thread_id << " -Is already flagged as naughty (content scanning)";
2354 else if (isbypass)
2355 std::cerr << thread_id << " -Is flagged as a bypass";
2356 else if (docheader->authRequired())
2357 std::cerr << thread_id << " -Is a set of auth required headers";
2358 else if (!docheader->isContentType("text",ldl->fg[filtergroup]))
2359 std::cerr << thread_id << " -Not text";
2360 std::cerr << thread_id << std::endl;
2361 }
2362 #endif
2363 }
2364
2365 // don't do phrase filtering or content replacement on exception/bypass accesses
2366 if (checkme->isException || isbypass) {
2367 // don't forget to swap back to compressed!
2368 docbody->swapbacktocompressed();
2369 return;
2370 }
2371
2372 if ((dblen <= o.max_content_filter_size) && !checkme->isItNaughty &&
2373 docheader->isContentType("text", ldl->fg[filtergroup])) {
2374 contentmodified = docbody->contentRegExp(ldl->fg[filtergroup]);
2375 // content modifying uses global variable
2376 }
2377 #ifdef DGDEBUG
2378 else {
2379 std::cerr << thread_id << " -Skipping content modification: ";
2380 if (dblen > o.max_content_filter_size)
2381 std::cerr << thread_id << " -Content too large";
2382 else if (!docheader->isContentType("text",ldl->fg[filtergroup]))
2383 std::cerr << thread_id << " -Not text";
2384 else if (checkme->isItNaughty)
2385 std::cerr << thread_id << " -Already flagged as naughty";
2386 std::cerr << thread_id << std::endl;
2387 }
2388 //rc = system("date");
2389 #endif
2390
2391 if (contentmodified) { // this would not include infected/cured files
2392 // if the content was modified then it must have fit in ram so no
2393 // need to worry about swapped to disk stuff
2394 #ifdef DGDEBUG
2395 std::cerr << thread_id << " -content modification made" << std::endl;
2396 #endif
2397 if (compressed) {
2398 docheader->removeEncoding(docbody->buffer_length);
2399 // need to modify header to mark as not compressed
2400 // it also modifies Content-Length as well
2401 } else {
2402 docheader->setContentLength(docbody->buffer_length);
2403 }
2404 } else {
2405 docbody->swapbacktocompressed();
2406 // if we've not modified it might as well go back to
2407 // the original compressed version (if there) and send
2408 // that to the browser
2409 }
2410 #ifdef DGDEBUG
2411 std::cerr << thread_id << " Returning from content checking" << std::endl;
2412 #endif
2413 }
2414
2415
sendProxyConnect(String & hostname,Socket * sock,NaughtyFilter * checkme)2416 int ConnectionHandler::sendProxyConnect(String &hostname, Socket *sock, NaughtyFilter *checkme) {
2417 String connect_request = "CONNECT " + hostname + ":";
2418 connect_request += "443 HTTP/1.1\r\n\r\n";
2419
2420 #ifdef DGDEBUG
2421 std::cerr << thread_id << " -creating tunnel through proxy to " << hostname << std::endl;
2422 #endif
2423
2424 //somewhere to hold the header from the proxy
2425 HTTPHeader header(__HEADER_RESPONSE);
2426 //header.setTimeout(o.pcon_timeout);
2427 header.setTimeout(o.proxy_timeout);
2428
2429 if (!(sock->writeString(connect_request.c_str()) && header.in(sock, true))) {
2430
2431 #ifdef DGDEBUG
2432 syslog(LOG_ERR, "Error creating tunnel through proxy\n");
2433 std::cerr << thread_id << " -Error creating tunnel through proxy" << strerror(errno) << std::endl;
2434 #endif
2435 //(*checkme).whatIsNaughty = "Unable to create tunnel through local proxy";
2436 checkme->message_no = 157;
2437 (*checkme).whatIsNaughty = o.language_list.getTranslation(157);
2438 (*checkme).whatIsNaughtyLog = (*checkme).whatIsNaughty;
2439 (*checkme).isItNaughty = true;
2440 (*checkme).whatIsNaughtyCategories = o.language_list.getTranslation(70);
2441
2442 return -1;
2443 }
2444 //do http connect
2445 if (header.returnCode() != 200) {
2446 //connection failed
2447 //(*checkme).whatIsNaughty = "Opening tunnel failed";
2448 checkme->message_no = 158;
2449 (*checkme).whatIsNaughty = o.language_list.getTranslation(158);
2450 (*checkme).whatIsNaughtyLog = (*checkme).whatIsNaughty + " with error code " + String(header.returnCode());
2451 (*checkme).isItNaughty = true;
2452 (*checkme).whatIsNaughtyCategories = o.language_list.getTranslation(70);
2453
2454 #ifdef DGDEBUG
2455 syslog(LOG_ERR, "Tunnel status not 200 ok aborting\n");
2456 std::cerr << thread_id << " -Tunnel status was " << header.returnCode() << " expecting 200 ok" << std::endl;
2457 #endif
2458
2459 return -1;
2460 }
2461
2462 return 0;
2463 }
2464
2465 #ifdef __SSLMITM
checkCertificate(String & hostname,Socket * sslsock,NaughtyFilter * checkme)2466 void ConnectionHandler::checkCertificate(String &hostname, Socket *sslsock, NaughtyFilter *checkme)
2467 {
2468
2469 #ifdef DGDEBUG
2470 std::cerr << thread_id << " -checking SSL certificate is valid" << std::endl;
2471 #endif
2472
2473
2474 long rc = sslsock->checkCertValid(hostname);
2475 //check that everything in this certificate is correct appart from the hostname
2476 if (rc < 0) {
2477 //no certificate
2478 if ( ldl->fg[filtergroup]->allow_empty_host_certs)
2479 return;
2480 checkme->isItNaughty = true;
2481 //(*checkme).whatIsNaughty = "No SSL certificate supplied by server";
2482 checkme->message_no = 155;
2483 (*checkme).whatIsNaughty = o.language_list.getTranslation(155);
2484 (*checkme).whatIsNaughtyLog = (*checkme).whatIsNaughty;
2485 (*checkme).whatIsNaughtyCategories = o.language_list.getTranslation(70);
2486 return;
2487 } else if (rc != X509_V_OK) {
2488 //something was wrong in the certificate
2489 checkme->isItNaughty = true;
2490 // (*checkme).whatIsNaughty = "Certificate supplied by server was not valid";
2491 checkme->message_no = 150;
2492 (*checkme).whatIsNaughty = o.language_list.getTranslation(150);
2493 (*checkme).whatIsNaughtyLog = (*checkme).whatIsNaughty + ": " + X509_verify_cert_error_string(rc);
2494 (*checkme).whatIsNaughtyCategories = o.language_list.getTranslation(70);
2495 return;
2496 }
2497
2498 #ifdef DGDEBUG
2499 std::cerr << thread_id << " -checking SSL certificate hostname" << std::endl;
2500 #endif
2501
2502 //check the common name and altnames of a certificate against hostname
2503 if (sslsock->checkCertHostname(hostname) < 0) {
2504 //hostname was not matched by the certificate
2505 checkme->isItNaughty = true;
2506 //(*checkme).whatIsNaughty = "Server's SSL certificate does not match domain name";
2507 checkme->message_no = 156;
2508 (*checkme).whatIsNaughty = o.language_list.getTranslation(156);
2509 (*checkme).whatIsNaughtyLog = (*checkme).whatIsNaughty;
2510 (*checkme).whatIsNaughtyCategories = o.language_list.getTranslation(70);
2511 return;
2512 }
2513 }
2514 #endif //__SSLMITM
2515
2516
getdnstxt(std::string & clientip,String & user)2517 bool ConnectionHandler::getdnstxt(std::string &clientip, String &user) {
2518
2519 String ippath;
2520
2521 ippath = clientip;
2522 // if (o.use_xforwardedfor) {
2523 // // grab the X-Forwarded-For IP if available
2524 // p2 = header.getXForwardedForIP();
2525 // if (p2.length() > 0) {
2526 // ippath = p1 + "-" + p2;
2527 // } else {
2528 // ippath = p1;
2529 // }
2530 // } else {
2531 // ippath = p1;
2532 // }
2533
2534 #ifdef DGDEBUG
2535 std::cerr << "IPPath is " << ippath << std::endl;
2536 #endif
2537
2538 // change '.' to '-'
2539 ippath.swapChar('.', '-');
2540 #ifdef DGDEBUG
2541 std::cerr << "IPPath is " << ippath << std::endl;
2542 #endif
2543 #ifdef PRT_DNSAUTH
2544 // get info from DNS
2545 union {
2546 HEADER hdr;
2547 u_char buf[NS_PACKETSZ];
2548 } response;
2549 int responseLen;
2550 ns_msg handle; /* handle for response message */
2551 responseLen = res_querydomain(ippath.c_str(), o.dns_user_logging_domain.c_str(), ns_c_in, ns_t_txt, (u_char *)&response, sizeof(response));
2552 if (responseLen < 0) {
2553 #ifdef DGDEBUG
2554 std::cerr << "DNS query returned error " << dns_error(h_errno) << std::endl;
2555 #endif
2556 return false;
2557 }
2558 if (ns_initparse(response.buf, responseLen, &handle) < 0) {
2559 #ifdef DGDEBUG
2560 std::cerr << "ns_initparse returned error " << strerror(errno) << std::endl;
2561 #endif
2562 return false;
2563 }
2564 //int rrnum; /* resource record number */
2565 ns_rr rr; /* expanded resource record */
2566 //u_char *cp;
2567 //char ans[MAXDNAME];
2568
2569 int i = ns_msg_count(handle, ns_s_an);
2570 if (i > 0) {
2571 if (ns_parserr(&handle, ns_s_an, 0, &rr)) {
2572 #ifdef DGDEBUG
2573 std::cerr << "ns_paserr returned error " << strerror(errno) << std::endl;
2574 #endif
2575 return false;
2576 } else {
2577 if (ns_rr_type(rr) == ns_t_txt) {
2578 #ifdef DGDEBUG
2579 std::cerr << "ns_rr_rdlen returned " << ns_rr_rdlen(rr) << std::endl;
2580 #endif
2581 u_char *k = (u_char *)ns_rr_rdata(rr);
2582 char p[400];
2583 unsigned int j = 0;
2584 for (unsigned int j1 = 1; j1 < ns_rr_rdlen(rr); j1++) {
2585 p[j++] = k[j1];
2586 }
2587 // p[j] = (char)NULL;
2588 p[j] = '\0';
2589 #ifdef DGDEBUG
2590 std::cerr << "ns_rr_data returned " << p << std::endl;
2591 #endif
2592 String dnstxt(p);
2593 user = dnstxt.before(",");
2594 return true;
2595 }
2596 }
2597 }
2598 #endif
2599 return false;
2600 }
2601
dns_error(int herror)2602 String ConnectionHandler::dns_error(int herror) {
2603
2604 String s;
2605
2606 switch (herror) {
2607 case HOST_NOT_FOUND:
2608 s = "HOST_NOT_FOUND";
2609 break;
2610 case TRY_AGAIN:
2611 s = "TRY_AGAIN - DNS server failure";
2612 break;
2613 case NO_DATA:
2614 s = "NO_DATA - unexpected DNS error";
2615 break;
2616 default:
2617 String S2(herror);
2618 s = "DNS - Unexpected error number " + S2;
2619 break;
2620 }
2621 return s;
2622 }
2623
2624 bool
gen_error_mess(Socket & peerconn,NaughtyFilter & cm,String & eheader,String & ebody,int mess_no1,int mess_no2,std::string mess)2625 ConnectionHandler::gen_error_mess(Socket &peerconn, NaughtyFilter &cm, String &eheader, String &ebody, int mess_no1,
2626 int mess_no2, std::string mess) {
2627 cm.message_no = mess_no1;
2628 eheader = "HTTP/1.1 " + mess + "\nContent-Type: text/html\r\nConnection: Close\r\n";
2629 if(mess_no1 > 0) {
2630 ebody = "<HTML><HEAD><TITLE>e2guardian - ";
2631 ebody += mess;
2632 ebody += "</TITLE></HEAD><BODY><H1>e2guardian - ";
2633 ebody += mess;
2634 ebody += "</H1>";
2635 if (mess_no1 > 0)
2636 ebody += o.language_list.getTranslation(mess_no1);
2637 if (mess_no2 > 0)
2638 ebody += o.language_list.getTranslation(mess_no2);
2639 ebody += "</BODY></HTML>\r\n";
2640 }
2641 return true;
2642 }
2643
2644 bool
writeback_error(NaughtyFilter & cm,Socket & cl_sock,int mess_no1,int mess_no2,std::string mess)2645 ConnectionHandler::writeback_error(NaughtyFilter &cm, Socket &cl_sock, int mess_no1, int mess_no2, std::string mess) {
2646 String eheader, ebody;
2647 gen_error_mess(cl_sock, cm, eheader, ebody, mess_no1, mess_no2, mess);
2648 cl_sock.writeString(eheader.c_str());
2649 cl_sock.writeString("\r\n");
2650 cl_sock.writeString(ebody.c_str());
2651 return true;
2652 }
2653
2654 #ifdef __SSLMITM
2655 bool
goMITM(NaughtyFilter & checkme,Socket & proxysock,Socket & peerconn,bool & persistProxy,bool & authed,bool & persistent_authed,String & ip,stat_rec * & dystat,std::string & clientip,bool transparent)2656 ConnectionHandler::goMITM(NaughtyFilter &checkme, Socket &proxysock, Socket &peerconn, bool &persistProxy, bool &authed,
2657 bool &persistent_authed, String &ip, stat_rec *&dystat, std::string &clientip,
2658 bool transparent) {
2659 #ifdef DGDEBUG
2660 std::cerr << thread_id << " Start goMITM nf " << checkme.isItNaughty <<
2661 " upfail " << checkme.upfailure << std::endl;
2662 #endif
2663
2664
2665 #ifdef DGDEBUG
2666 std::cerr << thread_id << " -Intercepting HTTPS connection" << std::endl;
2667 #endif
2668 HTTPHeader *header = checkme.request_header;
2669 HTTPHeader *docheader = checkme.response_header;
2670 bool justLog = false;
2671
2672 // CA intialisation now Moved into OptionContainer so now done once on start-up
2673 // instead of on every request
2674
2675 X509 *cert = NULL;
2676 struct ca_serial caser;
2677 caser.asn = NULL;
2678 caser.charhex = NULL;
2679 caser.filepath = NULL;
2680 caser.filename = NULL;
2681
2682 EVP_PKEY *pkey = NULL;
2683 bool certfromcache = false;
2684 //generate the cert
2685 #ifdef DGDEBUG
2686 std::cerr << thread_id << " -Getting ssl certificate for client connection" << std::endl;
2687 #endif
2688
2689 pkey = o.ca->getServerPkey();
2690
2691 //generate the certificate but dont write it to disk (avoid someone
2692 //requesting lots of places that dont exist causing the disk to fill
2693 //up / run out of inodes
2694 certfromcache = o.ca->getServerCertificate(checkme.urldomain.CN().c_str(), &cert,
2695 &caser);
2696 #ifdef DGDEBUG
2697 if (caser.asn == NULL) {
2698 std::cerr << "caser.asn is NULL" << std::endl;
2699 }
2700 // std::cerr << "serials are: " << (char) *caser.asn << " " < caser.charhex << std::endl;
2701 #endif
2702
2703 //check that the generated cert is not null and fillin checkme if it is
2704 if (cert == NULL) {
2705 checkme.isItNaughty = true;
2706 //checkme.whatIsNaughty = "Failed to get ssl certificate";
2707 checkme.message_no = 151;
2708 checkme.whatIsNaughty = o.language_list.getTranslation(151);
2709 checkme.whatIsNaughtyLog = checkme.whatIsNaughty;
2710 checkme.whatIsNaughtyCategories = o.language_list.getTranslation(70);
2711 justLog = true;
2712 } else if (pkey == NULL) {
2713 checkme.isItNaughty = true;
2714 //checkme.whatIsNaughty = "Failed to load ssl private key";
2715 checkme.message_no = 153;
2716 checkme.whatIsNaughty = o.language_list.getTranslation(153);
2717 checkme.whatIsNaughtyLog = checkme.whatIsNaughty;
2718 checkme.whatIsNaughtyCategories = o.language_list.getTranslation(70);
2719 justLog = true;
2720 X509_free(cert);
2721 cert = NULL;
2722 }
2723
2724 //startsslserver on the connection to the client
2725 //if (!checkme.isItNaughty)
2726 if (true)
2727 {
2728 #ifdef DGDEBUG
2729 std::cerr << thread_id << " -Going SSL on the peer connection" << std::endl;
2730 #endif
2731
2732 if (!transparent) {
2733 //send a 200 to the client no matter what because they managed to get a connection to us
2734 //and we can use it for a blockpage if nothing else
2735 std::string msg = "HTTP/1.1 200 Connection established\r\n\r\n";
2736 if (!peerconn.writeString(msg.c_str()))
2737 {
2738 peerDiag("Unable to send 200 connection established to client ", peerconn);
2739 if(cert != NULL) {
2740 X509_free(cert);
2741 cert = NULL;
2742 }
2743 return false;
2744 }
2745 }
2746
2747 if (peerconn.startSslServer(cert, pkey, o.set_cipher_list) < 0) {
2748 //make sure the ssl stuff is shutdown properly so we display the old ssl blockpage
2749 peerconn.stopSsl();
2750
2751 checkme.isItNaughty = true;
2752 //checkme.whatIsNaughty = "Failed to negotiate ssl connection to client";
2753 checkme.message_no = 154;
2754 checkme.whatIsNaughty = o.language_list.getTranslation(154);
2755 checkme.whatIsNaughtyLog = checkme.whatIsNaughty;
2756 checkme.whatIsNaughtyCategories = o.language_list.getTranslation(70);
2757 justLog = true;
2758 if(cert != NULL) {
2759 X509_free(cert);
2760 cert = NULL;
2761 }
2762
2763 }
2764 }
2765
2766 bool badcert = false;
2767
2768 if (proxysock.isOpen()) {
2769 // tsslclient connected to the proxy and check the certificate of the server
2770 #ifdef DGDEBUG
2771 std::cerr << thread_id << " nf " << checkme.isItNaughty <<
2772 " upfail " << checkme.upfailure << std::endl;
2773 #endif
2774 if (!checkme.isItNaughty) {
2775 #ifdef DGDEBUG
2776 std::cerr << thread_id << " -Going SSL on upstream connection " << std::endl;
2777 #endif
2778 std::string certpath = std::string(o.ssl_certificate_path);
2779 if (proxysock.startSslClient(certpath, checkme.urldomain)) {
2780 checkme.isItNaughty = true;
2781 //checkme.whatIsNaughty = "Failed to negotiate ssl connection to server";
2782 checkme.message_no = 160;
2783 checkme.whatIsNaughty = o.language_list.getTranslation(160);
2784 checkme.whatIsNaughtyLog = checkme.whatIsNaughty;
2785 checkme.whatIsNaughtyCategories = o.language_list.getTranslation(70);
2786 }
2787 }
2788 #ifdef DGDEBUG
2789 std::cerr << thread_id << " nf " << checkme.isItNaughty <<
2790 " upfail " << checkme.upfailure << std::endl;
2791 #endif
2792
2793 if (!checkme.isItNaughty) {
2794
2795 #ifdef DGDEBUG
2796 std::cerr << thread_id << " -Checking certificate" << std::endl;
2797 #endif
2798 //will fill in checkme of its own accord
2799 if (!checkme.nocheckcert) {
2800 checkCertificate(checkme.urldomain, &proxysock, &checkme);
2801 badcert = checkme.isItNaughty;
2802 }
2803 }
2804 }
2805
2806 #ifdef DGDEBUG
2807 std::cerr << thread_id << " nf " << checkme.isItNaughty <<
2808 " upfail " << checkme.upfailure << std::endl;
2809 #endif
2810 if ((!checkme.isItNaughty) && (!checkme.upfailure)) {
2811 bool writecert = true;
2812 if (!certfromcache) {
2813 writecert = o.ca->writeCertificate(checkme.urldomain.c_str(), cert,
2814 &caser);
2815 }
2816
2817 //if we cant write the certificate its not the end of the world but it is slow
2818 if (!writecert) {
2819 #ifdef DGDEBUG
2820 std::cerr << thread_id << " -Couldn't save certificate to on disk cache" << std::endl;
2821 #endif
2822 syslog(LOG_ERR, "Couldn't save certificate to on disk cache");
2823 }
2824 #ifdef DGDEBUG
2825 std::cerr << thread_id << " -Handling connections inside ssl tunnel" << std::endl;
2826 #endif
2827
2828 if (authed) {
2829 persistent_authed = true;
2830 }
2831
2832 //handleConnection inside the ssl tunnel
2833 handleConnection(peerconn, ip, true, proxysock, dystat);
2834 #ifdef DGDEBUG
2835 std::cerr << thread_id << " -Handling connections inside ssl tunnel: done" << std::endl;
2836 #endif
2837 }
2838 o.ca->free_ca_serial(&caser);
2839
2840 //stopssl on the proxy connection
2841 //if it was marked as naughty then show a deny page and close the connection
2842 if (checkme.isItNaughty || checkme.upfailure) {
2843 #ifdef DGDEBUG
2844 std::cerr << thread_id << " -SSL Interception failed " << checkme.whatIsNaughty << " nf " << checkme.isItNaughty <<
2845 " upfail " << checkme.upfailure << std::endl;
2846 #endif
2847
2848 doLog(clientuser, clientip, checkme);
2849
2850 if(!justLog)
2851 denyAccess(&peerconn, &proxysock, header, docheader, &checkme.logurl, &checkme, &clientuser,
2852 &clientip, filtergroup, checkme.ispostblock, checkme.headersent, checkme.wasinfected,
2853 checkme.scanerror, badcert);
2854 }
2855 #ifdef DGDEBUG
2856 std::cerr << thread_id << " -Shutting down ssl to proxy" << std::endl;
2857 #endif
2858 proxysock.stopSsl();
2859
2860 #ifdef DGDEBUG
2861 std::cerr << thread_id << " -Shutting down ssl to client" << std::endl;
2862 #endif
2863
2864 peerconn.stopSsl();
2865
2866 //tidy up key and cert
2867 if(cert != NULL) {
2868 X509_free(cert);
2869 cert = NULL;
2870 }
2871 if(pkey != NULL) {
2872 EVP_PKEY_free(pkey);
2873 pkey = NULL;
2874 }
2875
2876 persistProxy = false;
2877 proxysock.close();
2878
2879 return true;
2880 }
2881 #endif
2882
doAuth(int & auth_result,bool & authed,int & filtergroup,AuthPlugin * auth_plugin,Socket & peerconn,HTTPHeader & header,bool only_client_ip,bool isconnect_like)2883 bool ConnectionHandler::doAuth(int &auth_result, bool &authed, int &filtergroup, AuthPlugin *auth_plugin, Socket &peerconn,
2884 HTTPHeader &header, bool only_client_ip, bool isconnect_like) {
2885 Socket nullsock;
2886 return doAuth(auth_result, authed, filtergroup, auth_plugin, peerconn, nullsock, header, only_client_ip,
2887 isconnect_like);
2888 }
2889
doAuth(int & rc,bool & authed,int & filtergroup,AuthPlugin * auth_plugin,Socket & peerconn,Socket & proxysock,HTTPHeader & header,bool only_client_ip,bool isconnect_like)2890 bool ConnectionHandler::doAuth(int &rc, bool &authed, int &filtergroup, AuthPlugin *auth_plugin, Socket &peerconn,
2891 Socket &proxysock, HTTPHeader &header, bool only_client_ip, bool isconnect_like) {
2892
2893 #ifdef DGDEBUG
2894 std::cerr << thread_id << " -Not got persistent credentials for this connection - querying auth plugins" << std::endl;
2895 #endif
2896 bool dobreak = false;
2897 rc = 0;
2898 if (o.authplugins.size() != 0) {
2899 // We have some auth plugins load
2900 int authloop = 0;
2901 rc = 0;
2902 String tmp;
2903
2904 for (std::deque<Plugin *>::iterator i = o.authplugins_begin; i != o.authplugins_end; i++) {
2905 #ifdef DGDEBUG
2906 std::cerr << thread_id << " -Querying next auth plugin..." << std::endl;
2907 #endif
2908 // try to get the username & parse the return value
2909 auth_plugin = (AuthPlugin *) (*i);
2910 if (only_client_ip && !auth_plugin->client_ip_based)
2911 continue;
2912
2913 // auth plugin selection for multi ports
2914 //
2915 //
2916 // Logic changed to allow auth scan with multiple ports as option to auth-port
2917 // fixed mapping
2918 //
2919 if (o.map_auth_to_ports) {
2920 if (o.filter_ports.size() > 1) {
2921 tmp = o.auth_map[peerconn.getPort()];
2922 } else {
2923 // auth plugin selection for one port
2924 tmp = o.auth_map[authloop];
2925 authloop++;
2926 }
2927
2928 if (tmp.compare(auth_plugin->getPluginName().toCharArray()) == 0) {
2929 rc = auth_plugin->identify(peerconn, proxysock, header, clientuser, is_real_user);
2930 } else {
2931 rc = DGAUTH_NOMATCH;
2932 }
2933 } else {
2934 rc = auth_plugin->identify(peerconn, proxysock, header, clientuser, is_real_user);
2935 }
2936
2937 if (rc == DGAUTH_NOMATCH) {
2938 #ifdef DGDEBUG
2939 std::cerr << "Auth plugin did not find a match; querying remaining plugins" << std::endl;
2940 #endif
2941 continue;
2942 } else if (rc == DGAUTH_REDIRECT) {
2943 #ifdef DGDEBUG
2944 std::cerr << "Auth plugin told us to redirect client to \"" << clientuser << "\"; not querying remaining plugins" << std::endl;
2945 #endif
2946 if (isconnect_like) // it is connect or trans https so cannot send redirect
2947 {
2948 dobreak = true;
2949 break;
2950 } else {
2951 // ident plugin told us to redirect to a login page
2952 String writestring("HTTP/1.1 302 Redirect\r\nLocation: ");
2953 writestring += clientuser;
2954 writestring += "\r\n\r\n";
2955 peerconn.writeString(writestring.toCharArray()); // no action on failure
2956 dobreak = true;
2957 break;
2958 }
2959 } else if (rc == DGAUTH_OK_NOPERSIST) {
2960 #ifdef DGDEBUG
2961 std::cerr << "Auth plugin returned OK but no persist not setting persist auth" << std::endl;
2962 #endif
2963 overide_persist = true;
2964 } else if (rc < 0) {
2965 if (!is_daemonised)
2966 std::cerr << thread_id << "Auth plugin returned error code: " << rc << std::endl;
2967 syslog(LOG_ERR, "%sAuth plugin returned error code: %d", thread_id.c_str(), rc);
2968 dobreak = true;
2969 break;
2970 }
2971 #ifdef DGDEBUG
2972 std::cerr << thread_id << " -Auth plugin found username " << clientuser << " (" << oldclientuser << "), now determining group" << std::endl;
2973 #endif
2974 if (clientuser == oldclientuser) {
2975 #ifdef DGDEBUG
2976 std::cerr << thread_id << " -Same user as last time, re-using old group no." << std::endl;
2977 #endif
2978 authed = true;
2979 filtergroup = oldfg;
2980 break;
2981 }
2982 // try to get the filter group & parse the return value
2983 rc = auth_plugin->determineGroup(clientuser, filtergroup, ldl->filter_groups_list);
2984 if (rc == DGAUTH_OK) {
2985 #ifdef DGDEBUG
2986 std::cerr << thread_id << "Auth plugin found username & group; not querying remaining plugins" << std::endl;
2987 #endif
2988 authed = true;
2989 break;
2990 } else if (rc == DGAUTH_NOMATCH) {
2991 #ifdef DGDEBUG
2992 std::cerr << thread_id << "Auth plugin did not find a match; querying remaining plugins" << std::endl;
2993 #endif
2994 continue;
2995 } else if (rc == DGAUTH_NOUSER) {
2996 #ifdef DGDEBUG
2997 std::cerr << thread_id << "Auth plugin found username \"" << clientuser << "\" but no associated group; not querying remaining plugins" << std::endl;
2998 #endif
2999 if (o.auth_requires_user_and_group)
3000 continue;
3001 //filtergroup = 0; // default now set before call to doAuth
3002 authed = true;
3003 break;
3004 } else if (rc < 0) {
3005 if (!is_daemonised)
3006 std::cerr << thread_id << "Auth plugin returned error code: " << rc << std::endl;
3007 syslog(LOG_ERR, "%sAuth plugin returned error code: %d", thread_id.c_str(), rc);
3008 dobreak = true;
3009 break;
3010 }
3011 } // end of querying all plugins (for)
3012
3013 // break the peer loop
3014 if (dobreak)
3015 return false;
3016 //break;
3017
3018 if ((!authed) || (filtergroup < 0) || (filtergroup >= o.numfg)) {
3019 #ifdef DGDEBUG
3020 if (!authed)
3021 std::cerr << thread_id << " -No identity found; using defaults" << std::endl;
3022 else
3023 std::cerr << thread_id << " -Plugin returned out-of-range filter group number; using defaults" << std::endl;
3024 #endif
3025 // If none of the auth plugins currently loaded rely on querying the proxy,
3026 // such as 'ident' or 'ip', then pretend we're authed. What this flag
3027 // actually controls is whether or not the query should be forwarded to the
3028 // proxy (without pre-emptive blocking); we don't want this for 'ident' or
3029 // 'ip', because Squid isn't necessarily going to return 'auth required'.
3030 authed = !o.auth_needs_proxy_query;
3031 #ifdef DGDEBUG
3032 if (!o.auth_needs_proxy_query)
3033 std::cerr << thread_id << " -No loaded auth plugins require parent proxy queries; enabling pre-emptive blocking despite lack of authentication" << std::endl;
3034 #endif
3035 clientuser = "-";
3036 //filtergroup = 0; //default group - one day configurable? - default now set before call to doAuth
3037 } else {
3038 #ifdef DGDEBUG
3039 std::cerr << thread_id << " -Identity found; caching username & group" << std::endl;
3040 #endif
3041 if (auth_plugin->is_connection_based && !overide_persist) {
3042 #ifdef DGDEBUG
3043 std::cerr << "Auth plugin is for a connection-based auth method - keeping credentials for entire connection" << std::endl;
3044 #endif
3045 persistent_authed = true;
3046 }
3047 oldclientuser = clientuser;
3048 oldfg = filtergroup;
3049 }
3050 } else {
3051 // We don't have any auth plugins loaded
3052 #ifdef DGDEBUG
3053 std::cerr << thread_id << " -No auth plugins loaded; using defaults & feigning persistency" << std::endl;
3054 #endif
3055 authed = true;
3056 clientuser = "-";
3057 //filtergroup = 0; //default group - one day configurable? - default now set before call to doAuth
3058 persistent_authed = true;
3059 }
3060 return true;
3061 }
3062
checkByPass(NaughtyFilter & checkme,std::shared_ptr<LOptionContainer> & ldl,HTTPHeader & header,Socket & proxysock,Socket & peerconn,std::string & clientip)3063 bool ConnectionHandler::checkByPass(NaughtyFilter &checkme, std::shared_ptr<LOptionContainer> &ldl, HTTPHeader &header,
3064 Socket &proxysock, Socket &peerconn, std::string &clientip) {
3065
3066 //first check if bypass allowed and set isbypassallowed
3067 checkme.isbypassallowed = (ldl->fg[filtergroup]->bypass_mode != 0);
3068 checkme.isinfectionbypassallowed = (ldl->fg[filtergroup]->infection_bypass_mode != 0);
3069 if (!(checkme.isbypassallowed || checkme.isinfectionbypassallowed))
3070 return false;
3071
3072 // int bypasstimestamp = 0;
3073 if (isScanBypassURL(checkme.url, ldl->fg[filtergroup]->magic.c_str(), clientip.c_str())) {
3074 #ifdef DGDEBUG
3075 std::cerr << thread_id << " -Scan Bypass URL match" << std::endl;
3076 #endif
3077 checkme.isscanbypass = true;
3078 checkme.isbypass = true;
3079 checkme.message_no = 608;
3080 checkme.log_message_no = 608;
3081 checkme.exceptionreason = o.language_list.getTranslation(608);
3082 } else if ((ldl->fg[filtergroup]->bypass_mode != 0) || (ldl->fg[filtergroup]->infection_bypass_mode != 0)) {
3083 #ifdef DGDEBUG
3084 std::cerr << thread_id << " -About to check for bypass..." << std::endl;
3085 #endif
3086 if (ldl->fg[filtergroup]->bypass_mode != 0)
3087 checkme.bypasstimestamp = isBypassURL(checkme.url, ldl->fg[filtergroup]->magic.c_str(),
3088 clientip.c_str(), NULL, clientuser);
3089 if ((checkme.bypasstimestamp == 0) && (ldl->fg[filtergroup]->infection_bypass_mode != 0))
3090 checkme.bypasstimestamp = isBypassURL(checkme.url, ldl->fg[filtergroup]->imagic.c_str(),
3091 clientip.c_str(), &checkme.isvirusbypass,
3092 clientuser);
3093 if (checkme.bypasstimestamp > 0) {
3094 #ifdef DGDEBUG
3095 if (checkme.isvirusbypass)
3096 std::cerr << thread_id << " -Infection bypass URL match" << std::endl;
3097 else
3098 std::cerr << thread_id << " -Filter bypass URL match" << std::endl;
3099 #endif
3100 header.chopBypass(checkme.logurl, checkme.isvirusbypass);
3101 if (checkme.bypasstimestamp > 1) { // not expired
3102 checkme.isbypass = true;
3103 checkme.isexception = true;
3104 // checkme: need a TR string for virus bypass
3105 checkme.exceptionreason = o.language_list.getTranslation(606);
3106 checkme.message_no = 606;
3107 checkme.log_message_no = 606;
3108 }
3109 } else if (ldl->fg[filtergroup]->bypass_mode != 0) {
3110 if (header.isBypassCookie(checkme.urldomain, ldl->fg[filtergroup]->cookie_magic.c_str(),
3111 clientip.c_str(), clientuser.c_str())) {
3112 #ifdef DGDEBUG
3113 std::cerr << thread_id << " -Bypass cookie match" << std::endl;
3114 #endif
3115 checkme.iscookiebypass = true;
3116 checkme.isbypass = true;
3117 checkme.isexception = true;
3118 checkme.exceptionreason = o.language_list.getTranslation(607);
3119 }
3120 }
3121 #ifdef DGDEBUG
3122 std::cerr << thread_id << " -Finished bypass checks." << std::endl;
3123 #endif
3124 }
3125
3126 #ifdef DGDEBUG
3127 if (checkme.isbypass) {
3128 std::cerr << thread_id << " -bypass activated!" << std::endl;
3129 }
3130 #endif
3131 //
3132 // End of bypass
3133 //
3134 // Start of scan by pass
3135 //
3136
3137 if (checkme.isscanbypass) {
3138 //we need to decode the URL and send the temp file with the
3139 //correct header to the client then delete the temp file
3140 checkme.tempfilename = (checkme.url.after("GSBYPASS=").after("&N="));
3141 checkme.tempfilemime = (checkme.tempfilename.after("&M="));
3142 checkme.tempfiledis = (header.decode(checkme.tempfilemime.after("&D="), true));
3143 #ifdef DGDEBUG
3144 std::cerr << thread_id << " -Original filename: " << checkme.tempfiledis << std::endl;
3145 #endif
3146 String rtype(header.requestType());
3147 checkme.tempfilemime = checkme.tempfilemime.before("&D=");
3148 checkme.tempfilename = o.download_dir + "/tf" + checkme.tempfilename.before("&M=");
3149 return true;
3150 }
3151 return false;
3152 }
3153
sendScanFile(Socket & peerconn,NaughtyFilter & checkme,bool is_icap,ICAPHeader * icaphead)3154 bool ConnectionHandler::sendScanFile(Socket &peerconn, NaughtyFilter &checkme, bool is_icap, ICAPHeader *icaphead) {
3155 try {
3156 checkme.docsize = sendFile(&peerconn, checkme, checkme.url, is_icap, icaphead);
3157 checkme.request_header->chopScanBypass(checkme.url);
3158 checkme.logurl = checkme.request_header->getLogUrl();
3159
3160 doLog(clientuser, checkme.clientip, checkme);
3161
3162 if (o.delete_downloaded_temp_files) {
3163 unlink(checkme.tempfilename.toCharArray());
3164 }
3165 } catch (
3166 std::exception &e
3167 ) {
3168 }
3169 // persistProxy = false;
3170 // proxysock.close(); // close connection to proxy
3171 return true;
3172 }
3173
check_search_terms(NaughtyFilter & cm)3174 void ConnectionHandler::check_search_terms(NaughtyFilter &cm) {
3175 if (ldl->fg[filtergroup]->searchterm_limit > 0) {
3176 String terms;
3177 terms = cm.search_terms;
3178 // search terms are URL parameter type "0"
3179 urlparams.append("0=").append(terms).append(";");
3180 // Add spaces at beginning and end of block before filtering, so
3181 // that the quick & dirty trick of putting spaces around words
3182 // (Scunthorpe problem) can still be used, bearing in mind the block
3183 // of text here is usually very small.
3184 terms.insert(terms.begin(), ' ');
3185 terms.append(" ");
3186 cm.checkme(terms.c_str(), terms.length(), NULL, NULL, ldl->fg[filtergroup],
3187 (ldl->fg[filtergroup]->searchterm_flag ? ldl->fg[filtergroup]->searchterm_list
3188 : ldl->fg[filtergroup]->banned_phrase_list),
3189 ldl->fg[filtergroup]->searchterm_limit, true);
3190 if (cm.isItNaughty) {
3191 cm.blocktype = 2;
3192 }
3193 }
3194 return;
3195 }
3196
check_content(NaughtyFilter & cm,DataBuffer & docbody,Socket & proxysock,Socket & peerconn,std::deque<CSPlugin * > & responsescanners)3197 void ConnectionHandler::check_content(NaughtyFilter &cm, DataBuffer &docbody, Socket &proxysock, Socket &peerconn,
3198 std::deque<CSPlugin *> &responsescanners) {
3199 if (((cm.response_header->isContentType("text", ldl->fg[filtergroup]) ||
3200 cm.response_header->isContentType("-", ldl->fg[filtergroup])) && !cm.isexception) ||
3201 !responsescanners.empty()) {
3202 cm.waschecked = true;
3203 if (!responsescanners.empty()) {
3204 #ifdef DGDEBUG
3205 std::cerr << thread_id << " -Filtering with expectation of a possible csmessage" << " Line: " << __LINE__ << " Function: " << __func__ << std::endl;
3206 #endif
3207 String csmessage;
3208 contentFilter(cm.response_header, cm.request_header, &docbody, &proxysock, &peerconn, &cm.headersent,
3209 &cm.pausedtoobig,
3210 &cm.docsize, &cm, cm.wasclean, filtergroup, responsescanners, &clientuser, &cm.clientip,
3211 &cm.wasinfected, &cm.wasscanned, cm.isbypass, cm.urld, cm.urldomain, &cm.scanerror,
3212 cm.contentmodified, &csmessage);
3213 if (csmessage.length() > 0) {
3214 #ifdef DGDEBUG
3215 std::cerr << thread_id << " -csmessage found: " << csmessage << " Line: " << __LINE__ << " Function: " << __func__ << std::endl;
3216 #endif
3217 cm.exceptionreason = csmessage.toCharArray();
3218 }
3219 } else {
3220 #ifdef DGDEBUG
3221 std::cerr << thread_id << " -Calling contentFilter " << " Line: " << __LINE__ << " Function: " << __func__ << std::endl;
3222 #endif
3223 contentFilter(cm.response_header, cm.request_header, &docbody, &proxysock, &peerconn, &cm.headersent,
3224 &cm.pausedtoobig,
3225 &cm.docsize, &cm, cm.wasclean, filtergroup, responsescanners, &clientuser, &cm.clientip,
3226 &cm.wasinfected, &cm.wasscanned, cm.isbypass, cm.urld, cm.urldomain, &cm.scanerror,
3227 cm.contentmodified, NULL);
3228 }
3229 } else {
3230 cm.tunnel_rest = true;
3231 }
3232 #ifdef DGDEBUG
3233 std::cerr << thread_id << "End content check isitNaughty is " << cm.isItNaughty << std::endl;
3234 #endif
3235 }
3236
3237 #ifdef __SSLMITM
handleTHTTPSConnection(Socket & peerconn,String & ip,Socket & proxysock,stat_rec * & dystat)3238 int ConnectionHandler::handleTHTTPSConnection(Socket &peerconn, String &ip, Socket &proxysock, stat_rec* &dystat) {
3239 struct timeval thestart;
3240 gettimeofday(&thestart, NULL);
3241
3242 peerconn.setTimeout(o.pcon_timeout);
3243
3244 HTTPHeader docheader(__HEADER_RESPONSE); // to hold the returned page header from proxy
3245 HTTPHeader header(__HEADER_REQUEST); // to hold the incoming client request headeri(ldl)
3246
3247 NaughtyFilter checkme(header, docheader);
3248 checkme.reset();
3249
3250
3251 std::string clientip(ip.toCharArray()); // hold the clients ip
3252 docheader.setClientIP(ip);
3253
3254 if (clienthost) delete clienthost;
3255
3256 clienthost = NULL; // and the hostname, if available
3257 matchedip = false;
3258
3259
3260 #ifdef DGDEBUG // debug stuff surprisingly enough
3261 std::cerr << thread_id << " -got peer connection - clientip is " << clientip << std::endl;
3262 #endif
3263
3264 try {
3265 int rc;
3266
3267
3268 //int oldfg = 0;
3269 bool authed = false;
3270 bool isbanneduser = false;
3271 bool firsttime = true;
3272
3273 AuthPlugin *auth_plugin = NULL;
3274
3275 // RFC states that connections are persistent
3276 //bool persistOutgoing = true;
3277 //bool persistPeer = true;
3278 bool persistProxy = false;
3279 //bool direct = false;
3280
3281 char buff[2048];
3282 rc = peerconn.readFromSocket(buff, 5, (MSG_PEEK ), 20000, true);
3283 #ifdef DGDEBUG
3284 std::cerr << thread_id << "bytes peeked " << rc << std::endl;
3285 #endif
3286 unsigned short toread = 0;
3287 if (rc == 5) {
3288 if (buff[0] == 22 && buff[1] == 3 && buff[2] > 0 && buff[2] < 4 ) // has TLS hello signiture
3289 checkme.isTLS = true;
3290
3291 toread = ( buff[3] << (8*1) | buff[4]);
3292 if (toread > 2048) toread = 2048;
3293 }
3294
3295 #ifdef DGDEBUG
3296 std::cerr << thread_id << "hello length is " << toread << " magic is " << buff[0] << buff[1] << buff[2] << " isTLS is " << checkme.isTLS << std::endl;
3297 #endif
3298
3299 if(checkme.isTLS) {
3300 rc = peerconn.readFromSocket(buff, toread, (MSG_PEEK ), 10000, false);
3301 if (rc < 1 ) { // get header from client, allowing persistency
3302 if (o.logconerror) {
3303 if (peerconn.getFD() > -1) {
3304
3305 int err = peerconn.getErrno();
3306 //int pport = peerconn.getPeerSourcePort();
3307 std::string peerIP = peerconn.getPeerIP();
3308 if(peerconn.isTimedout())
3309 {
3310 #ifdef DGDEBUG
3311 std::cerr << thread_id << "Connection timed out" << std::endl;
3312 #endif
3313 }
3314 syslog(LOG_INFO, "%sNo header recd from client at %s - errno: %d", thread_id.c_str(), peerIP.c_str(), err);
3315 #ifdef DGDEBUG
3316 std::cerr << thread_id << "No header recd from client - errno: " << err << std::endl;
3317 #endif
3318 } else {
3319 syslog(LOG_INFO, "%sClient connection closed early - no TLS header received",
3320 thread_id.c_str() );
3321 }
3322 }
3323 firsttime = false;
3324 //persistPeer = false;
3325 } else {
3326 #ifdef DGDEBUG
3327 std::cerr << thread_id << "bytes peeked " << rc << std::endl;
3328 #endif
3329 char *ret = get_TLS_SNI(buff, &rc);
3330 if (ret != NULL) {
3331 checkme.url = ret;
3332 checkme.hasSNI = true;
3333 }
3334
3335 ++dystat->reqs;
3336 }
3337 }
3338 { // get original IP destination & port
3339
3340 sockaddr_in origaddr;
3341 socklen_t origaddrlen(sizeof(sockaddr_in));
3342 if (
3343 #ifdef SOL_IP // linux
3344 #define SO_ORIGINAL_DST 80
3345 getsockopt(peerconn.getFD(), SOL_IP, SO_ORIGINAL_DST, &origaddr, &origaddrlen ) < 0
3346 #else // BSD
3347 getsockname(peerconn.getFD(), (struct sockaddr *)&origaddr, &origaddrlen) < 0
3348 #endif
3349 ) {
3350 syslog(LOG_ERR, "%sFailed to get client's original destination IP: %s", thread_id.c_str(), strerror(errno));
3351 return -1;
3352 } else {
3353 char res[INET_ADDRSTRLEN];
3354 checkme.orig_ip = inet_ntop(AF_INET,&origaddr.sin_addr,res,sizeof(res));
3355 checkme.orig_port = ntohs(origaddr.sin_port);
3356 }
3357 }
3358
3359 if(!checkme.hasSNI) {
3360 checkme.url = checkme.orig_ip;
3361 }
3362 #ifdef DGDEBUG
3363 std::cerr << thread_id << "hasSNI = " << checkme.hasSNI << " SNI is " << checkme.url << " Orig IP " << checkme.orig_ip << " Orig port " << checkme.orig_port << std::endl;
3364 #endif
3365 //
3366 // End of set-up section
3367
3368 while (firsttime ) // do just the once
3369 {
3370 ldl = o.currentLists();
3371 //DataBuffer docbody;
3372 //docbody.setTimeout(o.exchange_timeout);
3373 FDTunnel fdt;
3374
3375 firsttime = false;
3376
3377 //
3378 // do all of this normalisation etc just the once at the start.
3379 checkme.url = "https://" + checkme.url;
3380 checkme.setURL(checkme.url);
3381 gettimeofday(&checkme.thestart, NULL);
3382
3383
3384 // Look up reverse DNS name of client if needed
3385 if (o.reverse_client_ip_lookups) {
3386 getClientFromIP(clientip.c_str(), checkme.clienthost);
3387 //std::unique_ptr<std::deque<String> > hostnames;
3388 // hostnames.reset(ipToHostname(clientip.c_str()));
3389 // checkme.clienthost = std::string(hostnames->front().toCharArray());
3390 }
3391
3392 filtergroup = o.default_trans_fg;
3393
3394 if(o.log_requests) {
3395 std::string fnt = "THTTPS";
3396 doRQLog(clientuser, clientip, checkme, fnt);
3397 }
3398
3399 //CALL SB pre-authcheck
3400 ldl->StoryA.runFunctEntry(ENT_STORYA_PRE_AUTH_THTTPS,checkme);
3401 #ifdef DGDEBUG
3402 std::cerr << thread_id << "After StoryA thttps-pre-authcheck" << checkme.isexception << " mess_no "
3403 << checkme.message_no << std::endl;
3404 #endif
3405 checkme.isItNaughty = checkme.isBlocked;
3406 bool isbannedip = checkme.isBlocked;
3407
3408 //
3409 //
3410 // Start of Authentication Checks
3411 //
3412 //
3413 // don't have credentials for this connection yet? get some!
3414 overide_persist = false;
3415 if(!(checkme.isItNaughty || checkme.isexception)) {
3416 if (!doAuth(checkme.auth_result, authed, filtergroup, auth_plugin, peerconn, proxysock, header, true, true))
3417 {
3418
3419 if((checkme.auth_result == DGAUTH_REDIRECT) && ldl->fg[filtergroup]->ssl_mitm)
3420 {
3421 checkme.gomitm = true;
3422 checkme.isdone = true;
3423 } else {
3424 break;
3425 }
3426 }
3427 }
3428 checkme.filtergroup = filtergroup;
3429
3430 #ifdef DGDEBUG
3431 std::cerr << thread_id << " -username: " << clientuser << " -filtergroup: " << filtergroup << std::endl;
3432 #endif
3433 //
3434 //
3435 // End of Authentication Checking
3436 //
3437 //
3438
3439 //
3440 //
3441 // Now check if user or machine is banned and room-based checking
3442 //
3443 //
3444
3445 // is this user banned?
3446 isbanneduser = false;
3447 checkme.clientip = clientip;
3448
3449
3450 if(checkme.hasSNI) checkme.ismitmcandidate = ldl->fg[filtergroup]->ssl_mitm;
3451
3452
3453 // TODO restore this for THTTPS ??
3454 //if (isbannedip) {
3455 // matchedip = clienthost == NULL;
3456 //} else {
3457 // / if (ldl->inRoom(clientip, room, clienthost, &isbannedip, &part_banned, &checkme.isexception,
3458 // / checkme.urld)) {
3459 #ifdef DGDEBUG
3460 // / std::cerr << " isbannedip = " << isbannedip << "ispart_banned = " << part_banned << " isexception = " << checkme.isexception << std::endl;
3461 #endif
3462 //. if (isbannedip) {
3463 // matchedip = clienthost == NULL;
3464 // checkme.isBlocked = checkme.isItNaughty = true;
3465 // / }
3466 // if (checkme.isexception) {
3467 // do reason codes etc
3468 //checkme.exceptionreason = o.language_list.getTranslation(630);
3469 //checkme.exceptionreason.append(room);
3470 //checkme.exceptionreason.append(o.language_list.getTranslation(631));
3471 //checkme.message_no = 632;
3472 //}
3473 //}
3474 //}
3475
3476 //
3477 // Start of exception checking
3478 //
3479 // being a banned user/IP overrides the fact that a site may be in the exception lists
3480 // needn't check these lists in bypass modes
3481 if (!(checkme.isdone || isbanneduser || isbannedip || checkme.isexception)) {
3482 ldl->fg[filtergroup]->StoryB.runFunctEntry(ENT_STORYB_THTTPS_REQUEST,checkme);
3483 #ifdef DGDEBUG
3484 std::cerr << thread_id << "After StoryB thttps-checkrequest " << checkme.isexception << " mess_no "
3485 << checkme.message_no << std::endl;
3486 #endif
3487
3488 if (ldl->fg[filtergroup]->reporting_level != -1){
3489 checkme.isItNaughty = checkme.isBlocked;
3490 } else {
3491 checkme.isItNaughty = false;
3492 checkme.isBlocked = false;
3493 }
3494 }
3495
3496 //now send upstream and get response
3497 if (!checkme.isItNaughty && !persistProxy) {
3498 if (connectUpstream(proxysock, checkme,443) > -1) {
3499 if(!checkme.isdirect) {
3500 if (sendProxyConnect(checkme.connect_site,&proxysock, &checkme) != 0) {
3501 checkme.upfailure = true;
3502 proxysock.close();
3503 }
3504 }
3505 } else {
3506 checkme.upfailure = true;
3507 }
3508 }
3509
3510 #ifdef DGDEBUG
3511 std::cerr << thread_id << " after connectUpstream nf " << checkme.isItNaughty <<
3512 " upfail " << checkme.upfailure << std::endl;
3513 #endif
3514
3515 if((checkme.isItNaughty ||checkme.upfailure) && ldl->fg[filtergroup]->ssl_mitm && ldl->fg[filtergroup]->automitm && checkme.hasSNI)
3516 checkme.gomitm = true; // allows us to send splash page
3517
3518 if (checkme.isexception && !checkme.upfailure) {
3519 checkme.tunnel_rest = true;
3520 } else {
3521
3522 //if ismitm - GO MITM
3523 if (checkme.gomitm)
3524 {
3525 #ifdef DGDEBUG
3526 std::cerr << thread_id << "Going MITM ...." << std::endl;
3527 #endif
3528 if(!ldl->fg[filtergroup]->mitm_check_cert)
3529 checkme.nocheckcert = true;
3530 goMITM(checkme, proxysock, peerconn, persistProxy, authed, persistent_authed, ip, dystat, clientip, true);
3531 //persistPeer = false;
3532 persistProxy = false;
3533 //if (!checkme.isItNaughty)
3534 break;
3535 } else {
3536 if (!checkme.upfailure)
3537 checkme.tunnel_rest = true;
3538 }
3539 }
3540
3541 //if not grey tunnel response
3542 if (!checkme.isItNaughty && checkme.tunnel_rest) {
3543 #ifdef DGDEBUG
3544 std::cerr << thread_id << " -Tunnelling to client" << std::endl;
3545 #endif
3546 if (!fdt.tunnel(proxysock, peerconn,true, -1 , true))
3547 persistProxy = false;
3548 checkme.docsize += fdt.throughput;
3549 }
3550
3551 // it is not possible to send splash page on Thttps without MITM so do not try!
3552
3553 //Log
3554 if (!checkme.isourwebserver) { // don't log requests to the web server
3555 doLog(clientuser, clientip, checkme);
3556 }
3557
3558
3559 proxysock.close(); // close connection to proxy
3560
3561
3562 }
3563 } catch (std::exception & e)
3564 {
3565 #ifdef DGDEBUG
3566 std::cerr << thread_id << " - THTTPS connection handler caught an exception: " << e.what() << " Line: " << __LINE__ << " Function: " << __func__ << std::endl;
3567 #endif
3568 if(o.logconerror)
3569 syslog(LOG_ERR, " - THTTPS connection handler caught an exception %s" , e.what());
3570
3571 // close connection to proxy
3572 proxysock.close();
3573 return -1;
3574 }
3575
3576 return 0;
3577 }
3578
3579
get_TLS_SNI(char * inbytes,int * len)3580 char *get_TLS_SNI(char *inbytes, int* len)
3581 {
3582 unsigned char *bytes = reinterpret_cast<unsigned char*>(inbytes);
3583 unsigned char *curr;
3584 unsigned char *ebytes;
3585 ebytes = bytes + *len;
3586 if (*len < 44) return NULL;
3587 unsigned char sidlen = bytes[43];
3588 curr = bytes + 1 + 43 + sidlen;
3589 if (curr > ebytes) return NULL;
3590 unsigned short cslen = ntohs(*(unsigned short*)curr);
3591 curr += 2 + cslen;
3592 if (curr > ebytes) return NULL;
3593 unsigned char cmplen = *curr;
3594 curr += 1 + cmplen;
3595 if (curr > ebytes) return NULL;
3596 unsigned char *maxchar = curr + 2 + ntohs(*(unsigned short*)curr);
3597 curr += 2;
3598 unsigned short ext_type = 1;
3599 unsigned short ext_len;
3600 while(curr < maxchar && ext_type != 0)
3601 {
3602 if (curr > ebytes) return NULL;
3603 ext_type = ntohs(*(unsigned short*)curr);
3604 curr += 2;
3605 if (curr > ebytes) return NULL;
3606 ext_len = ntohs(*(unsigned short*)curr);
3607 curr += 2;
3608 if(ext_type == 0)
3609 {
3610 curr += 3;
3611 if (curr > ebytes) return NULL;
3612 unsigned short namelen = ntohs(*(unsigned short*)curr);
3613 curr += 2;
3614 if ((curr + namelen) > ebytes) return NULL;
3615 //*len = namelen;
3616 *(curr +namelen) = (char)0;
3617 return (char*)curr;
3618 }
3619 else curr += ext_len;
3620 }
3621 //if (curr != maxchar) throw std::exception("incomplete SSL Client Hello");
3622 return NULL; //SNI was not present
3623 }
3624
3625 #endif
3626
3627
handleICAPConnection(Socket & peerconn,String & ip,Socket & proxysock,stat_rec * & dystat)3628 int ConnectionHandler::handleICAPConnection(Socket &peerconn, String &ip, Socket &proxysock, stat_rec *&dystat) {
3629
3630 int pcount = 0;
3631 bool ismitm = false;
3632
3633 struct timeval thestart;
3634 gettimeofday(&thestart, NULL);
3635
3636 peerconn.setTimeout(o.pcon_timeout);
3637
3638 std::string clientip(ip.toCharArray()); // hold the ICAP clients ip
3639
3640 if (clienthost) delete clienthost;
3641
3642 clienthost = NULL; // and the hostname, if available
3643 matchedip = false;
3644
3645 //try {
3646 //int rc;
3647 //bool authed = false;
3648 bool firsttime = true;
3649 //bool isbanneduser = true;
3650
3651 AuthPlugin *auth_plugin = NULL;
3652
3653 // RFC states that connections are persistent
3654 bool persistPeer = true;
3655
3656 //
3657 // End of set-up section
3658
3659 // Start of main loop
3660
3661 //
3662
3663 // maintain a persistent connection
3664 while ((firsttime || persistPeer) && !ttg)
3665 {
3666 ICAPHeader icaphead;
3667 ldl = o.currentLists();
3668 icaphead.ISTag = ldl->ISTag();
3669
3670 NaughtyFilter checkme(icaphead.HTTPrequest, icaphead.HTTPresponse);
3671 DataBuffer docbody;
3672 docbody.setTimeout(o.exchange_timeout);
3673 docbody.setICAP(true);
3674 FDTunnel fdt;
3675 String wline = "";
3676 if (firsttime) {
3677 // reset flags & objects next time round the loop
3678 firsttime = false;
3679 gettimeofday(&thestart, NULL);
3680 checkme.thestart = thestart;
3681 }
3682
3683 {
3684 // another round...
3685 #ifndef NEWDEBUG_OFF
3686 if(o.myDebug->ICAP)
3687 {
3688 std::ostringstream oss (std::ostringstream::out);
3689 oss << thread_id << " ICAP -persisting (count " << ++pcount << ")" << " Client IP: " << clientip << std::endl;
3690 o.myDebug->Debug("ICAP",oss.str());
3691 std::cerr << thread_id << " ICAP -persisting (count " << ++pcount << ")" << " - " << clientip << std::endl;
3692 }
3693 #endif
3694 icaphead.reset();
3695 if (!icaphead.in(&peerconn, true)) {
3696 if (peerconn.isTimedout()) {
3697 #ifndef NEWDEBUG_OFF
3698 if(o.myDebug->ICAP)
3699 {
3700 std::ostringstream oss (std::ostringstream::out);
3701 oss << thread_id << " -ICAP Persistent connection timed out" << std::endl;
3702 o.myDebug->Debug("ICAP",oss.str());
3703 std::cerr << thread_id << " -ICAP Persistent connection timed out" << std::endl;
3704 }
3705 #endif
3706 //send error response
3707 wline = "ICAP/1.0 408 Request timeout\r\n";
3708 wline += "Service: ";
3709 wline += PACKAGE_STRING;
3710 wline += "\r\n";
3711 wline += "Encapsulated: null-body=0\r\n";
3712 wline += "\r\n";
3713 peerconn.writeString(wline.toCharArray());
3714 } else {
3715
3716 #ifndef NEWDEBUG_OFF
3717 if(o.myDebug->ICAP)
3718 {
3719 std::ostringstream oss (std::ostringstream::out);
3720 oss << thread_id << " -ICAP Persistent connection closed" << std::endl;
3721 o.myDebug->Debug("ICAP",oss.str());
3722 std::cerr << thread_id << " -ICAP Persistent connection closed" << std::endl;
3723 }
3724 #endif
3725 // TODO: send error reply if needed
3726 break;
3727 }
3728 }
3729 ++dystat->reqs;
3730
3731 ip = icaphead.clientip;
3732
3733 // we will actually need to do *lots* of resetting of flags etc. here for pconns to work
3734 gettimeofday(&thestart, NULL);
3735 checkme.thestart = thestart;
3736
3737 //authed = false;
3738 //isbanneduser = false;
3739
3740 //requestscanners.clear();
3741 //responsescanners.clear();
3742
3743 matchedip = false;
3744 urlparams.clear();
3745 postparts.clear();
3746 checkme.mimetype = "-";
3747 //exceptionreason = "";
3748 //exceptioncat = "";
3749 //room = ""; // CHECK THIS - surely room is persistant?????
3750
3751 // reset docheader & docbody
3752 // headers *should* take care of themselves on the next in()
3753 // actually not entirely true for docheader - we may read
3754 // certain properties of it (in denyAccess) before we've
3755 // actually performed the next in(), so make sure we do a full
3756 // reset now.
3757 docbody.reset();
3758 docbody.setICAP(true);
3759 peerconn.resetChunk();
3760 }
3761 #ifndef NEWDEBUG_OFF
3762 if(o.myDebug->ICAP)
3763 {
3764 std::ostringstream oss (std::ostringstream::out);
3765 oss << thread_id << "service options enabled : " << wline << " icaphead.service_reqmod: "<< icaphead.service_reqmod << " icaphead.service_resmod: " << icaphead.service_resmod << " icaphead.service_options: " << " icaphead.icap_reqmod_service: " << icaphead.icap_reqmod_service << " icaphead.icap_resmod_service: " << icaphead.icap_resmod_service << " icaphead.icap_reqmod_service: " << icaphead.icap_reqmod_service << std::endl;
3766 o.myDebug->Debug("ICAP",oss.str());
3767 std::cerr << thread_id << "service options enabled : " << wline << " icaphead.service_reqmod: "<< icaphead.service_reqmod << " icaphead.service_resmod: " << icaphead.service_resmod << " icaphead.service_options: " << " icaphead.icap_reqmod_service: " << icaphead.icap_reqmod_service << " icaphead.icap_resmod_service: " << icaphead.icap_resmod_service << " icaphead.icap_reqmod_service: " << icaphead.icap_reqmod_service << std::endl;
3768 }
3769 #endif
3770 // Check service option REQMOD, RESMOD, OPTIONS and call appropreate function(s)
3771 //
3772 if (icaphead.service_reqmod && icaphead.icap_reqmod_service) {
3773 #ifndef NEWDEBUG_OFF
3774 if(o.myDebug->ICAP)
3775 {
3776 std::ostringstream oss (std::ostringstream::out);
3777 oss << thread_id << "Icap reqmod check " << std::endl;
3778 o.myDebug->Debug("ICAP",oss.str());
3779 std::cerr << thread_id << "Icap reqmod check " << std::endl;
3780 }
3781 #endif
3782 if (handleICAPreqmod(peerconn,ip, checkme, icaphead, auth_plugin) == 0){
3783 continue;
3784 }else{
3785 break;
3786 }
3787
3788 } else if (icaphead.service_resmod && icaphead.icap_resmod_service) {
3789 #ifndef NEWDEBUG_OFF
3790 if(o.myDebug->ICAP)
3791 {
3792 std::ostringstream oss (std::ostringstream::out);
3793 oss << thread_id << "Icap resmod check " << std::endl;
3794 o.myDebug->Debug("ICAP",oss.str());
3795 std::cerr << thread_id << "Icap resmod check " << std::endl;
3796 }
3797 #endif
3798 if (handleICAPresmod(peerconn,ip, checkme, icaphead, docbody) == 0)
3799 continue;
3800 else
3801 break;
3802 } else if (icaphead.service_options && icaphead.icap_reqmod_service) {
3803 // respond with option response
3804 wline = "ICAP/1.0 200 OK\r\n";
3805 wline += "Methods: REQMOD\r\n";
3806 wline += "Service: ";
3807 wline += PACKAGE_STRING;
3808 wline += "\r\n";
3809 wline += "ISTag: \"";
3810 wline += ldl->ISTag();
3811 wline += "\"\r\n";
3812 wline += "Encapsulated: null-body=0\r\n";
3813 wline += "Allow: 204\r\n";
3814 // wline += "Preview: 0\r\n";
3815 wline += "\r\n";
3816 peerconn.writeString(wline.toCharArray());
3817 #ifndef NEWDEBUG_OFF
3818 if(o.myDebug->ICAP)
3819 {
3820 std::ostringstream oss (std::ostringstream::out);
3821 oss << thread_id << "respmod service options response : " << wline << std::endl;
3822 o.myDebug->Debug("ICAP",oss.str());
3823 std::cerr << thread_id << "respmod service options response : " << wline << std::endl;
3824 }
3825 #endif
3826 } else if (icaphead.service_options && icaphead.icap_resmod_service) {
3827 // respond with option response
3828 wline = "ICAP/1.0 200 OK\r\n";
3829 wline += "Methods: RESPMOD\r\n";
3830 wline += "Service: ";
3831 wline += PACKAGE_STRING;
3832 wline += "\r\n";
3833 wline += "ISTag:";
3834 wline += ldl->ISTag();
3835 wline += "\r\n";
3836 wline += "Encapsulated: null-body=0\r\n";
3837 wline += "Allow: 204\r\n";
3838 // wline += "Preview: 0\r\n";
3839 wline += "\r\n";
3840 peerconn.writeString(wline.toCharArray());
3841 #ifndef NEWDEBUG_OFF
3842 if(o.myDebug->ICAP)
3843 {
3844 std::ostringstream oss (std::ostringstream::out);
3845 oss << thread_id << "respmod service options response : " << wline << std::endl;
3846 o.myDebug->Debug("ICAP",oss.str());
3847 std::cerr << thread_id << "respmod service options response : " << wline << std::endl;
3848 }
3849 #endif
3850 } else if ((icaphead.service_reqmod && !icaphead.icap_reqmod_service) ||
3851 (icaphead.service_resmod && !icaphead.icap_resmod_service)) {
3852 wline = "ICAP/1.0 405 Method not allowed for service\r\n";
3853 wline += "Service: ";
3854 wline += PACKAGE_STRING;
3855 wline += "\r\n";
3856 wline += "Encapsulated: null-body=0\r\n";
3857 wline += "\r\n";
3858 peerconn.writeString(wline.toCharArray());
3859 #ifndef NEWDEBUG_OFF
3860 if(o.myDebug->ICAP)
3861 {
3862 std::ostringstream oss (std::ostringstream::out);
3863 oss << thread_id << "ICAP/1.0 405 Method not allowed for service " << wline << std::endl;
3864 o.myDebug->Debug("ICAP",oss.str());
3865 std::cerr << thread_id << "ICAP/1.0 405 Method not allowed for service " << wline << std::endl;
3866 }
3867 #endif
3868 } else {
3869 //send error response
3870 wline = "ICAP/1.0 400 Bad request\r\n";
3871 wline += "Service: ";
3872 wline += PACKAGE_STRING;
3873 wline += "\r\n";
3874 peerconn.writeString(wline.toCharArray());
3875 #ifndef NEWDEBUG_OFF
3876 if(o.myDebug->ICAP)
3877 {
3878 std::ostringstream oss (std::ostringstream::out);
3879 oss << thread_id << "ICAP/1.0 400 Bad request : " << wline << std::endl;
3880 o.myDebug->Debug("ICAP",oss.str());
3881 std::cerr << thread_id << "ICAP/1.0 400 Bad request : " << wline << std::endl;
3882 }
3883 #endif
3884 }
3885 }
3886 // } //catch (std::exception & e)
3887
3888 if (!ismitm)
3889 try {
3890 #ifndef NEWDEBUG_OFF
3891 if(o.myDebug->ICAP)
3892 {
3893 std::ostringstream oss (std::ostringstream::out);
3894 oss << thread_id << "ICAP -Attempting graceful connection close" << std::endl;
3895 o.myDebug->Debug("ICAP",oss.str());
3896 std::cerr << thread_id << "ICAP -Attempting graceful connection close" << std::endl;
3897 }
3898 #endif
3899
3900 int fd = peerconn.getFD();
3901 if (fd > -1) {
3902 if (shutdown(fd, SHUT_WR) == 0) {
3903 char buff[2];
3904 peerconn.readFromSocket(buff, 2, 0, 5000);
3905 };
3906 };
3907
3908 // close connection to the client
3909 peerconn.close();
3910 } catch (std::exception &e) {
3911 #ifndef NEWDEBUG_OFF
3912 if(o.myDebug->ICAP)
3913 {
3914 std::ostringstream oss (std::ostringstream::out);
3915 oss << thread_id << " -ICAP connection handler caught an exception on connection closedown: " << e.what() << std::endl;
3916 o.myDebug->Debug("ICAP",oss.str());
3917 std::cerr << thread_id << " -ICAP connection handler caught an exception on connection closedown: " << e.what() << std::endl;
3918 }
3919 #endif
3920 // close connection to the client
3921 peerconn.close();
3922 }
3923
3924 return 0;
3925 }
3926
3927
handleICAPreqmod(Socket & peerconn,String & ip,NaughtyFilter & checkme,ICAPHeader & icaphead,AuthPlugin * auth_plugin)3928 int ConnectionHandler::handleICAPreqmod(Socket &peerconn, String &ip, NaughtyFilter &checkme, ICAPHeader &icaphead,
3929 AuthPlugin *auth_plugin) {
3930 bool authed = false;
3931 String clientip = ip;
3932 // do all of this normalisation etc just the once at the start.
3933 checkme.setURL();
3934 String res_hdr, res_body, req_hdr, req_body;
3935
3936 // checks for bad URLs to prevent security holes/domain obfuscation.
3937 if (icaphead.HTTPrequest.malformedURL(checkme.url)) {
3938 // The requested URL is malformed.
3939 gen_error_mess(peerconn, checkme, res_hdr, res_body, 200, 0, "400 Bad Request");
3940 checkme.isdone = true;
3941 icaphead.errorResponse(peerconn, res_hdr, res_body);
3942 if (icaphead.req_body_flag) {
3943 peerconn.drainChunk(peerconn.getTimeout()); // drains body
3944 }
3945 return 0;
3946 }
3947
3948 // do total block list checking here - now done in pre-auth story
3949 #ifdef NOTDEF
3950 if (o.use_total_block_list && o.inTotalBlockList(checkme.urld)) {
3951 res_hdr = "HTTP/1.1 200 OK\n";
3952 o.banned_image.display_hb(res_hdr, res_body);
3953 icaphead.errorResponse(peerconn, res_hdr, res_body);
3954 if (icaphead.req_body_flag) {
3955 peerconn.drainChunk(peerconn.getTimeout()); // drains body
3956 }
3957 return 0;
3958 }
3959 #endif
3960
3961 //
3962 //
3963 // Start of Authentication Checks
3964 //
3965 //
3966 // don't have credentials for this connection yet? get some!
3967 overide_persist = false;
3968 filtergroup = o.default_icap_fg;
3969 #ifdef DGBEBUG
3970 std::cerr << thread_id << "filtergroup set to ICAP default " << filtergroup << " " << std::endl;
3971 #endif
3972 clientuser = icaphead.username;
3973
3974 if(o.log_requests) {
3975 std::string fnt = "REQMOD";
3976 doRQLog(clientuser, clientip, checkme, fnt);
3977 }
3978
3979 int rc = DGAUTH_NOUSER;
3980 if (clientuser != "") {
3981 rc = determineGroup(clientuser, filtergroup, ldl->filter_groups_list);
3982 if (rc != DGAUTH_OK)
3983 {};
3984 }
3985 else { // TODO - NEED to allow alternate group checking when no match in filter_groups_list
3986 if (!doAuth(checkme.auth_result, authed, filtergroup, auth_plugin, peerconn, icaphead.HTTPrequest, true,
3987 true)) {
3988 //break; // TODO Error return????
3989 }
3990 }
3991
3992 authed = true;
3993 checkme.filtergroup = filtergroup;
3994
3995 #ifndef NEWDEBUG_OFF
3996 if(o.myDebug->ICAP)
3997 {
3998 std::ostringstream oss (std::ostringstream::out);
3999 int unrealgroup = filtergroup+1;
4000 oss << thread_id << "-username: " << clientuser << " ICAP -filtergroup: " << unrealgroup << std::endl;
4001 o.myDebug->Debug("ICAP",oss.str());
4002 std::cerr << thread_id << "-username: " << clientuser << " ICAP -filtergroup: " << unrealgroup << std::endl;
4003 }
4004 #endif
4005
4006 //
4007 //
4008 // End of Authentication Checking
4009 //
4010 //
4011
4012
4013 //
4014 //
4015 // Now check if user or machine is banned and room-based checking
4016 //
4017 //
4018
4019 // is this user banned?
4020 bool isbanneduser = false;
4021 checkme.clientip = clientip;
4022
4023 // Look up reverse DNS name of client if needed
4024 if (o.reverse_client_ip_lookups) {
4025 getClientFromIP(clientip.c_str(),checkme.clienthost);
4026 // std::unique_ptr<std::deque<String> > hostnames;
4027 // hostnames.reset(ipToHostname(clientip.c_str()));
4028 // checkme.clienthost = std::string(hostnames->front().toCharArray());
4029 }
4030
4031 //CALL SB pre-authcheck
4032 ldl->StoryA.runFunctEntry(ENT_STORYA_PRE_AUTH_ICAP, checkme);
4033 #ifdef DGDEBUG
4034 std::cerr << thread_id << "After StoryA icap-pre-authcheck" << checkme.isexception << " mess_no "
4035 << checkme.message_no << std::endl;
4036 #endif
4037 checkme.isItNaughty = checkme.isBlocked;
4038 bool isbannedip = checkme.isBlocked;
4039 //bool part_banned;
4040 if (isbannedip) {
4041 // matchedip = clienthost == NULL;
4042 } else {
4043 #ifdef NOTDEF // TODO does this need restoring???
4044 if (ldl->inRoom(clientip, room, clienthost, &isbannedip, &part_banned, &checkme.isexception,
4045 checkme.urld)) {
4046 #ifdef DGDEBUG
4047 std::cerr << thread_id << "ICAP isbannedip = " << isbannedip << "ispart_banned = " << part_banned << " isexception = " << checkme.isexception << std::endl;
4048 #endif
4049 if (isbannedip) {
4050 // matchedip = clienthost == NULL;
4051 checkme.isBlocked = checkme.isItNaughty = true;
4052 }
4053 if (checkme.isexception) {
4054 // do reason codes etc
4055 checkme.exceptionreason = o.language_list.getTranslation(630);
4056 checkme.exceptionreason.append(room);
4057 checkme.exceptionreason.append(o.language_list.getTranslation(631));
4058 checkme.message_no = 632;
4059 }
4060 }
4061 #endif
4062 }
4063
4064 //
4065 // Start of by pass
4066 if (!checkme.is_ssl) {
4067
4068 if (checkByPass(checkme, ldl, icaphead.HTTPrequest, peerconn, peerconn, clientip)
4069 && sendScanFile(peerconn, checkme, true, &icaphead)) {
4070 return 0; // returns only when Scanfile sent. Sets checkme.isbypass if it is a bypass.
4071 }
4072 }
4073 //
4074 // End of scan by pass
4075 //
4076
4077 bool done = false;
4078
4079 //
4080 // Start of exception checking
4081 //
4082 // being a banned user/IP overrides the fact that a site may be in the exception lists
4083 // needn't check these lists in bypass modes
4084 if (!(isbanneduser || isbannedip || checkme.isexception)) {
4085 // Main checking is now done in Storyboard function(s)
4086 ldl->fg[filtergroup]->StoryB.runFunctEntry(ENT_STORYB_ICAP_REQMOD, checkme);
4087 #ifdef DGDEBUG
4088 std::cerr << thread_id << "After StoryB checkreqmod" << checkme.isexception << " mess_no "
4089 << checkme.message_no << " allow_204 : " << icaphead.allow_204 << std::endl;
4090 #endif
4091
4092 if (ldl->fg[filtergroup]->reporting_level != -1){
4093 checkme.isItNaughty = checkme.isBlocked;
4094 } else {
4095 checkme.isItNaughty = false;
4096 checkme.isBlocked = false;
4097 }
4098 }
4099
4100 if (checkme.isbypass && !(checkme.iscookiebypass || checkme.isvirusbypass)) {
4101 #ifdef DGDEBUG
4102 std::cout << thread_id << "Setting GBYPASS cookie; bypasstimestamp = " << checkme.bypasstimestamp << __func__ << std::endl;
4103 #endif
4104 String ud(checkme.urldomain);
4105 if (ud.startsWith("www.")) {
4106 ud = ud.after("www.");
4107 }
4108
4109 String outhead = "HTTP/1.1 302 Redirect\r\n";
4110 outhead += "Set-Cookie: GBYPASS=";
4111 outhead += hashedCookie(&ud, ldl->fg[filtergroup]->cookie_magic.c_str(), &clientip,
4112 checkme.bypasstimestamp).toCharArray();
4113 outhead += "; path=/; domain=.";
4114 outhead += ud;
4115 outhead += "\r\n";
4116 outhead += "Location: ";
4117 outhead += icaphead.HTTPrequest.getUrl(true);
4118 outhead += "\r\n";
4119 outhead += "\r\n";
4120 icaphead.out_res_header = outhead;
4121 icaphead.out_res_hdr_flag = true;
4122 icaphead.respond(peerconn);
4123 return 0;
4124 }
4125
4126 // TODO add logic for 204 response etc.
4127 if (checkme.isexception) {
4128 std::string code;
4129 if (checkme.isvirusbypass)
4130 code = "V";
4131 else if (checkme.isbypass)
4132 code = "Y";
4133 else
4134 code = "E";
4135
4136 icaphead.set_icap_com(clientuser,code, filtergroup, checkme.message_no, checkme.log_message_no,
4137 checkme.whatIsNaughtyLog);
4138 if (icaphead.allow_204) {
4139 icaphead.respond(peerconn, "204 No Content", false, false);
4140 if (icaphead.req_body_flag) {
4141 peerconn.drainChunk(peerconn.getTimeout()); // drains any body
4142 }
4143 done = true;
4144 } else {
4145 // pipe through headers and body
4146 icaphead.respond(peerconn, "200 OK", true);
4147 if (icaphead.req_body_flag) {
4148 peerconn.loopChunk(peerconn.getTimeout()); // echos any body
4149 }
4150 done = true;
4151 }
4152 }
4153
4154 //check for redirect
4155 // URL regexp search and edirect
4156 if (checkme.urlredirect) {
4157 checkme.url = icaphead.HTTPrequest.redirecturl();
4158 String writestring("HTTP/1.1 302 Redirect\r\nLocation: ");
4159 writestring += checkme.url;
4160 writestring += "\r\n\r\n";
4161 res_hdr = writestring;
4162 icaphead.errorResponse(peerconn, res_hdr, res_body);
4163 if (icaphead.req_body_flag) {
4164 peerconn.drainChunk(peerconn.getTimeout()); // drains any body
4165 }
4166 done = true;
4167 }
4168
4169 //if is a search - content check search terms
4170 if (!done && !checkme.isdone && checkme.isGrey && checkme.isSearch)
4171 check_search_terms(checkme); // will set isItNaughty if needed
4172
4173
4174 // TODO V5 call POST scanning code New NaughtyFilter function????
4175
4176 if (!done && checkme.isItNaughty) {
4177 if (genDenyAccess(peerconn, res_hdr, res_body, &icaphead.HTTPrequest, &icaphead.HTTPresponse,
4178 &checkme.logurl, &checkme, &clientuser, &clientip,
4179 filtergroup, checkme.ispostblock, checkme.headersent, checkme.wasinfected,
4180 checkme.scanerror)) {
4181 icaphead.errorResponse(peerconn, res_hdr, res_body);
4182 if (icaphead.req_body_flag) {
4183 peerconn.drainChunk(peerconn.getTimeout()); // drains any body
4184 }
4185 done = true;
4186 #ifdef DGDEBUG
4187 std::cerr << thread_id << "ICAP Naughty" << std::endl;
4188 #endif
4189 // break loop "// maintain a persistent connection"
4190 // return 1;
4191 };
4192 }
4193
4194
4195 if (!done) {
4196 icaphead.set_icap_com(clientuser, "G", filtergroup, checkme.message_no, checkme.log_message_no,
4197 checkme.whatIsNaughtyLog);
4198 icaphead.respond(peerconn, "200 OK", true);
4199 if (icaphead.req_body_flag) {
4200 peerconn.loopChunk(peerconn.getTimeout()); // echoes any body
4201 }
4202 }
4203 //Log
4204 if (!(checkme.isourwebserver || checkme.nolog)) { // don't log requests to the web server
4205 doLog(clientuser, clientip, checkme);
4206 }
4207 return 0;
4208 }
4209
handleICAPresmod(Socket & peerconn,String & ip,NaughtyFilter & checkme,ICAPHeader & icaphead,DataBuffer & docbody)4210 int ConnectionHandler::handleICAPresmod(Socket &peerconn, String &ip, NaughtyFilter &checkme, ICAPHeader &icaphead,
4211 DataBuffer &docbody) {
4212
4213 //bool authed = false;
4214 bool persistPeer = true;
4215 bool done = false;
4216 String clientip = ip;
4217 String res_hdr, res_body;
4218 std::deque<CSPlugin *> responsescanners;
4219
4220 // do all of this normalisation etc just the once at the start.
4221 checkme.setURL();
4222
4223 overide_persist = false;
4224 if (icaphead.icap_com.filtergroup < 0) //i.e. no X-ICAP-E2G
4225 {
4226 String wline = "ICAP/1.0 418 Bad composition - X-ICAP-E2G header not present\r\n";
4227 wline += "Service: ";
4228 wline += PACKAGE_STRING;
4229 wline += "\r\n";
4230 wline += "Encapsulated: null-body=0\r\n";
4231 wline += "\r\n";
4232 peerconn.writeString(wline.toCharArray());
4233 #ifndef NEWDEBUG_OFF
4234 if(o.myDebug->ICAP)
4235 {
4236 std::ostringstream oss (std::ostringstream::out);
4237 oss << thread_id << " ICAP Error: " << wline << std::endl;
4238 o.myDebug->Debug("ICAP",oss.str());
4239 std::cerr << thread_id << " ICAP Error: " << wline << std::endl;
4240 }
4241 #endif
4242 return 1;
4243 }
4244
4245 filtergroup = icaphead.icap_com.filtergroup;
4246 checkme.filtergroup = icaphead.icap_com.filtergroup;
4247 clientuser = icaphead.icap_com.user;
4248
4249 if (icaphead.icap_com.EBG == "E") { // exception
4250 checkme.isexception = true;
4251 checkme.message_no = icaphead.icap_com.mess_no;
4252 checkme.log_message_no = icaphead.icap_com.log_mess_no;
4253 checkme.whatIsNaughtyLog = icaphead.icap_com.mess_string;
4254 } else if (icaphead.icap_com.EBG == "G") // grey - content check
4255 checkme.isGrey = true;
4256 else if (icaphead.icap_com.EBG == "Y") { // ordinary bypass
4257 checkme.isbypass = true;
4258 checkme.isexception = true;
4259 checkme.message_no = icaphead.icap_com.mess_no;
4260 checkme.log_message_no = icaphead.icap_com.log_mess_no;
4261 checkme.whatIsNaughtyLog = icaphead.icap_com.mess_string;
4262 } else if (icaphead.icap_com.EBG == "V") { // virus bypass
4263 checkme.isvirusbypass = true;
4264 checkme.isbypass = true;
4265 checkme.isexception = true;
4266 checkme.message_no = icaphead.icap_com.mess_no;
4267 checkme.log_message_no = icaphead.icap_com.log_mess_no;
4268 checkme.whatIsNaughtyLog = icaphead.icap_com.mess_string;
4269 }
4270
4271 #ifndef NEWDEBUG_OFF
4272 if(o.myDebug->ICAP)
4273 {
4274 int unrealfiltergroup = filtergroup + 1;
4275 std::ostringstream oss (std::ostringstream::out);
4276 oss << thread_id << "ICAP Respmod enabled - username: " << clientuser << " -filtergroup: " << unrealfiltergroup << " icaphead.icap_com.EBG: " << icaphead.icap_com.EBG << " icaphead.res_body_flag: " << icaphead.res_body_flag << std::endl;
4277 o.myDebug->Debug("ICAP",oss.str());
4278 std::cerr << thread_id << "ICAP Respmod enabled -username: " << clientuser << " -filtergroup: " << unrealfiltergroup << " icaphead.icap_com.EBG: " << icaphead.icap_com.EBG << " icaphead.res_body_flag: " << icaphead.res_body_flag << std::endl;
4279 }
4280 #endif
4281
4282 checkme.clientip = ip;
4283 checkme.filtergroup = filtergroup;
4284 if(o.log_requests) {
4285 std::string fnt = "RESPMOD";
4286 doRQLog(clientuser, clientip, checkme, fnt);
4287 }
4288 // Look up reverse DNS name of client if needed
4289 if (o.reverse_client_ip_lookups) {
4290 getClientFromIP(clientip.c_str(), checkme.clienthost);
4291 }
4292
4293 //bool part_banned;
4294
4295 // virus checkichurchillng candidate?
4296 // checkme.noviruscheck defaults to true
4297 #ifndef NEWDEBUG_OFF
4298 if(o.myDebug->ICAP)
4299 {
4300 std::ostringstream oss (std::ostringstream::out);
4301 oss << thread_id << "Virus scan checkme.isexception: " << checkme.isexception << " checkme.noviruscheck: " << checkme.noviruscheck << " content_scan_exceptions: " << ldl->fg[filtergroup]->content_scan_exceptions << " checkme.isBlocked: " << checkme.isBlocked << " disable_content_scan: " << ldl->fg[filtergroup]->disable_content_scan << " csplugins: " << o.csplugins.size() << std::endl;
4302 o.myDebug->Debug("ICAP",oss.str());
4303 std::cerr << thread_id << "Virus scan checkme.isexception: " << checkme.isexception << " checkme.noviruscheck: " << checkme.noviruscheck << " content_scan_exceptions: " << ldl->fg[filtergroup]->content_scan_exceptions << " checkme.isBlocked: " << checkme.isBlocked << " disable_content_scan: " << ldl->fg[filtergroup]->disable_content_scan << " csplugins: " << o.csplugins.size() << std::endl;
4304 }
4305 #endif
4306
4307 if (icaphead.res_body_flag // can only scan if body present
4308 && !(checkme.isBlocked) // or not already blocked
4309 && (o.csplugins.size() > 0) // and we have scan plugins
4310 && !ldl->fg[filtergroup]->disable_content_scan // and is not disabled
4311 && !(checkme.isexception && !ldl->fg[filtergroup]->content_scan_exceptions)
4312 && !checkme.isvirusbypass // and is not virus bypass
4313 // and not exception unless scan exceptions enable
4314 ) {
4315 checkme.noviruscheck = false; // note this may be reset by Storyboard to enable exceptions
4316 }
4317
4318 //
4319 // being a banned user/IP overrides the fact that a site may be in the exception lists
4320 // needn't check these lists in bypass modes
4321 if (!(checkme.isexception) || !checkme.noviruscheck) {
4322 // Main checking is done in Storyboard function(s)
4323 ldl->fg[filtergroup]->StoryB.runFunctEntry(ENT_STORYB_ICAP_RESMOD,checkme);
4324 #ifndef NEWDEBUG_OFF
4325 if(o.myDebug->ICAP)
4326 {
4327 std::ostringstream oss (std::ostringstream::out);
4328 oss << thread_id << "After StoryB icapcheckresmod" << checkme.isexception << " mess_no " << checkme.message_no << " checkme.noviruscheck: "<< checkme.noviruscheck << " content_scan_exceptions: " << ldl->fg[filtergroup]->content_scan_exceptions << std::endl;
4329 o.myDebug->Debug("ICAP",oss.str());
4330 std::cerr << thread_id << "After StoryB icapcheckresmod" << checkme.isexception << " mess_no " << checkme.message_no << " checkme.noviruscheck: "<< checkme.noviruscheck << " content_scan_exceptions: " << ldl->fg[filtergroup]->content_scan_exceptions << std::endl;
4331 }
4332 #endif
4333
4334 if (ldl->fg[filtergroup]->reporting_level != -1){
4335 checkme.isItNaughty = checkme.isBlocked;
4336 } else {
4337 checkme.isItNaughty = false;
4338 checkme.isBlocked = false;
4339 }
4340 }
4341
4342 if (checkme.isexception && !checkme.noviruscheck && !ldl->fg[filtergroup]->content_scan_exceptions)
4343 checkme.noviruscheck = true;
4344
4345 if (ldl->fg[filtergroup]->content_scan_exceptions && checkme.isexception)
4346 checkme.noviruscheck = false;
4347
4348 if ((checkme.isexception && checkme.noviruscheck)|| !icaphead.res_body_flag) {
4349 if (icaphead.allow_204) {
4350 icaphead.respond(peerconn, "204 No Content", false, false);
4351 if (icaphead.res_body_flag) {
4352 peerconn.drainChunk(peerconn.getTimeout()); // drains any body
4353 }
4354 } else {
4355 // pipe through headers and body
4356 icaphead.respond(peerconn, "200 OK", true);
4357 if (icaphead.res_body_flag) {
4358 peerconn.loopChunk(peerconn.getTimeout()); // echos any body
4359 }
4360 }
4361 done = true;
4362 }
4363
4364 // should now only be left with grey which has content body
4365
4366 //- if grey content check
4367 // can't do content filtering on HEAD or redirections (no content)
4368 // actually, redirections CAN have content
4369
4370 if (!done && !checkme.isItNaughty) {
4371 if(!checkme.noviruscheck)
4372 {
4373 for (std::deque<Plugin *>::iterator i = o.csplugins_begin; i != o.csplugins_end; ++i) {
4374 int csrc = ((CSPlugin *)(*i))->willScanRequest(checkme.url, clientuser.c_str(), ldl->fg[filtergroup], clientip.c_str(), false, false, checkme.isexception, checkme.isbypass);
4375 if (csrc > 0)
4376 responsescanners.push_back((CSPlugin *)(*i));
4377 else if (csrc < 0)
4378 syslog(LOG_ERR, "%swillScanRequest returned error: %d", thread_id.c_str(), csrc);
4379 }
4380 }
4381 check_content(checkme, docbody,peerconn, peerconn,responsescanners);
4382 }
4383
4384 //send response header to client
4385 if (!done && !checkme.isItNaughty) {
4386 icaphead.respond(peerconn, "200 OK", true);
4387 if(checkme.waschecked) {
4388 if (!docbody.out(&peerconn))
4389 checkme.pausedtoobig = false;
4390 if (checkme.pausedtoobig)
4391 checkme.tunnel_rest = true;
4392 }
4393 if (checkme.tunnel_rest){
4394 peerconn.loopChunk(peerconn.getTimeout()); // echos any body
4395 done = true;
4396 }
4397 }
4398
4399 if(checkme.isItNaughty) {
4400 if(genDenyAccess(peerconn,res_hdr, res_body, &icaphead.HTTPrequest, &icaphead.HTTPresponse, &checkme.logurl, &checkme, &clientuser, &ip,
4401 filtergroup, checkme.ispostblock,checkme.headersent, checkme.wasinfected, checkme.scanerror))
4402 {
4403 icaphead.errorResponse(peerconn, res_hdr, res_body);
4404 if (icaphead.res_body_flag) {
4405 peerconn.drainChunk(peerconn.getTimeout()); // drains any body
4406 }
4407 done = true;
4408 }
4409 persistPeer = false;
4410 }
4411
4412 //Log
4413 if (!checkme.isourwebserver && checkme.isItNaughty) { // don't log requests to the web server & and normal response
4414 checkme.whatIsNaughtyLog = "ICAP Response filtering: ";
4415 checkme.whatIsNaughtyLog += checkme.whatIsNaughty;
4416 doLog(clientuser, clientip, checkme);
4417 }
4418 if (persistPeer)
4419 return 0;
4420 else
4421 return 1;
4422 }
4423
4424 // determine what filter group the given username is in
4425 // return -1 when user not found
determineGroup(std::string & user,int & fg,ListContainer & uglc)4426 int ConnectionHandler::determineGroup(std::string &user, int &fg, ListContainer &uglc) {
4427 if (user.length() < 1 || user == "-") {
4428 return DGAUTH_NOMATCH;
4429 }
4430 String u(user);
4431 String lastcategory;
4432 u.toLower(); // since the filtergroupslist is read in in lowercase, we should do this.
4433 user = u.toCharArray(); // also pass back to ConnectionHandler, so appears lowercase in logs
4434 String ue(u);
4435 ue += "=";
4436
4437 //char *i = ldl->filter_groups_list.findStartsWithPartial(ue.toCharArray(), lastcategory);
4438 char *i = uglc.findStartsWithPartial(ue.toCharArray(), lastcategory);
4439
4440 if (i == NULL) {
4441 #ifdef DGDEBUG
4442 std::cerr << thread_id << "User not in filter groups list: " << ue << std::endl;
4443 #endif
4444 return DGAUTH_NOUSER;
4445 }
4446 #ifdef DGDEBUG
4447 std::cerr << thread_id << "User found in filter group list: " << i << std::endl;
4448 #endif
4449 ue = i;
4450 if (ue.before("=") == u) {
4451 ue = ue.after("=filter");
4452 int l = ue.length();
4453 if (l < 1 || l > 2) {
4454 return DGAUTH_NOUSER;
4455 }
4456 int t;
4457 t = ue.toInteger();
4458 if (t > o.numfg) {
4459 return DGAUTH_NOUSER;
4460 }
4461 if (t > 0) {
4462 fg = --t;
4463 #ifdef DGDEBUG
4464 std::cerr << thread_id << "determineGroup gives fg: " << fg << std::endl;
4465 #endif
4466 return DGAUTH_OK;
4467 }
4468 }
4469 return DGAUTH_NOUSER;
4470 }
4471