1 /*
2 * http.c
3 *
4 * Copyright (C) 2011-21 - ntop.org
5 *
6 * This file is part of nDPI, an open source deep packet inspection
7 * library based on the OpenDPI and PACE technology by ipoque GmbH
8 *
9 * nDPI is free software: you can redistribute it and/or modify
10 * it under the terms of the GNU Lesser General Public License as published by
11 * the Free Software Foundation, either version 3 of the License, or
12 * (at your option) any later version.
13 *
14 * nDPI is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 * GNU Lesser General Public License for more details.
18 *
19 * You should have received a copy of the GNU Lesser General Public License
20 * along with nDPI. If not, see <http://www.gnu.org/licenses/>.
21 *
22 */
23
24 #include "ndpi_protocol_ids.h"
25
26 #define NDPI_CURRENT_PROTO NDPI_PROTOCOL_HTTP
27
28 #include "ndpi_api.h"
29
30 static const char* binary_file_mimes_e[] = { "exe", NULL };
31 static const char* binary_file_mimes_v[] = { "vnd.ms-cab-compressed", "vnd.microsoft.portable-executable", NULL };
32 static const char* binary_file_mimes_x[] = { "x-msdownload", "x-dosexec", NULL };
33
34 static const char* download_file_mimes_b[] = { "bz", "bz2", NULL };
35 static const char* download_file_mimes_o[] = { "octet-stream", NULL };
36 static const char* download_file_mimes_x[] = { "x-tar", "x-zip", "x-bzip", NULL };
37
38 #define ATTACHMENT_LEN 3
39 static const char* binary_file_ext[] = {
40 "exe",
41 "msi",
42 "cab",
43 NULL
44 };
45
46 static void ndpi_search_http_tcp(struct ndpi_detection_module_struct *ndpi_struct,
47 struct ndpi_flow_struct *flow);
48
49 /* *********************************************** */
50
ndpi_analyze_content_signature(struct ndpi_detection_module_struct * ndpi_struct,struct ndpi_flow_struct * flow)51 static void ndpi_analyze_content_signature(struct ndpi_detection_module_struct *ndpi_struct,
52 struct ndpi_flow_struct *flow) {
53 if((flow->initial_binary_bytes_len >= 2) && (flow->initial_binary_bytes[0] == 0x4D) && (flow->initial_binary_bytes[1] == 0x5A))
54 ndpi_set_risk(ndpi_struct, flow, NDPI_BINARY_APPLICATION_TRANSFER); /* Win executable */
55 else if((flow->initial_binary_bytes_len >= 4) && (flow->initial_binary_bytes[0] == 0x7F) && (flow->initial_binary_bytes[1] == 'E')
56 && (flow->initial_binary_bytes[2] == 'L') && (flow->initial_binary_bytes[3] == 'F'))
57 ndpi_set_risk(ndpi_struct, flow, NDPI_BINARY_APPLICATION_TRANSFER); /* Linux executable */
58 else if((flow->initial_binary_bytes_len >= 4) && (flow->initial_binary_bytes[0] == 0xCF) && (flow->initial_binary_bytes[1] == 0xFA)
59 && (flow->initial_binary_bytes[2] == 0xED) && (flow->initial_binary_bytes[3] == 0xFE))
60 ndpi_set_risk(ndpi_struct, flow, NDPI_BINARY_APPLICATION_TRANSFER); /* Linux executable */
61 else if((flow->initial_binary_bytes_len >= 3)
62 && (flow->initial_binary_bytes[0] == '#')
63 && (flow->initial_binary_bytes[1] == '!')
64 && (flow->initial_binary_bytes[2] == '/'))
65 ndpi_set_risk(ndpi_struct, flow, NDPI_BINARY_APPLICATION_TRANSFER); /* Unix script (e.g. #!/bin/sh) */
66 else if(flow->initial_binary_bytes_len >= 8) {
67 u_int8_t exec_pattern[] = { 0x64, 0x65, 0x78, 0x0A, 0x30, 0x33, 0x35, 0x00 };
68
69 if(memcmp(flow->initial_binary_bytes, exec_pattern, 8) == 0)
70 ndpi_set_risk(ndpi_struct, flow, NDPI_BINARY_APPLICATION_TRANSFER); /* Dalvik Executable (Android) */
71 }
72 }
73
74 /* *********************************************** */
75
ndpi_search_http_tcp_again(struct ndpi_detection_module_struct * ndpi_struct,struct ndpi_flow_struct * flow)76 static int ndpi_search_http_tcp_again(struct ndpi_detection_module_struct *ndpi_struct,
77 struct ndpi_flow_struct *flow) {
78
79 ndpi_search_http_tcp(ndpi_struct, flow);
80
81 #ifdef HTTP_DEBUG
82 printf("=> %s()\n", __FUNCTION__);
83 #endif
84
85 if((flow->host_server_name[0] != '\0')
86 && (flow->http.response_status_code != 0)
87 ) {
88 /* stop extra processing */
89
90 if(flow->initial_binary_bytes_len) ndpi_analyze_content_signature(ndpi_struct, flow);
91 flow->extra_packets_func = NULL; /* We're good now */
92 return(0);
93 }
94
95 /* Possibly more processing */
96 return(1);
97 }
98
99 /* *********************************************** */
100
ndpi_http_is_print(char c)101 static int ndpi_http_is_print(char c) {
102 if(isprint(c) || (c == '\t') || (c == '\r') || (c == '\n'))
103 return(1);
104 else
105 return(0);
106 }
107
108 /* *********************************************** */
109
ndpi_http_check_human_redeable_content(struct ndpi_detection_module_struct * ndpi_struct,struct ndpi_flow_struct * flow,const u_int8_t * content,u_int16_t content_len)110 static void ndpi_http_check_human_redeable_content(struct ndpi_detection_module_struct *ndpi_struct,
111 struct ndpi_flow_struct *flow,
112 const u_int8_t *content, u_int16_t content_len) {
113 if(content_len >= 4) {
114 NDPI_LOG_DBG(ndpi_struct, " [len: %u] [%02X %02X %02X %02X][%c%c%c%c]", content_len,
115 content[0], content[1], content[2], content[3],
116 content[0], content[1], content[2], content[3]
117 );
118
119 if(ndpi_http_is_print(content[0]) && ndpi_http_is_print(content[1])
120 && ndpi_http_is_print(content[2]) && ndpi_http_is_print(content[3])) {
121 /* OK */
122 } else {
123 /* Looks bad: last resort check if it's gzipped [1F 8B 08 00] */
124
125 if((content[0] == 0x1F)
126 && (content[1] == 0x8B)
127 && (content[2] == 0x08)
128 && (content[3] == 0x00)) {
129 /* Looks like compressed data */
130 } else
131 ndpi_set_risk(ndpi_struct, flow, NDPI_HTTP_SUSPICIOUS_CONTENT);
132 }
133 }
134 }
135
136 /* *********************************************** */
137
ndpi_validate_http_content(struct ndpi_detection_module_struct * ndpi_struct,struct ndpi_flow_struct * flow)138 static void ndpi_validate_http_content(struct ndpi_detection_module_struct *ndpi_struct,
139 struct ndpi_flow_struct *flow) {
140 struct ndpi_packet_struct *packet = &flow->packet;
141 const u_int8_t *double_ret = (const u_int8_t *)ndpi_strnstr((const char *)packet->payload, "\r\n\r\n", packet->payload_packet_len);
142
143 NDPI_LOG_DBG(ndpi_struct, "==>>> [len: %u] ", packet->payload_packet_len);
144 NDPI_LOG_DBG(ndpi_struct, "->> %.*s\n", packet->content_line.len, (const char *)packet->content_line.ptr);
145
146 if(double_ret) {
147 u_int len;
148
149 len = packet->payload_packet_len - (double_ret - packet->payload);
150
151 if(ndpi_strnstr((const char *)packet->content_line.ptr, "text/", packet->content_line.len)
152 || ndpi_strnstr((const char *)packet->content_line.ptr, "/json", packet->content_line.len)
153 || ndpi_strnstr((const char *)packet->content_line.ptr, "x-www-form-urlencoded", packet->content_line.len)
154 ) {
155 /* This is supposed to be a human-readeable text file */
156
157 packet->http_check_content = 1;
158
159 if(len >= 8 /* 4 chars for \r\n\r\n and at least 4 charts for content guess */) {
160 double_ret += 4;
161
162 ndpi_http_check_human_redeable_content(ndpi_struct, flow, double_ret, len);
163 }
164 }
165
166 NDPI_LOG_DBG(ndpi_struct, "\n");
167 }
168 }
169
170 /* *********************************************** */
171
172 /* https://www.freeformatter.com/mime-types-list.html */
ndpi_http_check_content(struct ndpi_detection_module_struct * ndpi_struct,struct ndpi_flow_struct * flow)173 static ndpi_protocol_category_t ndpi_http_check_content(struct ndpi_detection_module_struct *ndpi_struct,
174 struct ndpi_flow_struct *flow) {
175 struct ndpi_packet_struct *packet = &flow->packet;
176
177 if(packet->content_line.len > 0) {
178 u_int app_len = sizeof("application");
179
180 if(packet->content_line.len > app_len) {
181 const char *app = (const char *)&packet->content_line.ptr[app_len];
182 u_int app_len_avail = packet->content_line.len-app_len;
183
184 if(strncasecmp(app, "mpeg", app_len_avail) == 0) {
185 flow->guessed_category = flow->category = NDPI_PROTOCOL_CATEGORY_STREAMING;
186 return(flow->category);
187 } else {
188 if(app_len_avail > 3) {
189 const char** cmp_mimes = NULL;
190
191 switch(app[0]) {
192 case 'b': cmp_mimes = download_file_mimes_b; break;
193 case 'o': cmp_mimes = download_file_mimes_o; break;
194 case 'x': cmp_mimes = download_file_mimes_x; break;
195 }
196
197 if(cmp_mimes != NULL) {
198 u_int8_t i;
199
200 for(i = 0; cmp_mimes[i] != NULL; i++) {
201 if(strncasecmp(app, cmp_mimes[i], app_len_avail) == 0) {
202 flow->guessed_category = flow->category = NDPI_PROTOCOL_CATEGORY_DOWNLOAD_FT;
203 NDPI_LOG_INFO(ndpi_struct, "found executable HTTP transfer");
204 break;
205 }
206 }
207 }
208
209 /* ***************************************** */
210
211 switch(app[0]) {
212 case 'e': cmp_mimes = binary_file_mimes_e; break;
213 case 'v': cmp_mimes = binary_file_mimes_v; break;
214 case 'x': cmp_mimes = binary_file_mimes_x; break;
215 }
216
217 if(cmp_mimes != NULL) {
218 u_int8_t i;
219
220 for(i = 0; cmp_mimes[i] != NULL; i++) {
221 if(strncasecmp(app, cmp_mimes[i], app_len_avail) == 0) {
222 flow->guessed_category = flow->category = NDPI_PROTOCOL_CATEGORY_DOWNLOAD_FT;
223 ndpi_set_risk(ndpi_struct, flow, NDPI_BINARY_APPLICATION_TRANSFER);
224 NDPI_LOG_INFO(ndpi_struct, "found executable HTTP transfer");
225 return(flow->category);
226 }
227 }
228 }
229 }
230
231 ndpi_validate_http_content(ndpi_struct, flow);
232 }
233 }
234
235 /* check for attachment */
236 if(packet->content_disposition_line.len > 0) {
237 u_int8_t attachment_len = sizeof("attachment; filename");
238
239 if(packet->content_disposition_line.len > attachment_len) {
240 u_int8_t filename_len = packet->content_disposition_line.len - attachment_len;
241
242 if(filename_len > ATTACHMENT_LEN) {
243 attachment_len += filename_len-ATTACHMENT_LEN-1;
244
245 if((attachment_len+ATTACHMENT_LEN) <= packet->content_disposition_line.len) {
246 for(int i = 0; binary_file_ext[i] != NULL; i++) {
247 /* Use memcmp in case content-disposition contains binary data */
248 if(memcmp((const char*)&packet->content_disposition_line.ptr[attachment_len],
249 binary_file_ext[i], ATTACHMENT_LEN) == 0) {
250 flow->guessed_category = flow->category = NDPI_PROTOCOL_CATEGORY_DOWNLOAD_FT;
251 ndpi_set_risk(ndpi_struct, flow, NDPI_BINARY_APPLICATION_TRANSFER);
252 NDPI_LOG_INFO(ndpi_struct, "found executable HTTP transfer");
253 return(flow->category);
254 }
255 }
256 }
257 }
258 }
259 }
260
261 switch(packet->content_line.ptr[0]) {
262 case 'a':
263 if(strncasecmp((const char *)packet->content_line.ptr, "audio",
264 ndpi_min(packet->content_line.len, 5)) == 0)
265 flow->guessed_category = flow->category = NDPI_PROTOCOL_CATEGORY_MEDIA;
266 break;
267
268 case 'v':
269 if(strncasecmp((const char *)packet->content_line.ptr, "video",
270 ndpi_min(packet->content_line.len, 5)) == 0)
271 flow->guessed_category = flow->category = NDPI_PROTOCOL_CATEGORY_MEDIA;
272 break;
273 }
274 }
275
276 return(flow->category);
277 }
278
279 /* *********************************************** */
280
ndpi_int_http_add_connection(struct ndpi_detection_module_struct * ndpi_struct,struct ndpi_flow_struct * flow,u_int16_t http_protocol,ndpi_protocol_category_t category)281 static void ndpi_int_http_add_connection(struct ndpi_detection_module_struct *ndpi_struct,
282 struct ndpi_flow_struct *flow,
283 u_int16_t http_protocol,
284 ndpi_protocol_category_t category) {
285 #ifdef HTTP_DEBUG
286 printf("=> %s()\n", __FUNCTION__);
287 #endif
288
289 if(flow->extra_packets_func && (flow->guessed_host_protocol_id == NDPI_PROTOCOL_UNKNOWN))
290 return; /* Nothing new to add */
291
292 /* This is HTTP and it is not a sub protocol (e.g. skype or dropbox) */
293 ndpi_search_tcp_or_udp(ndpi_struct, flow);
294
295 /* If no custom protocol has been detected */
296 if((flow->guessed_host_protocol_id == NDPI_PROTOCOL_UNKNOWN) || (http_protocol != NDPI_PROTOCOL_HTTP))
297 flow->guessed_host_protocol_id = http_protocol;
298
299 // ndpi_int_reset_protocol(flow);
300 ndpi_set_detected_protocol(ndpi_struct, flow, flow->guessed_host_protocol_id, NDPI_PROTOCOL_HTTP);
301
302 /* This is necessary to inform the core to call this dissector again */
303 flow->check_extra_packets = 1;
304 flow->max_extra_packets_to_check = 8;
305 flow->extra_packets_func = ndpi_search_http_tcp_again;
306 flow->http_detected = 1;
307 }
308
309 /* ************************************************************* */
310
rtsp_parse_packet_acceptline(struct ndpi_detection_module_struct * ndpi_struct,struct ndpi_flow_struct * flow)311 static void rtsp_parse_packet_acceptline(struct ndpi_detection_module_struct
312 *ndpi_struct, struct ndpi_flow_struct *flow)
313 {
314 struct ndpi_packet_struct *packet = &flow->packet;
315
316 if((packet->accept_line.len >= 28)
317 && (memcmp(packet->accept_line.ptr, "application/x-rtsp-tunnelled", 28) == 0)) {
318 NDPI_LOG_INFO(ndpi_struct, "found RTSP accept line\n");
319 ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_RTSP, NDPI_PROTOCOL_CATEGORY_MEDIA);
320 }
321 }
322
323 /* ************************************************************* */
324
setHttpUserAgent(struct ndpi_detection_module_struct * ndpi_struct,struct ndpi_flow_struct * flow,char * ua)325 static void setHttpUserAgent(struct ndpi_detection_module_struct *ndpi_struct,
326 struct ndpi_flow_struct *flow, char *ua) {
327 if( !strcmp(ua, "Windows NT 5.0")) ua = "Windows 2000";
328 else if(!strcmp(ua, "Windows NT 5.1")) ua = "Windows XP";
329 else if(!strcmp(ua, "Windows NT 5.2")) ua = "Windows Server 2003";
330 else if(!strcmp(ua, "Windows NT 6.0")) ua = "Windows Vista";
331 else if(!strcmp(ua, "Windows NT 6.1")) ua = "Windows 7";
332 else if(!strcmp(ua, "Windows NT 6.2")) ua = "Windows 8";
333 else if(!strcmp(ua, "Windows NT 6.3")) ua = "Windows 8.1";
334 else if(!strcmp(ua, "Windows NT 10.0")) ua = "Windows 10";
335
336 /* Good reference for future implementations:
337 * https://github.com/ua-parser/uap-core/blob/master/regexes.yaml */
338
339 snprintf((char*)flow->http.detected_os,
340 sizeof(flow->http.detected_os), "%s", ua);
341 }
342
343 /* ************************************************************* */
344
ndpi_http_parse_subprotocol(struct ndpi_detection_module_struct * ndpi_struct,struct ndpi_flow_struct * flow)345 static void ndpi_http_parse_subprotocol(struct ndpi_detection_module_struct *ndpi_struct,
346 struct ndpi_flow_struct *flow) {
347 if((flow->l4.tcp.http_stage == 0) || (flow->http.url && flow->http_detected)) {
348 char *double_col = strchr((char*)flow->host_server_name, ':');
349
350 if(double_col) double_col[0] = '\0';
351
352 if(ndpi_match_hostname_protocol(ndpi_struct, flow, NDPI_PROTOCOL_HTTP,
353 (char *)flow->host_server_name,
354 strlen((const char *)flow->host_server_name)) == 0) {
355 if(flow->http.url &&
356 ((strstr(flow->http.url, ":8080/downloading?n=0.") != NULL)
357 || (strstr(flow->http.url, ":8080/upload?n=0.") != NULL))) {
358 /* This looks like Ookla speedtest */
359 ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_OOKLA, NDPI_PROTOCOL_HTTP);
360 }
361 }
362 }
363 }
364
365 /* ************************************************************* */
366
ndpi_check_user_agent(struct ndpi_detection_module_struct * ndpi_struct,struct ndpi_flow_struct * flow,char * ua)367 static void ndpi_check_user_agent(struct ndpi_detection_module_struct *ndpi_struct,
368 struct ndpi_flow_struct *flow,
369 char *ua) {
370 if((!ua) || (ua[0] == '\0')) return;
371
372 if((strlen(ua) < 4)
373 || (!strncmp(ua, "test", 4))
374 || (!strncmp(ua, "<?", 2))
375 || strchr(ua, '{')
376 || strchr(ua, '}')
377 // || ndpi_check_dga_name(ndpi_struct, NULL, ua, 0)
378 // || ndpi_match_bigram(ndpi_struct, &ndpi_struct->impossible_bigrams_automa, ua)
379 ) {
380 ndpi_set_risk(ndpi_struct, flow, NDPI_HTTP_SUSPICIOUS_USER_AGENT);
381 }
382 }
383
http_process_user_agent(struct ndpi_detection_module_struct * ndpi_struct,struct ndpi_flow_struct * flow,const u_int8_t * ua_ptr,u_int16_t ua_ptr_len)384 int http_process_user_agent(struct ndpi_detection_module_struct *ndpi_struct,
385 struct ndpi_flow_struct *flow,
386 const u_int8_t *ua_ptr, u_int16_t ua_ptr_len)
387 {
388 /**
389 Format examples:
390 Mozilla/5.0 (iPad; U; CPU OS 3_2 like Mac OS X; en-us) AppleWebKit/531.21.10 (KHTML, like Gecko) ....
391 Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:54.0) Gecko/20100101 Firefox/54.0
392 */
393 if(ua_ptr_len > 7) {
394 char ua[256];
395 u_int mlen = ndpi_min(ua_ptr_len, sizeof(ua)-1);
396
397 strncpy(ua, (const char *)ua_ptr, mlen);
398 ua[mlen] = '\0';
399
400 if(strncmp(ua, "Mozilla", 7) == 0) {
401 char *parent = strchr(ua, '(');
402
403 if(parent) {
404 char *token, *end;
405
406 parent++;
407 end = strchr(parent, ')');
408 if(end) end[0] = '\0';
409
410 token = strsep(&parent, ";");
411 if(token) {
412 if((strcmp(token, "X11") == 0)
413 || (strcmp(token, "compatible") == 0)
414 || (strcmp(token, "Linux") == 0)
415 || (strcmp(token, "Macintosh") == 0)
416 ) {
417 token = strsep(&parent, ";");
418 if(token && (token[0] == ' ')) token++; /* Skip space */
419
420 if(token
421 && ((strcmp(token, "U") == 0)
422 || (strncmp(token, "MSIE", 4) == 0))) {
423 token = strsep(&parent, ";");
424 if(token && (token[0] == ' ')) token++; /* Skip space */
425
426 if(token && (strncmp(token, "Update", 6) == 0)) {
427 token = strsep(&parent, ";");
428
429 if(token && (token[0] == ' ')) token++; /* Skip space */
430
431 if(token && (strncmp(token, "AOL", 3) == 0)) {
432
433 token = strsep(&parent, ";");
434 if(token && (token[0] == ' ')) token++; /* Skip space */
435 }
436 }
437 }
438 }
439
440 if(token)
441 setHttpUserAgent(ndpi_struct, flow, token);
442 }
443 }
444 }
445 }
446
447 if(flow->http.user_agent == NULL) {
448 int len = ua_ptr_len + 1;
449
450 flow->http.user_agent = ndpi_malloc(len);
451 if(flow->http.user_agent) {
452 memcpy(flow->http.user_agent, (char*)ua_ptr, ua_ptr_len);
453 flow->http.user_agent[ua_ptr_len] = '\0';
454
455 ndpi_check_user_agent(ndpi_struct, flow, flow->http.user_agent);
456 }
457 }
458
459 NDPI_LOG_DBG2(ndpi_struct, "User Agent Type line found %.*s\n",
460 ua_ptr_len, ua_ptr);
461 return 0;
462 }
463
464 /* ************************************************************* */
465
ndpi_check_numeric_ip(struct ndpi_detection_module_struct * ndpi_struct,struct ndpi_flow_struct * flow,char * ip,u_int ip_len)466 static void ndpi_check_numeric_ip(struct ndpi_detection_module_struct *ndpi_struct,
467 struct ndpi_flow_struct *flow,
468 char *ip, u_int ip_len) {
469 char buf[22], *double_dot;
470 struct in_addr ip_addr;
471
472 strncpy(buf, ip, ip_len);
473 buf[ip_len] = '\0';
474
475 if((double_dot = strchr(buf, ':')) != NULL)
476 double_dot[0] = '\0';
477
478 ip_addr.s_addr = inet_addr(buf);
479 if(strcmp(inet_ntoa(ip_addr), buf) == 0)
480 ndpi_set_risk(ndpi_struct, flow, NDPI_HTTP_NUMERIC_IP_HOST);
481 }
482
483 /* ************************************************************* */
484
ndpi_check_http_url(struct ndpi_detection_module_struct * ndpi_struct,struct ndpi_flow_struct * flow,char * url)485 static void ndpi_check_http_url(struct ndpi_detection_module_struct *ndpi_struct,
486 struct ndpi_flow_struct *flow,
487 char *url) {
488 /* Nothing to do */
489 }
490
491 /* ************************************************************* */
492
493 /**
494 NOTE
495 ndpi_parse_packet_line_info is in ndpi_main.c
496 */
check_content_type_and_change_protocol(struct ndpi_detection_module_struct * ndpi_struct,struct ndpi_flow_struct * flow)497 static void check_content_type_and_change_protocol(struct ndpi_detection_module_struct *ndpi_struct,
498 struct ndpi_flow_struct *flow) {
499 struct ndpi_packet_struct *packet = &flow->packet;
500 int ret;
501
502 if(flow->http_detected && (flow->http.response_status_code != 0))
503 return;
504
505 if((flow->http.url == NULL)
506 && (packet->http_url_name.len > 0)
507 && (packet->host_line.len > 0)) {
508 int len = packet->http_url_name.len + packet->host_line.len + 1;
509
510 if(isdigit(packet->host_line.ptr[0])
511 && (packet->host_line.len < 21))
512 ndpi_check_numeric_ip(ndpi_struct, flow, (char*)packet->host_line.ptr, packet->host_line.len);
513
514 flow->http.url = ndpi_malloc(len);
515 if(flow->http.url) {
516 strncpy(flow->http.url, (char*)packet->host_line.ptr, packet->host_line.len);
517 strncpy(&flow->http.url[packet->host_line.len], (char*)packet->http_url_name.ptr,
518 packet->http_url_name.len);
519 flow->http.url[len-1] = '\0';
520
521 ndpi_check_http_url(ndpi_struct, flow, &flow->http.url[packet->host_line.len]);
522 }
523
524 flow->http.method = ndpi_http_str2method((const char*)flow->packet.http_method.ptr,
525 (u_int16_t)flow->packet.http_method.len);
526 }
527
528 if(packet->server_line.ptr != NULL && (packet->server_line.len > 7)) {
529 if(strncmp((const char *)packet->server_line.ptr, "ntopng ", 7) == 0) {
530 ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_NTOP, NDPI_PROTOCOL_HTTP);
531 NDPI_CLR_BIT(flow->risk, NDPI_KNOWN_PROTOCOL_ON_NON_STANDARD_PORT);
532 }
533 }
534
535 if(packet->user_agent_line.ptr != NULL && packet->user_agent_line.len != 0) {
536 ret = http_process_user_agent(ndpi_struct, flow, packet->user_agent_line.ptr, packet->user_agent_line.len);
537 /* TODO: Is it correct to avoid setting ua, host_name,... if we have a (Netflix) subclassification? */
538 if(ret != 0)
539 return;
540 }
541
542 /* check for host line */
543 if(packet->host_line.ptr != NULL) {
544 u_int len;
545
546 NDPI_LOG_DBG2(ndpi_struct, "HOST line found %.*s\n",
547 packet->host_line.len, packet->host_line.ptr);
548
549 /* Copy result for nDPI apps */
550 len = ndpi_min(packet->host_line.len, sizeof(flow->host_server_name)-1);
551 strncpy((char*)flow->host_server_name, (char*)packet->host_line.ptr, len);
552 flow->host_server_name[len] = '\0';
553 flow->extra_packets_func = NULL; /* We're good now */
554
555 if(len > 0) ndpi_check_dga_name(ndpi_struct, flow, (char*)flow->host_server_name, 1);
556 flow->server_id = flow->dst;
557
558 if(packet->forwarded_line.ptr) {
559 len = ndpi_min(packet->forwarded_line.len, sizeof(flow->protos.http.nat_ip)-1);
560 strncpy((char*)flow->protos.http.nat_ip, (char*)packet->forwarded_line.ptr, len);
561 flow->protos.http.nat_ip[len] = '\0';
562 }
563
564 ndpi_http_parse_subprotocol(ndpi_struct, flow);
565
566 /**
567 check result of host subprotocol detection
568
569 if "detected" in flow == 0 then "detected" = "guess"
570 else "guess" = "detected"
571 **/
572 if(flow->detected_protocol_stack[1] == NDPI_PROTOCOL_UNKNOWN) {
573 /* Avoid putting as subprotocol a "core" protocol such as SSL or DNS */
574 if(ndpi_struct->proto_defaults[flow->guessed_protocol_id].subprotocol_count == 0) {
575 flow->detected_protocol_stack[1] = flow->guessed_protocol_id;
576 if(flow->detected_protocol_stack[0] == NDPI_PROTOCOL_UNKNOWN)
577 flow->detected_protocol_stack[0] = flow->guessed_host_protocol_id;
578 }
579 }
580 else {
581 if(flow->detected_protocol_stack[1] != flow->guessed_protocol_id)
582 flow->guessed_protocol_id = flow->detected_protocol_stack[1];
583 if(flow->detected_protocol_stack[0] != flow->guessed_host_protocol_id)
584 flow->guessed_host_protocol_id = flow->detected_protocol_stack[0];
585 }
586
587 if((flow->detected_protocol_stack[0] == NDPI_PROTOCOL_UNKNOWN)
588 && (flow->http_detected)
589 && (packet->http_origin.len > 0)) {
590 ndpi_protocol_match_result ret_match;
591
592 ndpi_match_host_subprotocol(ndpi_struct, flow,
593 (char *)packet->http_origin.ptr,
594 packet->http_origin.len,
595 &ret_match,
596 NDPI_PROTOCOL_HTTP);
597 }
598
599 if(flow->detected_protocol_stack[0] != NDPI_PROTOCOL_UNKNOWN) {
600 if(packet->detected_protocol_stack[0] != NDPI_PROTOCOL_HTTP) {
601 NDPI_LOG_INFO(ndpi_struct, "found HTTP/%s\n",
602 ndpi_get_proto_name(ndpi_struct, packet->detected_protocol_stack[0]));
603 ndpi_int_http_add_connection(ndpi_struct, flow, packet->detected_protocol_stack[0], NDPI_PROTOCOL_CATEGORY_WEB);
604 return; /* We have identified a sub-protocol so we're done */
605 }
606 }
607 }
608
609 #if 0
610 if(flow->http_detected)
611 ndpi_http_parse_subprotocol(ndpi_struct, flow);
612 #endif
613
614 if(flow->guessed_protocol_id == NDPI_PROTOCOL_UNKNOWN)
615 flow->guessed_protocol_id = NDPI_PROTOCOL_HTTP;
616
617 /* check for accept line */
618 if(packet->accept_line.ptr != NULL) {
619 NDPI_LOG_DBG2(ndpi_struct, "Accept line found %.*s\n",
620 packet->accept_line.len, packet->accept_line.ptr);
621 if(NDPI_COMPARE_PROTOCOL_TO_BITMASK(ndpi_struct->detection_bitmask,
622 NDPI_PROTOCOL_RTSP) != 0) {
623 rtsp_parse_packet_acceptline(ndpi_struct, flow);
624 }
625 }
626
627 if(packet->content_line.ptr != NULL && packet->content_line.len != 0) {
628 NDPI_LOG_DBG2(ndpi_struct, "Content Type line found %.*s\n",
629 packet->content_line.len, packet->content_line.ptr);
630
631 if(flow->http.response_status_code == 0) {
632 /* Request */
633 if((flow->http.request_content_type == NULL) && (packet->content_line.len > 0)) {
634 int len = packet->content_line.len + 1;
635
636 flow->http.request_content_type = ndpi_malloc(len);
637 if(flow->http.request_content_type) {
638 strncpy(flow->http.request_content_type, (char*)packet->content_line.ptr,
639 packet->content_line.len);
640 flow->http.request_content_type[packet->content_line.len] = '\0';
641 }
642 }
643 } else {
644 /* Response */
645 if((flow->http.content_type == NULL) && (packet->content_line.len > 0)) {
646 int len = packet->content_line.len + 1;
647
648 flow->http.content_type = ndpi_malloc(len);
649 if(flow->http.content_type) {
650 strncpy(flow->http.content_type, (char*)packet->content_line.ptr,
651 packet->content_line.len);
652 flow->http.content_type[packet->content_line.len] = '\0';
653
654 flow->guessed_category = flow->category = ndpi_http_check_content(ndpi_struct, flow);
655 }
656 }
657 }
658
659 if(flow->http_detected && packet->content_line.ptr && *(char*)packet->content_line.ptr) {
660 ndpi_protocol_match_result ret_match;
661
662 ndpi_match_content_subprotocol(ndpi_struct, flow,
663 (char*)packet->content_line.ptr, packet->content_line.len,
664 &ret_match, NDPI_PROTOCOL_HTTP);
665 }
666 }
667
668 if (ndpi_get_http_method(ndpi_struct, flow) != NDPI_HTTP_METHOD_UNKNOWN)
669 {
670 ndpi_int_http_add_connection(ndpi_struct, flow, packet->detected_protocol_stack[0], NDPI_PROTOCOL_CATEGORY_WEB);
671 }
672 }
673
674 /* ************************************************************* */
675
check_http_payload(struct ndpi_detection_module_struct * ndpi_struct,struct ndpi_flow_struct * flow)676 static void check_http_payload(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow) {
677 /* Add here your paylod code check */
678 }
679
680 /* ************************************************************* */
681
682 #ifdef NDPI_ENABLE_DEBUG_MESSAGES
non_ctrl(uint8_t c)683 static uint8_t non_ctrl(uint8_t c) {
684 return c < 32 ? '.':c;
685 }
686 #endif
687
688 /* ************************************************************* */
689
690 /**
691 * Functions to check whether the packet begins with a valid http request
692 * @param ndpi_struct
693 * @returnvalue 0 if no valid request has been found
694 * @returnvalue >0 indicates start of filename but not necessarily in packet limit
695 */
696
697 #define STATIC_STRING_L(a) {.str=a, .len=sizeof(a)-1 }
698
699 static struct l_string {
700 const char *str;
701 size_t len;
702 } http_methods[] = {
703 STATIC_STRING_L("GET "),
704 STATIC_STRING_L("POST "),
705 STATIC_STRING_L("OPTIONS "),
706 STATIC_STRING_L("HEAD "),
707 STATIC_STRING_L("PUT "),
708 STATIC_STRING_L("PATCH "),
709 STATIC_STRING_L("DELETE "),
710 STATIC_STRING_L("CONNECT "),
711 STATIC_STRING_L("PROPFIND "),
712 STATIC_STRING_L("REPORT ") };
713 static const char *http_fs = "CDGHOPR";
714
http_request_url_offset(struct ndpi_detection_module_struct * ndpi_struct,struct ndpi_flow_struct * flow)715 static u_int16_t http_request_url_offset(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow)
716 {
717 struct ndpi_packet_struct *packet = &flow->packet;
718 int i;
719
720 NDPI_LOG_DBG2(ndpi_struct, "====>>>> HTTP: %c%c%c%c [len: %u]\n",
721 packet->payload_packet_len > 0 ? non_ctrl(packet->payload[0]) : '.',
722 packet->payload_packet_len > 1 ? non_ctrl(packet->payload[1]) : '.',
723 packet->payload_packet_len > 2 ? non_ctrl(packet->payload[2]) : '.',
724 packet->payload_packet_len > 3 ? non_ctrl(packet->payload[3]) : '.',
725 packet->payload_packet_len);
726
727 /* Check first char */
728 if(!packet->payload_packet_len || !strchr(http_fs,packet->payload[0]))
729 return 0;
730
731 /**
732 FIRST PAYLOAD PACKET FROM CLIENT
733 **/
734 for(i=0; i < sizeof(http_methods)/sizeof(http_methods[0]); i++) {
735 if(packet->payload_packet_len >= http_methods[i].len &&
736 memcmp(packet->payload,http_methods[i].str,http_methods[i].len) == 0) {
737 NDPI_LOG_DBG2(ndpi_struct, "HTTP: %sFOUND\n",http_methods[i].str);
738 return http_methods[i].len;
739 }
740 }
741 return 0;
742 }
743
http_bitmask_exclude_other(struct ndpi_flow_struct * flow)744 static void http_bitmask_exclude_other(struct ndpi_flow_struct *flow)
745 {
746 NDPI_ADD_PROTOCOL_TO_BITMASK(flow->excluded_protocol_bitmask, NDPI_PROTOCOL_XBOX);
747 }
748
749 /* *********************************************************************************************** */
750
751 /* Trick to speed-up detection */
752 static const char* suspicious_http_header_keys_A[] = { "Arch", NULL};
753 static const char* suspicious_http_header_keys_C[] = { "Cores", NULL};
754 static const char* suspicious_http_header_keys_M[] = { "Mem", NULL};
755 static const char* suspicious_http_header_keys_O[] = { "Os", "Osname", "Osversion", NULL};
756 static const char* suspicious_http_header_keys_R[] = { "Root", NULL};
757 static const char* suspicious_http_header_keys_S[] = { "S", NULL};
758 static const char* suspicious_http_header_keys_T[] = { "TLS_version", NULL};
759 static const char* suspicious_http_header_keys_U[] = { "Uuid", NULL};
760 static const char* suspicious_http_header_keys_X[] = { "X-Hire-Me", NULL};
761
is_a_suspicious_header(const char * suspicious_headers[],struct ndpi_int_one_line_struct packet_line)762 static int is_a_suspicious_header(const char* suspicious_headers[], struct ndpi_int_one_line_struct packet_line){
763 int i;
764 unsigned int header_len;
765 const u_int8_t* header_limit;
766
767 if((header_limit = memchr(packet_line.ptr, ':', packet_line.len))) {
768 header_len = header_limit - packet_line.ptr;
769 for(i=0; suspicious_headers[i] != NULL; i++){
770 if(!strncasecmp((const char*) packet_line.ptr,
771 suspicious_headers[i], header_len))
772 return 1;
773 }
774 }
775
776 return 0;
777 }
778
779 /* *********************************************************************************************** */
780
ndpi_check_http_header(struct ndpi_detection_module_struct * ndpi_struct,struct ndpi_flow_struct * flow)781 static void ndpi_check_http_header(struct ndpi_detection_module_struct *ndpi_struct,
782 struct ndpi_flow_struct *flow) {
783 u_int32_t i;
784 struct ndpi_packet_struct *packet = &flow->packet;
785
786 for(i=0; (i < packet->parsed_lines)
787 && (packet->line[i].ptr != NULL)
788 && (packet->line[i].len > 0); i++) {
789 switch(packet->line[i].ptr[0]){
790 case 'A':
791 if(is_a_suspicious_header(suspicious_http_header_keys_A, packet->line[i])) {
792 ndpi_set_risk(ndpi_struct, flow, NDPI_HTTP_SUSPICIOUS_HEADER);
793 return;
794 }
795 break;
796 case 'C':
797 if(is_a_suspicious_header(suspicious_http_header_keys_C, packet->line[i])) {
798 ndpi_set_risk(ndpi_struct, flow, NDPI_HTTP_SUSPICIOUS_HEADER);
799 return;
800 }
801 break;
802 case 'M':
803 if(is_a_suspicious_header(suspicious_http_header_keys_M, packet->line[i])) {
804 ndpi_set_risk(ndpi_struct, flow, NDPI_HTTP_SUSPICIOUS_HEADER);
805 return;
806 }
807 break;
808 case 'O':
809 if(is_a_suspicious_header(suspicious_http_header_keys_O, packet->line[i])) {
810 ndpi_set_risk(ndpi_struct, flow, NDPI_HTTP_SUSPICIOUS_HEADER);
811 return;
812 }
813 break;
814 case 'R':
815 if(is_a_suspicious_header(suspicious_http_header_keys_R, packet->line[i])) {
816 ndpi_set_risk(ndpi_struct, flow, NDPI_HTTP_SUSPICIOUS_HEADER);
817 return;
818 }
819 break;
820 case 'S':
821 if(is_a_suspicious_header(suspicious_http_header_keys_S, packet->line[i])) {
822 ndpi_set_risk(ndpi_struct, flow, NDPI_HTTP_SUSPICIOUS_HEADER);
823 return;
824 }
825 break;
826 case 'T':
827 if(is_a_suspicious_header(suspicious_http_header_keys_T, packet->line[i])) {
828 ndpi_set_risk(ndpi_struct, flow, NDPI_HTTP_SUSPICIOUS_HEADER);
829 return;
830 }
831 break;
832 case 'U':
833 if(is_a_suspicious_header(suspicious_http_header_keys_U, packet->line[i])) {
834 ndpi_set_risk(ndpi_struct, flow, NDPI_HTTP_SUSPICIOUS_HEADER);
835 return;
836 }
837 break;
838 case 'X':
839 if(is_a_suspicious_header(suspicious_http_header_keys_X, packet->line[i])) {
840 ndpi_set_risk(ndpi_struct, flow, NDPI_HTTP_SUSPICIOUS_HEADER);
841 return;
842 }
843
844 break;
845 }
846 }
847 }
848 /*************************************************************************************************/
849
ndpi_check_http_tcp(struct ndpi_detection_module_struct * ndpi_struct,struct ndpi_flow_struct * flow)850 static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct,
851 struct ndpi_flow_struct *flow) {
852 struct ndpi_packet_struct *packet = &flow->packet;
853 u_int16_t filename_start; /* the filename in the request method line, e.g., "GET filename_start..."*/
854
855 packet->packet_lines_parsed_complete = 0;
856
857 if(packet->http_check_content && (packet->payload_packet_len > 0)) {
858 ndpi_http_check_human_redeable_content(ndpi_struct, flow, packet->payload, packet->payload_packet_len);
859 packet->http_check_content = 0; /* One packet is enough */
860 }
861
862 /* Check if we so far detected the protocol in the request or not. */
863 if((packet->payload_packet_len > 0) /* Needed in case of extra packet processing */
864 && (flow->l4.tcp.http_stage == 0)) {
865 /* Expected a request */
866 flow->http_detected = 0;
867
868 NDPI_LOG_DBG2(ndpi_struct, "HTTP stage %d: \n", flow->l4.tcp.http_stage);
869
870 filename_start = http_request_url_offset(ndpi_struct, flow);
871
872 if(filename_start == 0) { /* not a regular request. In the HTTP first stage, may be a truncated flow or other protocols */
873 NDPI_LOG_DBG2(ndpi_struct, "Filename HTTP not found, we look for possible truncate flow..\n");
874
875 if(packet->payload_packet_len >= 7 && memcmp(packet->payload, "HTTP/1.", 7) == 0) {
876 NDPI_LOG_INFO(ndpi_struct, "found HTTP response\n");
877
878 if(packet->payload_packet_len >= 12) {
879 char buf[4];
880
881 /* Set server HTTP response code */
882 strncpy(buf, (char*)&packet->payload[9], 3);
883 buf[3] = '\0';
884
885 flow->http.response_status_code = atoi(buf);
886 /* https://en.wikipedia.org/wiki/List_of_HTTP_status_codes */
887 if((flow->http.response_status_code < 100) || (flow->http.response_status_code > 509))
888 flow->http.response_status_code = 0; /* Out of range */
889 }
890
891 ndpi_parse_packet_line_info(ndpi_struct, flow);
892 check_content_type_and_change_protocol(ndpi_struct, flow);
893 ndpi_validate_http_content(ndpi_struct, flow);
894 return;
895 }
896
897 if((packet->payload_packet_len == 3) && memcmp(packet->payload, "HI\n", 3) == 0) {
898 /* This looks like Ookla: we don't give up with HTTP yet */
899 flow->l4.tcp.http_stage = 1;
900 return;
901 }
902
903 if((packet->payload_packet_len == 40) && (flow->l4.tcp.http_stage == 0)) {
904 /*
905 -> QR O06L0072-6L91-4O43-857J-K8OO172L6L51
906 <- QNUUX 2.5 2017-08-15.1314.4jn12m5
907 -> MXFWUXJM 31625365
908 */
909
910 if((packet->payload[2] == ' ')
911 && (packet->payload[11] == '-')
912 && (packet->payload[16] == '-')
913 && (packet->payload[21] == '-')
914 && (packet->payload[26] == '-')
915 && (packet->payload[39] == 0x0A)
916 )
917 flow->l4.tcp.http_stage = 1;
918 return;
919 }
920
921 if((packet->payload_packet_len == 23) && (memcmp(packet->payload, "<policy-file-request/>", 23) == 0)) {
922 /*
923 <policy-file-request/>
924 <cross-domain-policy>
925 <allow-access-from domain="*.ookla.com" to-ports="8080"/>
926 <allow-access-from domain="*.speedtest.net" to-ports="8080"/>
927 </cross-domain-policy>
928 */
929 ookla_found:
930 ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_OOKLA, NDPI_PROTOCOL_CATEGORY_WEB);
931
932 if(ndpi_struct->ookla_cache == NULL)
933 ndpi_struct->ookla_cache = ndpi_lru_cache_init(1024);
934
935 if(ndpi_struct->ookla_cache != NULL) {
936 if(packet->iph != NULL) {
937 if(packet->tcp->source == htons(8080))
938 ndpi_lru_add_to_cache(ndpi_struct->ookla_cache, packet->iph->saddr, 1 /* dummy */);
939 else
940 ndpi_lru_add_to_cache(ndpi_struct->ookla_cache, packet->iph->daddr, 1 /* dummy */);
941 } else if(packet->iphv6 != NULL) {
942 u_int32_t h;
943
944 if(packet->tcp->source == htons(8080))
945 h = ndpi_quick_hash((unsigned char *)&packet->iphv6->ip6_src, sizeof(packet->iphv6->ip6_src));
946 else
947 h = ndpi_quick_hash((unsigned char *)&packet->iphv6->ip6_dst, sizeof(packet->iphv6->ip6_dst));
948
949 ndpi_lru_add_to_cache(ndpi_struct->ookla_cache, h, 1 /* dummy */);
950 }
951 }
952
953 return;
954 }
955
956 /* try to get some additional request header info even if the packet may not be HTTP */
957 ndpi_parse_packet_line_info(ndpi_struct, flow);
958 if(packet->http_num_headers > 0) {
959 check_content_type_and_change_protocol(ndpi_struct, flow);
960 return;
961 }
962
963 NDPI_EXCLUDE_PROTO(ndpi_struct, flow);
964 http_bitmask_exclude_other(flow);
965 return;
966 } else
967 ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_HTTP, NDPI_PROTOCOL_CATEGORY_WEB);
968
969 NDPI_LOG_DBG2(ndpi_struct,
970 "Filename HTTP found: %d, we look for line info..\n", filename_start);
971
972 ndpi_parse_packet_line_info(ndpi_struct, flow);
973 ndpi_check_http_header(ndpi_struct, flow);
974
975 if(packet->parsed_lines <= 1) {
976 NDPI_LOG_DBG2(ndpi_struct,
977 "Found just one line, we will look further for the next packet...\n");
978
979 packet->http_method.ptr = packet->line[0].ptr;
980 packet->http_method.len = filename_start - 1;
981
982 /* Encode the direction of the packet in the stage, so we will know when we need to look for the response packet. */
983 flow->l4.tcp.http_stage = packet->packet_direction + 1; // packet_direction 0: stage 1, packet_direction 1: stage 2
984 return;
985 }
986
987 NDPI_LOG_DBG2(ndpi_struct,
988 "Found more than one line, we look further for the next packet...\n");
989
990 if(packet->line[0].len >= (9 + filename_start)
991 && memcmp(&packet->line[0].ptr[packet->line[0].len - 9], " HTTP/1.", 8) == 0) {
992 /* Request line complete. Ex. "GET / HTTP/1.1" */
993
994 packet->http_url_name.ptr = &packet->payload[filename_start];
995 packet->http_url_name.len = packet->line[0].len - (filename_start + 9);
996
997 packet->http_method.ptr = packet->line[0].ptr;
998 packet->http_method.len = filename_start - 1;
999
1000 // Set the HTTP requested version: 0=HTTP/1.0 and 1=HTTP/1.1
1001 if(memcmp(&packet->line[0].ptr[packet->line[0].len - 1], "1", 1) == 0)
1002 flow->http.request_version = 1;
1003 else
1004 flow->http.request_version = 0;
1005
1006 /* Set the first found headers in request */
1007 flow->http.num_request_headers = packet->http_num_headers;
1008
1009 /* Check for Ookla */
1010 if((packet->referer_line.len > 0)
1011 && ndpi_strnstr((const char *)packet->referer_line.ptr, "www.speedtest.net", packet->referer_line.len)) {
1012 goto ookla_found;
1013 }
1014
1015 if((packet->http_url_name.len > 7)
1016 && (!strncmp((const char*) packet->http_url_name.ptr, "http://", 7))) {
1017 NDPI_LOG_INFO(ndpi_struct, "found HTTP_PROXY\n");
1018 ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_HTTP_PROXY, NDPI_PROTOCOL_CATEGORY_WEB);
1019 check_content_type_and_change_protocol(ndpi_struct, flow);
1020 }
1021
1022 if(filename_start == 8 && (memcmp(packet->payload, "CONNECT ", 8) == 0)) {
1023 /* nathan@getoffmalawn.com */
1024 NDPI_LOG_INFO(ndpi_struct, "found HTTP_CONNECT\n");
1025 ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_HTTP_CONNECT, NDPI_PROTOCOL_CATEGORY_WEB);
1026 check_content_type_and_change_protocol(ndpi_struct, flow);
1027 }
1028
1029 NDPI_LOG_DBG2(ndpi_struct,
1030 "HTTP START Found, we will look for sub-protocols (content and host)...\n");
1031
1032 if(packet->host_line.ptr != NULL) {
1033 /**
1034 nDPI is pretty scrupulous about HTTP so it waits until the
1035 HTTP response is received just to check that it conforms
1036 with the HTTP specs. However this might be a waste of time as
1037 in 99.99% of the cases is like that.
1038 */
1039
1040 ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_HTTP, NDPI_PROTOCOL_CATEGORY_WEB);
1041 flow->http_detected = 1;
1042 NDPI_LOG_DBG2(ndpi_struct,
1043 "HTTP START Found, we will look further for the response...\n");
1044 flow->l4.tcp.http_stage = packet->packet_direction + 1; // packet_direction 0: stage 1, packet_direction 1: stage 2
1045 check_content_type_and_change_protocol(ndpi_struct, flow);
1046 return;
1047 }
1048 }
1049
1050 NDPI_EXCLUDE_PROTO(ndpi_struct, flow);
1051 http_bitmask_exclude_other(flow);
1052 } else if((flow->l4.tcp.http_stage == 1) || (flow->l4.tcp.http_stage == 2)) {
1053 NDPI_LOG_DBG2(ndpi_struct, "HTTP stage %u: \n", flow->l4.tcp.http_stage);
1054
1055 if((packet->payload_packet_len == 34) && (flow->l4.tcp.http_stage == 1)) {
1056 if((packet->payload[5] == ' ') && (packet->payload[9] == ' ')) {
1057 goto ookla_found;
1058 }
1059 }
1060
1061 if((packet->payload_packet_len > 6) && memcmp(packet->payload, "HELLO ", 6) == 0) {
1062 /* This looks like Ookla */
1063 goto ookla_found;
1064 } else
1065 NDPI_ADD_PROTOCOL_TO_BITMASK(flow->excluded_protocol_bitmask, NDPI_PROTOCOL_OOKLA);
1066
1067 /**
1068 At first check, if this is for sure a response packet
1069 (in another direction. If not, if HTTP is detected do nothing now and return,
1070 otherwise check the second packet for the HTTP request
1071 */
1072 if((flow->l4.tcp.http_stage - packet->packet_direction) == 1) { /* Expected a response package */
1073
1074 if(flow->http_detected)
1075 return;
1076
1077 NDPI_LOG_DBG2(ndpi_struct,
1078 " SECOND PAYLOAD TRAFFIC FROM CLIENT, FIRST PACKET MIGHT HAVE BEEN HTTP...UNKNOWN TRAFFIC, HERE FOR HTTP again.. \n");
1079
1080 ndpi_parse_packet_line_info(ndpi_struct, flow);
1081
1082 // Add more found HTTP request headers.
1083 flow->http.num_request_headers+=packet->http_num_headers;
1084
1085 if(packet->parsed_lines <= 1) {
1086 /* wait some packets in case request is split over more than 2 packets */
1087 if(flow->packet_counter < 5) {
1088 NDPI_LOG_DBG2(ndpi_struct, "line still not finished, search next packet\n");
1089 return;
1090 } else {
1091 /* stop parsing here */
1092 NDPI_LOG_DBG2(ndpi_struct, "exclude HTTP: PACKET DOES NOT HAVE A LINE STRUCTURE\n");
1093 NDPI_EXCLUDE_PROTO(ndpi_struct, flow);
1094 http_bitmask_exclude_other(flow);
1095 return;
1096 }
1097 }
1098 // http://www.slideshare.net/DSPIP/rtsp-analysis-wireshark
1099 if(packet->line[0].len >= 9
1100 && memcmp(&packet->line[0].ptr[packet->line[0].len - 9], " HTTP/1.", 8) == 0) {
1101
1102 NDPI_LOG_INFO(ndpi_struct, "found HTTP\n");
1103 ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_HTTP, NDPI_PROTOCOL_CATEGORY_WEB);
1104 check_content_type_and_change_protocol(ndpi_struct, flow);
1105
1106 NDPI_LOG_DBG2(ndpi_struct,
1107 "HTTP START Found in 2. packet, we will look further for the response....\n");
1108 flow->http_detected = 1;
1109 }
1110
1111 return;
1112 }
1113
1114 /**
1115 This is a packet in another direction. Check if we find the proper response.
1116 We have received a response for a previously identified partial HTTP request
1117 */
1118
1119 /* response without headers
1120 * TODO: Shouldn't it be below ndpi_parse_packet_line_info, line ~825 ?
1121 */
1122 if((packet->parsed_lines == 1) && (packet->packet_direction == 1 /* server -> client */)) {
1123 /* In Apache if you do "GET /\n\n" the response comes without any header */
1124 NDPI_LOG_INFO(ndpi_struct, "found HTTP. (apache)\n");
1125 ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_HTTP, NDPI_PROTOCOL_CATEGORY_WEB);
1126 check_content_type_and_change_protocol(ndpi_struct, flow);
1127 return;
1128 }
1129
1130 /* If we already detected the HTTP request, we can add the connection and then check for the sub-protocol */
1131 if(flow->http_detected) {
1132 NDPI_LOG_INFO(ndpi_struct, "found HTTP\n");
1133 ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_HTTP, NDPI_PROTOCOL_CATEGORY_WEB);
1134 }
1135
1136 /* Parse packet line and we look for the subprotocols */
1137 ndpi_parse_packet_line_info(ndpi_struct, flow);
1138 check_content_type_and_change_protocol(ndpi_struct, flow);
1139
1140 if(packet->packet_direction == 1 /* server -> client */)
1141 flow->http.num_response_headers += packet->http_num_headers; /* flow structs are initialized with zeros */
1142
1143 if(packet->empty_line_position_set != 0 || flow->l4.tcp.http_empty_line_seen == 1) {
1144 NDPI_LOG_DBG2(ndpi_struct, "empty line. check_http_payload\n");
1145 check_http_payload(ndpi_struct, flow);
1146 }
1147
1148 flow->l4.tcp.http_stage = 0;
1149 return;
1150 }
1151 }
1152
1153 /* ********************************* */
1154
ndpi_search_http_tcp(struct ndpi_detection_module_struct * ndpi_struct,struct ndpi_flow_struct * flow)1155 static void ndpi_search_http_tcp(struct ndpi_detection_module_struct *ndpi_struct,
1156 struct ndpi_flow_struct *flow) {
1157 /* Break after 20 packets. */
1158 if(flow->packet_counter > 20) {
1159 NDPI_EXCLUDE_PROTO(ndpi_struct, flow);
1160 http_bitmask_exclude_other(flow);
1161 return;
1162 }
1163
1164 NDPI_LOG_DBG(ndpi_struct, "search HTTP\n");
1165 ndpi_check_http_tcp(ndpi_struct, flow);
1166 }
1167
1168 /* ********************************* */
1169
ndpi_get_http_method(struct ndpi_detection_module_struct * ndpi_struct,struct ndpi_flow_struct * flow)1170 ndpi_http_method ndpi_get_http_method(struct ndpi_detection_module_struct *ndpi_struct,
1171 struct ndpi_flow_struct *flow) {
1172 if(!flow) {
1173 ndpi_set_risk(ndpi_struct, flow, NDPI_MALFORMED_PACKET);
1174 return(NDPI_HTTP_METHOD_UNKNOWN);
1175 } else
1176 return(flow->http.method);
1177 }
1178
1179 /* ********************************* */
1180
ndpi_get_http_url(struct ndpi_detection_module_struct * ndpi_struct,struct ndpi_flow_struct * flow)1181 char* ndpi_get_http_url(struct ndpi_detection_module_struct *ndpi_struct,
1182 struct ndpi_flow_struct *flow) {
1183 if((!flow) || (!flow->http.url))
1184 return("");
1185 else
1186 return(flow->http.url);
1187 }
1188
1189 /* ********************************* */
1190
ndpi_get_http_content_type(struct ndpi_detection_module_struct * ndpi_struct,struct ndpi_flow_struct * flow)1191 char* ndpi_get_http_content_type(struct ndpi_detection_module_struct *ndpi_struct,
1192 struct ndpi_flow_struct *flow) {
1193 if((!flow) || (!flow->http.content_type))
1194 return("");
1195 else
1196 return(flow->http.content_type);
1197 }
1198
1199
init_http_dissector(struct ndpi_detection_module_struct * ndpi_struct,u_int32_t * id,NDPI_PROTOCOL_BITMASK * detection_bitmask)1200 void init_http_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id,
1201 NDPI_PROTOCOL_BITMASK *detection_bitmask) {
1202 ndpi_set_bitmask_protocol_detection("HTTP",ndpi_struct, detection_bitmask, *id,
1203 NDPI_PROTOCOL_HTTP,
1204 ndpi_search_http_tcp,
1205 NDPI_SELECTION_BITMASK_PROTOCOL_V4_V6_TCP_WITH_PAYLOAD,
1206 SAVE_DETECTION_BITMASK_AS_UNKNOWN,
1207 ADD_TO_DETECTION_BITMASK);
1208 *id += 1;
1209 }
1210