1 /* Copyright (C) 2007-2020 Open Information Security Foundation
2 *
3 * You can copy, redistribute or modify this Program under the terms of
4 * the GNU General Public License version 2 as published by the Free
5 * Software Foundation.
6 *
7 * This program is distributed in the hope that it will be useful,
8 * but WITHOUT ANY WARRANTY; without even the implied warranty of
9 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 * GNU General Public License for more details.
11 *
12 * You should have received a copy of the GNU General Public License
13 * version 2 along with this program; if not, write to the Free Software
14 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
15 * 02110-1301, USA.
16 */
17
18 /**
19 * \file
20 *
21 * \author Gurvinder Singh <gurvindersinghdahiya@gmail.com>
22 * \author Victor Julien <victor@inliniac.net>
23 *
24 * Reference:
25 * Judy Novak, Steve Sturges: Target-Based TCP Stream Reassembly August, 2007
26 *
27 */
28
29 #include "suricata-common.h"
30 #include "suricata.h"
31 #include "debug.h"
32 #include "detect.h"
33 #include "flow.h"
34 #include "threads.h"
35 #include "conf.h"
36
37 #include "flow-util.h"
38
39 #include "threadvars.h"
40 #include "tm-threads.h"
41
42 #include "util-pool.h"
43 #include "util-unittest.h"
44 #include "util-print.h"
45 #include "util-host-os-info.h"
46 #include "util-unittest-helper.h"
47 #include "util-byte.h"
48 #include "util-device.h"
49
50 #include "stream-tcp.h"
51 #include "stream-tcp-private.h"
52 #include "stream-tcp-reassemble.h"
53 #include "stream-tcp-inline.h"
54 #include "stream-tcp-list.h"
55 #include "stream-tcp-util.h"
56
57 #include "stream.h"
58
59 #include "util-debug.h"
60 #include "app-layer-protos.h"
61 #include "app-layer.h"
62 #include "app-layer-events.h"
63 #include "app-layer-parser.h"
64
65 #include "detect-engine-state.h"
66
67 #include "util-profiling.h"
68 #include "util-validate.h"
69
70 #ifdef DEBUG
71 static SCMutex segment_pool_memuse_mutex;
72 static uint64_t segment_pool_memuse = 0;
73 static uint64_t segment_pool_memcnt = 0;
74 #endif
75
76 static PoolThread *segment_thread_pool = NULL;
77 /* init only, protect initializing and growing pool */
78 static SCMutex segment_thread_pool_mutex = SCMUTEX_INITIALIZER;
79
80 /* Memory use counter */
81 SC_ATOMIC_DECLARE(uint64_t, ra_memuse);
82
83 /* prototypes */
84 TcpSegment *StreamTcpGetSegment(ThreadVars *tv, TcpReassemblyThreadCtx *);
85 void StreamTcpCreateTestPacket(uint8_t *, uint8_t, uint8_t, uint8_t);
86
StreamTcpReassembleInitMemuse(void)87 void StreamTcpReassembleInitMemuse(void)
88 {
89 SC_ATOMIC_INIT(ra_memuse);
90 }
91
92 /**
93 * \brief Function to Increment the memory usage counter for the TCP reassembly
94 * segments
95 *
96 * \param size Size of the TCP segment and its payload length memory allocated
97 */
StreamTcpReassembleIncrMemuse(uint64_t size)98 void StreamTcpReassembleIncrMemuse(uint64_t size)
99 {
100 (void) SC_ATOMIC_ADD(ra_memuse, size);
101 SCLogDebug("REASSEMBLY %"PRIu64", incr %"PRIu64, StreamTcpReassembleMemuseGlobalCounter(), size);
102 return;
103 }
104
105 /**
106 * \brief Function to Decrease the memory usage counter for the TCP reassembly
107 * segments
108 *
109 * \param size Size of the TCP segment and its payload length memory allocated
110 */
StreamTcpReassembleDecrMemuse(uint64_t size)111 void StreamTcpReassembleDecrMemuse(uint64_t size)
112 {
113 #ifdef UNITTESTS
114 uint64_t presize = SC_ATOMIC_GET(ra_memuse);
115 if (RunmodeIsUnittests()) {
116 BUG_ON(presize > UINT_MAX);
117 }
118 #endif
119
120 (void) SC_ATOMIC_SUB(ra_memuse, size);
121
122 #ifdef UNITTESTS
123 if (RunmodeIsUnittests()) {
124 uint64_t postsize = SC_ATOMIC_GET(ra_memuse);
125 BUG_ON(postsize > presize);
126 }
127 #endif
128 SCLogDebug("REASSEMBLY %"PRIu64", decr %"PRIu64, StreamTcpReassembleMemuseGlobalCounter(), size);
129 return;
130 }
131
StreamTcpReassembleMemuseGlobalCounter(void)132 uint64_t StreamTcpReassembleMemuseGlobalCounter(void)
133 {
134 uint64_t smemuse = SC_ATOMIC_GET(ra_memuse);
135 return smemuse;
136 }
137
138 /**
139 * \brief Function to Check the reassembly memory usage counter against the
140 * allowed max memory usgae for TCP segments.
141 *
142 * \param size Size of the TCP segment and its payload length memory allocated
143 * \retval 1 if in bounds
144 * \retval 0 if not in bounds
145 */
StreamTcpReassembleCheckMemcap(uint64_t size)146 int StreamTcpReassembleCheckMemcap(uint64_t size)
147 {
148 uint64_t memcapcopy = SC_ATOMIC_GET(stream_config.reassembly_memcap);
149 if (memcapcopy == 0 ||
150 (uint64_t)((uint64_t)size + SC_ATOMIC_GET(ra_memuse)) <= memcapcopy)
151 return 1;
152 return 0;
153 }
154
155 /**
156 * \brief Update memcap value
157 *
158 * \param size new memcap value
159 */
StreamTcpReassembleSetMemcap(uint64_t size)160 int StreamTcpReassembleSetMemcap(uint64_t size)
161 {
162 if (size == 0 || (uint64_t)SC_ATOMIC_GET(ra_memuse) < size) {
163 SC_ATOMIC_SET(stream_config.reassembly_memcap, size);
164 return 1;
165 }
166
167 return 0;
168 }
169
170 /**
171 * \brief Return memcap value
172 *
173 * \return memcap memcap value
174 */
StreamTcpReassembleGetMemcap()175 uint64_t StreamTcpReassembleGetMemcap()
176 {
177 uint64_t memcapcopy = SC_ATOMIC_GET(stream_config.reassembly_memcap);
178 return memcapcopy;
179 }
180
181 /* memory functions for the streaming buffer API */
182
183 /*
184 void *(*Malloc)(size_t size);
185 */
ReassembleMalloc(size_t size)186 static void *ReassembleMalloc(size_t size)
187 {
188 if (StreamTcpReassembleCheckMemcap(size) == 0)
189 return NULL;
190 void *ptr = SCMalloc(size);
191 if (ptr == NULL)
192 return NULL;
193 StreamTcpReassembleIncrMemuse(size);
194 return ptr;
195 }
196
197 /*
198 void *(*Calloc)(size_t n, size_t size);
199 */
ReassembleCalloc(size_t n,size_t size)200 static void *ReassembleCalloc(size_t n, size_t size)
201 {
202 if (StreamTcpReassembleCheckMemcap(n * size) == 0)
203 return NULL;
204 void *ptr = SCCalloc(n, size);
205 if (ptr == NULL)
206 return NULL;
207 StreamTcpReassembleIncrMemuse(n * size);
208 return ptr;
209 }
210
211 /*
212 void *(*Realloc)(void *ptr, size_t orig_size, size_t size);
213 */
ReassembleRealloc(void * optr,size_t orig_size,size_t size)214 static void *ReassembleRealloc(void *optr, size_t orig_size, size_t size)
215 {
216 if (size > orig_size) {
217 if (StreamTcpReassembleCheckMemcap(size - orig_size) == 0)
218 return NULL;
219 }
220 void *nptr = SCRealloc(optr, size);
221 if (nptr == NULL)
222 return NULL;
223
224 if (size > orig_size) {
225 StreamTcpReassembleIncrMemuse(size - orig_size);
226 } else {
227 StreamTcpReassembleDecrMemuse(orig_size - size);
228 }
229 return nptr;
230 }
231
232 /*
233 void (*Free)(void *ptr, size_t size);
234 */
ReassembleFree(void * ptr,size_t size)235 static void ReassembleFree(void *ptr, size_t size)
236 {
237 SCFree(ptr);
238 StreamTcpReassembleDecrMemuse(size);
239 }
240
241 /** \brief alloc a tcp segment pool entry */
TcpSegmentPoolAlloc(void)242 static void *TcpSegmentPoolAlloc(void)
243 {
244 if (StreamTcpReassembleCheckMemcap((uint32_t)sizeof(TcpSegment)) == 0) {
245 return NULL;
246 }
247
248 TcpSegment *seg = NULL;
249
250 seg = SCMalloc(sizeof (TcpSegment));
251 if (unlikely(seg == NULL))
252 return NULL;
253 return seg;
254 }
255
TcpSegmentPoolInit(void * data,void * initdata)256 static int TcpSegmentPoolInit(void *data, void *initdata)
257 {
258 TcpSegment *seg = (TcpSegment *) data;
259
260 /* do this before the can bail, so TcpSegmentPoolCleanup
261 * won't have uninitialized memory to consider. */
262 memset(seg, 0, sizeof (TcpSegment));
263
264 if (StreamTcpReassembleCheckMemcap((uint32_t)sizeof(TcpSegment)) == 0) {
265 return 0;
266 }
267
268 #ifdef DEBUG
269 SCMutexLock(&segment_pool_memuse_mutex);
270 segment_pool_memuse += sizeof(TcpSegment);
271 segment_pool_memcnt++;
272 SCLogDebug("segment_pool_memcnt %"PRIu64"", segment_pool_memcnt);
273 SCMutexUnlock(&segment_pool_memuse_mutex);
274 #endif
275
276 StreamTcpReassembleIncrMemuse((uint32_t)sizeof(TcpSegment));
277 return 1;
278 }
279
280 /** \brief clean up a tcp segment pool entry */
TcpSegmentPoolCleanup(void * ptr)281 static void TcpSegmentPoolCleanup(void *ptr)
282 {
283 if (ptr == NULL)
284 return;
285
286 StreamTcpReassembleDecrMemuse((uint32_t)sizeof(TcpSegment));
287
288 #ifdef DEBUG
289 SCMutexLock(&segment_pool_memuse_mutex);
290 segment_pool_memuse -= sizeof(TcpSegment);
291 segment_pool_memcnt--;
292 SCLogDebug("segment_pool_memcnt %"PRIu64"", segment_pool_memcnt);
293 SCMutexUnlock(&segment_pool_memuse_mutex);
294 #endif
295 }
296
297 /**
298 * \brief Function to return the segment back to the pool.
299 *
300 * \param seg Segment which will be returned back to the pool.
301 */
StreamTcpSegmentReturntoPool(TcpSegment * seg)302 void StreamTcpSegmentReturntoPool(TcpSegment *seg)
303 {
304 if (seg == NULL)
305 return;
306
307 PoolThreadReturn(segment_thread_pool, seg);
308 }
309
310 /**
311 * \brief return all segments in this stream into the pool(s)
312 *
313 * \param stream the stream to cleanup
314 */
StreamTcpReturnStreamSegments(TcpStream * stream)315 void StreamTcpReturnStreamSegments (TcpStream *stream)
316 {
317 TcpSegment *seg = NULL, *safe = NULL;
318 RB_FOREACH_SAFE(seg, TCPSEG, &stream->seg_tree, safe)
319 {
320 RB_REMOVE(TCPSEG, &stream->seg_tree, seg);
321 StreamTcpSegmentReturntoPool(seg);
322 }
323 }
324
325 #ifdef UNITTESTS
326 /** \internal
327 * \brief check if segments falls before stream 'offset' */
SEGMENT_BEFORE_OFFSET(TcpStream * stream,TcpSegment * seg,uint64_t offset)328 static inline int SEGMENT_BEFORE_OFFSET(TcpStream *stream, TcpSegment *seg, uint64_t offset)
329 {
330 if (seg->sbseg.stream_offset + seg->sbseg.segment_len <= offset)
331 return 1;
332 return 0;
333 }
334 #endif
335
336 /** \param f locked flow */
StreamTcpDisableAppLayer(Flow * f)337 void StreamTcpDisableAppLayer(Flow *f)
338 {
339 if (f->protoctx == NULL)
340 return;
341
342 TcpSession *ssn = (TcpSession *)f->protoctx;
343 StreamTcpSetStreamFlagAppProtoDetectionCompleted(&ssn->client);
344 StreamTcpSetStreamFlagAppProtoDetectionCompleted(&ssn->server);
345 StreamTcpDisableAppLayerReassembly(ssn);
346 if (f->alparser) {
347 AppLayerParserStateSetFlag(f->alparser,
348 (APP_LAYER_PARSER_EOF_TS|APP_LAYER_PARSER_EOF_TC));
349 }
350 }
351
352 /** \param f locked flow */
StreamTcpAppLayerIsDisabled(Flow * f)353 int StreamTcpAppLayerIsDisabled(Flow *f)
354 {
355 if (f->protoctx == NULL || f->proto != IPPROTO_TCP)
356 return 0;
357
358 TcpSession *ssn = (TcpSession *)f->protoctx;
359 return (ssn->flags & STREAMTCP_FLAG_APP_LAYER_DISABLED);
360 }
361
StreamTcpReassemblyConfig(char quiet)362 static int StreamTcpReassemblyConfig(char quiet)
363 {
364 uint32_t segment_prealloc = 2048;
365 ConfNode *seg = ConfGetNode("stream.reassembly.segment-prealloc");
366 if (seg) {
367 uint32_t prealloc = 0;
368 if (StringParseUint32(&prealloc, 10, strlen(seg->val), seg->val) < 0)
369 {
370 SCLogError(SC_ERR_INVALID_ARGUMENT, "segment-prealloc of "
371 "%s is invalid", seg->val);
372 return -1;
373 }
374 segment_prealloc = prealloc;
375 }
376 if (!quiet)
377 SCLogConfig("stream.reassembly \"segment-prealloc\": %u", segment_prealloc);
378 stream_config.prealloc_segments = segment_prealloc;
379
380 int overlap_diff_data = 0;
381 ConfGetBool("stream.reassembly.check-overlap-different-data", &overlap_diff_data);
382 if (overlap_diff_data) {
383 StreamTcpReassembleConfigEnableOverlapCheck();
384 }
385 if (StreamTcpInlineMode() == TRUE) {
386 StreamTcpReassembleConfigEnableOverlapCheck();
387 }
388
389 stream_config.sbcnf.flags = STREAMING_BUFFER_NOFLAGS;
390 stream_config.sbcnf.buf_size = 2048;
391 stream_config.sbcnf.Malloc = ReassembleMalloc;
392 stream_config.sbcnf.Calloc = ReassembleCalloc;
393 stream_config.sbcnf.Realloc = ReassembleRealloc;
394 stream_config.sbcnf.Free = ReassembleFree;
395
396 return 0;
397 }
398
StreamTcpReassembleInit(char quiet)399 int StreamTcpReassembleInit(char quiet)
400 {
401 /* init the memcap/use tracker */
402 StreamTcpReassembleInitMemuse();
403
404 if (StreamTcpReassemblyConfig(quiet) < 0)
405 return -1;
406
407 #ifdef DEBUG
408 SCMutexInit(&segment_pool_memuse_mutex, NULL);
409 #endif
410 StatsRegisterGlobalCounter("tcp.reassembly_memuse",
411 StreamTcpReassembleMemuseGlobalCounter);
412 return 0;
413 }
414
StreamTcpReassembleFree(char quiet)415 void StreamTcpReassembleFree(char quiet)
416 {
417 SCMutexLock(&segment_thread_pool_mutex);
418 if (segment_thread_pool != NULL) {
419 PoolThreadFree(segment_thread_pool);
420 segment_thread_pool = NULL;
421 }
422 SCMutexUnlock(&segment_thread_pool_mutex);
423 SCMutexDestroy(&segment_thread_pool_mutex);
424
425 #ifdef DEBUG
426 if (segment_pool_memuse > 0)
427 SCLogInfo("segment_pool_memuse %"PRIu64"", segment_pool_memuse);
428 if (segment_pool_memcnt > 0)
429 SCLogInfo("segment_pool_memcnt %"PRIu64"", segment_pool_memcnt);
430 SCMutexDestroy(&segment_pool_memuse_mutex);
431 #endif
432 }
433
StreamTcpReassembleInitThreadCtx(ThreadVars * tv)434 TcpReassemblyThreadCtx *StreamTcpReassembleInitThreadCtx(ThreadVars *tv)
435 {
436 SCEnter();
437 TcpReassemblyThreadCtx *ra_ctx = SCMalloc(sizeof(TcpReassemblyThreadCtx));
438 if (unlikely(ra_ctx == NULL))
439 return NULL;
440
441 memset(ra_ctx, 0x00, sizeof(TcpReassemblyThreadCtx));
442
443 ra_ctx->app_tctx = AppLayerGetCtxThread(tv);
444
445 SCMutexLock(&segment_thread_pool_mutex);
446 if (segment_thread_pool == NULL) {
447 segment_thread_pool = PoolThreadInit(1, /* thread */
448 0, /* unlimited */
449 stream_config.prealloc_segments,
450 sizeof(TcpSegment),
451 TcpSegmentPoolAlloc,
452 TcpSegmentPoolInit, NULL,
453 TcpSegmentPoolCleanup, NULL);
454 ra_ctx->segment_thread_pool_id = 0;
455 SCLogDebug("pool size %d, thread segment_thread_pool_id %d",
456 PoolThreadSize(segment_thread_pool),
457 ra_ctx->segment_thread_pool_id);
458 } else {
459 /* grow segment_thread_pool until we have a element for our thread id */
460 ra_ctx->segment_thread_pool_id = PoolThreadExpand(segment_thread_pool);
461 SCLogDebug("pool size %d, thread segment_thread_pool_id %d",
462 PoolThreadSize(segment_thread_pool),
463 ra_ctx->segment_thread_pool_id);
464 }
465 SCMutexUnlock(&segment_thread_pool_mutex);
466 if (ra_ctx->segment_thread_pool_id < 0 || segment_thread_pool == NULL) {
467 SCLogError(SC_ERR_MEM_ALLOC, "failed to setup/expand stream segment pool. Expand stream.reassembly.memcap?");
468 StreamTcpReassembleFreeThreadCtx(ra_ctx);
469 SCReturnPtr(NULL, "TcpReassemblyThreadCtx");
470 }
471
472 SCReturnPtr(ra_ctx, "TcpReassemblyThreadCtx");
473 }
474
StreamTcpReassembleFreeThreadCtx(TcpReassemblyThreadCtx * ra_ctx)475 void StreamTcpReassembleFreeThreadCtx(TcpReassemblyThreadCtx *ra_ctx)
476 {
477 SCEnter();
478 if (ra_ctx) {
479 AppLayerDestroyCtxThread(ra_ctx->app_tctx);
480 SCFree(ra_ctx);
481 }
482 SCReturn;
483 }
484
485 /**
486 * \brief check if stream in pkt direction has depth reached
487 *
488 * \param p packet with *LOCKED* flow
489 *
490 * \retval 1 stream has depth reached
491 * \retval 0 stream does not have depth reached
492 */
StreamTcpReassembleDepthReached(Packet * p)493 int StreamTcpReassembleDepthReached(Packet *p)
494 {
495 if (p->flow != NULL && p->flow->protoctx != NULL) {
496 TcpSession *ssn = p->flow->protoctx;
497 TcpStream *stream;
498 if (p->flowflags & FLOW_PKT_TOSERVER) {
499 stream = &ssn->client;
500 } else {
501 stream = &ssn->server;
502 }
503
504 return (stream->flags & STREAMTCP_STREAM_FLAG_DEPTH_REACHED) ? 1 : 0;
505 }
506
507 return 0;
508 }
509
510 /**
511 * \internal
512 * \brief Function to Check the reassembly depth valuer against the
513 * allowed max depth of the stream reassembly for TCP streams.
514 *
515 * \param stream stream direction
516 * \param seq sequence number where "size" starts
517 * \param size size of the segment that is added
518 *
519 * \retval size Part of the size that fits in the depth, 0 if none
520 */
StreamTcpReassembleCheckDepth(TcpSession * ssn,TcpStream * stream,uint32_t seq,uint32_t size)521 static uint32_t StreamTcpReassembleCheckDepth(TcpSession *ssn, TcpStream *stream,
522 uint32_t seq, uint32_t size)
523 {
524 SCEnter();
525
526 /* if the configured depth value is 0, it means there is no limit on
527 reassembly depth. Otherwise carry on my boy ;) */
528 if (ssn->reassembly_depth == 0) {
529 SCReturnUInt(size);
530 }
531
532 /* if the final flag is set, we're not accepting anymore */
533 if (stream->flags & STREAMTCP_STREAM_FLAG_DEPTH_REACHED) {
534 SCReturnUInt(0);
535 }
536
537 uint64_t seg_depth;
538 if (SEQ_GT(stream->base_seq, seq)) {
539 if (SEQ_LEQ(seq+size, stream->base_seq)) {
540 SCLogDebug("segment entirely before base_seq, weird: base %u, seq %u, re %u",
541 stream->base_seq, seq, seq+size);
542 SCReturnUInt(0);
543 }
544
545 seg_depth = STREAM_BASE_OFFSET(stream) + size - (stream->base_seq - seq);
546 } else {
547 seg_depth = STREAM_BASE_OFFSET(stream) + ((seq + size) - stream->base_seq);
548 }
549
550 /* if the base_seq has moved passed the depth window we stop
551 * checking and just reject the rest of the packets including
552 * retransmissions. Saves us the hassle of dealing with sequence
553 * wraps as well */
554 SCLogDebug("seq + size %u, base %u, seg_depth %"PRIu64" limit %u", (seq + size),
555 stream->base_seq, seg_depth,
556 ssn->reassembly_depth);
557
558 if (seg_depth > (uint64_t)ssn->reassembly_depth) {
559 SCLogDebug("STREAMTCP_STREAM_FLAG_DEPTH_REACHED");
560 stream->flags |= STREAMTCP_STREAM_FLAG_DEPTH_REACHED;
561 SCReturnUInt(0);
562 }
563 SCLogDebug("NOT STREAMTCP_STREAM_FLAG_DEPTH_REACHED");
564 SCLogDebug("%"PRIu64" <= %u", seg_depth, ssn->reassembly_depth);
565 #if 0
566 SCLogDebug("full depth not yet reached: %"PRIu64" <= %"PRIu32,
567 (stream->base_seq_offset + stream->base_seq + size),
568 (stream->isn + ssn->reassembly_depth));
569 #endif
570 if (SEQ_GEQ(seq, stream->isn) && SEQ_LT(seq, (stream->isn + ssn->reassembly_depth))) {
571 /* packet (partly?) fits the depth window */
572
573 if (SEQ_LEQ((seq + size),(stream->isn + 1 + ssn->reassembly_depth))) {
574 /* complete fit */
575 SCReturnUInt(size);
576 } else {
577 stream->flags |= STREAMTCP_STREAM_FLAG_DEPTH_REACHED;
578 /* partial fit, return only what fits */
579 uint32_t part = (stream->isn + 1 + ssn->reassembly_depth) - seq;
580 DEBUG_VALIDATE_BUG_ON(part > size);
581 if (part > size)
582 part = size;
583 SCReturnUInt(part);
584 }
585 }
586
587 SCReturnUInt(0);
588 }
589
StreamDataAvailableForProtoDetect(TcpStream * stream)590 uint32_t StreamDataAvailableForProtoDetect(TcpStream *stream)
591 {
592 if (RB_EMPTY(&stream->sb.sbb_tree)) {
593 if (stream->sb.stream_offset != 0)
594 return 0;
595
596 return stream->sb.buf_offset;
597 } else {
598 DEBUG_VALIDATE_BUG_ON(stream->sb.head == NULL);
599 DEBUG_VALIDATE_BUG_ON(stream->sb.sbb_size == 0);
600 return stream->sb.sbb_size;
601 }
602 }
603
604 /**
605 * \brief Insert a packets TCP data into the stream reassembly engine.
606 *
607 * \retval 0 good segment, as far as we checked.
608 * \retval -1 badness, reason to drop in inline mode
609 *
610 * If the retval is 0 the segment is inserted correctly, or overlap is handled,
611 * or it wasn't added because of reassembly depth.
612 *
613 */
StreamTcpReassembleHandleSegmentHandleData(ThreadVars * tv,TcpReassemblyThreadCtx * ra_ctx,TcpSession * ssn,TcpStream * stream,Packet * p)614 int StreamTcpReassembleHandleSegmentHandleData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx,
615 TcpSession *ssn, TcpStream *stream, Packet *p)
616 {
617 SCEnter();
618
619 if (ssn->data_first_seen_dir == 0) {
620 if (PKT_IS_TOSERVER(p)) {
621 ssn->data_first_seen_dir = STREAM_TOSERVER;
622 } else {
623 ssn->data_first_seen_dir = STREAM_TOCLIENT;
624 }
625 }
626
627 /* If the OS policy is not set then set the OS policy for this stream */
628 if (stream->os_policy == 0) {
629 StreamTcpSetOSPolicy(stream, p);
630 }
631
632 if ((ssn->flags & STREAMTCP_FLAG_APP_LAYER_DISABLED) &&
633 (stream->flags & STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED)) {
634 SCLogDebug("ssn %p: both app and raw reassembly disabled, not reassembling", ssn);
635 SCReturnInt(0);
636 }
637
638 /* If we have reached the defined depth for either of the stream, then stop
639 reassembling the TCP session */
640 uint32_t size = StreamTcpReassembleCheckDepth(ssn, stream, TCP_GET_SEQ(p), p->payload_len);
641 SCLogDebug("ssn %p: check depth returned %"PRIu32, ssn, size);
642
643 if (stream->flags & STREAMTCP_STREAM_FLAG_DEPTH_REACHED) {
644 /* increment stream depth counter */
645 StatsIncr(tv, ra_ctx->counter_tcp_stream_depth);
646 }
647 if (size == 0) {
648 SCLogDebug("ssn %p: depth reached, not reassembling", ssn);
649 SCReturnInt(0);
650 }
651
652 DEBUG_VALIDATE_BUG_ON(size > p->payload_len);
653 if (size > p->payload_len)
654 size = p->payload_len;
655
656 TcpSegment *seg = StreamTcpGetSegment(tv, ra_ctx);
657 if (seg == NULL) {
658 SCLogDebug("segment_pool is empty");
659 StreamTcpSetEvent(p, STREAM_REASSEMBLY_NO_SEGMENT);
660 SCReturnInt(-1);
661 }
662
663 TCP_SEG_LEN(seg) = size;
664 seg->seq = TCP_GET_SEQ(p);
665
666 /* HACK: for TFO SYN packets the seq for data starts at + 1 */
667 if (TCP_HAS_TFO(p) && p->payload_len && p->tcph->th_flags == TH_SYN)
668 seg->seq += 1;
669
670 /* proto detection skipped, but now we do get data. Set event. */
671 if (RB_EMPTY(&stream->seg_tree) &&
672 stream->flags & STREAMTCP_STREAM_FLAG_APPPROTO_DETECTION_SKIPPED) {
673
674 AppLayerDecoderEventsSetEventRaw(&p->app_layer_events,
675 APPLAYER_PROTO_DETECTION_SKIPPED);
676 }
677
678 if (StreamTcpReassembleInsertSegment(tv, ra_ctx, stream, seg, p, TCP_GET_SEQ(p), p->payload, p->payload_len) != 0) {
679 SCLogDebug("StreamTcpReassembleInsertSegment failed");
680 SCReturnInt(-1);
681 }
682 SCReturnInt(0);
683 }
684
StreamGetAppLayerFlags(TcpSession * ssn,TcpStream * stream,Packet * p)685 static uint8_t StreamGetAppLayerFlags(TcpSession *ssn, TcpStream *stream,
686 Packet *p)
687 {
688 uint8_t flag = 0;
689
690 if (!(stream->flags & STREAMTCP_STREAM_FLAG_APPPROTO_DETECTION_COMPLETED)) {
691 flag |= STREAM_START;
692 }
693
694 if (ssn->state == TCP_CLOSED) {
695 flag |= STREAM_EOF;
696 }
697
698 if (ssn->flags & STREAMTCP_FLAG_MIDSTREAM) {
699 flag |= STREAM_MIDSTREAM;
700 }
701
702 if (p->flags & PKT_PSEUDO_STREAM_END) {
703 flag |= STREAM_EOF;
704 }
705
706 if (&ssn->client == stream) {
707 flag |= STREAM_TOSERVER;
708 } else {
709 flag |= STREAM_TOCLIENT;
710 }
711 if (stream->flags & STREAMTCP_STREAM_FLAG_DEPTH_REACHED) {
712 flag |= STREAM_DEPTH;
713 }
714 return flag;
715 }
716
717 /**
718 * \brief Check the minimum size limits for reassembly.
719 *
720 * \retval 0 don't reassemble yet
721 * \retval 1 do reassemble
722 */
StreamTcpReassembleRawCheckLimit(const TcpSession * ssn,const TcpStream * stream,const Packet * p)723 static int StreamTcpReassembleRawCheckLimit(const TcpSession *ssn,
724 const TcpStream *stream, const Packet *p)
725 {
726 SCEnter();
727
728 /* if any of these flags is set we always inspect immediately */
729 #define STREAMTCP_STREAM_FLAG_FLUSH_FLAGS \
730 ( STREAMTCP_STREAM_FLAG_DEPTH_REACHED \
731 | STREAMTCP_STREAM_FLAG_TRIGGER_RAW \
732 | STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED)
733
734 if (stream->flags & STREAMTCP_STREAM_FLAG_FLUSH_FLAGS) {
735 if (stream->flags & STREAMTCP_STREAM_FLAG_DEPTH_REACHED) {
736 SCLogDebug("reassembling now as STREAMTCP_STREAM_FLAG_DEPTH_REACHED "
737 "is set, so not expecting any new data segments");
738 }
739 if (stream->flags & STREAMTCP_STREAM_FLAG_TRIGGER_RAW) {
740 SCLogDebug("reassembling now as STREAMTCP_STREAM_FLAG_TRIGGER_RAW is set");
741 }
742 if (stream->flags & STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED) {
743 SCLogDebug("reassembling now as STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED is set, "
744 "so no new segments will be considered");
745 }
746 SCReturnInt(1);
747 }
748 #undef STREAMTCP_STREAM_FLAG_FLUSH_FLAGS
749
750 /* some states mean we reassemble no matter how much data we have */
751 if (ssn->state > TCP_TIME_WAIT)
752 SCReturnInt(1);
753
754 if (p->flags & PKT_PSEUDO_STREAM_END)
755 SCReturnInt(1);
756
757 /* check if we have enough data to do raw reassembly */
758 if (PKT_IS_TOSERVER(p)) {
759 if (STREAM_LASTACK_GT_BASESEQ(stream)) {
760 uint32_t delta = stream->last_ack - stream->base_seq;
761 /* get max absolute offset */
762 uint64_t max_offset = STREAM_BASE_OFFSET(stream) + delta;
763
764 int64_t diff = max_offset - STREAM_RAW_PROGRESS(stream);
765 if ((int64_t)stream_config.reassembly_toserver_chunk_size <= diff) {
766 SCReturnInt(1);
767 } else {
768 SCLogDebug("toserver min chunk len not yet reached: "
769 "last_ack %"PRIu32", ra_raw_base_seq %"PRIu32", %"PRIu32" < "
770 "%"PRIu32"", stream->last_ack, stream->base_seq,
771 (stream->last_ack - stream->base_seq),
772 stream_config.reassembly_toserver_chunk_size);
773 SCReturnInt(0);
774 }
775 }
776 } else {
777 if (STREAM_LASTACK_GT_BASESEQ(stream)) {
778 uint32_t delta = stream->last_ack - stream->base_seq;
779 /* get max absolute offset */
780 uint64_t max_offset = STREAM_BASE_OFFSET(stream) + delta;
781
782 int64_t diff = max_offset - STREAM_RAW_PROGRESS(stream);
783
784 if ((int64_t)stream_config.reassembly_toclient_chunk_size <= diff) {
785 SCReturnInt(1);
786 } else {
787 SCLogDebug("toclient min chunk len not yet reached: "
788 "last_ack %"PRIu32", base_seq %"PRIu32", %"PRIu32" < "
789 "%"PRIu32"", stream->last_ack, stream->base_seq,
790 (stream->last_ack - stream->base_seq),
791 stream_config.reassembly_toclient_chunk_size);
792 SCReturnInt(0);
793 }
794 }
795 }
796
797 SCReturnInt(0);
798 }
799
800 /**
801 * \brief see what if any work the TCP session still needs
802 */
StreamNeedsReassembly(const TcpSession * ssn,uint8_t direction)803 int StreamNeedsReassembly(const TcpSession *ssn, uint8_t direction)
804 {
805 const TcpStream *stream = NULL;
806 #ifdef DEBUG
807 const char *dirstr = NULL;
808 #endif
809 if (direction == STREAM_TOSERVER) {
810 stream = &ssn->client;
811 #ifdef DEBUG
812 dirstr = "client";
813 #endif
814 } else {
815 stream = &ssn->server;
816 #ifdef DEBUG
817 dirstr = "server";
818 #endif
819 }
820
821 int use_app = 1;
822 int use_raw = 1;
823
824 if (ssn->flags & STREAMTCP_FLAG_APP_LAYER_DISABLED) {
825 // app is dead
826 use_app = 0;
827 }
828
829 if (stream->flags & STREAMTCP_STREAM_FLAG_DISABLE_RAW) {
830 // raw is dead
831 use_raw = 0;
832 }
833
834 uint64_t right_edge = STREAM_BASE_OFFSET(stream) + stream->sb.buf_offset;
835
836 SCLogDebug("%s: app %"PRIu64" (use: %s), raw %"PRIu64" (use: %s). Stream right edge: %"PRIu64,
837 dirstr,
838 STREAM_APP_PROGRESS(stream), use_app ? "yes" : "no",
839 STREAM_RAW_PROGRESS(stream), use_raw ? "yes" : "no",
840 right_edge);
841 if (use_raw) {
842 if (right_edge > STREAM_RAW_PROGRESS(stream)) {
843 SCLogDebug("%s: STREAM_HAS_UNPROCESSED_SEGMENTS_NEED_ONLY_DETECTION", dirstr);
844 return STREAM_HAS_UNPROCESSED_SEGMENTS_NEED_ONLY_DETECTION;
845 }
846 }
847 if (use_app) {
848 if (right_edge > STREAM_APP_PROGRESS(stream)) {
849 SCLogDebug("%s: STREAM_HAS_UNPROCESSED_SEGMENTS_NEED_ONLY_DETECTION", dirstr);
850 return STREAM_HAS_UNPROCESSED_SEGMENTS_NEED_ONLY_DETECTION;
851 }
852 }
853
854 SCLogDebug("%s: STREAM_HAS_UNPROCESSED_SEGMENTS_NONE", dirstr);
855 return STREAM_HAS_UNPROCESSED_SEGMENTS_NONE;
856 }
857
858 #ifdef DEBUG
GetStreamSize(TcpStream * stream)859 static uint64_t GetStreamSize(TcpStream *stream)
860 {
861 if (stream) {
862 uint64_t size = 0;
863 uint32_t cnt = 0;
864
865 TcpSegment *seg;
866 RB_FOREACH(seg, TCPSEG, &stream->seg_tree) {
867 cnt++;
868 size += (uint64_t)TCP_SEG_LEN(seg);
869 }
870
871 SCLogDebug("size %"PRIu64", cnt %"PRIu32, size, cnt);
872 return size;
873 }
874 return (uint64_t)0;
875 }
876
GetSessionSize(TcpSession * ssn,Packet * p)877 static void GetSessionSize(TcpSession *ssn, Packet *p)
878 {
879 uint64_t size = 0;
880 if (ssn) {
881 size = GetStreamSize(&ssn->client);
882 size += GetStreamSize(&ssn->server);
883
884 //if (size > 900000)
885 // SCLogInfo("size %"PRIu64", packet %"PRIu64, size, p->pcap_cnt);
886 SCLogDebug("size %"PRIu64", packet %"PRIu64, size, p->pcap_cnt);
887 }
888 }
889 #endif
890
GetBlock(StreamingBuffer * sb,const uint64_t offset)891 static StreamingBufferBlock *GetBlock(StreamingBuffer *sb, const uint64_t offset)
892 {
893 StreamingBufferBlock *blk = sb->head;
894 if (blk == NULL)
895 return NULL;
896
897 for ( ; blk != NULL; blk = SBB_RB_NEXT(blk)) {
898 if (blk->offset >= offset)
899 return blk;
900 else if ((blk->offset + blk->len) > offset) {
901 return blk;
902 }
903 }
904 return NULL;
905 }
906
GetAbsLastAck(const TcpStream * stream)907 static inline uint64_t GetAbsLastAck(const TcpStream *stream)
908 {
909 if (STREAM_LASTACK_GT_BASESEQ(stream)) {
910 return STREAM_BASE_OFFSET(stream) +
911 (stream->last_ack - stream->base_seq);
912 } else {
913 return STREAM_BASE_OFFSET(stream);
914 }
915 }
916
GapAhead(TcpStream * stream,StreamingBufferBlock * cur_blk)917 static inline bool GapAhead(TcpStream *stream, StreamingBufferBlock *cur_blk)
918 {
919 StreamingBufferBlock *nblk = SBB_RB_NEXT(cur_blk);
920 if (nblk && (cur_blk->offset + cur_blk->len < nblk->offset) &&
921 GetAbsLastAck(stream) >= (cur_blk->offset + cur_blk->len)) {
922 return true;
923 }
924 return false;
925 }
926
927 /** \internal
928 *
929 * Get buffer, or first part of the buffer if data gaps exist.
930 *
931 * \brief get stream data from offset
932 * \param offset stream offset
933 * \param check_for_gap check if there is a gap ahead. Optional as it is only
934 * needed for app-layer incomplete support.
935 * \retval bool pkt loss ahead */
GetAppBuffer(TcpStream * stream,const uint8_t ** data,uint32_t * data_len,uint64_t offset,const bool check_for_gap)936 static bool GetAppBuffer(TcpStream *stream, const uint8_t **data, uint32_t *data_len,
937 uint64_t offset, const bool check_for_gap)
938 {
939 const uint8_t *mydata;
940 uint32_t mydata_len;
941 bool gap_ahead = false;
942
943 if (RB_EMPTY(&stream->sb.sbb_tree)) {
944 SCLogDebug("getting one blob");
945
946 StreamingBufferGetDataAtOffset(&stream->sb, &mydata, &mydata_len, offset);
947
948 *data = mydata;
949 *data_len = mydata_len;
950 } else {
951 StreamingBufferBlock *blk = GetBlock(&stream->sb, offset);
952 if (blk == NULL) {
953 *data = NULL;
954 *data_len = 0;
955 return false;
956 }
957
958 /* block at expected offset */
959 if (blk->offset == offset) {
960
961 StreamingBufferSBBGetData(&stream->sb, blk, data, data_len);
962
963 gap_ahead = check_for_gap && GapAhead(stream, blk);
964
965 /* block past out offset */
966 } else if (blk->offset > offset) {
967 SCLogDebug("gap, want data at offset %"PRIu64", "
968 "got data at %"PRIu64". GAP of size %"PRIu64,
969 offset, blk->offset, blk->offset - offset);
970 *data = NULL;
971 *data_len = blk->offset - offset;
972
973 /* block starts before offset, but ends after */
974 } else if (offset > blk->offset && offset <= (blk->offset + blk->len)) {
975 SCLogDebug("get data from offset %"PRIu64". SBB %"PRIu64"/%u",
976 offset, blk->offset, blk->len);
977 StreamingBufferSBBGetDataAtOffset(&stream->sb, blk, data, data_len, offset);
978 SCLogDebug("data %p, data_len %u", *data, *data_len);
979
980 gap_ahead = check_for_gap && GapAhead(stream, blk);
981
982 } else {
983 *data = NULL;
984 *data_len = 0;
985 }
986 }
987 return gap_ahead;
988 }
989
990 /** \internal
991 * \brief check to see if we should declare a GAP
992 * Call this when the app layer didn't get data at the requested
993 * offset.
994 */
CheckGap(TcpSession * ssn,TcpStream * stream,Packet * p)995 static inline bool CheckGap(TcpSession *ssn, TcpStream *stream, Packet *p)
996 {
997 const uint64_t app_progress = STREAM_APP_PROGRESS(stream);
998 uint64_t last_ack_abs = STREAM_BASE_OFFSET(stream);
999
1000 if (STREAM_LASTACK_GT_BASESEQ(stream)) {
1001 /* get window of data that is acked */
1002 const uint32_t delta = stream->last_ack - stream->base_seq;
1003 /* get max absolute offset */
1004 last_ack_abs += delta;
1005
1006 const int ackadded = (ssn->state >= TCP_FIN_WAIT1) ? 1 : 0;
1007 last_ack_abs -= ackadded;
1008
1009 SCLogDebug("last_ack %u abs %"PRIu64, stream->last_ack, last_ack_abs);
1010 SCLogDebug("next_seq %u", stream->next_seq);
1011
1012 /* if last_ack_abs is beyond the app_progress data that we haven't seen
1013 * has been ack'd. This looks like a GAP. */
1014 if (last_ack_abs > app_progress) {
1015 /* however, we can accept ACKs a bit too liberally. If last_ack
1016 * is beyond next_seq, we only consider it a gap now if we do
1017 * already have data beyond the gap. */
1018 if (SEQ_GT(stream->last_ack, stream->next_seq)) {
1019 if (RB_EMPTY(&stream->sb.sbb_tree)) {
1020 SCLogDebug("packet %"PRIu64": no GAP. "
1021 "next_seq %u < last_ack %u, but no data in list",
1022 p->pcap_cnt, stream->next_seq, stream->last_ack);
1023 return false;
1024 } else {
1025 const uint64_t next_seq_abs = STREAM_BASE_OFFSET(stream) + (stream->next_seq - stream->base_seq);
1026 const StreamingBufferBlock *blk = stream->sb.head;
1027 if (blk->offset > next_seq_abs && blk->offset < last_ack_abs) {
1028 /* ack'd data after the gap */
1029 SCLogDebug("packet %"PRIu64": GAP. "
1030 "next_seq %u < last_ack %u, but ACK'd data beyond gap.",
1031 p->pcap_cnt, stream->next_seq, stream->last_ack);
1032 return true;
1033 }
1034 }
1035 }
1036
1037 SCLogDebug("packet %"PRIu64": GAP! "
1038 "last_ack_abs %"PRIu64" > app_progress %"PRIu64", "
1039 "but we have no data.",
1040 p->pcap_cnt, last_ack_abs, app_progress);
1041 return true;
1042 }
1043 }
1044 SCLogDebug("packet %"PRIu64": no GAP. "
1045 "last_ack_abs %"PRIu64" <= app_progress %"PRIu64,
1046 p->pcap_cnt, last_ack_abs, app_progress);
1047 return false;
1048 }
1049
AdjustToAcked(const Packet * p,const TcpSession * ssn,const TcpStream * stream,const uint64_t app_progress,const uint32_t data_len)1050 static inline uint32_t AdjustToAcked(const Packet *p,
1051 const TcpSession *ssn, const TcpStream *stream,
1052 const uint64_t app_progress, const uint32_t data_len)
1053 {
1054 uint32_t adjusted = data_len;
1055
1056 /* get window of data that is acked */
1057 if (StreamTcpInlineMode() == FALSE) {
1058 SCLogDebug("ssn->state %s", StreamTcpStateAsString(ssn->state));
1059 if (data_len == 0 || ((ssn->state < TCP_CLOSED ||
1060 (ssn->state == TCP_CLOSED &&
1061 (ssn->flags & STREAMTCP_FLAG_CLOSED_BY_RST) != 0)) &&
1062 (p->flags & PKT_PSEUDO_STREAM_END))) {
1063 // fall through, we use all available data
1064 } else {
1065 uint64_t last_ack_abs = STREAM_BASE_OFFSET(stream);
1066 if (STREAM_LASTACK_GT_BASESEQ(stream)) {
1067 /* get window of data that is acked */
1068 uint32_t delta = stream->last_ack - stream->base_seq;
1069 /* get max absolute offset */
1070 last_ack_abs += delta;
1071 }
1072 DEBUG_VALIDATE_BUG_ON(app_progress > last_ack_abs);
1073
1074 /* see if the buffer contains unack'd data as well */
1075 if (app_progress <= last_ack_abs && app_progress + data_len > last_ack_abs) {
1076 uint32_t check = data_len;
1077 adjusted = last_ack_abs - app_progress;
1078 BUG_ON(adjusted > check);
1079 SCLogDebug("data len adjusted to %u to make sure only ACK'd "
1080 "data is considered", adjusted);
1081 }
1082 }
1083 }
1084 return adjusted;
1085 }
1086
1087 /** \internal
1088 * \brief get stream buffer and update the app-layer
1089 * \param stream pointer to pointer as app-layer can switch flow dir
1090 * \retval 0 success
1091 */
ReassembleUpdateAppLayer(ThreadVars * tv,TcpReassemblyThreadCtx * ra_ctx,TcpSession * ssn,TcpStream ** stream,Packet * p,enum StreamUpdateDir dir)1092 static int ReassembleUpdateAppLayer (ThreadVars *tv,
1093 TcpReassemblyThreadCtx *ra_ctx,
1094 TcpSession *ssn, TcpStream **stream,
1095 Packet *p, enum StreamUpdateDir dir)
1096 {
1097 uint64_t app_progress = STREAM_APP_PROGRESS(*stream);
1098
1099 SCLogDebug("app progress %"PRIu64, app_progress);
1100 SCLogDebug("last_ack %u, base_seq %u", (*stream)->last_ack, (*stream)->base_seq);
1101
1102 const uint8_t *mydata;
1103 uint32_t mydata_len;
1104 bool gap_ahead = false;
1105 bool last_was_gap = false;
1106
1107 while (1) {
1108 const uint8_t flags = StreamGetAppLayerFlags(ssn, *stream, p);
1109 bool check_for_gap_ahead = ((*stream)->data_required > 0);
1110 gap_ahead = GetAppBuffer(*stream, &mydata, &mydata_len,
1111 app_progress, check_for_gap_ahead);
1112 if (last_was_gap && mydata_len == 0) {
1113 break;
1114 }
1115 last_was_gap = false;
1116
1117 /* make sure to only deal with ACK'd data */
1118 mydata_len = AdjustToAcked(p, ssn, *stream, app_progress, mydata_len);
1119 DEBUG_VALIDATE_BUG_ON(mydata_len > (uint32_t)INT_MAX);
1120 if (mydata == NULL && mydata_len > 0 && CheckGap(ssn, *stream, p)) {
1121 SCLogDebug("sending GAP to app-layer (size: %u)", mydata_len);
1122
1123 int r = AppLayerHandleTCPData(tv, ra_ctx, p, p->flow, ssn, stream,
1124 NULL, mydata_len,
1125 StreamGetAppLayerFlags(ssn, *stream, p)|STREAM_GAP);
1126 AppLayerProfilingStore(ra_ctx->app_tctx, p);
1127
1128 StreamTcpSetEvent(p, STREAM_REASSEMBLY_SEQ_GAP);
1129 StatsIncr(tv, ra_ctx->counter_tcp_reass_gap);
1130
1131 /* AppLayerHandleTCPData has likely updated progress. */
1132 const bool no_progress_update = (app_progress == STREAM_APP_PROGRESS(*stream));
1133 app_progress = STREAM_APP_PROGRESS(*stream);
1134
1135 /* a GAP also consumes 'data required'. TODO perhaps we can use
1136 * this to skip post GAP data until the start of a next record. */
1137 if ((*stream)->data_required > 0) {
1138 if ((*stream)->data_required > mydata_len) {
1139 (*stream)->data_required -= mydata_len;
1140 } else {
1141 (*stream)->data_required = 0;
1142 }
1143 }
1144 if (r < 0)
1145 return 0;
1146 if (no_progress_update)
1147 break;
1148 last_was_gap = true;
1149 continue;
1150
1151 } else if (flags & STREAM_DEPTH) {
1152 // we're just called once with this flag, so make sure we pass it on
1153 if (mydata == NULL && mydata_len > 0) {
1154 mydata_len = 0;
1155 }
1156 } else if (mydata == NULL || (mydata_len == 0 && ((flags & STREAM_EOF) == 0))) {
1157 /* Possibly a gap, but no new data. */
1158 if ((p->flags & PKT_PSEUDO_STREAM_END) == 0 || ssn->state < TCP_CLOSED)
1159 SCReturnInt(0);
1160
1161 mydata = NULL;
1162 mydata_len = 0;
1163 SCLogDebug("%"PRIu64" got %p/%u", p->pcap_cnt, mydata, mydata_len);
1164 break;
1165 }
1166 DEBUG_VALIDATE_BUG_ON(mydata == NULL && mydata_len > 0);
1167
1168 SCLogDebug("stream %p data in buffer %p of len %u and offset %"PRIu64,
1169 *stream, &(*stream)->sb, mydata_len, app_progress);
1170
1171 if ((p->flags & PKT_PSEUDO_STREAM_END) == 0 || ssn->state < TCP_CLOSED) {
1172 if (mydata_len < (*stream)->data_required) {
1173 if (gap_ahead) {
1174 SCLogDebug("GAP while expecting more data (expect %u, gap size %u)",
1175 (*stream)->data_required, mydata_len);
1176 (*stream)->app_progress_rel += mydata_len;
1177 (*stream)->data_required -= mydata_len;
1178 // TODO send incomplete data to app-layer with special flag
1179 // indicating its all there is for this rec?
1180 } else {
1181 SCReturnInt(0);
1182 }
1183 app_progress = STREAM_APP_PROGRESS(*stream);
1184 continue;
1185 }
1186 }
1187 (*stream)->data_required = 0;
1188
1189 /* update the app-layer */
1190 (void)AppLayerHandleTCPData(tv, ra_ctx, p, p->flow, ssn, stream,
1191 (uint8_t *)mydata, mydata_len, flags);
1192 AppLayerProfilingStore(ra_ctx->app_tctx, p);
1193 uint64_t new_app_progress = STREAM_APP_PROGRESS(*stream);
1194 if (new_app_progress == app_progress || FlowChangeProto(p->flow))
1195 break;
1196 app_progress = new_app_progress;
1197 if (flags & STREAM_DEPTH)
1198 break;
1199 }
1200
1201 SCReturnInt(0);
1202 }
1203
1204 /**
1205 * \brief Update the stream reassembly upon receiving a packet.
1206 *
1207 * For IDS mode, the stream is in the opposite direction of the packet,
1208 * as the ACK-packet is ACK'ing the stream.
1209 *
1210 * One of the utilities call by this function AppLayerHandleTCPData(),
1211 * has a feature where it will call this very same function for the
1212 * stream opposing the stream it is called with. This shouldn't cause
1213 * any issues, since processing of each stream is independent of the
1214 * other stream.
1215 */
StreamTcpReassembleAppLayer(ThreadVars * tv,TcpReassemblyThreadCtx * ra_ctx,TcpSession * ssn,TcpStream * stream,Packet * p,enum StreamUpdateDir dir)1216 int StreamTcpReassembleAppLayer (ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx,
1217 TcpSession *ssn, TcpStream *stream,
1218 Packet *p, enum StreamUpdateDir dir)
1219 {
1220 SCEnter();
1221
1222 /* this function can be directly called by app layer protocol
1223 * detection. */
1224 if ((ssn->flags & STREAMTCP_FLAG_APP_LAYER_DISABLED) ||
1225 (stream->flags & STREAMTCP_STREAM_FLAG_NOREASSEMBLY)) {
1226 SCLogDebug("stream no reassembly flag set or app-layer disabled.");
1227 SCReturnInt(0);
1228 }
1229
1230 #ifdef DEBUG
1231 SCLogDebug("stream->seg_tree RB_MIN %p", RB_MIN(TCPSEG, &stream->seg_tree));
1232 GetSessionSize(ssn, p);
1233 #endif
1234 /* if no segments are in the list or all are already processed,
1235 * and state is beyond established, we send an empty msg */
1236 if (!STREAM_HAS_SEEN_DATA(stream) || STREAM_RIGHT_EDGE(stream) <= STREAM_APP_PROGRESS(stream))
1237 {
1238 /* send an empty EOF msg if we have no segments but TCP state
1239 * is beyond ESTABLISHED */
1240 if (ssn->state >= TCP_CLOSING || (p->flags & PKT_PSEUDO_STREAM_END)) {
1241 SCLogDebug("sending empty eof message");
1242 /* send EOF to app layer */
1243 AppLayerHandleTCPData(tv, ra_ctx, p, p->flow, ssn, &stream,
1244 NULL, 0,
1245 StreamGetAppLayerFlags(ssn, stream, p));
1246 AppLayerProfilingStore(ra_ctx->app_tctx, p);
1247
1248 SCReturnInt(0);
1249 }
1250 }
1251
1252 /* with all that out of the way, lets update the app-layer */
1253 return ReassembleUpdateAppLayer(tv, ra_ctx, ssn, &stream, p, dir);
1254 }
1255
1256 /** \internal
1257 * \brief get stream data from offset
1258 * \param offset stream offset */
GetRawBuffer(TcpStream * stream,const uint8_t ** data,uint32_t * data_len,StreamingBufferBlock ** iter,uint64_t offset,uint64_t * data_offset)1259 static int GetRawBuffer(TcpStream *stream, const uint8_t **data, uint32_t *data_len,
1260 StreamingBufferBlock **iter, uint64_t offset, uint64_t *data_offset)
1261 {
1262 const uint8_t *mydata;
1263 uint32_t mydata_len;
1264 if (RB_EMPTY(&stream->sb.sbb_tree)) {
1265 SCLogDebug("getting one blob for offset %"PRIu64, offset);
1266
1267 uint64_t roffset = offset;
1268 if (offset)
1269 StreamingBufferGetDataAtOffset(&stream->sb, &mydata, &mydata_len, offset);
1270 else {
1271 StreamingBufferGetData(&stream->sb, &mydata, &mydata_len, &roffset);
1272 }
1273
1274 *data = mydata;
1275 *data_len = mydata_len;
1276 *data_offset = roffset;
1277 } else {
1278 SCLogDebug("multiblob %s. Want offset %"PRIu64,
1279 *iter == NULL ? "starting" : "continuing", offset);
1280 if (*iter == NULL) {
1281 StreamingBufferBlock key = { .offset = offset, .len = 0 };
1282 *iter = SBB_RB_FIND_INCLUSIVE(&stream->sb.sbb_tree, &key);
1283 SCLogDebug("*iter %p", *iter);
1284 }
1285 if (*iter == NULL) {
1286 SCLogDebug("no data");
1287 *data = NULL;
1288 *data_len = 0;
1289 *data_offset = 0;
1290 return 0;
1291 }
1292 SCLogDebug("getting multiple blobs. Iter %p, %"PRIu64"/%u", *iter, (*iter)->offset, (*iter)->len);
1293
1294 StreamingBufferSBBGetData(&stream->sb, (*iter), &mydata, &mydata_len);
1295 SCLogDebug("mydata %p", mydata);
1296
1297 if ((*iter)->offset < offset) {
1298 uint64_t delta = offset - (*iter)->offset;
1299 if (delta < mydata_len) {
1300 *data = mydata + delta;
1301 *data_len = mydata_len - delta;
1302 *data_offset = offset;
1303 } else {
1304 SCLogDebug("no data (yet)");
1305 *data = NULL;
1306 *data_len = 0;
1307 *data_offset = 0;
1308 }
1309
1310 } else {
1311 *data = mydata;
1312 *data_len = mydata_len;
1313 *data_offset = (*iter)->offset;
1314 }
1315
1316 *iter = SBB_RB_NEXT(*iter);
1317 SCLogDebug("*iter %p", *iter);
1318 }
1319 return 0;
1320 }
1321
1322 /** \brief does the stream engine have data to inspect?
1323 *
1324 * Returns true if there is data to inspect. In IDS case this is
1325 * about ACK'd data in the packet's direction.
1326 *
1327 * In the IPS case this is about the packet itself.
1328 */
StreamReassembleRawHasDataReady(TcpSession * ssn,Packet * p)1329 bool StreamReassembleRawHasDataReady(TcpSession *ssn, Packet *p)
1330 {
1331 TcpStream *stream;
1332 if (PKT_IS_TOSERVER(p)) {
1333 stream = &ssn->client;
1334 } else {
1335 stream = &ssn->server;
1336 }
1337
1338 if (RB_EMPTY(&stream->seg_tree)) {
1339 return false;
1340 }
1341
1342 if (stream->flags & (STREAMTCP_STREAM_FLAG_NOREASSEMBLY|
1343 STREAMTCP_STREAM_FLAG_DISABLE_RAW))
1344 return false;
1345
1346 if (StreamTcpInlineMode() == FALSE) {
1347 if ((STREAM_RAW_PROGRESS(stream) == STREAM_BASE_OFFSET(stream) + stream->sb.buf_offset)) {
1348 return false;
1349 }
1350 if (StreamTcpReassembleRawCheckLimit(ssn, stream, p) == 1) {
1351 return true;
1352 }
1353 } else {
1354 if (p->payload_len > 0 && (p->flags & PKT_STREAM_ADD)) {
1355 return true;
1356 }
1357 }
1358 return false;
1359 }
1360
1361 /** \brief update stream engine after detection
1362 *
1363 * Tasked with progressing the 'progress' for Raw reassembly.
1364 * 2 main scenario's:
1365 * 1. progress is != 0, so we use this
1366 * 2. progress is 0, meaning the detect engine didn't touch
1367 * raw at all. In this case we need to look into progressing
1368 * raw anyway.
1369 *
1370 * Additionally, this function is tasked with disabling raw
1371 * reassembly if the app-layer requested to disable it.
1372 */
StreamReassembleRawUpdateProgress(TcpSession * ssn,Packet * p,uint64_t progress)1373 void StreamReassembleRawUpdateProgress(TcpSession *ssn, Packet *p, uint64_t progress)
1374 {
1375 TcpStream *stream;
1376 if (PKT_IS_TOSERVER(p)) {
1377 stream = &ssn->client;
1378 } else {
1379 stream = &ssn->server;
1380 }
1381
1382 if (progress > STREAM_RAW_PROGRESS(stream)) {
1383 uint32_t slide = progress - STREAM_RAW_PROGRESS(stream);
1384 stream->raw_progress_rel += slide;
1385 stream->flags &= ~STREAMTCP_STREAM_FLAG_TRIGGER_RAW;
1386
1387 /* if app is active and beyond raw, sync raw to app */
1388 } else if (progress == 0 && STREAM_APP_PROGRESS(stream) > STREAM_RAW_PROGRESS(stream) &&
1389 !(ssn->flags & STREAMTCP_FLAG_APP_LAYER_DISABLED)) {
1390 /* if trigger raw is set we sync the 2 trackers */
1391 if (stream->flags & STREAMTCP_STREAM_FLAG_TRIGGER_RAW)
1392 {
1393 uint32_t slide = STREAM_APP_PROGRESS(stream) - STREAM_RAW_PROGRESS(stream);
1394 stream->raw_progress_rel += slide;
1395 stream->flags &= ~STREAMTCP_STREAM_FLAG_TRIGGER_RAW;
1396
1397 /* otherwise mix in the tcp window */
1398 } else {
1399 uint64_t tcp_window = stream->window;
1400 if (tcp_window > 0 && STREAM_APP_PROGRESS(stream) > tcp_window) {
1401 uint64_t new_raw = STREAM_APP_PROGRESS(stream) - tcp_window;
1402 if (new_raw > STREAM_RAW_PROGRESS(stream)) {
1403 uint32_t slide = new_raw - STREAM_RAW_PROGRESS(stream);
1404 stream->raw_progress_rel += slide;
1405 }
1406 }
1407 }
1408 /* app is dead */
1409 } else if (progress == 0) {
1410 uint64_t tcp_window = stream->window;
1411 uint64_t stream_right_edge = STREAM_BASE_OFFSET(stream) + stream->sb.buf_offset;
1412 if (tcp_window < stream_right_edge) {
1413 uint64_t new_raw = stream_right_edge - tcp_window;
1414 if (new_raw > STREAM_RAW_PROGRESS(stream)) {
1415 uint32_t slide = new_raw - STREAM_RAW_PROGRESS(stream);
1416 stream->raw_progress_rel += slide;
1417 }
1418 }
1419 stream->flags &= ~STREAMTCP_STREAM_FLAG_TRIGGER_RAW;
1420
1421 } else {
1422 SCLogDebug("p->pcap_cnt %"PRIu64": progress %"PRIu64" app %"PRIu64" raw %"PRIu64" tcp win %"PRIu32,
1423 p->pcap_cnt, progress, STREAM_APP_PROGRESS(stream),
1424 STREAM_RAW_PROGRESS(stream), stream->window);
1425 }
1426
1427 /* if we were told to accept no more raw data, we can mark raw as
1428 * disabled now. */
1429 if (stream->flags & STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED) {
1430 stream->flags |= STREAMTCP_STREAM_FLAG_DISABLE_RAW;
1431 SCLogDebug("ssn %p: STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED set, "
1432 "now that detect ran also set STREAMTCP_STREAM_FLAG_DISABLE_RAW", ssn);
1433 }
1434
1435 SCLogDebug("stream raw progress now %"PRIu64, STREAM_RAW_PROGRESS(stream));
1436 }
1437
1438 /** \internal
1439 * \brief get a buffer around the current packet and run the callback on it
1440 *
1441 * The inline/IPS scanning method takes the current payload and wraps it in
1442 * data from other segments.
1443 *
1444 * How much data is inspected is controlled by the available data, chunk_size
1445 * and the payload size of the packet.
1446 *
1447 * Large packets: if payload size is close to the chunk_size, where close is
1448 * defined as more than 67% of the chunk_size, a larger chunk_size will be
1449 * used: payload_len + 33% of the chunk_size.
1450 * If the payload size if equal to or bigger than the chunk_size, we use
1451 * payload len + 33% of the chunk size.
1452 */
StreamReassembleRawInline(TcpSession * ssn,const Packet * p,StreamReassembleRawFunc Callback,void * cb_data,uint64_t * progress_out)1453 static int StreamReassembleRawInline(TcpSession *ssn, const Packet *p,
1454 StreamReassembleRawFunc Callback, void *cb_data, uint64_t *progress_out)
1455 {
1456 SCEnter();
1457 int r = 0;
1458
1459 TcpStream *stream;
1460 if (PKT_IS_TOSERVER(p)) {
1461 stream = &ssn->client;
1462 } else {
1463 stream = &ssn->server;
1464 }
1465
1466 if (p->payload_len == 0 || (p->flags & PKT_STREAM_ADD) == 0 ||
1467 (stream->flags & STREAMTCP_STREAM_FLAG_NOREASSEMBLY))
1468 {
1469 *progress_out = STREAM_RAW_PROGRESS(stream);
1470 return 0;
1471 }
1472
1473 uint32_t chunk_size = PKT_IS_TOSERVER(p) ?
1474 stream_config.reassembly_toserver_chunk_size :
1475 stream_config.reassembly_toclient_chunk_size;
1476 if (chunk_size <= p->payload_len) {
1477 chunk_size = p->payload_len + (chunk_size / 3);
1478 SCLogDebug("packet payload len %u, so chunk_size adjusted to %u",
1479 p->payload_len, chunk_size);
1480 } else if (((chunk_size / 3 ) * 2) < p->payload_len) {
1481 chunk_size = p->payload_len + ((chunk_size / 3));
1482 SCLogDebug("packet payload len %u, so chunk_size adjusted to %u",
1483 p->payload_len, chunk_size);
1484 }
1485
1486 uint64_t packet_leftedge_abs = STREAM_BASE_OFFSET(stream) + (TCP_GET_SEQ(p) - stream->base_seq);
1487 uint64_t packet_rightedge_abs = packet_leftedge_abs + p->payload_len;
1488 SCLogDebug("packet_leftedge_abs %"PRIu64", rightedge %"PRIu64,
1489 packet_leftedge_abs, packet_rightedge_abs);
1490
1491 const uint8_t *mydata = NULL;
1492 uint32_t mydata_len = 0;
1493 uint64_t mydata_offset = 0;
1494 /* simply return progress from the block we inspected. */
1495 bool return_progress = false;
1496
1497 if (RB_EMPTY(&stream->sb.sbb_tree)) {
1498 /* continues block */
1499 StreamingBufferGetData(&stream->sb, &mydata, &mydata_len, &mydata_offset);
1500 return_progress = true;
1501
1502 } else {
1503 SCLogDebug("finding our SBB from offset %"PRIu64, packet_leftedge_abs);
1504 /* find our block */
1505 StreamingBufferBlock key = { .offset = packet_leftedge_abs, .len = p->payload_len };
1506 StreamingBufferBlock *sbb = SBB_RB_FIND_INCLUSIVE(&stream->sb.sbb_tree, &key);
1507 if (sbb) {
1508 SCLogDebug("found %p offset %"PRIu64" len %u", sbb, sbb->offset, sbb->len);
1509 StreamingBufferSBBGetData(&stream->sb, sbb, &mydata, &mydata_len);
1510 mydata_offset = sbb->offset;
1511 }
1512 }
1513
1514 /* this can only happen if the segment insert of our current 'p' failed */
1515 uint64_t mydata_rightedge_abs = mydata_offset + mydata_len;
1516 if ((mydata == NULL || mydata_len == 0) || /* no data */
1517 (mydata_offset >= packet_rightedge_abs || /* data all to the right */
1518 packet_leftedge_abs >= mydata_rightedge_abs) || /* data all to the left */
1519 (packet_leftedge_abs < mydata_offset || /* data missing at the start */
1520 packet_rightedge_abs > mydata_rightedge_abs)) /* data missing at the end */
1521 {
1522 /* no data, or data is incomplete or wrong: use packet data */
1523 mydata = p->payload;
1524 mydata_len = p->payload_len;
1525 mydata_offset = packet_leftedge_abs;
1526 //mydata_rightedge_abs = packet_rightedge_abs;
1527 } else {
1528 /* adjust buffer to match chunk_size */
1529 SCLogDebug("chunk_size %u mydata_len %u", chunk_size, mydata_len);
1530 if (mydata_len > chunk_size) {
1531 uint32_t excess = mydata_len - chunk_size;
1532 SCLogDebug("chunk_size %u mydata_len %u excess %u", chunk_size, mydata_len, excess);
1533
1534 if (mydata_rightedge_abs == packet_rightedge_abs) {
1535 mydata += excess;
1536 mydata_len -= excess;
1537 mydata_offset += excess;
1538 SCLogDebug("cutting front of the buffer with %u", excess);
1539 } else if (mydata_offset == packet_leftedge_abs) {
1540 mydata_len -= excess;
1541 SCLogDebug("cutting tail of the buffer with %u", excess);
1542 } else {
1543 uint32_t before = (uint32_t)(packet_leftedge_abs - mydata_offset);
1544 uint32_t after = (uint32_t)(mydata_rightedge_abs - packet_rightedge_abs);
1545 SCLogDebug("before %u after %u", before, after);
1546
1547 if (after >= (chunk_size - p->payload_len) / 2) {
1548 // more trailing data than we need
1549
1550 if (before >= (chunk_size - p->payload_len) / 2) {
1551 // also more heading data, divide evenly
1552 before = after = (chunk_size - p->payload_len) / 2;
1553 } else {
1554 // heading data is less than requested, give the
1555 // rest to the trailing data
1556 after = (chunk_size - p->payload_len) - before;
1557 }
1558 } else {
1559 // less trailing data than requested
1560
1561 if (before >= (chunk_size - p->payload_len) / 2) {
1562 before = (chunk_size - p->payload_len) - after;
1563 } else {
1564 // both smaller than their requested size
1565 }
1566 }
1567
1568 /* adjust the buffer */
1569 uint32_t skip = (uint32_t)(packet_leftedge_abs - mydata_offset) - before;
1570 uint32_t cut = (uint32_t)(mydata_rightedge_abs - packet_rightedge_abs) - after;
1571 DEBUG_VALIDATE_BUG_ON(skip > mydata_len);
1572 DEBUG_VALIDATE_BUG_ON(cut > mydata_len);
1573 DEBUG_VALIDATE_BUG_ON(skip + cut > mydata_len);
1574 mydata += skip;
1575 mydata_len -= (skip + cut);
1576 mydata_offset += skip;
1577 }
1578 }
1579 }
1580
1581 /* run the callback */
1582 r = Callback(cb_data, mydata, mydata_len);
1583 BUG_ON(r < 0);
1584
1585 if (return_progress) {
1586 *progress_out = (mydata_offset + mydata_len);
1587 } else {
1588 /* several blocks of data, so we need to be a bit more careful:
1589 * - if last_ack is beyond last progress, move progress forward to last_ack
1590 * - if our block matches or starts before last ack, return right edge of
1591 * our block.
1592 */
1593 uint64_t last_ack_abs = STREAM_BASE_OFFSET(stream);
1594 if (STREAM_LASTACK_GT_BASESEQ(stream)) {
1595 uint32_t delta = stream->last_ack - stream->base_seq;
1596 /* get max absolute offset */
1597 last_ack_abs += delta;
1598 }
1599 SCLogDebug("last_ack_abs %"PRIu64, last_ack_abs);
1600
1601 if (STREAM_RAW_PROGRESS(stream) < last_ack_abs) {
1602 if (mydata_offset > last_ack_abs) {
1603 /* gap between us and last ack, set progress to last ack */
1604 *progress_out = last_ack_abs;
1605 } else {
1606 *progress_out = (mydata_offset + mydata_len);
1607 }
1608 } else {
1609 *progress_out = STREAM_RAW_PROGRESS(stream);
1610 }
1611 }
1612 return r;
1613 }
1614
1615 /** \brief access 'raw' reassembly data.
1616 *
1617 * Access data as tracked by 'raw' tracker. Data is made available to
1618 * callback that is passed to this function.
1619 *
1620 * In the case of IDS the callback may be run multiple times if data
1621 * contains gaps. It will then be run for each block of data that is
1622 * continuous.
1623 *
1624 * The callback should give on of 2 return values:
1625 * - 0 ok
1626 * - 1 done
1627 * The value 1 will break the loop if there is a block list that is
1628 * inspected.
1629 *
1630 * This function will return the 'progress' value that has been
1631 * consumed until now.
1632 *
1633 * \param ssn tcp session
1634 * \param stream tcp stream
1635 * \param Callback the function pointer to the callback function
1636 * \param cb_data callback data
1637 * \param[in] progress_in progress to work from
1638 * \param[out] progress_out absolute progress value of the data this
1639 * call handled.
1640 * \param eof we're wrapping up so inspect all data we have, incl unACKd
1641 * \param respect_inspect_depth use Stream::min_inspect_depth if set
1642 *
1643 * `respect_inspect_depth` is used to avoid useless inspection of too
1644 * much data.
1645 */
StreamReassembleRawDo(TcpSession * ssn,TcpStream * stream,StreamReassembleRawFunc Callback,void * cb_data,const uint64_t progress_in,uint64_t * progress_out,bool eof,bool respect_inspect_depth)1646 static int StreamReassembleRawDo(TcpSession *ssn, TcpStream *stream,
1647 StreamReassembleRawFunc Callback, void *cb_data,
1648 const uint64_t progress_in,
1649 uint64_t *progress_out, bool eof,
1650 bool respect_inspect_depth)
1651 {
1652 SCEnter();
1653 int r = 0;
1654
1655 StreamingBufferBlock *iter = NULL;
1656 uint64_t progress = progress_in;
1657 uint64_t last_ack_abs = STREAM_BASE_OFFSET(stream); /* absolute right edge of ack'd data */
1658
1659 /* if the app layer triggered a flush, and we're supposed to
1660 * use a minimal inspect depth, we actually take the app progress
1661 * as that is the right edge of the data. Then we take the window
1662 * of 'min_inspect_depth' before that. */
1663
1664 SCLogDebug("respect_inspect_depth %s STREAMTCP_STREAM_FLAG_TRIGGER_RAW %s stream->min_inspect_depth %u",
1665 respect_inspect_depth ? "true" : "false",
1666 (stream->flags & STREAMTCP_STREAM_FLAG_TRIGGER_RAW) ? "true" : "false",
1667 stream->min_inspect_depth);
1668
1669 if (respect_inspect_depth &&
1670 (stream->flags & STREAMTCP_STREAM_FLAG_TRIGGER_RAW)
1671 && stream->min_inspect_depth)
1672 {
1673 progress = STREAM_APP_PROGRESS(stream);
1674 if (stream->min_inspect_depth >= progress) {
1675 progress = 0;
1676 } else {
1677 progress -= stream->min_inspect_depth;
1678 }
1679
1680 SCLogDebug("stream app %"PRIu64", raw %"PRIu64, STREAM_APP_PROGRESS(stream), STREAM_RAW_PROGRESS(stream));
1681
1682 progress = MIN(progress, STREAM_RAW_PROGRESS(stream));
1683 SCLogDebug("applied min inspect depth due to STREAMTCP_STREAM_FLAG_TRIGGER_RAW: progress %"PRIu64, progress);
1684 }
1685
1686 SCLogDebug("progress %"PRIu64", min inspect depth %u %s", progress, stream->min_inspect_depth, stream->flags & STREAMTCP_STREAM_FLAG_TRIGGER_RAW ? "STREAMTCP_STREAM_FLAG_TRIGGER_RAW":"(no trigger)");
1687
1688 /* get window of data that is acked */
1689 if (STREAM_LASTACK_GT_BASESEQ(stream)) {
1690 SCLogDebug("last_ack %u, base_seq %u", stream->last_ack, stream->base_seq);
1691 uint32_t delta = stream->last_ack - stream->base_seq;
1692 /* get max absolute offset */
1693 last_ack_abs += delta;
1694 SCLogDebug("last_ack_abs %"PRIu64, last_ack_abs);
1695 }
1696
1697 /* loop through available buffers. On no packet loss we'll have a single
1698 * iteration. On missing data we'll walk the blocks */
1699 while (1) {
1700 const uint8_t *mydata;
1701 uint32_t mydata_len;
1702 uint64_t mydata_offset = 0;
1703
1704 GetRawBuffer(stream, &mydata, &mydata_len, &iter, progress, &mydata_offset);
1705 if (mydata_len == 0) {
1706 SCLogDebug("no data");
1707 break;
1708 }
1709 //PrintRawDataFp(stdout, mydata, mydata_len);
1710
1711 SCLogDebug("raw progress %"PRIu64, progress);
1712 SCLogDebug("stream %p data in buffer %p of len %u and offset %"PRIu64,
1713 stream, &stream->sb, mydata_len, progress);
1714
1715 if (eof) {
1716 // inspect all remaining data, ack'd or not
1717 } else {
1718 if (last_ack_abs < progress) {
1719 SCLogDebug("nothing to do");
1720 goto end;
1721 }
1722
1723 SCLogDebug("last_ack_abs %"PRIu64", raw_progress %"PRIu64, last_ack_abs, progress);
1724 SCLogDebug("raw_progress + mydata_len %"PRIu64", last_ack_abs %"PRIu64, progress + mydata_len, last_ack_abs);
1725
1726 /* see if the buffer contains unack'd data as well */
1727 if (progress + mydata_len > last_ack_abs) {
1728 uint32_t check = mydata_len;
1729 mydata_len = last_ack_abs - progress;
1730 BUG_ON(check < mydata_len);
1731 SCLogDebug("data len adjusted to %u to make sure only ACK'd "
1732 "data is considered", mydata_len);
1733 }
1734
1735 }
1736 if (mydata_len == 0)
1737 break;
1738
1739 SCLogDebug("data %p len %u", mydata, mydata_len);
1740
1741 /* we have data. */
1742 r = Callback(cb_data, mydata, mydata_len);
1743 BUG_ON(r < 0);
1744
1745 if (mydata_offset == progress) {
1746 SCLogDebug("progress %"PRIu64" increasing with data len %u to %"PRIu64,
1747 progress, mydata_len, progress_in + mydata_len);
1748
1749 progress += mydata_len;
1750 SCLogDebug("raw progress now %"PRIu64, progress);
1751
1752 /* data is beyond the progress we'd like, and before last ack. Gap. */
1753 } else if (mydata_offset > progress && mydata_offset < last_ack_abs) {
1754 SCLogDebug("GAP: data is missing from %"PRIu64" (%u bytes), setting to first data we have: %"PRIu64, progress, (uint32_t)(mydata_offset - progress), mydata_offset);
1755 SCLogDebug("last_ack_abs %"PRIu64, last_ack_abs);
1756 progress = mydata_offset;
1757 SCLogDebug("raw progress now %"PRIu64, progress);
1758
1759 } else {
1760 SCLogDebug("not increasing progress, data gap => mydata_offset "
1761 "%"PRIu64" != progress %"PRIu64, mydata_offset, progress);
1762 }
1763
1764 if (iter == NULL || r == 1)
1765 break;
1766 }
1767 end:
1768 *progress_out = progress;
1769 return r;
1770 }
1771
StreamReassembleRaw(TcpSession * ssn,const Packet * p,StreamReassembleRawFunc Callback,void * cb_data,uint64_t * progress_out,bool respect_inspect_depth)1772 int StreamReassembleRaw(TcpSession *ssn, const Packet *p,
1773 StreamReassembleRawFunc Callback, void *cb_data,
1774 uint64_t *progress_out, bool respect_inspect_depth)
1775 {
1776 /* handle inline separately as the logic is very different */
1777 if (StreamTcpInlineMode() == TRUE) {
1778 return StreamReassembleRawInline(ssn, p, Callback, cb_data, progress_out);
1779 }
1780
1781 TcpStream *stream;
1782 if (PKT_IS_TOSERVER(p)) {
1783 stream = &ssn->client;
1784 } else {
1785 stream = &ssn->server;
1786 }
1787
1788 if ((stream->flags & (STREAMTCP_STREAM_FLAG_NOREASSEMBLY|STREAMTCP_STREAM_FLAG_DISABLE_RAW)) ||
1789 StreamTcpReassembleRawCheckLimit(ssn, stream, p) == 0)
1790 {
1791 *progress_out = STREAM_RAW_PROGRESS(stream);
1792 return 0;
1793 }
1794
1795 return StreamReassembleRawDo(ssn, stream, Callback, cb_data,
1796 STREAM_RAW_PROGRESS(stream), progress_out,
1797 (p->flags & PKT_PSEUDO_STREAM_END), respect_inspect_depth);
1798 }
1799
StreamReassembleLog(TcpSession * ssn,TcpStream * stream,StreamReassembleRawFunc Callback,void * cb_data,uint64_t progress_in,uint64_t * progress_out,bool eof)1800 int StreamReassembleLog(TcpSession *ssn, TcpStream *stream,
1801 StreamReassembleRawFunc Callback, void *cb_data,
1802 uint64_t progress_in,
1803 uint64_t *progress_out, bool eof)
1804 {
1805 if (stream->flags & (STREAMTCP_STREAM_FLAG_NOREASSEMBLY))
1806 return 0;
1807
1808 return StreamReassembleRawDo(ssn, stream, Callback, cb_data,
1809 progress_in, progress_out, eof, false);
1810 }
1811
1812 /** \internal
1813 * \brief update app layer based on received ACK
1814 *
1815 * \retval r 0 on success, -1 on error
1816 */
StreamTcpReassembleHandleSegmentUpdateACK(ThreadVars * tv,TcpReassemblyThreadCtx * ra_ctx,TcpSession * ssn,TcpStream * stream,Packet * p)1817 static int StreamTcpReassembleHandleSegmentUpdateACK (ThreadVars *tv,
1818 TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p)
1819 {
1820 SCEnter();
1821
1822 if (StreamTcpReassembleAppLayer(tv, ra_ctx, ssn, stream, p, UPDATE_DIR_OPPOSING) < 0)
1823 SCReturnInt(-1);
1824
1825 SCReturnInt(0);
1826 }
1827
StreamTcpReassembleHandleSegment(ThreadVars * tv,TcpReassemblyThreadCtx * ra_ctx,TcpSession * ssn,TcpStream * stream,Packet * p,PacketQueueNoLock * pq)1828 int StreamTcpReassembleHandleSegment(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx,
1829 TcpSession *ssn, TcpStream *stream,
1830 Packet *p, PacketQueueNoLock *pq)
1831 {
1832 SCEnter();
1833
1834 DEBUG_VALIDATE_BUG_ON(p->tcph == NULL);
1835
1836 SCLogDebug("ssn %p, stream %p, p %p, p->payload_len %"PRIu16"",
1837 ssn, stream, p, p->payload_len);
1838
1839 /* default IDS: update opposing side (triggered by ACK) */
1840 enum StreamUpdateDir dir = UPDATE_DIR_OPPOSING;
1841 /* inline and stream end and flow timeout packets trigger same dir handling */
1842 if (StreamTcpInlineMode()) {
1843 dir = UPDATE_DIR_PACKET;
1844 } else if (p->flags & PKT_PSEUDO_STREAM_END) {
1845 dir = UPDATE_DIR_PACKET;
1846 } else if (p->tcph->th_flags & TH_RST) { // accepted rst
1847 dir = UPDATE_DIR_PACKET;
1848 } else if ((p->tcph->th_flags & TH_FIN) && ssn->state > TCP_TIME_WAIT) {
1849 dir = UPDATE_DIR_PACKET;
1850 } else if (ssn->state == TCP_CLOSED) {
1851 dir = UPDATE_DIR_BOTH;
1852 }
1853
1854 /* handle ack received */
1855 if ((dir == UPDATE_DIR_OPPOSING || dir == UPDATE_DIR_BOTH)) {
1856 /* we need to update the opposing stream in
1857 * StreamTcpReassembleHandleSegmentUpdateACK */
1858 TcpStream *opposing_stream = NULL;
1859 if (stream == &ssn->client) {
1860 opposing_stream = &ssn->server;
1861 } else {
1862 opposing_stream = &ssn->client;
1863 }
1864
1865 const bool reversed_before_ack_handling = (p->flow->flags & FLOW_DIR_REVERSED) != 0;
1866
1867 if (StreamTcpReassembleHandleSegmentUpdateACK(tv, ra_ctx, ssn, opposing_stream, p) != 0) {
1868 SCLogDebug("StreamTcpReassembleHandleSegmentUpdateACK error");
1869 SCReturnInt(-1);
1870 }
1871
1872 /* StreamTcpReassembleHandleSegmentUpdateACK
1873 * may swap content of ssn->server and ssn->client structures.
1874 * We have to continue with initial content of the stream in such case */
1875 const bool reversed_after_ack_handling = (p->flow->flags & FLOW_DIR_REVERSED) != 0;
1876 if (reversed_before_ack_handling != reversed_after_ack_handling) {
1877 SCLogDebug("TCP streams were swapped");
1878 stream = opposing_stream;
1879 }
1880 }
1881 /* if this segment contains data, insert it */
1882 if (p->payload_len > 0 && !(stream->flags & STREAMTCP_STREAM_FLAG_NOREASSEMBLY)) {
1883 SCLogDebug("calling StreamTcpReassembleHandleSegmentHandleData");
1884
1885 if (StreamTcpReassembleHandleSegmentHandleData(tv, ra_ctx, ssn, stream, p) != 0) {
1886 SCLogDebug("StreamTcpReassembleHandleSegmentHandleData error");
1887 SCReturnInt(-1);
1888 }
1889
1890 SCLogDebug("packet %"PRIu64" set PKT_STREAM_ADD", p->pcap_cnt);
1891 p->flags |= PKT_STREAM_ADD;
1892 } else {
1893 SCLogDebug("ssn %p / stream %p: not calling StreamTcpReassembleHandleSegmentHandleData:"
1894 " p->payload_len %u, STREAMTCP_STREAM_FLAG_NOREASSEMBLY %s",
1895 ssn, stream, p->payload_len,
1896 (stream->flags & STREAMTCP_STREAM_FLAG_NOREASSEMBLY) ? "true" : "false");
1897
1898 }
1899
1900 /* if the STREAMTCP_STREAM_FLAG_DEPTH_REACHED is set, but not the
1901 * STREAMTCP_STREAM_FLAG_NOREASSEMBLY flag, it means the DEPTH flag
1902 * was *just* set. In this case we trigger the AppLayer Truncate
1903 * logic, to inform the applayer no more data in this direction is
1904 * to be expected. */
1905 if ((stream->flags &
1906 (STREAMTCP_STREAM_FLAG_DEPTH_REACHED|STREAMTCP_STREAM_FLAG_NOREASSEMBLY)) ==
1907 STREAMTCP_STREAM_FLAG_DEPTH_REACHED)
1908 {
1909 SCLogDebug("STREAMTCP_STREAM_FLAG_DEPTH_REACHED, truncate applayer");
1910 if (dir != UPDATE_DIR_PACKET) {
1911 SCLogDebug("override: direction now UPDATE_DIR_PACKET so we "
1912 "can trigger Truncate");
1913 dir = UPDATE_DIR_PACKET;
1914 }
1915 }
1916
1917 /* in stream inline mode even if we have no data we call the reassembly
1918 * functions to handle EOF */
1919 if (dir == UPDATE_DIR_PACKET || dir == UPDATE_DIR_BOTH) {
1920 SCLogDebug("inline (%s) or PKT_PSEUDO_STREAM_END (%s)",
1921 StreamTcpInlineMode()?"true":"false",
1922 (p->flags & PKT_PSEUDO_STREAM_END) ?"true":"false");
1923 if (StreamTcpReassembleAppLayer(tv, ra_ctx, ssn, stream, p, dir) < 0) {
1924 SCReturnInt(-1);
1925 }
1926 }
1927
1928 SCReturnInt(0);
1929 }
1930
1931 /**
1932 * \brief get a segment from the pool
1933 *
1934 * \retval seg Segment from the pool or NULL
1935 */
StreamTcpGetSegment(ThreadVars * tv,TcpReassemblyThreadCtx * ra_ctx)1936 TcpSegment *StreamTcpGetSegment(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx)
1937 {
1938 TcpSegment *seg = (TcpSegment *) PoolThreadGetById(segment_thread_pool, ra_ctx->segment_thread_pool_id);
1939 SCLogDebug("seg we return is %p", seg);
1940 if (seg == NULL) {
1941 /* Increment the counter to show that we are not able to serve the
1942 segment request due to memcap limit */
1943 StatsIncr(tv, ra_ctx->counter_tcp_segment_memcap);
1944 } else {
1945 memset(&seg->sbseg, 0, sizeof(seg->sbseg));
1946 }
1947
1948 return seg;
1949 }
1950
1951 /**
1952 * \brief Trigger RAW stream reassembly
1953 *
1954 * Used by AppLayerTriggerRawStreamReassembly to trigger RAW stream
1955 * reassembly from the applayer, for example upon completion of a
1956 * HTTP request.
1957 *
1958 * It sets a flag in the stream so that the next Raw call will return
1959 * the data.
1960 *
1961 * \param ssn TcpSession
1962 */
StreamTcpReassembleTriggerRawReassembly(TcpSession * ssn,int direction)1963 void StreamTcpReassembleTriggerRawReassembly(TcpSession *ssn, int direction)
1964 {
1965 #ifdef DEBUG
1966 BUG_ON(ssn == NULL);
1967 #endif
1968
1969 if (ssn != NULL) {
1970 if (direction == STREAM_TOSERVER) {
1971 ssn->client.flags |= STREAMTCP_STREAM_FLAG_TRIGGER_RAW;
1972 } else {
1973 ssn->server.flags |= STREAMTCP_STREAM_FLAG_TRIGGER_RAW;
1974 }
1975
1976 SCLogDebug("flagged ssn %p for immediate raw reassembly", ssn);
1977 }
1978 }
1979
StreamTcpReassemblySetMinInspectDepth(TcpSession * ssn,int direction,uint32_t depth)1980 void StreamTcpReassemblySetMinInspectDepth(TcpSession *ssn, int direction, uint32_t depth)
1981 {
1982 #ifdef DEBUG
1983 BUG_ON(ssn == NULL);
1984 #endif
1985
1986 if (ssn != NULL) {
1987 if (direction == STREAM_TOSERVER) {
1988 ssn->client.min_inspect_depth = depth;
1989 SCLogDebug("ssn %p: set client.min_inspect_depth to %u", ssn, depth);
1990 } else {
1991 ssn->server.min_inspect_depth = depth;
1992 SCLogDebug("ssn %p: set server.min_inspect_depth to %u", ssn, depth);
1993 }
1994 }
1995 }
1996
1997 #ifdef UNITTESTS
1998 /** unit tests and it's support functions below */
1999
2000 #define SET_ISN(stream, setseq) \
2001 (stream)->isn = (setseq); \
2002 (stream)->base_seq = (setseq) + 1
2003
2004 /** \brief The Function to create the packet with given payload, which is used
2005 * to test the reassembly of the engine.
2006 *
2007 * \param payload The variable used to store the payload contents of the
2008 * current packet.
2009 * \param value The value which current payload will have for this packet
2010 * \param payload_len The length of the filed payload for current packet.
2011 * \param len Length of the payload array
2012 */
2013
StreamTcpCreateTestPacket(uint8_t * payload,uint8_t value,uint8_t payload_len,uint8_t len)2014 void StreamTcpCreateTestPacket(uint8_t *payload, uint8_t value,
2015 uint8_t payload_len, uint8_t len)
2016 {
2017 uint8_t i;
2018 for (i = 0; i < payload_len; i++)
2019 payload[i] = value;
2020 for (; i < len; i++)
2021 payload = NULL;
2022 }
2023
2024 /** \brief The Function Checks the reassembled stream contents against predefined
2025 * stream contents according to OS policy used.
2026 *
2027 * \param stream_policy Predefined value of stream for different OS policies
2028 * \param stream Reassembled stream returned from the reassembly functions
2029 */
2030
StreamTcpCheckStreamContents(uint8_t * stream_policy,uint16_t sp_size,TcpStream * stream)2031 int StreamTcpCheckStreamContents(uint8_t *stream_policy, uint16_t sp_size, TcpStream *stream)
2032 {
2033 if (StreamingBufferCompareRawData(&stream->sb, stream_policy,(uint32_t)sp_size) == 0)
2034 {
2035 //PrintRawDataFp(stdout, stream_policy, sp_size);
2036 return 0;
2037 }
2038 return 1;
2039 }
2040
VALIDATE(TcpStream * stream,uint8_t * data,uint32_t data_len)2041 static int VALIDATE(TcpStream *stream, uint8_t *data, uint32_t data_len)
2042 {
2043 if (StreamingBufferCompareRawData(&stream->sb,
2044 data, data_len) == 0)
2045 {
2046 SCReturnInt(0);
2047 }
2048 SCLogInfo("OK");
2049 PrintRawDataFp(stdout, data, data_len);
2050 return 1;
2051 }
2052
2053 #define MISSED_START(isn) \
2054 TcpReassemblyThreadCtx *ra_ctx = NULL; \
2055 TcpSession ssn; \
2056 ThreadVars tv; \
2057 memset(&tv, 0, sizeof(tv)); \
2058 \
2059 StreamTcpUTInit(&ra_ctx); \
2060 \
2061 StreamTcpUTSetupSession(&ssn); \
2062 StreamTcpUTSetupStream(&ssn.server, (isn)); \
2063 StreamTcpUTSetupStream(&ssn.client, (isn)); \
2064 \
2065 TcpStream *stream = &ssn.client;
2066
2067 #define MISSED_END \
2068 StreamTcpUTClearSession(&ssn); \
2069 StreamTcpUTDeinit(ra_ctx); \
2070 PASS
2071
2072 #define MISSED_STEP(seq, seg, seglen, buf, buflen) \
2073 StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, stream, (seq), (uint8_t *)(seg), (seglen)); \
2074 FAIL_IF(!(VALIDATE(stream, (uint8_t *)(buf), (buflen))));
2075
2076 /**
2077 * \test Test the handling of packets missed by both IDS and the end host.
2078 * The packet is missed in the starting of the stream.
2079 *
2080 * \retval On success it returns 1 and on failure 0.
2081 */
2082
StreamTcpReassembleTest25(void)2083 static int StreamTcpReassembleTest25 (void)
2084 {
2085 MISSED_START(6);
2086 MISSED_STEP(10, "BB", 2, "\0\0\0BB", 5);
2087 MISSED_STEP(12, "CC", 2, "\0\0\0BBCC", 7);
2088 MISSED_STEP(7, "AAA", 3, "AAABBCC", 7);
2089 MISSED_END;
2090 }
2091
2092 /**
2093 * \test Test the handling of packets missed by both IDS and the end host.
2094 * The packet is missed in the middle of the stream.
2095 *
2096 * \retval On success it returns 1 and on failure 0.
2097 */
2098
StreamTcpReassembleTest26(void)2099 static int StreamTcpReassembleTest26 (void)
2100 {
2101 MISSED_START(9);
2102 MISSED_STEP(10, "AAA", 3, "AAA", 3);
2103 MISSED_STEP(15, "CC", 2, "AAA\0\0CC", 7);
2104 MISSED_STEP(13, "BB", 2, "AAABBCC", 7);
2105 MISSED_END;
2106 }
2107
2108 /**
2109 * \test Test the handling of packets missed by both IDS and the end host.
2110 * The packet is missed in the end of the stream.
2111 *
2112 * \retval On success it returns 1 and on failure 0.
2113 */
2114
StreamTcpReassembleTest27(void)2115 static int StreamTcpReassembleTest27 (void)
2116 {
2117 MISSED_START(9);
2118 MISSED_STEP(10, "AAA", 3, "AAA", 3);
2119 MISSED_STEP(13, "BB", 2, "AAABB", 5);
2120 MISSED_STEP(15, "CC", 2, "AAABBCC", 7);
2121 MISSED_END;
2122 }
2123
2124 /**
2125 * \test Test the handling of packets missed by IDS, but the end host has
2126 * received it and send the acknowledgment of it. The packet is missed
2127 * in the starting of the stream.
2128 *
2129 * \retval On success it returns 1 and on failure 0.
2130 */
2131
StreamTcpReassembleTest28(void)2132 static int StreamTcpReassembleTest28 (void)
2133 {
2134 MISSED_START(6);
2135 MISSED_STEP(10, "AAA", 3, "\0\0\0AAA", 6);
2136 MISSED_STEP(13, "BB", 2, "\0\0\0AAABB", 8);
2137 ssn.state = TCP_TIME_WAIT;
2138 MISSED_STEP(15, "CC", 2, "\0\0\0AAABBCC", 10);
2139 MISSED_END;
2140 }
2141
2142 /**
2143 * \test Test the handling of packets missed by IDS, but the end host has
2144 * received it and send the acknowledgment of it. The packet is missed
2145 * in the middle of the stream.
2146 *
2147 * \retval On success it returns 1 and on failure 0.
2148 */
2149
StreamTcpReassembleTest29(void)2150 static int StreamTcpReassembleTest29 (void)
2151 {
2152 MISSED_START(9);
2153 MISSED_STEP(10, "AAA", 3, "AAA", 3);
2154 ssn.state = TCP_TIME_WAIT;
2155 MISSED_STEP(15, "CC", 2, "AAA\0\0CC", 7);
2156 MISSED_END;
2157 }
2158
StreamTcpReassembleTest33(void)2159 static int StreamTcpReassembleTest33(void)
2160 {
2161 TcpSession ssn;
2162 Packet *p = PacketGetFromAlloc();
2163 FAIL_IF(unlikely(p == NULL));
2164 Flow f;
2165 TCPHdr tcph;
2166 TcpReassemblyThreadCtx *ra_ctx = NULL;
2167 ssn.client.os_policy = OS_POLICY_BSD;
2168 uint8_t packet[1460] = "";
2169
2170 StreamTcpUTInit(&ra_ctx);
2171 StreamTcpUTSetupSession(&ssn);
2172
2173 PacketQueueNoLock pq;
2174 memset(&pq,0,sizeof(PacketQueueNoLock));
2175 memset(&f, 0, sizeof (Flow));
2176 memset(&tcph, 0, sizeof (TCPHdr));
2177 ThreadVars tv;
2178 memset(&tv, 0, sizeof (ThreadVars));
2179 FLOW_INITIALIZE(&f);
2180 f.protoctx = &ssn;
2181 f.proto = IPPROTO_TCP;
2182 p->src.family = AF_INET;
2183 p->dst.family = AF_INET;
2184 p->proto = IPPROTO_TCP;
2185 p->flow = &f;
2186 tcph.th_win = 5480;
2187 tcph.th_flags = TH_PUSH | TH_ACK;
2188 p->tcph = &tcph;
2189 p->flowflags = FLOW_PKT_TOSERVER;
2190 p->payload = packet;
2191
2192 p->tcph->th_seq = htonl(10);
2193 p->tcph->th_ack = htonl(31);
2194 p->payload_len = 10;
2195
2196 FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2197
2198 p->tcph->th_seq = htonl(20);
2199 p->tcph->th_ack = htonl(31);
2200 p->payload_len = 10;
2201
2202 FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2203
2204 p->tcph->th_seq = htonl(40);
2205 p->tcph->th_ack = htonl(31);
2206 p->payload_len = 10;
2207
2208 FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2209
2210 p->tcph->th_seq = htonl(5);
2211 p->tcph->th_ack = htonl(31);
2212 p->payload_len = 30;
2213
2214 FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2215
2216 StreamTcpUTClearSession(&ssn);
2217 StreamTcpUTDeinit(ra_ctx);
2218 SCFree(p);
2219 PASS;
2220 }
2221
StreamTcpReassembleTest34(void)2222 static int StreamTcpReassembleTest34(void)
2223 {
2224 TcpSession ssn;
2225 Packet *p = PacketGetFromAlloc();
2226 FAIL_IF(unlikely(p == NULL));
2227 Flow f;
2228 TCPHdr tcph;
2229 TcpReassemblyThreadCtx *ra_ctx = NULL;
2230 ssn.client.os_policy = OS_POLICY_BSD;
2231 uint8_t packet[1460] = "";
2232
2233 StreamTcpUTInit(&ra_ctx);
2234 StreamTcpUTSetupSession(&ssn);
2235 PacketQueueNoLock pq;
2236 memset(&pq,0,sizeof(PacketQueueNoLock));
2237 memset(&f, 0, sizeof (Flow));
2238 memset(&tcph, 0, sizeof (TCPHdr));
2239 ThreadVars tv;
2240 memset(&tv, 0, sizeof (ThreadVars));
2241 FLOW_INITIALIZE(&f);
2242 f.protoctx = &ssn;
2243 f.proto = IPPROTO_TCP;
2244 p->src.family = AF_INET;
2245 p->dst.family = AF_INET;
2246 p->proto = IPPROTO_TCP;
2247 p->flow = &f;
2248 tcph.th_win = 5480;
2249 tcph.th_flags = TH_PUSH | TH_ACK;
2250 p->tcph = &tcph;
2251 p->flowflags = FLOW_PKT_TOSERVER;
2252 p->payload = packet;
2253 SET_ISN(&ssn.client, 857961230);
2254
2255 p->tcph->th_seq = htonl(857961230);
2256 p->tcph->th_ack = htonl(31);
2257 p->payload_len = 304;
2258
2259 FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2260
2261 p->tcph->th_seq = htonl(857961534);
2262 p->tcph->th_ack = htonl(31);
2263 p->payload_len = 1460;
2264
2265 FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2266
2267 p->tcph->th_seq = htonl(857963582);
2268 p->tcph->th_ack = htonl(31);
2269 p->payload_len = 1460;
2270
2271 FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2272
2273 p->tcph->th_seq = htonl(857960946);
2274 p->tcph->th_ack = htonl(31);
2275 p->payload_len = 1460;
2276
2277 FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2278
2279 StreamTcpUTClearSession(&ssn);
2280 StreamTcpUTDeinit(ra_ctx);
2281 SCFree(p);
2282 PASS;
2283 }
2284
2285 /** \test Test the bug 76 condition */
StreamTcpReassembleTest37(void)2286 static int StreamTcpReassembleTest37(void)
2287 {
2288 TcpSession ssn;
2289 Flow f;
2290 TCPHdr tcph;
2291 TcpReassemblyThreadCtx *ra_ctx = NULL;
2292 uint8_t packet[1460] = "";
2293 PacketQueueNoLock pq;
2294 ThreadVars tv;
2295 memset(&tv, 0, sizeof (ThreadVars));
2296
2297 Packet *p = PacketGetFromAlloc();
2298 FAIL_IF(unlikely(p == NULL));
2299
2300 StreamTcpUTInit(&ra_ctx);
2301 StreamTcpUTSetupSession(&ssn);
2302 memset(&pq,0,sizeof(PacketQueueNoLock));
2303 memset(&f, 0, sizeof (Flow));
2304 memset(&tcph, 0, sizeof (TCPHdr));
2305 memset(&tv, 0, sizeof (ThreadVars));
2306
2307 FLOW_INITIALIZE(&f);
2308 f.protoctx = &ssn;
2309 f.proto = IPPROTO_TCP;
2310 p->src.family = AF_INET;
2311 p->dst.family = AF_INET;
2312 p->proto = IPPROTO_TCP;
2313 p->flow = &f;
2314 tcph.th_win = 5480;
2315 tcph.th_flags = TH_PUSH | TH_ACK;
2316 p->tcph = &tcph;
2317 p->flowflags = FLOW_PKT_TOSERVER;
2318 p->payload = packet;
2319 ssn.client.os_policy = OS_POLICY_BSD;
2320
2321 p->tcph->th_seq = htonl(3061088537UL);
2322 p->tcph->th_ack = htonl(1729548549UL);
2323 p->payload_len = 1391;
2324 ssn.client.last_ack = 3061091137UL;
2325 SET_ISN(&ssn.client, 3061091309UL);
2326
2327 /* pre base_seq, so should be rejected */
2328 FAIL_IF (StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) != -1);
2329
2330 p->tcph->th_seq = htonl(3061089928UL);
2331 p->tcph->th_ack = htonl(1729548549UL);
2332 p->payload_len = 1391;
2333 ssn.client.last_ack = 3061091137UL;
2334
2335 FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2336
2337 p->tcph->th_seq = htonl(3061091319UL);
2338 p->tcph->th_ack = htonl(1729548549UL);
2339 p->payload_len = 1391;
2340 ssn.client.last_ack = 3061091137UL;
2341
2342 FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &ssn.client, p, &pq) == -1);
2343
2344 StreamTcpUTClearSession(&ssn);
2345 StreamTcpUTDeinit(ra_ctx);
2346 SCFree(p);
2347 PASS;
2348 }
2349
2350 /**
2351 * \test Test to make sure that we don't return the segments until the app
2352 * layer proto has been detected and after that remove the processed
2353 * segments.
2354 *
2355 * \retval On success it returns 1 and on failure 0.
2356 */
2357
StreamTcpReassembleTest39(void)2358 static int StreamTcpReassembleTest39 (void)
2359 {
2360 Packet *p = PacketGetFromAlloc();
2361 FAIL_IF(unlikely(p == NULL));
2362 Flow f;
2363 ThreadVars tv;
2364 StreamTcpThread stt;
2365 TCPHdr tcph;
2366 PacketQueueNoLock pq;
2367 memset(&pq,0,sizeof(PacketQueueNoLock));
2368 memset (&f, 0, sizeof(Flow));
2369 memset(&tv, 0, sizeof (ThreadVars));
2370 memset(&stt, 0, sizeof (stt));
2371 memset(&tcph, 0, sizeof (TCPHdr));
2372
2373 FLOW_INITIALIZE(&f);
2374 f.flags = FLOW_IPV4;
2375 f.proto = IPPROTO_TCP;
2376 p->flow = &f;
2377 p->tcph = &tcph;
2378
2379 StreamTcpUTInit(&stt.ra_ctx);
2380
2381 /* handshake */
2382 tcph.th_win = htons(5480);
2383 tcph.th_flags = TH_SYN;
2384 p->flowflags = FLOW_PKT_TOSERVER;
2385 p->payload_len = 0;
2386 p->payload = NULL;
2387 FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2388
2389 TcpSession *ssn = (TcpSession *)f.protoctx;
2390 FAIL_IF_NULL(ssn);
2391
2392 FAIL_IF(StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(&ssn->server));
2393 FAIL_IF(StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(&ssn->client));
2394 FAIL_IF(f.alproto != ALPROTO_UNKNOWN);
2395 FAIL_IF(f.alproto_ts != ALPROTO_UNKNOWN);
2396 FAIL_IF(f.alproto_tc != ALPROTO_UNKNOWN);
2397 FAIL_IF(ssn->flags & STREAMTCP_FLAG_APP_LAYER_DISABLED);
2398 FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2399 FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2400 FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2401 FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2402 FAIL_IF(!RB_EMPTY(&ssn->client.seg_tree));
2403 FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2404 FAIL_IF(ssn->data_first_seen_dir != 0);
2405
2406 /* handshake */
2407 p->tcph->th_ack = htonl(1);
2408 p->tcph->th_flags = TH_SYN | TH_ACK;
2409 p->flowflags = FLOW_PKT_TOCLIENT;
2410 p->payload_len = 0;
2411 p->payload = NULL;
2412 FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2413 FAIL_IF(StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(&ssn->server));
2414 FAIL_IF(StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(&ssn->client));
2415 FAIL_IF(f.alproto != ALPROTO_UNKNOWN);
2416 FAIL_IF(f.alproto_ts != ALPROTO_UNKNOWN);
2417 FAIL_IF(f.alproto_tc != ALPROTO_UNKNOWN);
2418 FAIL_IF(ssn->flags & STREAMTCP_FLAG_APP_LAYER_DISABLED);
2419 FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2420 FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2421 FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2422 FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2423 FAIL_IF(!RB_EMPTY(&ssn->client.seg_tree));
2424 FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2425 FAIL_IF(ssn->data_first_seen_dir != 0);
2426
2427 /* handshake */
2428 p->tcph->th_ack = htonl(1);
2429 p->tcph->th_seq = htonl(1);
2430 p->tcph->th_flags = TH_ACK;
2431 p->flowflags = FLOW_PKT_TOSERVER;
2432 p->payload_len = 0;
2433 p->payload = NULL;
2434 FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2435 FAIL_IF(StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(&ssn->server));
2436 FAIL_IF(StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(&ssn->client));
2437 FAIL_IF(f.alproto != ALPROTO_UNKNOWN);
2438 FAIL_IF(f.alproto_ts != ALPROTO_UNKNOWN);
2439 FAIL_IF(f.alproto_tc != ALPROTO_UNKNOWN);
2440 FAIL_IF(ssn->flags & STREAMTCP_FLAG_APP_LAYER_DISABLED);
2441 FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2442 FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2443 FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2444 FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2445 FAIL_IF(!RB_EMPTY(&ssn->client.seg_tree));
2446 FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2447 FAIL_IF(ssn->data_first_seen_dir != 0);
2448
2449 /* partial request */
2450 uint8_t request1[] = { 0x47, 0x45, };
2451 p->tcph->th_ack = htonl(1);
2452 p->tcph->th_seq = htonl(1);
2453 p->tcph->th_flags = TH_PUSH | TH_ACK;
2454 p->flowflags = FLOW_PKT_TOSERVER;
2455 p->payload_len = sizeof(request1);
2456 p->payload = request1;
2457 FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2458 FAIL_IF(StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(&ssn->server));
2459 FAIL_IF(StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(&ssn->client));
2460 FAIL_IF(f.alproto != ALPROTO_UNKNOWN);
2461 FAIL_IF(f.alproto_ts != ALPROTO_UNKNOWN);
2462 FAIL_IF(f.alproto_tc != ALPROTO_UNKNOWN);
2463 FAIL_IF(ssn->flags & STREAMTCP_FLAG_APP_LAYER_DISABLED);
2464 FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2465 FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2466 FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2467 FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2468 FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2469 FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2470 FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2471 FAIL_IF(ssn->data_first_seen_dir != STREAM_TOSERVER);
2472
2473 /* response ack against partial request */
2474 p->tcph->th_ack = htonl(3);
2475 p->tcph->th_seq = htonl(1);
2476 p->tcph->th_flags = TH_ACK;
2477 p->flowflags = FLOW_PKT_TOCLIENT;
2478 p->payload_len = 0;
2479 p->payload = NULL;
2480 FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2481 FAIL_IF(StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(&ssn->server));
2482 FAIL_IF(StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(&ssn->client));
2483 FAIL_IF(f.alproto != ALPROTO_UNKNOWN);
2484 FAIL_IF(f.alproto_ts != ALPROTO_UNKNOWN);
2485 FAIL_IF(f.alproto_tc != ALPROTO_UNKNOWN);
2486 FAIL_IF(ssn->flags & STREAMTCP_FLAG_APP_LAYER_DISABLED);
2487 FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2488 FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2489 FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2490 FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2491 FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2492 FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2493 FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2494 FAIL_IF(ssn->data_first_seen_dir != STREAM_TOSERVER);
2495
2496 /* complete partial request */
2497 uint8_t request2[] = {
2498 0x54, 0x20, 0x2f, 0x69, 0x6e, 0x64,
2499 0x65, 0x78, 0x2e, 0x68, 0x74, 0x6d, 0x6c, 0x20,
2500 0x48, 0x54, 0x54, 0x50, 0x2f, 0x31, 0x2e, 0x30,
2501 0x0d, 0x0a, 0x48, 0x6f, 0x73, 0x74, 0x3a, 0x20,
2502 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x68, 0x6f, 0x73,
2503 0x74, 0x0d, 0x0a, 0x55, 0x73, 0x65, 0x72, 0x2d,
2504 0x41, 0x67, 0x65, 0x6e, 0x74, 0x3a, 0x20, 0x41,
2505 0x70, 0x61, 0x63, 0x68, 0x65, 0x42, 0x65, 0x6e,
2506 0x63, 0x68, 0x2f, 0x32, 0x2e, 0x33, 0x0d, 0x0a,
2507 0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x3a, 0x20,
2508 0x2a, 0x2f, 0x2a, 0x0d, 0x0a, 0x0d, 0x0a };
2509 p->tcph->th_ack = htonl(1);
2510 p->tcph->th_seq = htonl(3);
2511 p->tcph->th_flags = TH_PUSH | TH_ACK;
2512 p->flowflags = FLOW_PKT_TOSERVER;
2513 p->payload_len = sizeof(request2);
2514 p->payload = request2;
2515 FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2516 FAIL_IF(StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(&ssn->server));
2517 FAIL_IF(StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(&ssn->client));
2518 FAIL_IF(f.alproto != ALPROTO_UNKNOWN);
2519 FAIL_IF(f.alproto_ts != ALPROTO_UNKNOWN);
2520 FAIL_IF(f.alproto_tc != ALPROTO_UNKNOWN);
2521 FAIL_IF(ssn->flags & STREAMTCP_FLAG_APP_LAYER_DISABLED);
2522 FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2523 FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2524 FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2525 FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2526 FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2527 FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2528 FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2529 FAIL_IF(!RB_EMPTY(&ssn->server.seg_tree));
2530 FAIL_IF(ssn->data_first_seen_dir != STREAM_TOSERVER);
2531
2532 /* response - request ack */
2533 uint8_t response[] = {
2534 0x48, 0x54, 0x54, 0x50, 0x2f, 0x31, 0x2e, 0x31,
2535 0x20, 0x32, 0x30, 0x30, 0x20, 0x4f, 0x4b, 0x0d,
2536 0x0a, 0x44, 0x61, 0x74, 0x65, 0x3a, 0x20, 0x46,
2537 0x72, 0x69, 0x2c, 0x20, 0x32, 0x33, 0x20, 0x53,
2538 0x65, 0x70, 0x20, 0x32, 0x30, 0x31, 0x31, 0x20,
2539 0x30, 0x36, 0x3a, 0x32, 0x39, 0x3a, 0x33, 0x39,
2540 0x20, 0x47, 0x4d, 0x54, 0x0d, 0x0a, 0x53, 0x65,
2541 0x72, 0x76, 0x65, 0x72, 0x3a, 0x20, 0x41, 0x70,
2542 0x61, 0x63, 0x68, 0x65, 0x2f, 0x32, 0x2e, 0x32,
2543 0x2e, 0x31, 0x35, 0x20, 0x28, 0x55, 0x6e, 0x69,
2544 0x78, 0x29, 0x20, 0x44, 0x41, 0x56, 0x2f, 0x32,
2545 0x0d, 0x0a, 0x4c, 0x61, 0x73, 0x74, 0x2d, 0x4d,
2546 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x3a,
2547 0x20, 0x54, 0x68, 0x75, 0x2c, 0x20, 0x30, 0x34,
2548 0x20, 0x4e, 0x6f, 0x76, 0x20, 0x32, 0x30, 0x31,
2549 0x30, 0x20, 0x31, 0x35, 0x3a, 0x30, 0x34, 0x3a,
2550 0x34, 0x36, 0x20, 0x47, 0x4d, 0x54, 0x0d, 0x0a,
2551 0x45, 0x54, 0x61, 0x67, 0x3a, 0x20, 0x22, 0x61,
2552 0x62, 0x38, 0x39, 0x36, 0x35, 0x2d, 0x32, 0x63,
2553 0x2d, 0x34, 0x39, 0x34, 0x33, 0x62, 0x37, 0x61,
2554 0x37, 0x66, 0x37, 0x66, 0x38, 0x30, 0x22, 0x0d,
2555 0x0a, 0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x2d,
2556 0x52, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x3a, 0x20,
2557 0x62, 0x79, 0x74, 0x65, 0x73, 0x0d, 0x0a, 0x43,
2558 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x2d, 0x4c,
2559 0x65, 0x6e, 0x67, 0x74, 0x68, 0x3a, 0x20, 0x34,
2560 0x34, 0x0d, 0x0a, 0x43, 0x6f, 0x6e, 0x6e, 0x65,
2561 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x3a, 0x20, 0x63,
2562 0x6c, 0x6f, 0x73, 0x65, 0x0d, 0x0a, 0x43, 0x6f,
2563 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x2d, 0x54, 0x79,
2564 0x70, 0x65, 0x3a, 0x20, 0x74, 0x65, 0x78, 0x74,
2565 0x2f, 0x68, 0x74, 0x6d, 0x6c, 0x0d, 0x0a, 0x58,
2566 0x2d, 0x50, 0x61, 0x64, 0x3a, 0x20, 0x61, 0x76,
2567 0x6f, 0x69, 0x64, 0x20, 0x62, 0x72, 0x6f, 0x77,
2568 0x73, 0x65, 0x72, 0x20, 0x62, 0x75, 0x67, 0x0d,
2569 0x0a, 0x0d, 0x0a, 0x3c, 0x68, 0x74, 0x6d, 0x6c,
2570 0x3e, 0x3c, 0x62, 0x6f, 0x64, 0x79, 0x3e, 0x3c,
2571 0x68, 0x31, 0x3e, 0x49, 0x74, 0x20, 0x77, 0x6f,
2572 0x72, 0x6b, 0x73, 0x21, 0x3c, 0x2f, 0x68, 0x31,
2573 0x3e, 0x3c, 0x2f, 0x62, 0x6f, 0x64, 0x79, 0x3e,
2574 0x3c, 0x2f, 0x68, 0x74, 0x6d, 0x6c, 0x3e };
2575 p->tcph->th_ack = htonl(88);
2576 p->tcph->th_seq = htonl(1);
2577 p->tcph->th_flags = TH_PUSH | TH_ACK;
2578 p->flowflags = FLOW_PKT_TOCLIENT;
2579 p->payload_len = sizeof(response);
2580 p->payload = response;
2581
2582 FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2583 FAIL_IF(StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(&ssn->server));
2584 FAIL_IF(!StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(&ssn->client));
2585 FAIL_IF(f.alproto != ALPROTO_HTTP);
2586 FAIL_IF(f.alproto_ts != ALPROTO_HTTP);
2587 FAIL_IF(f.alproto_tc != ALPROTO_UNKNOWN);
2588 FAIL_IF(ssn->flags & STREAMTCP_FLAG_APP_LAYER_DISABLED);
2589 FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2590 FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2591 FAIL_IF(FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2592 FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2593 FAIL_IF(ssn->data_first_seen_dir != APP_LAYER_DATA_ALREADY_SENT_TO_APP_LAYER);
2594 FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2595 FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2596 FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2597 FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2598
2599 /* response ack from request */
2600 p->tcph->th_ack = htonl(328);
2601 p->tcph->th_seq = htonl(88);
2602 p->tcph->th_flags = TH_ACK;
2603 p->flowflags = FLOW_PKT_TOSERVER;
2604 p->payload_len = 0;
2605 p->payload = NULL;
2606 FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2607 FAIL_IF(!StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(&ssn->server));
2608 FAIL_IF(!StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(&ssn->client));
2609 FAIL_IF(f.alproto != ALPROTO_HTTP);
2610 FAIL_IF(f.alproto_ts != ALPROTO_HTTP);
2611 FAIL_IF(f.alproto_tc != ALPROTO_HTTP);
2612 FAIL_IF(ssn->flags & STREAMTCP_FLAG_APP_LAYER_DISABLED);
2613 FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2614 FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2615 FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2616 FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2617 FAIL_IF(ssn->data_first_seen_dir != APP_LAYER_DATA_ALREADY_SENT_TO_APP_LAYER);
2618 FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2619 FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2620 FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2621 FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2622 FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2623
2624 /* response - acking */
2625 p->tcph->th_ack = htonl(88);
2626 p->tcph->th_seq = htonl(328);
2627 p->tcph->th_flags = TH_PUSH | TH_ACK;
2628 p->flowflags = FLOW_PKT_TOCLIENT;
2629 p->payload_len = 0;
2630 p->payload = NULL;
2631 FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2632 FAIL_IF(!StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(&ssn->server));
2633 FAIL_IF(!StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(&ssn->client));
2634 FAIL_IF(f.alproto != ALPROTO_HTTP);
2635 FAIL_IF(f.alproto_ts != ALPROTO_HTTP);
2636 FAIL_IF(f.alproto_tc != ALPROTO_HTTP);
2637 FAIL_IF(ssn->flags & STREAMTCP_FLAG_APP_LAYER_DISABLED);
2638 FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2639 FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2640 FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2641 FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2642 FAIL_IF(ssn->data_first_seen_dir != APP_LAYER_DATA_ALREADY_SENT_TO_APP_LAYER);
2643 FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2644 FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2645 FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2646 FAIL_IF(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2647 FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2648
2649 /* response ack from request */
2650 p->tcph->th_ack = htonl(328);
2651 p->tcph->th_seq = htonl(88);
2652 p->tcph->th_flags = TH_ACK;
2653 p->flowflags = FLOW_PKT_TOSERVER;
2654 p->payload_len = 0;
2655 p->payload = NULL;
2656 FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2657 FAIL_IF(!StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(&ssn->server));
2658 FAIL_IF(!StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(&ssn->client));
2659 FAIL_IF(f.alproto != ALPROTO_HTTP);
2660 FAIL_IF(f.alproto_ts != ALPROTO_HTTP);
2661 FAIL_IF(f.alproto_tc != ALPROTO_HTTP);
2662 FAIL_IF(ssn->flags & STREAMTCP_FLAG_APP_LAYER_DISABLED);
2663 FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2664 FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2665 FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2666 FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2667 FAIL_IF(ssn->data_first_seen_dir != APP_LAYER_DATA_ALREADY_SENT_TO_APP_LAYER);
2668 FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2669 FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2670 FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2671 FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2672
2673 /* response - acking the request again*/
2674 p->tcph->th_ack = htonl(88);
2675 p->tcph->th_seq = htonl(328);
2676 p->tcph->th_flags = TH_PUSH | TH_ACK;
2677 p->flowflags = FLOW_PKT_TOCLIENT;
2678 p->payload_len = 0;
2679 p->payload = NULL;
2680 FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2681 FAIL_IF(!StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(&ssn->server));
2682 FAIL_IF(!StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(&ssn->client));
2683 FAIL_IF(f.alproto != ALPROTO_HTTP);
2684 FAIL_IF(f.alproto_ts != ALPROTO_HTTP);
2685 FAIL_IF(f.alproto_tc != ALPROTO_HTTP);
2686 FAIL_IF(ssn->flags & STREAMTCP_FLAG_APP_LAYER_DISABLED);
2687 FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2688 FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2689 FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2690 FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2691 FAIL_IF(ssn->data_first_seen_dir != APP_LAYER_DATA_ALREADY_SENT_TO_APP_LAYER);
2692 FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2693 FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2694 FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2695 FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2696
2697 /*** New Request ***/
2698
2699 /* partial request */
2700 p->tcph->th_ack = htonl(328);
2701 p->tcph->th_seq = htonl(88);
2702 p->tcph->th_flags = TH_PUSH | TH_ACK;
2703 p->flowflags = FLOW_PKT_TOSERVER;
2704 p->payload_len = sizeof(request1);
2705 p->payload = request1;
2706 FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2707 FAIL_IF(!StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(&ssn->server));
2708 FAIL_IF(!StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(&ssn->client));
2709 FAIL_IF(f.alproto != ALPROTO_HTTP);
2710 FAIL_IF(f.alproto_ts != ALPROTO_HTTP);
2711 FAIL_IF(f.alproto_tc != ALPROTO_HTTP);
2712 FAIL_IF(ssn->flags & STREAMTCP_FLAG_APP_LAYER_DISABLED);
2713 FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2714 FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2715 FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2716 FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2717 FAIL_IF(ssn->data_first_seen_dir != APP_LAYER_DATA_ALREADY_SENT_TO_APP_LAYER);
2718 FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2719 FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2720 FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2721 FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2722 FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2723
2724 /* response ack against partial request */
2725 p->tcph->th_ack = htonl(90);
2726 p->tcph->th_seq = htonl(328);
2727 p->tcph->th_flags = TH_ACK;
2728 p->flowflags = FLOW_PKT_TOCLIENT;
2729 p->payload_len = 0;
2730 p->payload = NULL;
2731 FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2732 FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2733 FAIL_IF(!StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(&ssn->server));
2734 FAIL_IF(!StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(&ssn->client));
2735 FAIL_IF(f.alproto != ALPROTO_HTTP);
2736 FAIL_IF(f.alproto_ts != ALPROTO_HTTP);
2737 FAIL_IF(f.alproto_tc != ALPROTO_HTTP);
2738 FAIL_IF(ssn->flags & STREAMTCP_FLAG_APP_LAYER_DISABLED);
2739 FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2740 FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2741 FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2742 FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2743 FAIL_IF(ssn->data_first_seen_dir != APP_LAYER_DATA_ALREADY_SENT_TO_APP_LAYER);
2744 FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2745 FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2746 FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2747 FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2748 FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2749
2750 /* complete request */
2751 p->tcph->th_ack = htonl(328);
2752 p->tcph->th_seq = htonl(90);
2753 p->tcph->th_flags = TH_PUSH | TH_ACK;
2754 p->flowflags = FLOW_PKT_TOSERVER;
2755 p->payload_len = sizeof(request2);
2756 p->payload = request2;
2757 FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2758 FAIL_IF(!StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(&ssn->server));
2759 FAIL_IF(!StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(&ssn->client));
2760 FAIL_IF(f.alproto != ALPROTO_HTTP);
2761 FAIL_IF(f.alproto_ts != ALPROTO_HTTP);
2762 FAIL_IF(f.alproto_tc != ALPROTO_HTTP);
2763 FAIL_IF(ssn->flags & STREAMTCP_FLAG_APP_LAYER_DISABLED);
2764 FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2765 FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2766 FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2767 FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2768 FAIL_IF(ssn->data_first_seen_dir != APP_LAYER_DATA_ALREADY_SENT_TO_APP_LAYER);
2769 FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2770 FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2771 FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2772 FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)))));
2773 FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2774 FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2775
2776 /* response ack against second partial request */
2777 p->tcph->th_ack = htonl(175);
2778 p->tcph->th_seq = htonl(328);
2779 p->tcph->th_flags = TH_ACK;
2780 p->flowflags = FLOW_PKT_TOCLIENT;
2781 p->payload_len = 0;
2782 p->payload = NULL;
2783
2784 FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2785 FAIL_IF(!StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(&ssn->server));
2786 FAIL_IF(!StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(&ssn->client));
2787 FAIL_IF(f.alproto != ALPROTO_HTTP);
2788 FAIL_IF(f.alproto_ts != ALPROTO_HTTP);
2789 FAIL_IF(f.alproto_tc != ALPROTO_HTTP);
2790 FAIL_IF(ssn->flags & STREAMTCP_FLAG_APP_LAYER_DISABLED);
2791 FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOSERVER));
2792 FAIL_IF(!FLOW_IS_PP_DONE(&f, STREAM_TOSERVER));
2793 FAIL_IF(!FLOW_IS_PM_DONE(&f, STREAM_TOCLIENT));
2794 FAIL_IF(FLOW_IS_PP_DONE(&f, STREAM_TOCLIENT));
2795 FAIL_IF(ssn->data_first_seen_dir != APP_LAYER_DATA_ALREADY_SENT_TO_APP_LAYER);
2796 FAIL_IF(RB_EMPTY(&ssn->client.seg_tree));
2797 FAIL_IF(!TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)));
2798 FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree))));
2799 FAIL_IF(!TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->client.seg_tree)))));
2800 FAIL_IF(RB_EMPTY(&ssn->server.seg_tree));
2801 FAIL_IF(TCPSEG_RB_NEXT(RB_MIN(TCPSEG, &ssn->server.seg_tree)));
2802
2803 /* response acking a request */
2804 p->tcph->th_ack = htonl(175);
2805 p->tcph->th_seq = htonl(328);
2806 p->tcph->th_flags = TH_ACK;
2807 p->flowflags = FLOW_PKT_TOCLIENT;
2808 p->payload_len = 0;
2809 p->payload = NULL;
2810 FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2811 FAIL_IF(!StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(&ssn->server));
2812 FAIL_IF(!StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(&ssn->client));
2813 FAIL_IF(f.alproto != ALPROTO_HTTP);
2814 FAIL_IF(f.alproto_ts != ALPROTO_HTTP);
2815 FAIL_IF(f.alproto_tc != ALPROTO_HTTP);
2816
2817 StreamTcpPruneSession(&f, STREAM_TOSERVER);
2818 StreamTcpPruneSession(&f, STREAM_TOCLIENT);
2819
2820 /* request acking a response */
2821 p->tcph->th_ack = htonl(328);
2822 p->tcph->th_seq = htonl(175);
2823 p->tcph->th_flags = TH_ACK;
2824 p->flowflags = FLOW_PKT_TOSERVER;
2825 p->payload_len = 0;
2826 p->payload = NULL;
2827 FAIL_IF(StreamTcpPacket(&tv, p, &stt, &pq) == -1);
2828
2829 StreamTcpSessionClear(ssn);
2830 StreamTcpUTDeinit(stt.ra_ctx);
2831 SCFree(p);
2832 PASS;
2833 }
2834
2835 /**
2836 * \test Test to make sure that we sent all the segments from the initial
2837 * segments to app layer until we have detected the app layer proto.
2838 *
2839 * \retval On success it returns 1 and on failure 0.
2840 */
2841
StreamTcpReassembleTest40(void)2842 static int StreamTcpReassembleTest40 (void)
2843 {
2844 Packet *p = PacketGetFromAlloc();
2845 FAIL_IF_NULL(p);
2846 Flow *f = NULL;
2847 TCPHdr tcph;
2848 TcpSession ssn;
2849 PacketQueueNoLock pq;
2850 memset(&pq,0,sizeof(PacketQueueNoLock));
2851 memset(&tcph, 0, sizeof (TCPHdr));
2852 ThreadVars tv;
2853 memset(&tv, 0, sizeof (ThreadVars));
2854
2855 StreamTcpInitConfig(TRUE);
2856 StreamTcpUTSetupSession(&ssn);
2857
2858 TcpReassemblyThreadCtx *ra_ctx = StreamTcpReassembleInitThreadCtx(&tv);
2859 FAIL_IF_NULL(ra_ctx);
2860
2861 uint8_t httpbuf1[] = "P";
2862 uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
2863 uint8_t httpbuf3[] = "O";
2864 uint32_t httplen3 = sizeof(httpbuf3) - 1; /* minus the \0 */
2865 uint8_t httpbuf4[] = "S";
2866 uint32_t httplen4 = sizeof(httpbuf4) - 1; /* minus the \0 */
2867 uint8_t httpbuf5[] = "T \r\n";
2868 uint32_t httplen5 = sizeof(httpbuf5) - 1; /* minus the \0 */
2869
2870 uint8_t httpbuf2[] = "HTTP/1.0 200 OK\r\nServer: VictorServer/1.0\r\n\r\n";
2871 uint32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */
2872
2873 SET_ISN(&ssn.server, 9);
2874 ssn.server.last_ack = 10;
2875 SET_ISN(&ssn.client, 9);
2876 ssn.client.isn = 9;
2877
2878 f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 200, 220);
2879 FAIL_IF_NULL(f);
2880 f->protoctx = &ssn;
2881 f->proto = IPPROTO_TCP;
2882 p->flow = f;
2883
2884 tcph.th_win = htons(5480);
2885 tcph.th_seq = htonl(10);
2886 tcph.th_ack = htonl(10);
2887 tcph.th_flags = TH_ACK|TH_PUSH;
2888 p->tcph = &tcph;
2889 p->flowflags = FLOW_PKT_TOSERVER;
2890 p->payload = httpbuf1;
2891 p->payload_len = httplen1;
2892 ssn.state = TCP_ESTABLISHED;
2893 TcpStream *s = &ssn.client;
2894 SCLogDebug("1 -- start");
2895 FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2896
2897 p->flowflags = FLOW_PKT_TOCLIENT;
2898 p->payload = httpbuf2;
2899 p->payload_len = httplen2;
2900 tcph.th_seq = htonl(10);
2901 tcph.th_ack = htonl(11);
2902 s = &ssn.server;
2903 ssn.server.last_ack = 11;
2904 SCLogDebug("2 -- start");
2905 FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2906
2907 p->flowflags = FLOW_PKT_TOSERVER;
2908 p->payload = httpbuf3;
2909 p->payload_len = httplen3;
2910 tcph.th_seq = htonl(11);
2911 tcph.th_ack = htonl(55);
2912 s = &ssn.client;
2913 ssn.client.last_ack = 55;
2914 SCLogDebug("3 -- start");
2915 FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2916
2917 p->flowflags = FLOW_PKT_TOCLIENT;
2918 p->payload = httpbuf2;
2919 p->payload_len = httplen2;
2920 tcph.th_seq = htonl(55);
2921 tcph.th_ack = htonl(12);
2922 s = &ssn.server;
2923 ssn.server.last_ack = 12;
2924 SCLogDebug("4 -- start");
2925 FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2926
2927 /* check is have the segment in the list and flagged or not */
2928 TcpSegment *seg = RB_MIN(TCPSEG, &ssn.client.seg_tree);
2929 FAIL_IF_NULL(seg);
2930 FAIL_IF(SEGMENT_BEFORE_OFFSET(&ssn.client, seg, STREAM_APP_PROGRESS(&ssn.client)));
2931
2932 p->flowflags = FLOW_PKT_TOSERVER;
2933 p->payload = httpbuf4;
2934 p->payload_len = httplen4;
2935 tcph.th_seq = htonl(12);
2936 tcph.th_ack = htonl(100);
2937 s = &ssn.client;
2938 ssn.client.last_ack = 100;
2939 SCLogDebug("5 -- start");
2940 FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2941
2942 p->flowflags = FLOW_PKT_TOCLIENT;
2943 p->payload = httpbuf2;
2944 p->payload_len = httplen2;
2945 tcph.th_seq = htonl(100);
2946 tcph.th_ack = htonl(13);
2947 s = &ssn.server;
2948 ssn.server.last_ack = 13;
2949 SCLogDebug("6 -- start");
2950 FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2951
2952 p->flowflags = FLOW_PKT_TOSERVER;
2953 p->payload = httpbuf5;
2954 p->payload_len = httplen5;
2955 tcph.th_seq = htonl(13);
2956 tcph.th_ack = htonl(145);
2957 s = &ssn.client;
2958 ssn.client.last_ack = 145;
2959 SCLogDebug("7 -- start");
2960 FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2961
2962 p->flowflags = FLOW_PKT_TOCLIENT;
2963 p->payload = httpbuf2;
2964 p->payload_len = httplen2;
2965 tcph.th_seq = htonl(145);
2966 tcph.th_ack = htonl(16);
2967 s = &ssn.server;
2968 ssn.server.last_ack = 16;
2969 SCLogDebug("8 -- start");
2970 FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
2971 FAIL_IF(f->alproto != ALPROTO_HTTP);
2972
2973 StreamTcpUTClearSession(&ssn);
2974 StreamTcpReassembleFreeThreadCtx(ra_ctx);
2975 StreamTcpFreeConfig(TRUE);
2976 SCFree(p);
2977 UTHFreeFlow(f);
2978 PASS;
2979 }
2980
2981 /** \test Test the memcap incrementing/decrementing and memcap check */
StreamTcpReassembleTest44(void)2982 static int StreamTcpReassembleTest44(void)
2983 {
2984 StreamTcpInitConfig(TRUE);
2985 uint32_t memuse = SC_ATOMIC_GET(ra_memuse);
2986 StreamTcpReassembleIncrMemuse(500);
2987 FAIL_IF(SC_ATOMIC_GET(ra_memuse) != (memuse+500));
2988 StreamTcpReassembleDecrMemuse(500);
2989 FAIL_IF(SC_ATOMIC_GET(ra_memuse) != memuse);
2990 FAIL_IF(StreamTcpReassembleCheckMemcap(500) != 1);
2991 FAIL_IF(StreamTcpReassembleCheckMemcap((1 + memuse + SC_ATOMIC_GET(stream_config.reassembly_memcap))) != 0);
2992 StreamTcpFreeConfig(TRUE);
2993 FAIL_IF(SC_ATOMIC_GET(ra_memuse) != 0);
2994 PASS;
2995 }
2996
2997 /**
2998 * \test Test to make sure that reassembly_depth is enforced.
2999 *
3000 * \retval On success it returns 1 and on failure 0.
3001 */
3002
StreamTcpReassembleTest45(void)3003 static int StreamTcpReassembleTest45 (void)
3004 {
3005 TcpReassemblyThreadCtx *ra_ctx = NULL;
3006 TcpSession ssn;
3007 ThreadVars tv;
3008 memset(&tv, 0, sizeof(tv));
3009 uint8_t payload[100] = {0};
3010 uint16_t payload_size = 100;
3011
3012 StreamTcpUTInit(&ra_ctx);
3013 stream_config.reassembly_depth = 100;
3014
3015 StreamTcpUTSetupSession(&ssn);
3016 ssn.reassembly_depth = 100;
3017 StreamTcpUTSetupStream(&ssn.server, 100);
3018 StreamTcpUTSetupStream(&ssn.client, 100);
3019
3020 int r = StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 101, payload, payload_size);
3021 FAIL_IF(r != 0);
3022 FAIL_IF(ssn.client.flags & STREAMTCP_STREAM_FLAG_DEPTH_REACHED);
3023
3024 r = StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 201, payload, payload_size);
3025 FAIL_IF(r != 0);
3026 FAIL_IF(!(ssn.client.flags & STREAMTCP_STREAM_FLAG_DEPTH_REACHED));
3027
3028 StreamTcpUTClearStream(&ssn.server);
3029 StreamTcpUTClearStream(&ssn.client);
3030 StreamTcpUTClearSession(&ssn);
3031 StreamTcpUTDeinit(ra_ctx);
3032 PASS;
3033 }
3034
3035 /**
3036 * \test Test the unlimited config value of reassembly depth.
3037 *
3038 * \retval On success it returns 1 and on failure 0.
3039 */
3040
StreamTcpReassembleTest46(void)3041 static int StreamTcpReassembleTest46 (void)
3042 {
3043 int result = 0;
3044 TcpReassemblyThreadCtx *ra_ctx = NULL;
3045 TcpSession ssn;
3046 ThreadVars tv;
3047 memset(&tv, 0, sizeof(tv));
3048 uint8_t payload[100] = {0};
3049 uint16_t payload_size = 100;
3050
3051 StreamTcpUTInit(&ra_ctx);
3052 stream_config.reassembly_depth = 0;
3053
3054 StreamTcpUTSetupSession(&ssn);
3055 StreamTcpUTSetupStream(&ssn.server, 100);
3056 StreamTcpUTSetupStream(&ssn.client, 100);
3057
3058 int r = StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 101, payload, payload_size);
3059 if (r != 0)
3060 goto end;
3061 if (ssn.client.flags & STREAMTCP_STREAM_FLAG_NOREASSEMBLY) {
3062 printf("STREAMTCP_STREAM_FLAG_NOREASSEMBLY set: ");
3063 goto end;
3064 }
3065
3066 r = StreamTcpUTAddPayload(&tv, ra_ctx, &ssn, &ssn.client, 201, payload, payload_size);
3067 if (r != 0)
3068 goto end;
3069 if (ssn.client.flags & STREAMTCP_STREAM_FLAG_NOREASSEMBLY) {
3070 printf("STREAMTCP_STREAM_FLAG_NOREASSEMBLY set: ");
3071 goto end;
3072 }
3073
3074 result = 1;
3075 end:
3076 StreamTcpUTClearStream(&ssn.server);
3077 StreamTcpUTClearStream(&ssn.client);
3078 StreamTcpUTClearSession(&ssn);
3079 StreamTcpUTDeinit(ra_ctx);
3080 return result;
3081 }
3082
3083 /**
3084 * \test Test to make sure we detect the sequence wrap around and continue
3085 * stream reassembly properly.
3086 *
3087 * \retval On success it returns 1 and on failure 0.
3088 */
3089
StreamTcpReassembleTest47(void)3090 static int StreamTcpReassembleTest47 (void)
3091 {
3092 Packet *p = PacketGetFromAlloc();
3093 FAIL_IF(unlikely(p == NULL));
3094 Flow *f = NULL;
3095 TCPHdr tcph;
3096 TcpSession ssn;
3097 ThreadVars tv;
3098 PacketQueueNoLock pq;
3099 memset(&pq,0,sizeof(PacketQueueNoLock));
3100 memset(&tcph, 0, sizeof (TCPHdr));
3101 memset(&tv, 0, sizeof (ThreadVars));
3102 StreamTcpInitConfig(TRUE);
3103 StreamTcpUTSetupSession(&ssn);
3104 TcpReassemblyThreadCtx *ra_ctx = StreamTcpReassembleInitThreadCtx(&tv);
3105
3106 uint8_t httpbuf1[] = "GET /EVILSUFF HTTP/1.1\r\n\r\n";
3107 uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
3108
3109 SET_ISN(&ssn.server, 572799781UL);
3110 ssn.server.last_ack = 572799782UL;
3111
3112 SET_ISN(&ssn.client, 4294967289UL);
3113 ssn.client.last_ack = 21;
3114
3115 f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 200, 220);
3116 FAIL_IF(f == NULL);
3117 f->protoctx = &ssn;
3118 f->proto = IPPROTO_TCP;
3119 p->flow = f;
3120
3121 tcph.th_win = htons(5480);
3122 ssn.state = TCP_ESTABLISHED;
3123 TcpStream *s = NULL;
3124 uint8_t cnt = 0;
3125
3126 for (cnt=0; cnt < httplen1; cnt++) {
3127 tcph.th_seq = htonl(ssn.client.isn + 1 + cnt);
3128 tcph.th_ack = htonl(572799782UL);
3129 tcph.th_flags = TH_ACK|TH_PUSH;
3130 p->tcph = &tcph;
3131 p->flowflags = FLOW_PKT_TOSERVER;
3132 p->payload = &httpbuf1[cnt];
3133 p->payload_len = 1;
3134 s = &ssn.client;
3135
3136 FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
3137
3138 p->flowflags = FLOW_PKT_TOCLIENT;
3139 p->payload = NULL;
3140 p->payload_len = 0;
3141 tcph.th_seq = htonl(572799782UL);
3142 tcph.th_ack = htonl(ssn.client.isn + 1 + cnt);
3143 tcph.th_flags = TH_ACK;
3144 p->tcph = &tcph;
3145 s = &ssn.server;
3146
3147 FAIL_IF(StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1);
3148 }
3149
3150 FAIL_IF(f->alproto != ALPROTO_HTTP);
3151
3152 StreamTcpUTClearSession(&ssn);
3153 StreamTcpReassembleFreeThreadCtx(ra_ctx);
3154 StreamTcpFreeConfig(TRUE);
3155 SCFree(p);
3156 UTHFreeFlow(f);
3157 PASS;
3158 }
3159
3160 /** \test 3 in order segments in inline reassembly */
StreamTcpReassembleInlineTest01(void)3161 static int StreamTcpReassembleInlineTest01(void)
3162 {
3163 int ret = 0;
3164 TcpReassemblyThreadCtx *ra_ctx = NULL;
3165 ThreadVars tv;
3166 TcpSession ssn;
3167 Flow f;
3168
3169 memset(&tv, 0x00, sizeof(tv));
3170
3171 StreamTcpUTInit(&ra_ctx);
3172 StreamTcpUTInitInline();
3173 StreamTcpUTSetupSession(&ssn);
3174 StreamTcpUTSetupStream(&ssn.client, 1);
3175 FLOW_INITIALIZE(&f);
3176
3177 uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3178 Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3179 if (p == NULL) {
3180 printf("couldn't get a packet: ");
3181 goto end;
3182 }
3183 p->tcph->th_seq = htonl(12);
3184 p->flow = &f;
3185
3186 FLOWLOCK_WRLOCK(&f);
3187 if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3188 printf("failed to add segment 1: ");
3189 goto end;
3190 }
3191 if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3192 printf("failed to add segment 2: ");
3193 goto end;
3194 }
3195 if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3196 printf("failed to add segment 3: ");
3197 goto end;
3198 }
3199 ssn.client.next_seq = 17;
3200 ret = 1;
3201 end:
3202 FLOWLOCK_UNLOCK(&f);
3203 FLOW_DESTROY(&f);
3204 UTHFreePacket(p);
3205 StreamTcpUTClearSession(&ssn);
3206 StreamTcpUTDeinit(ra_ctx);
3207 return ret;
3208 }
3209
3210 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3211 * test the sliding window reassembly.
3212 */
StreamTcpReassembleInlineTest02(void)3213 static int StreamTcpReassembleInlineTest02(void)
3214 {
3215 int ret = 0;
3216 TcpReassemblyThreadCtx *ra_ctx = NULL;
3217 ThreadVars tv;
3218 TcpSession ssn;
3219 Flow f;
3220
3221 memset(&tv, 0x00, sizeof(tv));
3222
3223 StreamTcpUTInit(&ra_ctx);
3224 StreamTcpUTInitInline();
3225 StreamTcpUTSetupSession(&ssn);
3226 StreamTcpUTSetupStream(&ssn.client, 1);
3227 FLOW_INITIALIZE(&f);
3228
3229 uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3230 Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3231 if (p == NULL) {
3232 printf("couldn't get a packet: ");
3233 goto end;
3234 }
3235 p->tcph->th_seq = htonl(12);
3236 p->flow = &f;
3237
3238 FLOWLOCK_WRLOCK(&f);
3239 if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3240 printf("failed to add segment 1: ");
3241 goto end;
3242 }
3243 if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3244 printf("failed to add segment 2: ");
3245 goto end;
3246 }
3247 if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3248 printf("failed to add segment 3: ");
3249 goto end;
3250 }
3251 ssn.client.next_seq = 17;
3252 if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) {
3253 printf("failed to add segment 4: ");
3254 goto end;
3255 }
3256 ssn.client.next_seq = 22;
3257 ret = 1;
3258 end:
3259 FLOWLOCK_UNLOCK(&f);
3260 FLOW_DESTROY(&f);
3261 UTHFreePacket(p);
3262 StreamTcpUTClearSession(&ssn);
3263 StreamTcpUTDeinit(ra_ctx);
3264 return ret;
3265 }
3266
3267 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3268 * test the sliding window reassembly with a small window size so that we
3269 * cutting off at the start (left edge)
3270 */
StreamTcpReassembleInlineTest03(void)3271 static int StreamTcpReassembleInlineTest03(void)
3272 {
3273 int ret = 0;
3274 TcpReassemblyThreadCtx *ra_ctx = NULL;
3275 ThreadVars tv;
3276 TcpSession ssn;
3277 Flow f;
3278
3279 memset(&tv, 0x00, sizeof(tv));
3280
3281 StreamTcpUTInit(&ra_ctx);
3282 StreamTcpUTInitInline();
3283 StreamTcpUTSetupSession(&ssn);
3284 StreamTcpUTSetupStream(&ssn.client, 1);
3285 FLOW_INITIALIZE(&f);
3286
3287 stream_config.reassembly_toserver_chunk_size = 15;
3288
3289 uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3290 Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3291 if (p == NULL) {
3292 printf("couldn't get a packet: ");
3293 goto end;
3294 }
3295 p->tcph->th_seq = htonl(12);
3296 p->flow = &f;
3297 p->flowflags |= FLOW_PKT_TOSERVER;
3298
3299 FLOWLOCK_WRLOCK(&f);
3300 if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3301 printf("failed to add segment 1: ");
3302 goto end;
3303 }
3304 if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3305 printf("failed to add segment 2: ");
3306 goto end;
3307 }
3308 if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3309 printf("failed to add segment 3: ");
3310 goto end;
3311 }
3312 ssn.client.next_seq = 17;
3313 if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) {
3314 printf("failed to add segment 4: ");
3315 goto end;
3316 }
3317 ssn.client.next_seq = 22;
3318
3319 p->tcph->th_seq = htonl(17);
3320 ret = 1;
3321 end:
3322 FLOWLOCK_UNLOCK(&f);
3323 FLOW_DESTROY(&f);
3324 UTHFreePacket(p);
3325 StreamTcpUTClearSession(&ssn);
3326 StreamTcpUTDeinit(ra_ctx);
3327 return ret;
3328 }
3329
3330 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3331 * test the sliding window reassembly with a small window size so that we
3332 * cutting off at the start (left edge) with small packet overlap.
3333 */
StreamTcpReassembleInlineTest04(void)3334 static int StreamTcpReassembleInlineTest04(void)
3335 {
3336 int ret = 0;
3337 TcpReassemblyThreadCtx *ra_ctx = NULL;
3338 ThreadVars tv;
3339 TcpSession ssn;
3340 Flow f;
3341
3342 memset(&tv, 0x00, sizeof(tv));
3343
3344 StreamTcpUTInit(&ra_ctx);
3345 StreamTcpUTInitInline();
3346 StreamTcpUTSetupSession(&ssn);
3347 StreamTcpUTSetupStream(&ssn.client, 1);
3348 FLOW_INITIALIZE(&f);
3349
3350 stream_config.reassembly_toserver_chunk_size = 16;
3351
3352 uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3353 Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3354 if (p == NULL) {
3355 printf("couldn't get a packet: ");
3356 goto end;
3357 }
3358 p->tcph->th_seq = htonl(12);
3359 p->flow = &f;
3360 p->flowflags |= FLOW_PKT_TOSERVER;
3361
3362 FLOWLOCK_WRLOCK(&f);
3363 if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3364 printf("failed to add segment 1: ");
3365 goto end;
3366 }
3367 if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3368 printf("failed to add segment 2: ");
3369 goto end;
3370 }
3371 if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3372 printf("failed to add segment 3: ");
3373 goto end;
3374 }
3375 ssn.client.next_seq = 17;
3376 if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) {
3377 printf("failed to add segment 4: ");
3378 goto end;
3379 }
3380 ssn.client.next_seq = 22;
3381
3382 p->tcph->th_seq = htonl(17);
3383 ret = 1;
3384 end:
3385 FLOWLOCK_UNLOCK(&f);
3386 FLOW_DESTROY(&f);
3387 UTHFreePacket(p);
3388 StreamTcpUTClearSession(&ssn);
3389 StreamTcpUTDeinit(ra_ctx);
3390 return ret;
3391 }
3392
3393 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3394 * test the sliding window reassembly with a small window size so that we
3395 * cutting off at the start (left edge). Test if the first segment is
3396 * removed from the list.
3397 */
StreamTcpReassembleInlineTest08(void)3398 static int StreamTcpReassembleInlineTest08(void)
3399 {
3400 TcpReassemblyThreadCtx *ra_ctx = NULL;
3401 ThreadVars tv;
3402 memset(&tv, 0x00, sizeof(tv));
3403 TcpSession ssn;
3404 Flow f;
3405 StreamTcpUTInit(&ra_ctx);
3406 StreamTcpUTInitInline();
3407 StreamTcpUTSetupSession(&ssn);
3408 StreamTcpUTSetupStream(&ssn.client, 1);
3409 FLOW_INITIALIZE(&f);
3410
3411 stream_config.reassembly_toserver_chunk_size = 15;
3412 f.protoctx = &ssn;
3413
3414 uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3415 Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3416 FAIL_IF(p == NULL);
3417 p->tcph->th_seq = htonl(12);
3418 p->flow = &f;
3419 p->flowflags |= FLOW_PKT_TOSERVER;
3420
3421 FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1);
3422 FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1);
3423 FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1);
3424 ssn.client.next_seq = 17;
3425 FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1);
3426 ssn.client.next_seq = 22;
3427 p->tcph->th_seq = htonl(17);
3428 StreamTcpPruneSession(&f, STREAM_TOSERVER);
3429
3430 TcpSegment *seg = RB_MIN(TCPSEG, &ssn.client.seg_tree);
3431 FAIL_IF_NULL(seg);
3432 FAIL_IF_NOT(seg->seq == 2);
3433
3434 FLOW_DESTROY(&f);
3435 UTHFreePacket(p);
3436 StreamTcpUTClearSession(&ssn);
3437 StreamTcpUTDeinit(ra_ctx);
3438 PASS;
3439 }
3440
3441 /** \test 3 in order segments, then reassemble, add one more and reassemble again.
3442 * test the sliding window reassembly with a small window size so that we
3443 * cutting off at the start (left edge). Test if the first segment is
3444 * removed from the list.
3445 */
StreamTcpReassembleInlineTest09(void)3446 static int StreamTcpReassembleInlineTest09(void)
3447 {
3448 int ret = 0;
3449 TcpReassemblyThreadCtx *ra_ctx = NULL;
3450 ThreadVars tv;
3451 TcpSession ssn;
3452 Flow f;
3453
3454 memset(&tv, 0x00, sizeof(tv));
3455
3456 StreamTcpUTInit(&ra_ctx);
3457 StreamTcpUTInitInline();
3458 StreamTcpUTSetupSession(&ssn);
3459 StreamTcpUTSetupStream(&ssn.client, 1);
3460 FLOW_INITIALIZE(&f);
3461
3462 stream_config.reassembly_toserver_chunk_size = 20;
3463
3464 uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3465 Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3466 if (p == NULL) {
3467 printf("couldn't get a packet: ");
3468 goto end;
3469 }
3470 p->tcph->th_seq = htonl(17);
3471 p->flow = &f;
3472 p->flowflags |= FLOW_PKT_TOSERVER;
3473
3474 FLOWLOCK_WRLOCK(&f);
3475 if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) {
3476 printf("failed to add segment 1: ");
3477 goto end;
3478 }
3479 if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) {
3480 printf("failed to add segment 2: ");
3481 goto end;
3482 }
3483 if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) {
3484 printf("failed to add segment 3: ");
3485 goto end;
3486 }
3487 ssn.client.next_seq = 12;
3488 ssn.client.last_ack = 10;
3489
3490 /* close the GAP and see if we properly reassemble and update base_seq */
3491 if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) {
3492 printf("failed to add segment 4: ");
3493 goto end;
3494 }
3495 ssn.client.next_seq = 22;
3496
3497 p->tcph->th_seq = htonl(12);
3498
3499 TcpSegment *seg = RB_MIN(TCPSEG, &ssn.client.seg_tree);
3500 FAIL_IF_NULL(seg);
3501 FAIL_IF_NOT(seg->seq == 2);
3502
3503 ret = 1;
3504 end:
3505 FLOWLOCK_UNLOCK(&f);
3506 FLOW_DESTROY(&f);
3507 UTHFreePacket(p);
3508 StreamTcpUTClearSession(&ssn);
3509 StreamTcpUTDeinit(ra_ctx);
3510 return ret;
3511 }
3512
3513 /** \test App Layer reassembly.
3514 */
StreamTcpReassembleInlineTest10(void)3515 static int StreamTcpReassembleInlineTest10(void)
3516 {
3517 int ret = 0;
3518 TcpReassemblyThreadCtx *ra_ctx = NULL;
3519 ThreadVars tv;
3520 TcpSession ssn;
3521 Flow *f = NULL;
3522 Packet *p = NULL;
3523
3524 memset(&tv, 0x00, sizeof(tv));
3525
3526 StreamTcpUTInit(&ra_ctx);
3527 StreamTcpUTInitInline();
3528 StreamTcpUTSetupSession(&ssn);
3529 StreamTcpUTSetupStream(&ssn.server, 1);
3530 ssn.server.last_ack = 2;
3531 StreamTcpUTSetupStream(&ssn.client, 1);
3532 ssn.client.last_ack = 2;
3533 ssn.data_first_seen_dir = STREAM_TOSERVER;
3534
3535 f = UTHBuildFlow(AF_INET, "1.1.1.1", "2.2.2.2", 1024, 80);
3536 if (f == NULL)
3537 goto end;
3538 f->protoctx = &ssn;
3539 f->proto = IPPROTO_TCP;
3540
3541 uint8_t stream_payload1[] = "GE";
3542 uint8_t stream_payload2[] = "T /";
3543 uint8_t stream_payload3[] = "HTTP/1.0\r\n\r\n";
3544
3545 p = UTHBuildPacketReal(stream_payload3, 12, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3546 if (p == NULL) {
3547 printf("couldn't get a packet: ");
3548 goto end;
3549 }
3550 p->tcph->th_seq = htonl(7);
3551 p->flow = f;
3552 p->flowflags = FLOW_PKT_TOSERVER;
3553
3554 FLOWLOCK_WRLOCK(f);
3555 if (StreamTcpUTAddSegmentWithPayload(&tv, ra_ctx, &ssn.client, 2, stream_payload1, 2) == -1) {
3556 printf("failed to add segment 1: ");
3557 goto end;
3558 }
3559 ssn.client.next_seq = 4;
3560
3561 int r = StreamTcpReassembleAppLayer(&tv, ra_ctx, &ssn, &ssn.client, p, UPDATE_DIR_PACKET);
3562 if (r < 0) {
3563 printf("StreamTcpReassembleAppLayer failed: ");
3564 goto end;
3565 }
3566
3567 /* ssn.server.ra_app_base_seq should be isn here. */
3568 if (ssn.client.base_seq != 2 || ssn.client.base_seq != ssn.client.isn+1) {
3569 printf("expected ra_app_base_seq 1, got %u: ", ssn.client.base_seq);
3570 goto end;
3571 }
3572
3573 if (StreamTcpUTAddSegmentWithPayload(&tv, ra_ctx, &ssn.client, 4, stream_payload2, 3) == -1) {
3574 printf("failed to add segment 2: ");
3575 goto end;
3576 }
3577 if (StreamTcpUTAddSegmentWithPayload(&tv, ra_ctx, &ssn.client, 7, stream_payload3, 12) == -1) {
3578 printf("failed to add segment 3: ");
3579 goto end;
3580 }
3581 ssn.client.next_seq = 19;
3582
3583 r = StreamTcpReassembleAppLayer(&tv, ra_ctx, &ssn, &ssn.client, p, UPDATE_DIR_PACKET);
3584 if (r < 0) {
3585 printf("StreamTcpReassembleAppLayer failed: ");
3586 goto end;
3587 }
3588
3589 FAIL_IF_NOT(STREAM_APP_PROGRESS(&ssn.client) == 17);
3590
3591 ret = 1;
3592 end:
3593 UTHFreePacket(p);
3594 StreamTcpUTClearSession(&ssn);
3595 StreamTcpUTDeinit(ra_ctx);
3596 FLOWLOCK_UNLOCK(f);
3597 UTHFreeFlow(f);
3598 return ret;
3599 }
3600
3601 /** \test test insert with overlap
3602 */
StreamTcpReassembleInsertTest01(void)3603 static int StreamTcpReassembleInsertTest01(void)
3604 {
3605 TcpReassemblyThreadCtx *ra_ctx = NULL;
3606 ThreadVars tv;
3607 TcpSession ssn;
3608 Flow f;
3609
3610 memset(&tv, 0x00, sizeof(tv));
3611
3612 StreamTcpUTInit(&ra_ctx);
3613 StreamTcpUTSetupSession(&ssn);
3614 StreamTcpUTSetupStream(&ssn.client, 1);
3615 ssn.client.os_policy = OS_POLICY_LAST;
3616 FLOW_INITIALIZE(&f);
3617
3618 uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' };
3619 Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
3620 FAIL_IF(p == NULL);
3621 p->tcph->th_seq = htonl(12);
3622 p->flow = &f;
3623
3624 FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1);
3625 FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1);
3626 FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 14, 'D', 2) == -1);
3627 FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 16, 'D', 6) == -1);
3628 FAIL_IF(StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1);
3629 ssn.client.next_seq = 21;
3630
3631 FLOW_DESTROY(&f);
3632 UTHFreePacket(p);
3633 StreamTcpUTClearSession(&ssn);
3634 StreamTcpUTDeinit(ra_ctx);
3635 PASS;
3636 }
3637
3638 /** \test test insert with overlaps
3639 */
StreamTcpReassembleInsertTest02(void)3640 static int StreamTcpReassembleInsertTest02(void)
3641 {
3642 int ret = 0;
3643 TcpReassemblyThreadCtx *ra_ctx = NULL;
3644 ThreadVars tv;
3645 TcpSession ssn;
3646
3647 memset(&tv, 0x00, sizeof(tv));
3648
3649 StreamTcpUTInit(&ra_ctx);
3650 StreamTcpUTSetupSession(&ssn);
3651 StreamTcpUTSetupStream(&ssn.client, 1);
3652
3653 int i;
3654 for (i = 2; i < 10; i++) {
3655 int len;
3656 len = i % 2;
3657 if (len == 0)
3658 len = 1;
3659 int seq;
3660 seq = i * 10;
3661 if (seq < 2)
3662 seq = 2;
3663
3664 if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, seq, 'A', len) == -1) {
3665 printf("failed to add segment 1: ");
3666 goto end;
3667 }
3668 }
3669 if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'B', 1024) == -1) {
3670 printf("failed to add segment 2: ");
3671 goto end;
3672 }
3673
3674 ret = 1;
3675 end:
3676 StreamTcpUTClearSession(&ssn);
3677 StreamTcpUTDeinit(ra_ctx);
3678 return ret;
3679 }
3680
3681 /** \test test insert with overlaps
3682 */
StreamTcpReassembleInsertTest03(void)3683 static int StreamTcpReassembleInsertTest03(void)
3684 {
3685 int ret = 0;
3686 TcpReassemblyThreadCtx *ra_ctx = NULL;
3687 ThreadVars tv;
3688 TcpSession ssn;
3689
3690 memset(&tv, 0x00, sizeof(tv));
3691
3692 StreamTcpUTInit(&ra_ctx);
3693 StreamTcpUTSetupSession(&ssn);
3694 StreamTcpUTSetupStream(&ssn.client, 1);
3695
3696 if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 1024) == -1) {
3697 printf("failed to add segment 2: ");
3698 goto end;
3699 }
3700
3701 int i;
3702 for (i = 2; i < 10; i++) {
3703 int len;
3704 len = i % 2;
3705 if (len == 0)
3706 len = 1;
3707 int seq;
3708 seq = i * 10;
3709 if (seq < 2)
3710 seq = 2;
3711
3712 if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, seq, 'B', len) == -1) {
3713 printf("failed to add segment 2: ");
3714 goto end;
3715 }
3716 }
3717 ret = 1;
3718 end:
3719 StreamTcpUTClearSession(&ssn);
3720 StreamTcpUTDeinit(ra_ctx);
3721 return ret;
3722 }
3723
3724 #include "tests/stream-tcp-reassemble.c"
3725 #endif /* UNITTESTS */
3726
3727 /** \brief The Function Register the Unit tests to test the reassembly engine
3728 * for various OS policies.
3729 */
3730
StreamTcpReassembleRegisterTests(void)3731 void StreamTcpReassembleRegisterTests(void)
3732 {
3733 #ifdef UNITTESTS
3734 UtRegisterTest("StreamTcpReassembleTest25 -- Gap at Start Reassembly Test",
3735 StreamTcpReassembleTest25);
3736 UtRegisterTest("StreamTcpReassembleTest26 -- Gap at middle Reassembly Test",
3737 StreamTcpReassembleTest26);
3738 UtRegisterTest("StreamTcpReassembleTest27 -- Gap at after Reassembly Test",
3739 StreamTcpReassembleTest27);
3740 UtRegisterTest("StreamTcpReassembleTest28 -- Gap at Start IDS missed packet Reassembly Test",
3741 StreamTcpReassembleTest28);
3742 UtRegisterTest("StreamTcpReassembleTest29 -- Gap at Middle IDS missed packet Reassembly Test",
3743 StreamTcpReassembleTest29);
3744 UtRegisterTest("StreamTcpReassembleTest33 -- Bug test",
3745 StreamTcpReassembleTest33);
3746 UtRegisterTest("StreamTcpReassembleTest34 -- Bug test",
3747 StreamTcpReassembleTest34);
3748 UtRegisterTest("StreamTcpReassembleTest37 -- Bug76 test",
3749 StreamTcpReassembleTest37);
3750 UtRegisterTest("StreamTcpReassembleTest39 -- app proto test",
3751 StreamTcpReassembleTest39);
3752 UtRegisterTest("StreamTcpReassembleTest40 -- app proto test",
3753 StreamTcpReassembleTest40);
3754 UtRegisterTest("StreamTcpReassembleTest44 -- Memcap Test",
3755 StreamTcpReassembleTest44);
3756 UtRegisterTest("StreamTcpReassembleTest45 -- Depth Test",
3757 StreamTcpReassembleTest45);
3758 UtRegisterTest("StreamTcpReassembleTest46 -- Depth Test",
3759 StreamTcpReassembleTest46);
3760 UtRegisterTest("StreamTcpReassembleTest47 -- TCP Sequence Wraparound Test",
3761 StreamTcpReassembleTest47);
3762
3763 UtRegisterTest("StreamTcpReassembleInlineTest01 -- inline RAW ra",
3764 StreamTcpReassembleInlineTest01);
3765 UtRegisterTest("StreamTcpReassembleInlineTest02 -- inline RAW ra 2",
3766 StreamTcpReassembleInlineTest02);
3767 UtRegisterTest("StreamTcpReassembleInlineTest03 -- inline RAW ra 3",
3768 StreamTcpReassembleInlineTest03);
3769 UtRegisterTest("StreamTcpReassembleInlineTest04 -- inline RAW ra 4",
3770 StreamTcpReassembleInlineTest04);
3771 UtRegisterTest("StreamTcpReassembleInlineTest08 -- inline RAW ra 8 cleanup",
3772 StreamTcpReassembleInlineTest08);
3773 UtRegisterTest("StreamTcpReassembleInlineTest09 -- inline RAW ra 9 GAP cleanup",
3774 StreamTcpReassembleInlineTest09);
3775
3776 UtRegisterTest("StreamTcpReassembleInlineTest10 -- inline APP ra 10",
3777 StreamTcpReassembleInlineTest10);
3778
3779 UtRegisterTest("StreamTcpReassembleInsertTest01 -- insert with overlap",
3780 StreamTcpReassembleInsertTest01);
3781 UtRegisterTest("StreamTcpReassembleInsertTest02 -- insert with overlap",
3782 StreamTcpReassembleInsertTest02);
3783 UtRegisterTest("StreamTcpReassembleInsertTest03 -- insert with overlap",
3784 StreamTcpReassembleInsertTest03);
3785
3786 StreamTcpInlineRegisterTests();
3787 StreamTcpUtilRegisterTests();
3788 StreamTcpListRegisterTests();
3789 StreamTcpReassembleRawRegisterTests();
3790 #endif /* UNITTESTS */
3791 }
3792