1 /*-
2 ***********************************************************************
3 *
4 * $Id: analyze.c,v 1.62 2014/07/30 07:07:30 mavrik Exp $
5 *
6 ***********************************************************************
7 *
8 * Copyright 2000-2014 The FTimes Project, All Rights Reserved.
9 *
10 ***********************************************************************
11 */
12 #include "all-includes.h"
13
14 /*-
15 ***********************************************************************
16 *
17 * Defines
18 *
19 ***********************************************************************
20 */
21 #define ANALYZE_BLOCK_MULTIPLIER 3
22 #define ANALYZE_BLOCK_SIZE 0x8000
23 #define ANALYZE_CARRY_SIZE 0x0400
24 #define ANALYZE_FIRST_BLOCK 1
25 #define ANALYZE_FINAL_BLOCK 2
26
27 static APP_UI32 gui32Files;
28 static APP_UI64 gui64ByteCount;
29 static APP_UI64 gui64Bytes;
30 static APP_UI64 gui64StartOffset;
31 static double gdDps;
32 static double gdAnalysisTime;
33 static int giAnalyzeBlockSize = ANALYZE_BLOCK_SIZE;
34 static int giAnalyzeCarrySize = ANALYZE_CARRY_SIZE;
35 #ifdef USE_XMAGIC
36 static int giAnalyzeStepSize = ANALYZE_BLOCK_SIZE;
37 #endif
38
39 #ifdef WINNT
40 static HANDLE ghFile; /* Needed for memory mapped XMagic. */
41 #else
42 static int giFile; /* Needed for memory mapped XMagic. */
43 #endif
44
45 /*-
46 ***********************************************************************
47 *
48 * AnalyzeGetAnalysisTime
49 *
50 ***********************************************************************
51 */
52 double
AnalyzeGetAnalysisTime(void)53 AnalyzeGetAnalysisTime(void)
54 {
55 return gdAnalysisTime;
56 }
57
58
59 /*-
60 ***********************************************************************
61 *
62 * AnalyzeGetBlockSize
63 *
64 ***********************************************************************
65 */
66 int
AnalyzeGetBlockSize(void)67 AnalyzeGetBlockSize(void)
68 {
69 return giAnalyzeBlockSize;
70 }
71
72
73 /*-
74 ***********************************************************************
75 *
76 * AnalyzeGetByteCount
77 *
78 ***********************************************************************
79 */
80 APP_UI64
AnalyzeGetByteCount(void)81 AnalyzeGetByteCount(void)
82 {
83 return gui64Bytes;
84 }
85
86
87 /*-
88 ***********************************************************************
89 *
90 * AnalyzeGetCarrySize
91 *
92 ***********************************************************************
93 */
94 int
AnalyzeGetCarrySize(void)95 AnalyzeGetCarrySize(void)
96 {
97 return giAnalyzeCarrySize;
98 }
99
100
101 /*-
102 ***********************************************************************
103 *
104 * AnalyzeGetDigSaveBuffer
105 *
106 ***********************************************************************
107 */
108 unsigned char *
AnalyzeGetDigSaveBuffer(int iCarrySize,char * pcError)109 AnalyzeGetDigSaveBuffer(int iCarrySize, char *pcError)
110 {
111 const char acRoutine[] = "AnalyzeGetDigSaveBuffer()";
112 static unsigned char *pucBuffer = NULL;
113
114 /*-
115 *********************************************************************
116 *
117 * Allocate iCarrySize bytes of memory for use as a save buffer.
118 * This memory should not be freed -- i.e., it should be allocated
119 * once and remain active until the program exits.
120 *
121 *********************************************************************
122 */
123 if (pucBuffer == NULL)
124 {
125 pucBuffer = calloc(iCarrySize, 1);
126 if (pucBuffer == NULL)
127 {
128 snprintf(pcError, MESSAGE_SIZE, "%s: calloc(): %s", acRoutine, strerror(errno));
129 return NULL;
130 }
131 }
132
133 return pucBuffer;
134 }
135
136
137 /*-
138 ***********************************************************************
139 *
140 * AnalyzeGetDps
141 *
142 ***********************************************************************
143 */
144 double
AnalyzeGetDps(void)145 AnalyzeGetDps(void)
146 {
147 return gdDps;
148 }
149
150
151 /*-
152 ***********************************************************************
153 *
154 * AnalyzeGetFileCount
155 *
156 ***********************************************************************
157 */
158 APP_UI32
AnalyzeGetFileCount(void)159 AnalyzeGetFileCount(void)
160 {
161 return gui32Files;
162 }
163
164
165 /*-
166 ***********************************************************************
167 *
168 * AnalyzeGetStartOffset
169 *
170 ***********************************************************************
171 */
172 APP_UI64
AnalyzeGetStartOffset(void)173 AnalyzeGetStartOffset(void)
174 {
175 return gui64StartOffset;
176 }
177
178
179 #ifdef USE_XMAGIC
180 /*-
181 ***********************************************************************
182 *
183 * AnalyzeGetStepSize
184 *
185 ***********************************************************************
186 */
187 int
AnalyzeGetStepSize(void)188 AnalyzeGetStepSize(void)
189 {
190 return giAnalyzeStepSize;
191 }
192 #endif
193
194
195 /*-
196 ***********************************************************************
197 *
198 * AnalyzeGetWorkBuffer
199 *
200 ***********************************************************************
201 */
202 unsigned char *
AnalyzeGetWorkBuffer(int iBlockSize,char * pcError)203 AnalyzeGetWorkBuffer(int iBlockSize, char *pcError)
204 {
205 const char acRoutine[] = "AnalyzeGetWorkBuffer()";
206 static unsigned char *pucBuffer = NULL;
207
208 /*-
209 *********************************************************************
210 *
211 * Allocate (ANALYZE_BLOCK_MULTIPLIER * iBlockSize) bytes of memory
212 * for use as a work buffer. If the memory was previously allocated,
213 * then just zero it out. This memory should not be freed -- i.e.,
214 * it should be allocated once and remain active until the program
215 * exits.
216 *
217 *********************************************************************
218 */
219 if (pucBuffer == NULL)
220 {
221 pucBuffer = calloc((ANALYZE_BLOCK_MULTIPLIER * iBlockSize), 1);
222 if (pucBuffer == NULL)
223 {
224 snprintf(pcError, MESSAGE_SIZE, "%s: calloc(): %s", acRoutine, strerror(errno));
225 return NULL;
226 }
227 }
228 else
229 {
230 memset(pucBuffer, 0, (ANALYZE_BLOCK_MULTIPLIER * iBlockSize));
231 }
232
233 return pucBuffer;
234 }
235
236
237 /*-
238 ***********************************************************************
239 *
240 * AnalyzeFile
241 *
242 ***********************************************************************
243 *
244 * With each read, AnalyzeFile() places the data into the second
245 * third of a buffer that is three times the defined read size.
246 * This provides each analysis stage with some overhead and a safety
247 * zone. Typically, a stage would use overhead to prepend leftover
248 * data from the last read to the data for the current read. In
249 * this way each stage sees a continuous stream of data. The safety
250 * zone helps to ensure that pattern searching algorithms don't
251 * slide into restricted memory. All data is initialized to zero
252 * once per file.
253 *
254 * +----------------+ +----------------+
255 * | | | |
256 * | | | |
257 * | OVERHEAD | | OVERHEAD |
258 * | | | |
259 * | | +---> |xxxxx SAVE xxxxx| +---> ...
260 * +----------------+ | +----------------+ |
261 * | | | | | |
262 * | | | | | |
263 * | 1st READ | | | 2nd READ | |
264 * | | | | | |
265 * |xxxxx SAVE xxxxx| ---+ | | ---+
266 * +----------------+ +----------------+
267 * |00000 ZERO 00000| |00000 ZERO 00000|
268 * | | | |
269 * | SAFETYZONE | | SAFETYZONE |
270 * | | | |
271 * | | | |
272 * +----------------+ +----------------+
273 *
274 ***********************************************************************
275 */
276 int
AnalyzeFile(FTIMES_PROPERTIES * psProperties,FTIMES_FILE_DATA * psFTFileData,char * pcError)277 AnalyzeFile(FTIMES_PROPERTIES *psProperties, FTIMES_FILE_DATA *psFTFileData, char *pcError)
278 {
279 const char acRoutine[] = "AnalyzeFile()";
280 char acLocalError[MESSAGE_SIZE] = "";
281 int i;
282 int iBlockSize = AnalyzeGetBlockSize();
283 int iBlockTag;
284 int iError;
285 int iNRead;
286 int iNToSeek = 0;
287 unsigned char *pucBuffer = NULL;
288 #ifdef WINNT
289 char *pcMessage;
290 HANDLE hFile;
291 int iFile;
292 #endif
293 FILE *pFile;
294 APP_UI64 ui64FileSize;
295 APP_UI64 ui64NToSeek = 0;
296
297 /*-
298 *********************************************************************
299 *
300 * If the size of the specified file is greater than FileSizeLimit,
301 * set its hash to a predefined value and return.
302 *
303 *********************************************************************
304 */
305 #ifdef WINNT
306 ui64FileSize = (((APP_UI64) psFTFileData->dwFileSizeHigh) << 32) | psFTFileData->dwFileSizeLow;
307 #else
308 ui64FileSize = (APP_UI64) psFTFileData->sStatEntry.st_size;
309 #endif
310 if (psProperties->ulFileSizeLimit != 0 && ui64FileSize > (APP_UI64) psProperties->ulFileSizeLimit)
311 {
312 if (MASK_BIT_IS_SET(psProperties->psFieldMask->ulMask, MAP_MD5))
313 {
314 memset(psFTFileData->aucFileMd5, 0xff, MD5_HASH_SIZE);
315 }
316 if (MASK_BIT_IS_SET(psProperties->psFieldMask->ulMask, MAP_SHA1))
317 {
318 memset(psFTFileData->aucFileSha1, 0xff, SHA1_HASH_SIZE);
319 }
320 if (MASK_BIT_IS_SET(psProperties->psFieldMask->ulMask, MAP_SHA256))
321 {
322 memset(psFTFileData->aucFileSha256, 0xff, SHA256_HASH_SIZE);
323 }
324 return ER_OK;
325 }
326
327 /*-
328 *********************************************************************
329 *
330 * Get a pointer to the work buffer.
331 *
332 *********************************************************************
333 */
334 pucBuffer = AnalyzeGetWorkBuffer(iBlockSize, acLocalError);
335 if (pucBuffer == NULL)
336 {
337 snprintf(pcError, MESSAGE_SIZE, "%s: %s", acRoutine, acLocalError);
338 return ER;
339 }
340
341 /*-
342 *********************************************************************
343 *
344 * Open the specified file. Since CreateFile() yields better file
345 * access, it is the preferred open method on WINNT platforms. Once
346 * a valid handle has been obtained, convert it to a file descriptor
347 * so fread() may use it. The major benefit of this approach is that
348 * it leads to simplified logic -- i.e., it eliminates the need for
349 * multipe #ifdef statements to support platform-specific mechanics.
350 * It also simplifies the process for detecting EOF, which wasn't
351 * an issue until PCRE support was added.
352 *
353 * Note that MapGetFileHandle() is not used here because the desired
354 * access flags differ. Here, we only need to read the file's data.
355 * However, the flags used in MapGetFileHandle() were set to obtain
356 * attributes for as many files as possible. More importantly, they
357 * were not set to read the file's data. Unfortunately, this means
358 * that the same file must be opened twice. We should look for ways
359 * to combine the code so that a given file is only opened once. To
360 * that end, we should look at DuplicateHandle(), which allows the
361 * caller to specify desired access flags.
362 *
363 *********************************************************************
364 */
365 #ifdef WINNT
366 hFile = CreateFileW
367 (
368 psFTFileData->pwcRawPath,
369 GENERIC_READ,
370 FILE_SHARE_READ,
371 NULL,
372 OPEN_EXISTING,
373 FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_SEQUENTIAL_SCAN,
374 NULL
375 );
376 if (hFile == INVALID_HANDLE_VALUE)
377 {
378 ErrorFormatWinxError(GetLastError(), &pcMessage);
379 snprintf(pcError, MESSAGE_SIZE, "%s: %s", acRoutine, pcMessage);
380 return ER;
381 }
382 ghFile = hFile; /* Needed for memory mapped XMagic. */
383 iFile = _open_osfhandle((intptr_t) hFile, 0);
384 if (iFile == ER)
385 {
386 snprintf(pcError, MESSAGE_SIZE, "%s: open_osfhandle(): Handle association failed.", acRoutine);
387 return ER;
388 }
389 pFile = fdopen(iFile, "rb");
390 if (pFile == NULL)
391 {
392 snprintf(pcError, MESSAGE_SIZE, "%s: fdopen(): %s", acRoutine, strerror(errno));
393 return ER;
394 }
395 #else
396 pFile = fopen(psFTFileData->pcRawPath, "rb");
397 if (pFile == NULL)
398 {
399 snprintf(pcError, MESSAGE_SIZE, "%s: fopen(): %s", acRoutine, strerror(errno));
400 return ER;
401 }
402 giFile = fileno(pFile); /* Needed for memory mapped XMagic. */
403 #endif
404
405 /*-
406 *********************************************************************
407 *
408 * Conditionally seek to the specified start offset. Return an error
409 * if the file is regular and its size is less than this offset.
410 *
411 *********************************************************************
412 */
413 gui64StartOffset = 0;
414 if (psProperties->ui64AnalyzeStartOffset)
415 {
416 if
417 (
418 #ifdef UNIX
419 S_ISREG(psFTFileData->sStatEntry.st_mode) &&
420 #endif
421 ui64FileSize < psProperties->ui64AnalyzeStartOffset
422 )
423 {
424 snprintf(pcError, MESSAGE_SIZE, "%s: File is smaller than the specified start offset.", acRoutine);
425 fclose(pFile);
426 #ifdef WINNT
427 CloseHandle(hFile);
428 #endif
429 return ER;
430 }
431 ui64NToSeek = psProperties->ui64AnalyzeStartOffset;
432 do
433 {
434 iNToSeek = (ui64NToSeek > (APP_UI64) 0x7fffffff) ? 0x7fffffff : (int) ui64NToSeek;
435 #ifdef HAVE_FSEEKO
436 iError = fseeko(pFile, (off_t) iNToSeek, SEEK_CUR);
437 #else
438 iError = fseek(pFile, iNToSeek, SEEK_CUR);
439 #endif
440 if (iError == ER)
441 {
442 #ifdef HAVE_FSEEKO
443 snprintf(pcError, MESSAGE_SIZE, "%s: fseeko(): %s", acRoutine, strerror(errno));
444 #else
445 snprintf(pcError, MESSAGE_SIZE, "%s: fseek(): %s", acRoutine, strerror(errno));
446 #endif
447 fclose(pFile);
448 #ifdef WINNT
449 CloseHandle(hFile);
450 #endif
451 return ER;
452 }
453 ui64NToSeek -= iNToSeek;
454 gui64StartOffset += iNToSeek;
455 } while (ui64NToSeek > 0);
456 }
457
458 /*-
459 *********************************************************************
460 *
461 * Update the global files counter.
462 *
463 *********************************************************************
464 */
465 gui32Files++;
466
467 /*-
468 *********************************************************************
469 *
470 * Initialize the block tag. Each analysis stage executes different
471 * logic based on the value of this tag.
472 *
473 *********************************************************************
474 */
475 iBlockTag = ANALYZE_FIRST_BLOCK;
476 gui64ByteCount = 0;
477
478 while ((iBlockTag & ANALYZE_FINAL_BLOCK) != ANALYZE_FINAL_BLOCK)
479 {
480 /*-
481 *******************************************************************
482 *
483 * Read a block of data, and insert it in the middle of our buffer.
484 *
485 *******************************************************************
486 */
487 iNRead = fread(&pucBuffer[iBlockSize], 1, iBlockSize, pFile);
488
489 /*-
490 *******************************************************************
491 *
492 * Update the global byte counters.
493 *
494 *******************************************************************
495 */
496 gui64Bytes += iNRead;
497 gui64ByteCount += iNRead;
498
499 /*-
500 *******************************************************************
501 *
502 * Determine the current DPS, and apply the brakes as needed.
503 *
504 *******************************************************************
505 */
506 AnalyzeThrottleDps(gui64Bytes, psProperties->iAnalyzeMaxDps);
507
508 /*-
509 *******************************************************************
510 *
511 * If there was a read error, close the file and return an error.
512 *
513 *******************************************************************
514 */
515 if (ferror(pFile))
516 {
517 snprintf(pcError, MESSAGE_SIZE, "%s: %s", acRoutine, strerror(errno));
518 fclose(pFile);
519 #ifdef WINNT
520 CloseHandle(hFile);
521 #endif
522 return ER;
523 }
524
525 /*-
526 *******************************************************************
527 *
528 * If EOF or the specified byte count was reached, update the block
529 * tag. If specified byte count was exceeded, reduce the read count
530 * by the difference to ensure that extra bytes are not analyzed.
531 *
532 *******************************************************************
533 */
534 if (feof(pFile))
535 {
536 iBlockTag |= ANALYZE_FINAL_BLOCK;
537 }
538 else
539 {
540 if (psProperties->ui64AnalyzeByteCount && gui64ByteCount >= psProperties->ui64AnalyzeByteCount)
541 {
542 APP_UI64 ui64Delta = gui64ByteCount - psProperties->ui64AnalyzeByteCount;
543 if (ui64Delta > (APP_UI64) iNRead)
544 {
545 snprintf(pcError, MESSAGE_SIZE, "%s: Byte count delta exceeds the read count [%d]. That shouldn't happen.", acRoutine, iNRead);
546 fclose(pFile);
547 #ifdef WINNT
548 CloseHandle(hFile);
549 #endif
550 return ER;
551 }
552 iNRead -= (int) ui64Delta;
553 iBlockTag |= ANALYZE_FINAL_BLOCK;
554 }
555 }
556
557 /*-
558 *******************************************************************
559 *
560 * Run through the defined analysis stages. Warn the user if a
561 * stage fails, but keep going.
562 *
563 *******************************************************************
564 */
565 for (i = 0; i < psProperties->iLastAnalysisStage; i++)
566 {
567 iError = psProperties->asAnalysisStages[i].piRoutine(&pucBuffer[iBlockSize], iNRead, iBlockTag, iBlockSize, psFTFileData, acLocalError);
568 if (iError != ER_OK)
569 {
570 snprintf(pcError, MESSAGE_SIZE, "%s: %s", acRoutine, acLocalError);
571 ErrorHandler(psProperties->asAnalysisStages[i].iError, pcError, ERROR_FAILURE);
572 }
573 }
574
575 /*-
576 *******************************************************************
577 *
578 * Update the block tag.
579 *
580 *******************************************************************
581 */
582 if ((iBlockTag & ANALYZE_FIRST_BLOCK) == ANALYZE_FIRST_BLOCK)
583 {
584 iBlockTag ^= ANALYZE_FIRST_BLOCK;
585 #ifdef USE_XMAGIC
586 /*-
587 *****************************************************************
588 *
589 * If XMagic was the only type of analysis requested, we're done.
590 *
591 *****************************************************************
592 */
593 if (psProperties->iLastAnalysisStage == 1 && psProperties->asAnalysisStages[0].piRoutine == AnalyzeDoXMagic)
594 {
595 iBlockTag |= ANALYZE_FINAL_BLOCK;
596 }
597 #endif
598 }
599 }
600 fclose(pFile);
601 #ifdef WINNT
602 CloseHandle(hFile);
603 #endif
604
605 return ER_OK;
606 }
607
608
609 /*-
610 ***********************************************************************
611 *
612 * AnalyzeEnableDigestEngine
613 *
614 ***********************************************************************
615 */
616 void
AnalyzeEnableDigestEngine(FTIMES_PROPERTIES * psProperties)617 AnalyzeEnableDigestEngine(FTIMES_PROPERTIES *psProperties)
618 {
619 if (MASK_BIT_IS_SET(psProperties->psFieldMask->ulMask, MAP_MD5))
620 {
621 strcpy(psProperties->asAnalysisStages[psProperties->iLastAnalysisStage].acDescription, "Md5Digest");
622 psProperties->asAnalysisStages[psProperties->iLastAnalysisStage].iError = ER_DoDigest;
623 psProperties->asAnalysisStages[psProperties->iLastAnalysisStage++].piRoutine = AnalyzeDoMd5Digest;
624 }
625 if (MASK_BIT_IS_SET(psProperties->psFieldMask->ulMask, MAP_SHA1))
626 {
627 strcpy(psProperties->asAnalysisStages[psProperties->iLastAnalysisStage].acDescription, "Sha1Digest");
628 psProperties->asAnalysisStages[psProperties->iLastAnalysisStage].iError = ER_DoDigest;
629 psProperties->asAnalysisStages[psProperties->iLastAnalysisStage++].piRoutine = AnalyzeDoSha1Digest;
630 }
631 if (MASK_BIT_IS_SET(psProperties->psFieldMask->ulMask, MAP_SHA256))
632 {
633 strcpy(psProperties->asAnalysisStages[psProperties->iLastAnalysisStage].acDescription, "Sha256Digest");
634 psProperties->asAnalysisStages[psProperties->iLastAnalysisStage].iError = ER_DoDigest;
635 psProperties->asAnalysisStages[psProperties->iLastAnalysisStage++].piRoutine = AnalyzeDoSha256Digest;
636 }
637 }
638
639
640 /*-
641 ***********************************************************************
642 *
643 * AnalyzeDoMd5Digest
644 *
645 ***********************************************************************
646 */
647 int
AnalyzeDoMd5Digest(unsigned char * pucBuffer,int iBufferLength,int iBlockTag,int iBufferOverhead,FTIMES_FILE_DATA * psFTFileData,char * pcError)648 AnalyzeDoMd5Digest(unsigned char *pucBuffer, int iBufferLength, int iBlockTag, int iBufferOverhead, FTIMES_FILE_DATA *psFTFileData, char *pcError)
649 {
650 static MD5_CONTEXT sFileMD5Context;
651
652 if ((iBlockTag & ANALYZE_FIRST_BLOCK) == ANALYZE_FIRST_BLOCK)
653 {
654 MD5Alpha(&sFileMD5Context);
655 }
656
657 MD5Cycle(&sFileMD5Context, pucBuffer, iBufferLength);
658
659 if ((iBlockTag & ANALYZE_FINAL_BLOCK) == ANALYZE_FINAL_BLOCK)
660 {
661 MD5Omega(&sFileMD5Context, psFTFileData->aucFileMd5);
662 psFTFileData->ulAttributeMask |= MAP_MD5;
663 }
664
665 return ER_OK;
666 }
667
668
669 /*-
670 ***********************************************************************
671 *
672 * AnalyzeDoSha1Digest
673 *
674 ***********************************************************************
675 */
676 int
AnalyzeDoSha1Digest(unsigned char * pucBuffer,int iBufferLength,int iBlockTag,int iBufferOverhead,FTIMES_FILE_DATA * psFTFileData,char * pcError)677 AnalyzeDoSha1Digest(unsigned char *pucBuffer, int iBufferLength, int iBlockTag, int iBufferOverhead, FTIMES_FILE_DATA *psFTFileData, char *pcError)
678 {
679 static SHA1_CONTEXT sFileSha1Context;
680
681 if ((iBlockTag & ANALYZE_FIRST_BLOCK) == ANALYZE_FIRST_BLOCK)
682 {
683 SHA1Alpha(&sFileSha1Context);
684 }
685
686 SHA1Cycle(&sFileSha1Context, pucBuffer, iBufferLength);
687
688 if ((iBlockTag & ANALYZE_FINAL_BLOCK) == ANALYZE_FINAL_BLOCK)
689 {
690 SHA1Omega(&sFileSha1Context, psFTFileData->aucFileSha1);
691 psFTFileData->ulAttributeMask |= MAP_SHA1;
692 }
693
694 return ER_OK;
695 }
696
697
698 /*-
699 ***********************************************************************
700 *
701 * AnalyzeDoSha256Digest
702 *
703 ***********************************************************************
704 */
705 int
AnalyzeDoSha256Digest(unsigned char * pucBuffer,int iBufferLength,int iBlockTag,int iBufferOverhead,FTIMES_FILE_DATA * psFTFileData,char * pcError)706 AnalyzeDoSha256Digest(unsigned char *pucBuffer, int iBufferLength, int iBlockTag, int iBufferOverhead, FTIMES_FILE_DATA *psFTFileData, char *pcError)
707 {
708 static SHA256_CONTEXT sFileSha256Context;
709
710 if ((iBlockTag & ANALYZE_FIRST_BLOCK) == ANALYZE_FIRST_BLOCK)
711 {
712 SHA256Alpha(&sFileSha256Context);
713 }
714
715 SHA256Cycle(&sFileSha256Context, pucBuffer, iBufferLength);
716
717 if ((iBlockTag & ANALYZE_FINAL_BLOCK) == ANALYZE_FINAL_BLOCK)
718 {
719 SHA256Omega(&sFileSha256Context, psFTFileData->aucFileSha256);
720 psFTFileData->ulAttributeMask |= MAP_SHA256;
721 }
722
723 return ER_OK;
724 }
725
726
727 /*-
728 ***********************************************************************
729 *
730 * AnalyzeEnableDigEngine
731 *
732 ***********************************************************************
733 */
734 void
AnalyzeEnableDigEngine(FTIMES_PROPERTIES * psProperties)735 AnalyzeEnableDigEngine(FTIMES_PROPERTIES *psProperties)
736 {
737 strcpy(psProperties->asAnalysisStages[psProperties->iLastAnalysisStage].acDescription, "Search");
738 psProperties->asAnalysisStages[psProperties->iLastAnalysisStage].iError = ER_DoDig;
739 psProperties->asAnalysisStages[psProperties->iLastAnalysisStage++].piRoutine = AnalyzeDoDig;
740 }
741
742
743 /*-
744 ***********************************************************************
745 *
746 * AnalyzeDoDig
747 *
748 ***********************************************************************
749 */
750 int
AnalyzeDoDig(unsigned char * pucBuffer,int iBufferLength,int iBlockTag,int iBufferOverhead,FTIMES_FILE_DATA * psFTFileData,char * pcError)751 AnalyzeDoDig(unsigned char *pucBuffer, int iBufferLength, int iBlockTag, int iBufferOverhead, FTIMES_FILE_DATA *psFTFileData, char *pcError)
752 {
753 const char acRoutine[] = "AnalyzeDoDig()";
754 char acLocalError[MESSAGE_SIZE] = "";
755 unsigned char *pucToSearch;
756 #ifdef USE_PCRE
757 int iBlockSize = AnalyzeGetBlockSize();
758 #endif
759 int iCarrySize = AnalyzeGetCarrySize();
760 int iError;
761 int iNToSearch;
762 int iStopShort;
763 int iType;
764 unsigned char *pucSaveBuffer = NULL;
765 static int iNToSave;
766 static int iSaveOffset;
767 static APP_UI64 ui64SearchOffset;
768
769 /*-
770 *********************************************************************
771 *
772 * Get a pointer to the save buffer.
773 *
774 *********************************************************************
775 */
776 pucSaveBuffer = AnalyzeGetDigSaveBuffer(iCarrySize, acLocalError);
777 if (pucSaveBuffer == NULL)
778 {
779 snprintf(pcError, MESSAGE_SIZE, "%s: %s", acRoutine, acLocalError);
780 return ER;
781 }
782
783 /*-
784 *********************************************************************
785 *
786 * If this is the final block, clear the stop short flag. Otherwise,
787 * set the flag and make sure that we have a nonzero length.
788 *
789 *********************************************************************
790 */
791 if ((iBlockTag & ANALYZE_FINAL_BLOCK) == ANALYZE_FINAL_BLOCK)
792 {
793 iStopShort = 0;
794 }
795 else
796 {
797 if (iBufferLength == 0)
798 {
799 snprintf(pcError, MESSAGE_SIZE, "%s: A zero length block is illegal unless it is tagged as the final block.", acRoutine);
800 return ER;
801 }
802 iStopShort = 1;
803 }
804
805 /*-
806 *********************************************************************
807 *
808 * If this is the first block, initialize. Otherwise, prepend any
809 * saved data to the incoming buffer.
810 *
811 *********************************************************************
812 */
813 if ((iBlockTag & ANALYZE_FIRST_BLOCK) == ANALYZE_FIRST_BLOCK)
814 {
815 DigClearCounts();
816
817 /*-
818 *******************************************************************
819 *
820 * Make sure that the max string length is less than or equal to
821 * the size of the save buffer. The dig algorithm stops searching
822 * after x bytes have been processed. Where
823 *
824 * x = (bufsize - (maxstring - 1))
825 *
826 * For example, if bufsize = 16 and maxstring = 6, then all but
827 * the last 5 bytes may be searched (i.e., to stay in bounds). Thus,
828 * the search routine needs to stop searching when it reaches byte
829 * 11.
830 *
831 * If no search strings were defined, maxstring should be zero.
832 *
833 * There is no case where iNToSave should be set to a negative
834 * value.
835 *
836 *******************************************************************
837 */
838 if (DigGetMaxStringLength() <= iCarrySize)
839 {
840 iNToSave = iCarrySize;
841 DigSetSaveLength(iCarrySize);
842 }
843 else
844 {
845 snprintf(pcError, MESSAGE_SIZE, "%s: Not enough overhead in the save buffer. The carry size (%d) must be no less than %d bytes for this job to work as intended.", acRoutine, iCarrySize, DigGetMaxStringLength());
846 return ER;
847 }
848
849 ui64SearchOffset = 0;
850 pucToSearch = pucBuffer;
851 iNToSearch = iBufferLength;
852 }
853 else
854 {
855 pucToSearch = pucBuffer - iNToSave;
856 memcpy(pucToSearch, pucSaveBuffer, iNToSave);
857 iNToSearch = iBufferLength + iNToSave;
858 }
859
860 /*-
861 *********************************************************************
862 *
863 * Search the input.
864 *
865 *********************************************************************
866 */
867 for (iType = DIG_STRING_TYPE_NORMAL; iType < DIG_STRING_TYPE_NOMORE; iType++)
868 {
869 iError = DigSearchData(pucToSearch, iNToSearch, iStopShort, iType, ui64SearchOffset, psFTFileData->pcNeuteredPath, acLocalError);
870 if (iError != ER_OK)
871 {
872 snprintf(pcError, MESSAGE_SIZE, "%s: %s", acRoutine, acLocalError);
873 return iError;
874 }
875 }
876
877 /*-
878 *********************************************************************
879 *
880 * Save any data that could not be fully searched.
881 *
882 *********************************************************************
883 */
884 iSaveOffset = iNToSearch - iNToSave;
885 memcpy(pucSaveBuffer, &pucToSearch[iSaveOffset], iNToSave);
886
887 /*-
888 *********************************************************************
889 *
890 * Update the search offset. This ensures that search routine is
891 * passed the offset it expects. In other words, subtract off the
892 * unsearched portion of the prior search buffer, and keep it for
893 * next time.
894 *
895 *********************************************************************
896 */
897 if ((iBlockTag & ANALYZE_FIRST_BLOCK) == ANALYZE_FIRST_BLOCK)
898 {
899 ui64SearchOffset += iBufferLength - iNToSave;
900 }
901 else
902 {
903 ui64SearchOffset += iBufferLength;
904 }
905
906 #ifdef USE_PCRE
907 /*-
908 *********************************************************************
909 *
910 * Update dig offsets. The trim size is equal to the block size in
911 * all but the first block.
912 *
913 *********************************************************************
914 */
915 if ((iBlockTag & ANALYZE_FIRST_BLOCK) == ANALYZE_FIRST_BLOCK)
916 {
917 DigAdjustRegExpOffsets(iBlockSize - iCarrySize);
918 }
919 else
920 {
921 DigAdjustRegExpOffsets(iBlockSize);
922 }
923 #endif
924
925 return ER_OK;
926 }
927
928
929 #ifdef USE_XMAGIC
930 /*-
931 ***********************************************************************
932 *
933 * AnalyzeEnableXMagicEngine
934 *
935 ***********************************************************************
936 */
937 int
AnalyzeEnableXMagicEngine(FTIMES_PROPERTIES * psProperties,char * pcError)938 AnalyzeEnableXMagicEngine(FTIMES_PROPERTIES *psProperties, char *pcError)
939 {
940 const char acRoutine[] = "AnalyzeEnableXMagicEngine()";
941 char acLocalError[MESSAGE_SIZE] = "";
942 unsigned char aucMD5[MD5_HASH_SIZE];
943 int i;
944 int iError;
945 FILE *pFile;
946 APP_UI64 ui64Size;
947
948 /*-
949 *********************************************************************
950 *
951 * Set up the XMagic analysis function pointer.
952 *
953 *********************************************************************
954 */
955 strcpy(psProperties->asAnalysisStages[psProperties->iLastAnalysisStage].acDescription, "XMagic");
956 psProperties->asAnalysisStages[psProperties->iLastAnalysisStage].iError = ER_DoXMagic;
957 psProperties->asAnalysisStages[psProperties->iLastAnalysisStage++].piRoutine = AnalyzeDoXMagic;
958
959 /*-
960 *********************************************************************
961 *
962 * Locate and load magic. If a file was specified, it must exist.
963 *
964 *********************************************************************
965 */
966 for (i = 0; i < 3; i++)
967 {
968 switch (i)
969 {
970 case 0:
971 if (psProperties->acMagicFileName[0])
972 {
973 iError = SupportExpandPath(psProperties->acMagicFileName, psProperties->acMagicFileName, FTIMES_MAX_PATH, 1, acLocalError);
974 if (iError != ER_OK)
975 {
976 snprintf(pcError, MESSAGE_SIZE, "%s: %s", acRoutine, acLocalError);
977 return ER_XMagic;
978 }
979 }
980 else
981 {
982 continue;
983 }
984 break;
985 case 1:
986 iError = SupportExpandPath(XMAGIC_DEFAULT_LOCATION, psProperties->acMagicFileName, FTIMES_MAX_PATH, 1, acLocalError);
987 break;
988 case 2:
989 iError = SupportExpandPath(XMAGIC_CURRENT_LOCATION, psProperties->acMagicFileName, FTIMES_MAX_PATH, 1, acLocalError);
990 break;
991 default:
992 iError = ER_BadValue;
993 snprintf(acLocalError, MESSAGE_SIZE, "bad loop index");
994 break;
995 }
996 if (iError == ER_OK)
997 {
998 psProperties->psXMagic = XMagicLoadMagic(psProperties->acMagicFileName, acLocalError);
999 if (psProperties->psXMagic == NULL)
1000 {
1001 snprintf(pcError, MESSAGE_SIZE, "%s: %s", acRoutine, acLocalError);
1002 return ER_XMagic;
1003 }
1004 else
1005 {
1006 if ((pFile = fopen(psProperties->acMagicFileName, "rb")) != NULL && MD5HashStream(pFile, aucMD5, &ui64Size) == ER_OK)
1007 {
1008 MD5HashToHex(aucMD5, psProperties->acMagicHash);
1009 fclose(pFile);
1010 }
1011 else
1012 {
1013 strcpy(psProperties->acMagicHash, "NONE");
1014 }
1015 break; /* Very important. This get's us out of the for loop. */
1016 }
1017 }
1018 psProperties->acMagicFileName[0] = 0;
1019 }
1020
1021 if (psProperties->acMagicFileName[0] == 0)
1022 {
1023 strcpy(psProperties->acMagicFileName, "NA");
1024 strcpy(psProperties->acMagicHash, "NA");
1025 }
1026
1027 return ER_OK;
1028 }
1029
1030
1031 /*-
1032 ***********************************************************************
1033 *
1034 * AnalyzeDoXMagic
1035 *
1036 ***********************************************************************
1037 */
1038 int
AnalyzeDoXMagic(unsigned char * pucBuffer,int iBufferLength,int iBlockTag,int iBufferOverhead,FTIMES_FILE_DATA * psFTFileData,char * pcError)1039 AnalyzeDoXMagic(unsigned char *pucBuffer, int iBufferLength, int iBlockTag, int iBufferOverhead, FTIMES_FILE_DATA *psFTFileData, char *pcError)
1040 {
1041 const char acRoutine[] = "AnalyzeDoXMagic()";
1042 char acLocalError[MESSAGE_SIZE] = "";
1043 char acMessage[MESSAGE_SIZE] = "";
1044 int iError = 0;
1045 int iMemoryMapped = 0;
1046 int iMemoryMapSize = 0;
1047 void *pvMemoryMap = NULL;
1048 static int iFirst = 1;
1049 static int iMemoryMapEnable = 0;
1050 static XMAGIC *psXMagic = NULL;
1051
1052 /*-
1053 *********************************************************************
1054 *
1055 * Obtain a reference to the XMagic, and set other static variables
1056 * on the first pass.
1057 *
1058 *********************************************************************
1059 */
1060 if (iFirst)
1061 {
1062 FTIMES_PROPERTIES *psProperties = FTimesGetPropertiesReference();
1063 psXMagic = psProperties->psXMagic;
1064 iMemoryMapEnable = psProperties->iMemoryMapEnable;
1065 iFirst = 0;
1066 }
1067
1068 /*-
1069 *********************************************************************
1070 *
1071 * XMagic tests are only done once -- either on the first block or
1072 * on a memory mapped view of the file.
1073 *
1074 *********************************************************************
1075 */
1076 if ((iBlockTag & ANALYZE_FIRST_BLOCK) != ANALYZE_FIRST_BLOCK)
1077 {
1078 return ER_OK;
1079 }
1080
1081 /*-
1082 *********************************************************************
1083 *
1084 * Conditionally map the file into memory. If the file's size is
1085 * zero, do not attempt to map it as this will lead to EINVAL errors
1086 * on some platforms -- either for mmap() or munmap(). If the memory
1087 * can't be mapped for whatever reason, fallback to using the block
1088 * of data provided by the caller.
1089 *
1090 *********************************************************************
1091 */
1092 if (iMemoryMapEnable)
1093 {
1094 #ifdef WINNT
1095 if (psFTFileData->dwFileSizeHigh == 0 && psFTFileData->dwFileSizeLow < FTIMES_MAX_MMAP_SIZE)
1096 {
1097 iMemoryMapSize = psFTFileData->dwFileSizeLow;
1098 }
1099 #else
1100 if (psFTFileData->sStatEntry.st_size < FTIMES_MAX_MMAP_SIZE)
1101 {
1102 iMemoryMapSize = psFTFileData->sStatEntry.st_size;
1103 }
1104 #endif
1105 else
1106 {
1107 iMemoryMapSize = FTIMES_MAX_MMAP_SIZE;
1108 }
1109 if (iMemoryMapSize > 0 && (pvMemoryMap = AnalyzeMapMemory(iMemoryMapSize)) != NULL)
1110 {
1111 pucBuffer = (unsigned char *) pvMemoryMap;
1112 iBufferLength = iMemoryMapSize;
1113 iMemoryMapped = 1;
1114 }
1115 }
1116 snprintf(acMessage, MESSAGE_SIZE, "AnalysisStage=XMagic MemoryMapped=%d BufferLength=%d", iMemoryMapped, iBufferLength);
1117 MessageHandler(MESSAGE_FLUSH_IT, MESSAGE_DEBUGGER, MESSAGE_DEBUGGER_STRING, acMessage);
1118
1119 /*-
1120 *********************************************************************
1121 *
1122 * Execute XMagic tests.
1123 *
1124 *********************************************************************
1125 */
1126 iError = XMagicTestBuffer(psXMagic, pucBuffer, iBufferLength, psFTFileData->acType, FTIMES_FILETYPE_BUFSIZE, acLocalError);
1127 if (iError == ER)
1128 {
1129 snprintf(pcError, MESSAGE_SIZE, "%s: %s", acRoutine, acLocalError);
1130 if (iMemoryMapped)
1131 {
1132 AnalyzeUnmapMemory(pvMemoryMap, iMemoryMapSize);
1133 }
1134 return ER_XMagic;
1135 }
1136 psFTFileData->ulAttributeMask |= MAP_MAGIC;
1137 if (iMemoryMapped)
1138 {
1139 AnalyzeUnmapMemory(pvMemoryMap, iMemoryMapSize);
1140 }
1141
1142 return ER_OK;
1143 }
1144 #endif
1145
1146
1147 /*-
1148 ***********************************************************************
1149 *
1150 * AnalyzeMapMemory
1151 *
1152 ***********************************************************************
1153 */
1154 void *
AnalyzeMapMemory(int iMemoryMapSize)1155 AnalyzeMapMemory(int iMemoryMapSize)
1156 {
1157 #ifdef WINNT
1158 HANDLE hMemoryMap = NULL;
1159 #endif
1160 void *pvMemoryMap = NULL;
1161
1162 /*-
1163 *********************************************************************
1164 *
1165 * This routine does not detect or return errors because its caller
1166 * (AnalyzeDoXMagic) is designed to use an alternate data buffer if
1167 * the file can't be mapped into memory.
1168 *
1169 *********************************************************************
1170 */
1171 #ifdef WINNT
1172 hMemoryMap = CreateFileMapping(ghFile, NULL, PAGE_READONLY, 0, 0, 0);
1173 if (hMemoryMap)
1174 {
1175 pvMemoryMap = MapViewOfFile(hMemoryMap, FILE_MAP_READ, 0, 0, 0);
1176 CloseHandle(hMemoryMap);
1177 }
1178 #else
1179 pvMemoryMap = mmap(NULL, iMemoryMapSize, PROT_READ, MAP_PRIVATE, giFile, 0);
1180 #if defined(FTimes_HPUX) && !defined(MAP_FAILED)
1181 #define MAP_FAILED ((void *)-1)
1182 #endif
1183 if (pvMemoryMap == MAP_FAILED)
1184 {
1185 return NULL;
1186 }
1187 #endif
1188
1189 return pvMemoryMap;
1190 }
1191
1192
1193 /*-
1194 ***********************************************************************
1195 *
1196 * AnalyzeSetBlockSize
1197 *
1198 ***********************************************************************
1199 */
1200 void
AnalyzeSetBlockSize(int iBlockSize)1201 AnalyzeSetBlockSize(int iBlockSize)
1202 {
1203 giAnalyzeBlockSize = iBlockSize;
1204 }
1205
1206
1207 /*-
1208 ***********************************************************************
1209 *
1210 * AnalyzeSetCarrySize
1211 *
1212 ***********************************************************************
1213 */
1214 void
AnalyzeSetCarrySize(int iCarrySize)1215 AnalyzeSetCarrySize(int iCarrySize)
1216 {
1217 giAnalyzeCarrySize = iCarrySize;
1218 }
1219
1220
1221 #ifdef USE_XMAGIC
1222 /*-
1223 ***********************************************************************
1224 *
1225 * AnalyzeSetStepSize
1226 *
1227 ***********************************************************************
1228 */
1229 void
AnalyzeSetStepSize(int iStepSize)1230 AnalyzeSetStepSize(int iStepSize)
1231 {
1232 giAnalyzeStepSize = iStepSize;
1233 }
1234 #endif
1235
1236
1237 /*-
1238 ***********************************************************************
1239 *
1240 * AnalyzeThrottleDps
1241 *
1242 ***********************************************************************
1243 */
1244 void
AnalyzeThrottleDps(APP_UI64 ui64Bytes,int iMaxDps)1245 AnalyzeThrottleDps(APP_UI64 ui64Bytes, int iMaxDps)
1246 {
1247 static double dNow = 0;
1248 static double dStartTime = 0;
1249 static int iFirst = 1;
1250 double dKBytes = 0;
1251 double dSleepTime = 0;
1252 int iSleepTime = 0;
1253 #ifdef WINNT
1254 APP_SI64 i64Bytes = (APP_SI64) ui64Bytes;
1255 #endif
1256
1257 /*-
1258 *********************************************************************
1259 *
1260 * Get the current time. Record the start time on the first pass.
1261 *
1262 *********************************************************************
1263 */
1264 dNow = TimeGetTimeValueAsDouble();
1265 if (iFirst)
1266 {
1267 dStartTime = dNow;
1268 iFirst = 0;
1269 }
1270
1271 /*-
1272 *********************************************************************
1273 *
1274 * Calculate the amount of time spent thus far. Force the result to
1275 * be at least one microsecond to thwart divide by zero issues.
1276 *
1277 *********************************************************************
1278 */
1279 gdAnalysisTime = (double) (dNow - dStartTime);
1280 if (gdAnalysisTime <= 0)
1281 {
1282 gdAnalysisTime = 0.000001;
1283 }
1284
1285 /*-
1286 *********************************************************************
1287 *
1288 * Calculate the Data Processing Speed (DPS). If the maximum DPS is
1289 * zero, do not apply the brakes. Otherwise, set the sleep time to a
1290 * value in the range [1,60], and sleep. If the DPS value has a
1291 * fractional component (of any amount), round it up.
1292 *
1293 *********************************************************************
1294 */
1295 #ifdef WINNT
1296 dKBytes = (double) ((double) i64Bytes / (double) 1024);
1297 #else
1298 dKBytes = (double) ((double) ui64Bytes / (double) 1024);
1299 #endif
1300 gdDps = dKBytes / gdAnalysisTime;
1301 if (iMaxDps && gdDps > (double) iMaxDps)
1302 {
1303 dSleepTime = (double) ((dKBytes / (double) iMaxDps) - gdAnalysisTime);
1304 if (dSleepTime < 1)
1305 {
1306 iSleepTime = 1;
1307 }
1308 else if (dSleepTime > 60)
1309 {
1310 iSleepTime = 60;
1311 }
1312 else
1313 {
1314 iSleepTime = (int) dSleepTime;
1315 if ((double) (dSleepTime / (double) iSleepTime))
1316 {
1317 iSleepTime += 1;
1318 }
1319 }
1320 #ifdef WINNT
1321 Sleep(iSleepTime * 1000);
1322 #else
1323 sleep(iSleepTime);
1324 #endif
1325 }
1326 }
1327
1328
1329 /*-
1330 ***********************************************************************
1331 *
1332 * AnalyzeUnmapMemory
1333 *
1334 ***********************************************************************
1335 */
1336 void
AnalyzeUnmapMemory(void * pvMemoryMap,int iMemoryMapSize)1337 AnalyzeUnmapMemory(void *pvMemoryMap, int iMemoryMapSize)
1338 {
1339 #ifdef WINNT
1340 UnmapViewOfFile(pvMemoryMap);
1341 #else
1342 munmap(pvMemoryMap, iMemoryMapSize);
1343 #endif
1344 return;
1345 }
1346