1 /*-
2  ***********************************************************************
3  *
4  * $Id: analyze.c,v 1.62 2014/07/30 07:07:30 mavrik Exp $
5  *
6  ***********************************************************************
7  *
8  * Copyright 2000-2014 The FTimes Project, All Rights Reserved.
9  *
10  ***********************************************************************
11  */
12 #include "all-includes.h"
13 
14 /*-
15  ***********************************************************************
16  *
17  * Defines
18  *
19  ***********************************************************************
20  */
21 #define ANALYZE_BLOCK_MULTIPLIER  3
22 #define ANALYZE_BLOCK_SIZE   0x8000
23 #define ANALYZE_CARRY_SIZE   0x0400
24 #define ANALYZE_FIRST_BLOCK       1
25 #define ANALYZE_FINAL_BLOCK       2
26 
27 static APP_UI32       gui32Files;
28 static APP_UI64       gui64ByteCount;
29 static APP_UI64       gui64Bytes;
30 static APP_UI64       gui64StartOffset;
31 static double         gdDps;
32 static double         gdAnalysisTime;
33 static int            giAnalyzeBlockSize = ANALYZE_BLOCK_SIZE;
34 static int            giAnalyzeCarrySize = ANALYZE_CARRY_SIZE;
35 #ifdef USE_XMAGIC
36 static int            giAnalyzeStepSize = ANALYZE_BLOCK_SIZE;
37 #endif
38 
39 #ifdef WINNT
40 static HANDLE         ghFile; /* Needed for memory mapped XMagic. */
41 #else
42 static int            giFile; /* Needed for memory mapped XMagic. */
43 #endif
44 
45 /*-
46  ***********************************************************************
47  *
48  * AnalyzeGetAnalysisTime
49  *
50  ***********************************************************************
51  */
52 double
AnalyzeGetAnalysisTime(void)53 AnalyzeGetAnalysisTime(void)
54 {
55   return gdAnalysisTime;
56 }
57 
58 
59 /*-
60  ***********************************************************************
61  *
62  * AnalyzeGetBlockSize
63  *
64  ***********************************************************************
65  */
66 int
AnalyzeGetBlockSize(void)67 AnalyzeGetBlockSize(void)
68 {
69   return giAnalyzeBlockSize;
70 }
71 
72 
73 /*-
74  ***********************************************************************
75  *
76  * AnalyzeGetByteCount
77  *
78  ***********************************************************************
79  */
80 APP_UI64
AnalyzeGetByteCount(void)81 AnalyzeGetByteCount(void)
82 {
83   return gui64Bytes;
84 }
85 
86 
87 /*-
88  ***********************************************************************
89  *
90  * AnalyzeGetCarrySize
91  *
92  ***********************************************************************
93  */
94 int
AnalyzeGetCarrySize(void)95 AnalyzeGetCarrySize(void)
96 {
97   return giAnalyzeCarrySize;
98 }
99 
100 
101 /*-
102  ***********************************************************************
103  *
104  * AnalyzeGetDigSaveBuffer
105  *
106  ***********************************************************************
107  */
108 unsigned char *
AnalyzeGetDigSaveBuffer(int iCarrySize,char * pcError)109 AnalyzeGetDigSaveBuffer(int iCarrySize, char *pcError)
110 {
111   const char          acRoutine[] = "AnalyzeGetDigSaveBuffer()";
112   static unsigned char *pucBuffer = NULL;
113 
114   /*-
115    *********************************************************************
116    *
117    * Allocate iCarrySize bytes of memory for use as a save buffer.
118    * This memory should not be freed -- i.e., it should be allocated
119    * once and remain active until the program exits.
120    *
121    *********************************************************************
122    */
123   if (pucBuffer == NULL)
124   {
125     pucBuffer = calloc(iCarrySize, 1);
126     if (pucBuffer == NULL)
127     {
128       snprintf(pcError, MESSAGE_SIZE, "%s: calloc(): %s", acRoutine, strerror(errno));
129       return NULL;
130     }
131   }
132 
133   return pucBuffer;
134 }
135 
136 
137 /*-
138  ***********************************************************************
139  *
140  * AnalyzeGetDps
141  *
142  ***********************************************************************
143  */
144 double
AnalyzeGetDps(void)145 AnalyzeGetDps(void)
146 {
147   return gdDps;
148 }
149 
150 
151 /*-
152  ***********************************************************************
153  *
154  * AnalyzeGetFileCount
155  *
156  ***********************************************************************
157  */
158 APP_UI32
AnalyzeGetFileCount(void)159 AnalyzeGetFileCount(void)
160 {
161   return gui32Files;
162 }
163 
164 
165 /*-
166  ***********************************************************************
167  *
168  * AnalyzeGetStartOffset
169  *
170  ***********************************************************************
171  */
172 APP_UI64
AnalyzeGetStartOffset(void)173 AnalyzeGetStartOffset(void)
174 {
175   return gui64StartOffset;
176 }
177 
178 
179 #ifdef USE_XMAGIC
180 /*-
181  ***********************************************************************
182  *
183  * AnalyzeGetStepSize
184  *
185  ***********************************************************************
186  */
187 int
AnalyzeGetStepSize(void)188 AnalyzeGetStepSize(void)
189 {
190   return giAnalyzeStepSize;
191 }
192 #endif
193 
194 
195 /*-
196  ***********************************************************************
197  *
198  * AnalyzeGetWorkBuffer
199  *
200  ***********************************************************************
201  */
202 unsigned char *
AnalyzeGetWorkBuffer(int iBlockSize,char * pcError)203 AnalyzeGetWorkBuffer(int iBlockSize, char *pcError)
204 {
205   const char          acRoutine[] = "AnalyzeGetWorkBuffer()";
206   static unsigned char *pucBuffer = NULL;
207 
208   /*-
209    *********************************************************************
210    *
211    * Allocate (ANALYZE_BLOCK_MULTIPLIER * iBlockSize) bytes of memory
212    * for use as a work buffer. If the memory was previously allocated,
213    * then just zero it out. This memory should not be freed -- i.e.,
214    * it should be allocated once and remain active until the program
215    * exits.
216    *
217    *********************************************************************
218    */
219   if (pucBuffer == NULL)
220   {
221     pucBuffer = calloc((ANALYZE_BLOCK_MULTIPLIER * iBlockSize), 1);
222     if (pucBuffer == NULL)
223     {
224       snprintf(pcError, MESSAGE_SIZE, "%s: calloc(): %s", acRoutine, strerror(errno));
225       return NULL;
226     }
227   }
228   else
229   {
230     memset(pucBuffer, 0, (ANALYZE_BLOCK_MULTIPLIER * iBlockSize));
231   }
232 
233   return pucBuffer;
234 }
235 
236 
237 /*-
238  ***********************************************************************
239  *
240  * AnalyzeFile
241  *
242  ***********************************************************************
243  *
244  * With each read, AnalyzeFile() places the data into the second
245  * third of a buffer that is three times the defined read size.
246  * This provides each analysis stage with some overhead and a safety
247  * zone. Typically, a stage would use overhead to prepend leftover
248  * data from the last read to the data for the current read. In
249  * this way each stage sees a continuous stream of data. The safety
250  * zone helps to ensure that pattern searching algorithms don't
251  * slide into restricted memory. All data is initialized to zero
252  * once per file.
253  *
254  *   +----------------+          +----------------+
255  *   |                |          |                |
256  *   |                |          |                |
257  *   |    OVERHEAD    |          |    OVERHEAD    |
258  *   |                |          |                |
259  *   |                |    +---> |xxxxx SAVE xxxxx|    +---> ...
260  *   +----------------+    |     +----------------+    |
261  *   |                |    |     |                |    |
262  *   |                |    |     |                |    |
263  *   |    1st READ    |    |     |    2nd READ    |    |
264  *   |                |    |     |                |    |
265  *   |xxxxx SAVE xxxxx| ---+     |                | ---+
266  *   +----------------+          +----------------+
267  *   |00000 ZERO 00000|          |00000 ZERO 00000|
268  *   |                |          |                |
269  *   |   SAFETYZONE   |          |   SAFETYZONE   |
270  *   |                |          |                |
271  *   |                |          |                |
272  *   +----------------+          +----------------+
273  *
274  ***********************************************************************
275  */
276 int
AnalyzeFile(FTIMES_PROPERTIES * psProperties,FTIMES_FILE_DATA * psFTFileData,char * pcError)277 AnalyzeFile(FTIMES_PROPERTIES *psProperties, FTIMES_FILE_DATA *psFTFileData, char *pcError)
278 {
279   const char          acRoutine[] = "AnalyzeFile()";
280   char                acLocalError[MESSAGE_SIZE] = "";
281   int                 i;
282   int                 iBlockSize = AnalyzeGetBlockSize();
283   int                 iBlockTag;
284   int                 iError;
285   int                 iNRead;
286   int                 iNToSeek = 0;
287   unsigned char      *pucBuffer = NULL;
288 #ifdef WINNT
289   char               *pcMessage;
290   HANDLE              hFile;
291   int                 iFile;
292 #endif
293   FILE               *pFile;
294   APP_UI64            ui64FileSize;
295   APP_UI64            ui64NToSeek = 0;
296 
297   /*-
298    *********************************************************************
299    *
300    * If the size of the specified file is greater than FileSizeLimit,
301    * set its hash to a predefined value and return.
302    *
303    *********************************************************************
304    */
305 #ifdef WINNT
306   ui64FileSize = (((APP_UI64) psFTFileData->dwFileSizeHigh) << 32) | psFTFileData->dwFileSizeLow;
307 #else
308   ui64FileSize = (APP_UI64) psFTFileData->sStatEntry.st_size;
309 #endif
310   if (psProperties->ulFileSizeLimit != 0 && ui64FileSize > (APP_UI64) psProperties->ulFileSizeLimit)
311   {
312     if (MASK_BIT_IS_SET(psProperties->psFieldMask->ulMask, MAP_MD5))
313     {
314       memset(psFTFileData->aucFileMd5, 0xff, MD5_HASH_SIZE);
315     }
316     if (MASK_BIT_IS_SET(psProperties->psFieldMask->ulMask, MAP_SHA1))
317     {
318       memset(psFTFileData->aucFileSha1, 0xff, SHA1_HASH_SIZE);
319     }
320     if (MASK_BIT_IS_SET(psProperties->psFieldMask->ulMask, MAP_SHA256))
321     {
322       memset(psFTFileData->aucFileSha256, 0xff, SHA256_HASH_SIZE);
323     }
324     return ER_OK;
325   }
326 
327   /*-
328    *********************************************************************
329    *
330    * Get a pointer to the work buffer.
331    *
332    *********************************************************************
333    */
334   pucBuffer = AnalyzeGetWorkBuffer(iBlockSize, acLocalError);
335   if (pucBuffer == NULL)
336   {
337     snprintf(pcError, MESSAGE_SIZE, "%s: %s", acRoutine, acLocalError);
338     return ER;
339   }
340 
341   /*-
342    *********************************************************************
343    *
344    * Open the specified file. Since CreateFile() yields better file
345    * access, it is the preferred open method on WINNT platforms. Once
346    * a valid handle has been obtained, convert it to a file descriptor
347    * so fread() may use it. The major benefit of this approach is that
348    * it leads to simplified logic -- i.e., it eliminates the need for
349    * multipe #ifdef statements to support platform-specific mechanics.
350    * It also simplifies the process for detecting EOF, which wasn't
351    * an issue until PCRE support was added.
352    *
353    * Note that MapGetFileHandle() is not used here because the desired
354    * access flags differ. Here, we only need to read the file's data.
355    * However, the flags used in MapGetFileHandle() were set to obtain
356    * attributes for as many files as possible. More importantly, they
357    * were not set to read the file's data. Unfortunately, this means
358    * that the same file must be opened twice. We should look for ways
359    * to combine the code so that a given file is only opened once. To
360    * that end, we should look at DuplicateHandle(), which allows the
361    * caller to specify desired access flags.
362    *
363    *********************************************************************
364    */
365 #ifdef WINNT
366   hFile = CreateFileW
367           (
368             psFTFileData->pwcRawPath,
369             GENERIC_READ,
370             FILE_SHARE_READ,
371             NULL,
372             OPEN_EXISTING,
373             FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_SEQUENTIAL_SCAN,
374             NULL
375           );
376   if (hFile == INVALID_HANDLE_VALUE)
377   {
378     ErrorFormatWinxError(GetLastError(), &pcMessage);
379     snprintf(pcError, MESSAGE_SIZE, "%s: %s", acRoutine, pcMessage);
380     return ER;
381   }
382   ghFile = hFile; /* Needed for memory mapped XMagic. */
383   iFile = _open_osfhandle((intptr_t) hFile, 0);
384   if (iFile == ER)
385   {
386     snprintf(pcError, MESSAGE_SIZE, "%s: open_osfhandle(): Handle association failed.", acRoutine);
387     return ER;
388   }
389   pFile = fdopen(iFile, "rb");
390   if (pFile == NULL)
391   {
392     snprintf(pcError, MESSAGE_SIZE, "%s: fdopen(): %s", acRoutine, strerror(errno));
393     return ER;
394   }
395 #else
396   pFile = fopen(psFTFileData->pcRawPath, "rb");
397   if (pFile == NULL)
398   {
399     snprintf(pcError, MESSAGE_SIZE, "%s: fopen(): %s", acRoutine, strerror(errno));
400     return ER;
401   }
402   giFile = fileno(pFile); /* Needed for memory mapped XMagic. */
403 #endif
404 
405   /*-
406    *********************************************************************
407    *
408    * Conditionally seek to the specified start offset. Return an error
409    * if the file is regular and its size is less than this offset.
410    *
411    *********************************************************************
412    */
413   gui64StartOffset = 0;
414   if (psProperties->ui64AnalyzeStartOffset)
415   {
416     if
417     (
418 #ifdef UNIX
419       S_ISREG(psFTFileData->sStatEntry.st_mode) &&
420 #endif
421       ui64FileSize < psProperties->ui64AnalyzeStartOffset
422     )
423     {
424       snprintf(pcError, MESSAGE_SIZE, "%s: File is smaller than the specified start offset.", acRoutine);
425       fclose(pFile);
426 #ifdef WINNT
427       CloseHandle(hFile);
428 #endif
429       return ER;
430     }
431     ui64NToSeek = psProperties->ui64AnalyzeStartOffset;
432     do
433     {
434       iNToSeek = (ui64NToSeek > (APP_UI64) 0x7fffffff) ? 0x7fffffff : (int) ui64NToSeek;
435 #ifdef HAVE_FSEEKO
436       iError = fseeko(pFile, (off_t) iNToSeek, SEEK_CUR);
437 #else
438       iError = fseek(pFile, iNToSeek, SEEK_CUR);
439 #endif
440       if (iError == ER)
441       {
442 #ifdef HAVE_FSEEKO
443         snprintf(pcError, MESSAGE_SIZE, "%s: fseeko(): %s", acRoutine, strerror(errno));
444 #else
445         snprintf(pcError, MESSAGE_SIZE, "%s: fseek(): %s", acRoutine, strerror(errno));
446 #endif
447         fclose(pFile);
448 #ifdef WINNT
449         CloseHandle(hFile);
450 #endif
451         return ER;
452       }
453       ui64NToSeek -= iNToSeek;
454       gui64StartOffset += iNToSeek;
455     } while (ui64NToSeek > 0);
456   }
457 
458   /*-
459    *********************************************************************
460    *
461    * Update the global files counter.
462    *
463    *********************************************************************
464    */
465   gui32Files++;
466 
467   /*-
468    *********************************************************************
469    *
470    * Initialize the block tag. Each analysis stage executes different
471    * logic based on the value of this tag.
472    *
473    *********************************************************************
474    */
475   iBlockTag = ANALYZE_FIRST_BLOCK;
476   gui64ByteCount = 0;
477 
478   while ((iBlockTag & ANALYZE_FINAL_BLOCK) != ANALYZE_FINAL_BLOCK)
479   {
480     /*-
481      *******************************************************************
482      *
483      * Read a block of data, and insert it in the middle of our buffer.
484      *
485      *******************************************************************
486      */
487     iNRead = fread(&pucBuffer[iBlockSize], 1, iBlockSize, pFile);
488 
489     /*-
490      *******************************************************************
491      *
492      * Update the global byte counters.
493      *
494      *******************************************************************
495      */
496     gui64Bytes += iNRead;
497     gui64ByteCount += iNRead;
498 
499     /*-
500      *******************************************************************
501      *
502      * Determine the current DPS, and apply the brakes as needed.
503      *
504      *******************************************************************
505      */
506     AnalyzeThrottleDps(gui64Bytes, psProperties->iAnalyzeMaxDps);
507 
508     /*-
509      *******************************************************************
510      *
511      * If there was a read error, close the file and return an error.
512      *
513      *******************************************************************
514      */
515     if (ferror(pFile))
516     {
517       snprintf(pcError, MESSAGE_SIZE, "%s: %s", acRoutine, strerror(errno));
518       fclose(pFile);
519 #ifdef WINNT
520       CloseHandle(hFile);
521 #endif
522       return ER;
523     }
524 
525     /*-
526      *******************************************************************
527      *
528      * If EOF or the specified byte count was reached, update the block
529      * tag. If specified byte count was exceeded, reduce the read count
530      * by the difference to ensure that extra bytes are not analyzed.
531      *
532      *******************************************************************
533      */
534     if (feof(pFile))
535     {
536       iBlockTag |= ANALYZE_FINAL_BLOCK;
537     }
538     else
539     {
540       if (psProperties->ui64AnalyzeByteCount && gui64ByteCount >= psProperties->ui64AnalyzeByteCount)
541       {
542         APP_UI64 ui64Delta = gui64ByteCount - psProperties->ui64AnalyzeByteCount;
543         if (ui64Delta > (APP_UI64) iNRead)
544         {
545           snprintf(pcError, MESSAGE_SIZE, "%s: Byte count delta exceeds the read count [%d]. That shouldn't happen.", acRoutine, iNRead);
546           fclose(pFile);
547 #ifdef WINNT
548           CloseHandle(hFile);
549 #endif
550           return ER;
551         }
552         iNRead -= (int) ui64Delta;
553         iBlockTag |= ANALYZE_FINAL_BLOCK;
554       }
555     }
556 
557     /*-
558      *******************************************************************
559      *
560      * Run through the defined analysis stages. Warn the user if a
561      * stage fails, but keep going.
562      *
563      *******************************************************************
564      */
565     for (i = 0; i < psProperties->iLastAnalysisStage; i++)
566     {
567       iError = psProperties->asAnalysisStages[i].piRoutine(&pucBuffer[iBlockSize], iNRead, iBlockTag, iBlockSize, psFTFileData, acLocalError);
568       if (iError != ER_OK)
569       {
570         snprintf(pcError, MESSAGE_SIZE, "%s: %s", acRoutine, acLocalError);
571         ErrorHandler(psProperties->asAnalysisStages[i].iError, pcError, ERROR_FAILURE);
572       }
573     }
574 
575     /*-
576      *******************************************************************
577      *
578      * Update the block tag.
579      *
580      *******************************************************************
581      */
582     if ((iBlockTag & ANALYZE_FIRST_BLOCK) == ANALYZE_FIRST_BLOCK)
583     {
584       iBlockTag ^= ANALYZE_FIRST_BLOCK;
585 #ifdef USE_XMAGIC
586       /*-
587        *****************************************************************
588        *
589        * If XMagic was the only type of analysis requested, we're done.
590        *
591        *****************************************************************
592        */
593       if (psProperties->iLastAnalysisStage == 1 && psProperties->asAnalysisStages[0].piRoutine == AnalyzeDoXMagic)
594       {
595         iBlockTag |= ANALYZE_FINAL_BLOCK;
596       }
597 #endif
598     }
599   }
600   fclose(pFile);
601 #ifdef WINNT
602   CloseHandle(hFile);
603 #endif
604 
605   return ER_OK;
606 }
607 
608 
609 /*-
610  ***********************************************************************
611  *
612  * AnalyzeEnableDigestEngine
613  *
614  ***********************************************************************
615  */
616 void
AnalyzeEnableDigestEngine(FTIMES_PROPERTIES * psProperties)617 AnalyzeEnableDigestEngine(FTIMES_PROPERTIES *psProperties)
618 {
619   if (MASK_BIT_IS_SET(psProperties->psFieldMask->ulMask, MAP_MD5))
620   {
621     strcpy(psProperties->asAnalysisStages[psProperties->iLastAnalysisStage].acDescription, "Md5Digest");
622     psProperties->asAnalysisStages[psProperties->iLastAnalysisStage].iError = ER_DoDigest;
623     psProperties->asAnalysisStages[psProperties->iLastAnalysisStage++].piRoutine = AnalyzeDoMd5Digest;
624   }
625   if (MASK_BIT_IS_SET(psProperties->psFieldMask->ulMask, MAP_SHA1))
626   {
627     strcpy(psProperties->asAnalysisStages[psProperties->iLastAnalysisStage].acDescription, "Sha1Digest");
628     psProperties->asAnalysisStages[psProperties->iLastAnalysisStage].iError = ER_DoDigest;
629     psProperties->asAnalysisStages[psProperties->iLastAnalysisStage++].piRoutine = AnalyzeDoSha1Digest;
630   }
631   if (MASK_BIT_IS_SET(psProperties->psFieldMask->ulMask, MAP_SHA256))
632   {
633     strcpy(psProperties->asAnalysisStages[psProperties->iLastAnalysisStage].acDescription, "Sha256Digest");
634     psProperties->asAnalysisStages[psProperties->iLastAnalysisStage].iError = ER_DoDigest;
635     psProperties->asAnalysisStages[psProperties->iLastAnalysisStage++].piRoutine = AnalyzeDoSha256Digest;
636   }
637 }
638 
639 
640 /*-
641  ***********************************************************************
642  *
643  * AnalyzeDoMd5Digest
644  *
645  ***********************************************************************
646  */
647 int
AnalyzeDoMd5Digest(unsigned char * pucBuffer,int iBufferLength,int iBlockTag,int iBufferOverhead,FTIMES_FILE_DATA * psFTFileData,char * pcError)648 AnalyzeDoMd5Digest(unsigned char *pucBuffer, int iBufferLength, int iBlockTag, int iBufferOverhead, FTIMES_FILE_DATA *psFTFileData, char *pcError)
649 {
650   static MD5_CONTEXT sFileMD5Context;
651 
652   if ((iBlockTag & ANALYZE_FIRST_BLOCK) == ANALYZE_FIRST_BLOCK)
653   {
654     MD5Alpha(&sFileMD5Context);
655   }
656 
657   MD5Cycle(&sFileMD5Context, pucBuffer, iBufferLength);
658 
659   if ((iBlockTag & ANALYZE_FINAL_BLOCK) == ANALYZE_FINAL_BLOCK)
660   {
661     MD5Omega(&sFileMD5Context, psFTFileData->aucFileMd5);
662     psFTFileData->ulAttributeMask |= MAP_MD5;
663   }
664 
665   return ER_OK;
666 }
667 
668 
669 /*-
670  ***********************************************************************
671  *
672  * AnalyzeDoSha1Digest
673  *
674  ***********************************************************************
675  */
676 int
AnalyzeDoSha1Digest(unsigned char * pucBuffer,int iBufferLength,int iBlockTag,int iBufferOverhead,FTIMES_FILE_DATA * psFTFileData,char * pcError)677 AnalyzeDoSha1Digest(unsigned char *pucBuffer, int iBufferLength, int iBlockTag, int iBufferOverhead, FTIMES_FILE_DATA *psFTFileData, char *pcError)
678 {
679   static SHA1_CONTEXT sFileSha1Context;
680 
681   if ((iBlockTag & ANALYZE_FIRST_BLOCK) == ANALYZE_FIRST_BLOCK)
682   {
683     SHA1Alpha(&sFileSha1Context);
684   }
685 
686   SHA1Cycle(&sFileSha1Context, pucBuffer, iBufferLength);
687 
688   if ((iBlockTag & ANALYZE_FINAL_BLOCK) == ANALYZE_FINAL_BLOCK)
689   {
690     SHA1Omega(&sFileSha1Context, psFTFileData->aucFileSha1);
691     psFTFileData->ulAttributeMask |= MAP_SHA1;
692   }
693 
694   return ER_OK;
695 }
696 
697 
698 /*-
699  ***********************************************************************
700  *
701  * AnalyzeDoSha256Digest
702  *
703  ***********************************************************************
704  */
705 int
AnalyzeDoSha256Digest(unsigned char * pucBuffer,int iBufferLength,int iBlockTag,int iBufferOverhead,FTIMES_FILE_DATA * psFTFileData,char * pcError)706 AnalyzeDoSha256Digest(unsigned char *pucBuffer, int iBufferLength, int iBlockTag, int iBufferOverhead, FTIMES_FILE_DATA *psFTFileData, char *pcError)
707 {
708   static SHA256_CONTEXT sFileSha256Context;
709 
710   if ((iBlockTag & ANALYZE_FIRST_BLOCK) == ANALYZE_FIRST_BLOCK)
711   {
712     SHA256Alpha(&sFileSha256Context);
713   }
714 
715   SHA256Cycle(&sFileSha256Context, pucBuffer, iBufferLength);
716 
717   if ((iBlockTag & ANALYZE_FINAL_BLOCK) == ANALYZE_FINAL_BLOCK)
718   {
719     SHA256Omega(&sFileSha256Context, psFTFileData->aucFileSha256);
720     psFTFileData->ulAttributeMask |= MAP_SHA256;
721   }
722 
723   return ER_OK;
724 }
725 
726 
727 /*-
728  ***********************************************************************
729  *
730  * AnalyzeEnableDigEngine
731  *
732  ***********************************************************************
733  */
734 void
AnalyzeEnableDigEngine(FTIMES_PROPERTIES * psProperties)735 AnalyzeEnableDigEngine(FTIMES_PROPERTIES *psProperties)
736 {
737   strcpy(psProperties->asAnalysisStages[psProperties->iLastAnalysisStage].acDescription, "Search");
738   psProperties->asAnalysisStages[psProperties->iLastAnalysisStage].iError = ER_DoDig;
739   psProperties->asAnalysisStages[psProperties->iLastAnalysisStage++].piRoutine = AnalyzeDoDig;
740 }
741 
742 
743 /*-
744  ***********************************************************************
745  *
746  * AnalyzeDoDig
747  *
748  ***********************************************************************
749  */
750 int
AnalyzeDoDig(unsigned char * pucBuffer,int iBufferLength,int iBlockTag,int iBufferOverhead,FTIMES_FILE_DATA * psFTFileData,char * pcError)751 AnalyzeDoDig(unsigned char *pucBuffer, int iBufferLength, int iBlockTag, int iBufferOverhead, FTIMES_FILE_DATA *psFTFileData, char *pcError)
752 {
753   const char          acRoutine[] = "AnalyzeDoDig()";
754   char                acLocalError[MESSAGE_SIZE] = "";
755   unsigned char      *pucToSearch;
756 #ifdef USE_PCRE
757   int                 iBlockSize = AnalyzeGetBlockSize();
758 #endif
759   int                 iCarrySize = AnalyzeGetCarrySize();
760   int                 iError;
761   int                 iNToSearch;
762   int                 iStopShort;
763   int                 iType;
764   unsigned char      *pucSaveBuffer = NULL;
765   static int          iNToSave;
766   static int          iSaveOffset;
767   static APP_UI64     ui64SearchOffset;
768 
769   /*-
770    *********************************************************************
771    *
772    * Get a pointer to the save buffer.
773    *
774    *********************************************************************
775    */
776   pucSaveBuffer = AnalyzeGetDigSaveBuffer(iCarrySize, acLocalError);
777   if (pucSaveBuffer == NULL)
778   {
779     snprintf(pcError, MESSAGE_SIZE, "%s: %s", acRoutine, acLocalError);
780     return ER;
781   }
782 
783   /*-
784    *********************************************************************
785    *
786    * If this is the final block, clear the stop short flag. Otherwise,
787    * set the flag and make sure that we have a nonzero length.
788    *
789    *********************************************************************
790    */
791   if ((iBlockTag & ANALYZE_FINAL_BLOCK) == ANALYZE_FINAL_BLOCK)
792   {
793     iStopShort = 0;
794   }
795   else
796   {
797     if (iBufferLength == 0)
798     {
799       snprintf(pcError, MESSAGE_SIZE, "%s: A zero length block is illegal unless it is tagged as the final block.", acRoutine);
800       return ER;
801     }
802     iStopShort = 1;
803   }
804 
805   /*-
806    *********************************************************************
807    *
808    * If this is the first block, initialize. Otherwise, prepend any
809    * saved data to the incoming buffer.
810    *
811    *********************************************************************
812    */
813   if ((iBlockTag & ANALYZE_FIRST_BLOCK) == ANALYZE_FIRST_BLOCK)
814   {
815     DigClearCounts();
816 
817     /*-
818      *******************************************************************
819      *
820      * Make sure that the max string length is less than or equal to
821      * the size of the save buffer. The dig algorithm stops searching
822      * after x bytes have been processed. Where
823      *
824      *   x = (bufsize - (maxstring - 1))
825      *
826      * For example, if bufsize = 16 and maxstring = 6, then all but
827      * the last 5 bytes may be searched (i.e., to stay in bounds). Thus,
828      * the search routine needs to stop searching when it reaches byte
829      * 11.
830      *
831      * If no search strings were defined, maxstring should be zero.
832      *
833      * There is no case where iNToSave should be set to a negative
834      * value.
835      *
836      *******************************************************************
837      */
838     if (DigGetMaxStringLength() <= iCarrySize)
839     {
840       iNToSave = iCarrySize;
841       DigSetSaveLength(iCarrySize);
842     }
843     else
844     {
845       snprintf(pcError, MESSAGE_SIZE, "%s: Not enough overhead in the save buffer. The carry size (%d) must be no less than %d bytes for this job to work as intended.", acRoutine, iCarrySize, DigGetMaxStringLength());
846       return ER;
847     }
848 
849     ui64SearchOffset = 0;
850     pucToSearch = pucBuffer;
851     iNToSearch = iBufferLength;
852   }
853   else
854   {
855     pucToSearch = pucBuffer - iNToSave;
856     memcpy(pucToSearch, pucSaveBuffer, iNToSave);
857     iNToSearch = iBufferLength + iNToSave;
858   }
859 
860   /*-
861    *********************************************************************
862    *
863    * Search the input.
864    *
865    *********************************************************************
866    */
867   for (iType = DIG_STRING_TYPE_NORMAL; iType < DIG_STRING_TYPE_NOMORE; iType++)
868   {
869     iError = DigSearchData(pucToSearch, iNToSearch, iStopShort, iType, ui64SearchOffset, psFTFileData->pcNeuteredPath, acLocalError);
870     if (iError != ER_OK)
871     {
872       snprintf(pcError, MESSAGE_SIZE, "%s: %s", acRoutine, acLocalError);
873       return iError;
874     }
875   }
876 
877   /*-
878    *********************************************************************
879    *
880    * Save any data that could not be fully searched.
881    *
882    *********************************************************************
883    */
884   iSaveOffset = iNToSearch - iNToSave;
885   memcpy(pucSaveBuffer, &pucToSearch[iSaveOffset], iNToSave);
886 
887   /*-
888    *********************************************************************
889    *
890    * Update the search offset. This ensures that search routine is
891    * passed the offset it expects. In other words, subtract off the
892    * unsearched portion of the prior search buffer, and keep it for
893    * next time.
894    *
895    *********************************************************************
896    */
897   if ((iBlockTag & ANALYZE_FIRST_BLOCK) == ANALYZE_FIRST_BLOCK)
898   {
899     ui64SearchOffset += iBufferLength - iNToSave;
900   }
901   else
902   {
903     ui64SearchOffset += iBufferLength;
904   }
905 
906 #ifdef USE_PCRE
907   /*-
908    *********************************************************************
909    *
910    * Update dig offsets. The trim size is equal to the block size in
911    * all but the first block.
912    *
913    *********************************************************************
914    */
915   if ((iBlockTag & ANALYZE_FIRST_BLOCK) == ANALYZE_FIRST_BLOCK)
916   {
917     DigAdjustRegExpOffsets(iBlockSize - iCarrySize);
918   }
919   else
920   {
921     DigAdjustRegExpOffsets(iBlockSize);
922   }
923 #endif
924 
925   return ER_OK;
926 }
927 
928 
929 #ifdef USE_XMAGIC
930 /*-
931  ***********************************************************************
932  *
933  * AnalyzeEnableXMagicEngine
934  *
935  ***********************************************************************
936  */
937 int
AnalyzeEnableXMagicEngine(FTIMES_PROPERTIES * psProperties,char * pcError)938 AnalyzeEnableXMagicEngine(FTIMES_PROPERTIES *psProperties, char *pcError)
939 {
940   const char          acRoutine[] = "AnalyzeEnableXMagicEngine()";
941   char                acLocalError[MESSAGE_SIZE] = "";
942   unsigned char       aucMD5[MD5_HASH_SIZE];
943   int                 i;
944   int                 iError;
945   FILE               *pFile;
946   APP_UI64            ui64Size;
947 
948   /*-
949    *********************************************************************
950    *
951    * Set up the XMagic analysis function pointer.
952    *
953    *********************************************************************
954    */
955   strcpy(psProperties->asAnalysisStages[psProperties->iLastAnalysisStage].acDescription, "XMagic");
956   psProperties->asAnalysisStages[psProperties->iLastAnalysisStage].iError = ER_DoXMagic;
957   psProperties->asAnalysisStages[psProperties->iLastAnalysisStage++].piRoutine = AnalyzeDoXMagic;
958 
959   /*-
960    *********************************************************************
961    *
962    * Locate and load magic. If a file was specified, it must exist.
963    *
964    *********************************************************************
965    */
966   for (i = 0; i < 3; i++)
967   {
968     switch (i)
969     {
970     case 0:
971       if (psProperties->acMagicFileName[0])
972       {
973         iError = SupportExpandPath(psProperties->acMagicFileName, psProperties->acMagicFileName, FTIMES_MAX_PATH, 1, acLocalError);
974         if (iError != ER_OK)
975         {
976           snprintf(pcError, MESSAGE_SIZE, "%s: %s", acRoutine, acLocalError);
977           return ER_XMagic;
978         }
979       }
980       else
981       {
982         continue;
983       }
984       break;
985     case 1:
986       iError = SupportExpandPath(XMAGIC_DEFAULT_LOCATION, psProperties->acMagicFileName, FTIMES_MAX_PATH, 1, acLocalError);
987       break;
988     case 2:
989       iError = SupportExpandPath(XMAGIC_CURRENT_LOCATION, psProperties->acMagicFileName, FTIMES_MAX_PATH, 1, acLocalError);
990       break;
991     default:
992       iError = ER_BadValue;
993       snprintf(acLocalError, MESSAGE_SIZE, "bad loop index");
994       break;
995     }
996     if (iError == ER_OK)
997     {
998       psProperties->psXMagic = XMagicLoadMagic(psProperties->acMagicFileName, acLocalError);
999       if (psProperties->psXMagic == NULL)
1000       {
1001         snprintf(pcError, MESSAGE_SIZE, "%s: %s", acRoutine, acLocalError);
1002         return ER_XMagic;
1003       }
1004       else
1005       {
1006         if ((pFile = fopen(psProperties->acMagicFileName, "rb")) != NULL && MD5HashStream(pFile, aucMD5, &ui64Size) == ER_OK)
1007         {
1008           MD5HashToHex(aucMD5, psProperties->acMagicHash);
1009           fclose(pFile);
1010         }
1011         else
1012         {
1013           strcpy(psProperties->acMagicHash, "NONE");
1014         }
1015         break; /* Very important. This get's us out of the for loop. */
1016       }
1017     }
1018     psProperties->acMagicFileName[0] = 0;
1019   }
1020 
1021   if (psProperties->acMagicFileName[0] == 0)
1022   {
1023     strcpy(psProperties->acMagicFileName, "NA");
1024     strcpy(psProperties->acMagicHash, "NA");
1025   }
1026 
1027   return ER_OK;
1028 }
1029 
1030 
1031 /*-
1032  ***********************************************************************
1033  *
1034  * AnalyzeDoXMagic
1035  *
1036  ***********************************************************************
1037  */
1038 int
AnalyzeDoXMagic(unsigned char * pucBuffer,int iBufferLength,int iBlockTag,int iBufferOverhead,FTIMES_FILE_DATA * psFTFileData,char * pcError)1039 AnalyzeDoXMagic(unsigned char *pucBuffer, int iBufferLength, int iBlockTag, int iBufferOverhead, FTIMES_FILE_DATA *psFTFileData, char *pcError)
1040 {
1041   const char          acRoutine[] = "AnalyzeDoXMagic()";
1042   char                acLocalError[MESSAGE_SIZE] = "";
1043   char                acMessage[MESSAGE_SIZE] = "";
1044   int                 iError = 0;
1045   int                 iMemoryMapped = 0;
1046   int                 iMemoryMapSize = 0;
1047   void               *pvMemoryMap = NULL;
1048   static int          iFirst = 1;
1049   static int          iMemoryMapEnable = 0;
1050   static XMAGIC      *psXMagic = NULL;
1051 
1052   /*-
1053    *********************************************************************
1054    *
1055    * Obtain a reference to the XMagic, and set other static variables
1056    * on the first pass.
1057    *
1058    *********************************************************************
1059    */
1060   if (iFirst)
1061   {
1062     FTIMES_PROPERTIES *psProperties = FTimesGetPropertiesReference();
1063     psXMagic = psProperties->psXMagic;
1064     iMemoryMapEnable = psProperties->iMemoryMapEnable;
1065     iFirst = 0;
1066   }
1067 
1068   /*-
1069    *********************************************************************
1070    *
1071    * XMagic tests are only done once -- either on the first block or
1072    * on a memory mapped view of the file.
1073    *
1074    *********************************************************************
1075    */
1076   if ((iBlockTag & ANALYZE_FIRST_BLOCK) != ANALYZE_FIRST_BLOCK)
1077   {
1078     return ER_OK;
1079   }
1080 
1081   /*-
1082    *********************************************************************
1083    *
1084    * Conditionally map the file into memory. If the file's size is
1085    * zero, do not attempt to map it as this will lead to EINVAL errors
1086    * on some platforms -- either for mmap() or munmap(). If the memory
1087    * can't be mapped for whatever reason, fallback to using the block
1088    * of data provided by the caller.
1089    *
1090    *********************************************************************
1091    */
1092   if (iMemoryMapEnable)
1093   {
1094 #ifdef WINNT
1095     if (psFTFileData->dwFileSizeHigh == 0 && psFTFileData->dwFileSizeLow < FTIMES_MAX_MMAP_SIZE)
1096     {
1097       iMemoryMapSize = psFTFileData->dwFileSizeLow;
1098     }
1099 #else
1100     if (psFTFileData->sStatEntry.st_size < FTIMES_MAX_MMAP_SIZE)
1101     {
1102       iMemoryMapSize = psFTFileData->sStatEntry.st_size;
1103     }
1104 #endif
1105     else
1106     {
1107       iMemoryMapSize = FTIMES_MAX_MMAP_SIZE;
1108     }
1109     if (iMemoryMapSize > 0 && (pvMemoryMap = AnalyzeMapMemory(iMemoryMapSize)) != NULL)
1110     {
1111       pucBuffer = (unsigned char *) pvMemoryMap;
1112       iBufferLength = iMemoryMapSize;
1113       iMemoryMapped = 1;
1114     }
1115   }
1116   snprintf(acMessage, MESSAGE_SIZE, "AnalysisStage=XMagic MemoryMapped=%d BufferLength=%d", iMemoryMapped, iBufferLength);
1117   MessageHandler(MESSAGE_FLUSH_IT, MESSAGE_DEBUGGER, MESSAGE_DEBUGGER_STRING, acMessage);
1118 
1119   /*-
1120    *********************************************************************
1121    *
1122    * Execute XMagic tests.
1123    *
1124    *********************************************************************
1125    */
1126   iError = XMagicTestBuffer(psXMagic, pucBuffer, iBufferLength, psFTFileData->acType, FTIMES_FILETYPE_BUFSIZE, acLocalError);
1127   if (iError == ER)
1128   {
1129     snprintf(pcError, MESSAGE_SIZE, "%s: %s", acRoutine, acLocalError);
1130     if (iMemoryMapped)
1131     {
1132       AnalyzeUnmapMemory(pvMemoryMap, iMemoryMapSize);
1133     }
1134     return ER_XMagic;
1135   }
1136   psFTFileData->ulAttributeMask |= MAP_MAGIC;
1137   if (iMemoryMapped)
1138   {
1139     AnalyzeUnmapMemory(pvMemoryMap, iMemoryMapSize);
1140   }
1141 
1142   return ER_OK;
1143 }
1144 #endif
1145 
1146 
1147 /*-
1148  ***********************************************************************
1149  *
1150  * AnalyzeMapMemory
1151  *
1152  ***********************************************************************
1153  */
1154 void *
AnalyzeMapMemory(int iMemoryMapSize)1155 AnalyzeMapMemory(int iMemoryMapSize)
1156 {
1157 #ifdef WINNT
1158   HANDLE              hMemoryMap = NULL;
1159 #endif
1160   void               *pvMemoryMap = NULL;
1161 
1162   /*-
1163    *********************************************************************
1164    *
1165    * This routine does not detect or return errors because its caller
1166    * (AnalyzeDoXMagic) is designed to use an alternate data buffer if
1167    * the file can't be mapped into memory.
1168    *
1169    *********************************************************************
1170    */
1171 #ifdef WINNT
1172   hMemoryMap = CreateFileMapping(ghFile, NULL, PAGE_READONLY, 0, 0, 0);
1173   if (hMemoryMap)
1174   {
1175     pvMemoryMap = MapViewOfFile(hMemoryMap, FILE_MAP_READ, 0, 0, 0);
1176     CloseHandle(hMemoryMap);
1177   }
1178 #else
1179   pvMemoryMap = mmap(NULL, iMemoryMapSize, PROT_READ, MAP_PRIVATE, giFile, 0);
1180 #if defined(FTimes_HPUX) && !defined(MAP_FAILED)
1181 #define MAP_FAILED ((void *)-1)
1182 #endif
1183   if (pvMemoryMap == MAP_FAILED)
1184   {
1185     return NULL;
1186   }
1187 #endif
1188 
1189   return pvMemoryMap;
1190 }
1191 
1192 
1193 /*-
1194  ***********************************************************************
1195  *
1196  * AnalyzeSetBlockSize
1197  *
1198  ***********************************************************************
1199  */
1200 void
AnalyzeSetBlockSize(int iBlockSize)1201 AnalyzeSetBlockSize(int iBlockSize)
1202 {
1203   giAnalyzeBlockSize = iBlockSize;
1204 }
1205 
1206 
1207 /*-
1208  ***********************************************************************
1209  *
1210  * AnalyzeSetCarrySize
1211  *
1212  ***********************************************************************
1213  */
1214 void
AnalyzeSetCarrySize(int iCarrySize)1215 AnalyzeSetCarrySize(int iCarrySize)
1216 {
1217   giAnalyzeCarrySize = iCarrySize;
1218 }
1219 
1220 
1221 #ifdef USE_XMAGIC
1222 /*-
1223  ***********************************************************************
1224  *
1225  * AnalyzeSetStepSize
1226  *
1227  ***********************************************************************
1228  */
1229 void
AnalyzeSetStepSize(int iStepSize)1230 AnalyzeSetStepSize(int iStepSize)
1231 {
1232   giAnalyzeStepSize = iStepSize;
1233 }
1234 #endif
1235 
1236 
1237 /*-
1238  ***********************************************************************
1239  *
1240  * AnalyzeThrottleDps
1241  *
1242  ***********************************************************************
1243  */
1244 void
AnalyzeThrottleDps(APP_UI64 ui64Bytes,int iMaxDps)1245 AnalyzeThrottleDps(APP_UI64 ui64Bytes, int iMaxDps)
1246 {
1247   static double       dNow = 0;
1248   static double       dStartTime = 0;
1249   static int          iFirst = 1;
1250   double              dKBytes = 0;
1251   double              dSleepTime = 0;
1252   int                 iSleepTime = 0;
1253 #ifdef WINNT
1254   APP_SI64            i64Bytes = (APP_SI64) ui64Bytes;
1255 #endif
1256 
1257   /*-
1258    *********************************************************************
1259    *
1260    * Get the current time. Record the start time on the first pass.
1261    *
1262    *********************************************************************
1263    */
1264   dNow = TimeGetTimeValueAsDouble();
1265   if (iFirst)
1266   {
1267     dStartTime = dNow;
1268     iFirst = 0;
1269   }
1270 
1271   /*-
1272    *********************************************************************
1273    *
1274    * Calculate the amount of time spent thus far. Force the result to
1275    * be at least one microsecond to thwart divide by zero issues.
1276    *
1277    *********************************************************************
1278    */
1279   gdAnalysisTime = (double) (dNow - dStartTime);
1280   if (gdAnalysisTime <= 0)
1281   {
1282     gdAnalysisTime = 0.000001;
1283   }
1284 
1285   /*-
1286    *********************************************************************
1287    *
1288    * Calculate the Data Processing Speed (DPS). If the maximum DPS is
1289    * zero, do not apply the brakes. Otherwise, set the sleep time to a
1290    * value in the range [1,60], and sleep. If the DPS value has a
1291    * fractional component (of any amount), round it up.
1292    *
1293    *********************************************************************
1294    */
1295 #ifdef WINNT
1296   dKBytes = (double) ((double)  i64Bytes / (double) 1024);
1297 #else
1298   dKBytes = (double) ((double) ui64Bytes / (double) 1024);
1299 #endif
1300   gdDps = dKBytes / gdAnalysisTime;
1301   if (iMaxDps && gdDps > (double) iMaxDps)
1302   {
1303     dSleepTime = (double) ((dKBytes / (double) iMaxDps) - gdAnalysisTime);
1304     if (dSleepTime < 1)
1305     {
1306       iSleepTime = 1;
1307     }
1308     else if (dSleepTime > 60)
1309     {
1310       iSleepTime = 60;
1311     }
1312     else
1313     {
1314       iSleepTime = (int) dSleepTime;
1315       if ((double) (dSleepTime / (double) iSleepTime))
1316       {
1317         iSleepTime += 1;
1318       }
1319     }
1320 #ifdef WINNT
1321     Sleep(iSleepTime * 1000);
1322 #else
1323     sleep(iSleepTime);
1324 #endif
1325   }
1326 }
1327 
1328 
1329 /*-
1330  ***********************************************************************
1331  *
1332  * AnalyzeUnmapMemory
1333  *
1334  ***********************************************************************
1335  */
1336 void
AnalyzeUnmapMemory(void * pvMemoryMap,int iMemoryMapSize)1337 AnalyzeUnmapMemory(void *pvMemoryMap, int iMemoryMapSize)
1338 {
1339 #ifdef WINNT
1340   UnmapViewOfFile(pvMemoryMap);
1341 #else
1342   munmap(pvMemoryMap, iMemoryMapSize);
1343 #endif
1344   return;
1345 }
1346