1 /*
2  * Argus Software
3  * Copyright (c) 2000-2016 QoSient, LLC
4  * All rights reserved.
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License as published by
8  * the Free Software Foundation; either version 2, or (at your option)
9  * any later version.
10 
11  * This program is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14  * GNU General Public License for more details.
15 
16  * You should have received a copy of the GNU General Public License
17  * along with this program; if not, write to the Free Software
18  * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
19  *
20  */
21 
22 /*
23  * $Id: //depot/argus/clients/include/argus_parser.h#112 $
24  * $DateTime: 2016/06/01 15:17:28 $
25  * $Change: 3148 $
26  */
27 
28 
29 #ifndef ArgusParser_h
30 #define ArgusParser_h
31 
32 #ifdef __cplusplus
33 extern "C" {
34 #endif
35 
36 #if defined(ARGUS_PCRE)
37 #include "pcreposix.h"
38 #else
39 #include <regex.h>
40 #endif
41 
42 #include <argus_def.h>
43 #include <argus_out.h>
44 #include <argus_util.h>
45 
46 #include <net/nff.h>
47 
48 #if defined(ARGUS_THREADS)
49 #include <pthread.h>
50 #endif
51 
52 #define ARGUS_RECORD_WRITTEN	0x0001
53 
54 #define ARGUS_PRINTGMT		0x0020
55 #define ARGUS_PRINTNET		0x0022
56 
57 #define RA_ENABLE_CIDR_ADDRESS_FORMAT	1
58 #define RA_STRICT_CIDR_ADDRESS_FORMAT	2
59 
60 #define ARGUS_ENCODE_ASCII	0
61 #define ARGUS_ENCODE_64		1
62 #define ARGUS_ENCODE_32		2
63 #define ARGUS_HEXDUMP		3
64 #define ARGUS_ENCODE_OBFUSCATE	4
65 
66 #define ARGUS_MAX_S_OPTIONS      256
67 
68 #define RA_PRINTPROTO		1
69 #define RA_PRINTSRCID		2
70 
71 #define RAMON_TOPN		1
72 #define RAMON_MATRIX		2
73 #define RAMON_SVCS		3
74 #define RAMON_SINGLE		4
75 
76 #define RA_FIXED_WIDTH		1
77 #define RA_VARIABLE_WIDTH	2
78 
79 #define RA_SINGLE_QUOTED	'\''
80 #define RA_DOUBLE_QUOTED	'\"'
81 
82 #define ARGUS_MAX_REMOTE	256
83 #define ARGUS_MAX_REGEX		16384
84 
85 #define RABINS_HASHTABLESIZE	0x100
86 
87 #define RAMAXWILDCARDFIELDS     6
88 
89 #define RAWILDCARDYEAR          0
90 #define RAWILDCARDMONTH         1
91 #define RAWILDCARDDAY           2
92 #define RAWILDCARDHOUR          3
93 #define RAWILDCARDMIN           4
94 #define RAWILDCARDSEC           5
95 
96 #define ARGUS_ASN_ASPLAIN	0
97 #define ARGUS_ASN_ASDOTPLUS	1
98 #define ARGUS_ASN_ASDOT    	2
99 
100 /* the ArgusRecordStruct (ns) is a single point data structure
101    for clients to use to process and report on ARGUS flow data.
102    To support this rather nebulous function, the ns  provides
103    a canonical record buffer which supports a completely
104    parsed and formatted record (exploded view), and indexes
105    to the various sections of the ns structure.
106 
107    A queue header struct is supported so that the strucutre
108    can be placed in a queue, and there is an hstruct pointer
109    provided so that if the record is hashed, the hash entry
110    can be found quickly, ie for removal.
111 
112    Because ns data is time series data, there is a desire to
113    support an array style data structure to provide the
114    ability to  maintain some aspects fo the time-series
115    qualities.  This struct was enabled in earlier versions
116    using the 'H' (histogram) option.  In order to support the
117    concept of the Stream Block Processor, however, this data
118    structure needs to be embeeded in the ns itself.
119 
120    The concept is that the ns itself holds the aggregate
121    stats for the time series array.  Each  member is an
122    ns, which can further subdivided into additional time series
123    structures.  For most applications, this structure will
124    be NULL, indicating that subdivision is not being done.
125 
126 */
127 
128 
129 struct ArgusHashStruct {
130    unsigned int len, hash;
131    unsigned int *buf;
132 };
133 
134 struct ArgusHashTableHdr {
135    struct ArgusHashTableHdr *nxt, *prv;
136    struct ArgusHashTable *htbl;
137    struct ArgusHashStruct hstruct;
138    void *object;
139 };
140 
141 struct ArgusHashTable {
142    unsigned int size, count;
143 #if defined(ARGUS_THREADS)
144    pthread_mutex_t lock;
145 #endif /* ARGUS_THREADS */
146    struct ArgusHashTableHdr **array;
147 };
148 
149 struct ArgusAdjustStruct {
150    int mode, turns, modify, hard, zero;
151    int count, qual, slen, len;
152 
153    double stperiod, dtperiod, trperiod;
154    double stduration, dtduration, trduration;
155 
156    double sploss, dploss;
157 
158    float spkts, sbytes, sappbytes;
159    float scpkts, scbytes, scappbytes;
160    float dpkts, dbytes, dappbytes;
161    float dcpkts, dcbytes, dcappbytes;
162 
163    struct timeval start, end;
164    struct tm RaStartTmStruct, RaEndTmStruct;
165 
166    double value;
167    long long startuSecs, enduSecs, size;
168 
169    char *filename, *filterstr;
170    struct nff_program filter;
171 };
172 
173 struct RaBinStruct {
174    int status;
175    long long value, size;
176    struct timeval stime, etime, timeout;
177    struct ArgusAggregatorStruct *agg;
178 
179    unsigned char ArgusSrcDataMask[16],ArgusDstDataMask[16];
180 };
181 
182 struct RaBinProcessStruct {
183    int status;
184 #if defined(ARGUS_THREADS)
185    pthread_mutex_t lock;
186 #endif
187    long long start, end, size;
188    struct timeval startpt, endpt, rtime;
189    int arraylen, len, max, count, index;
190    int scalesecs;
191    struct RaBinStruct **array;
192    struct ArgusAdjustStruct nadp;
193 };
194 
195 struct ArgusCorStruct {
196    int count, size;
197    struct ArgusRecordStruct **array;
198 };
199 
200 struct ArgusDisplayStruct {
201    int type, status;
202    char *str;
203 };
204 
205 #define ARGUS_NSR_STICKY		0x00100000
206 #define ARGUS_RECORD_MODIFIED		0x0100
207 #define ARGUS_RECORD_NEW		0x0200
208 
209 struct ArgusRecordStruct {
210    struct ArgusQueueHeader qhdr;
211    struct ArgusDisplayStruct disp;
212    struct ArgusAggregatorStruct *agg;
213    unsigned int status, dsrindex, rank, autoid;
214    unsigned short timeout, idle;
215    struct RaBinProcessStruct *bins;
216    struct ArgusHashTableHdr *htblhdr;
217    struct ArgusHashTableHdr *hinthdr;
218    struct ArgusQueueStruct *nsq;
219    struct ArgusInput *input;
220    struct ArgusRecordHeader hdr;
221    struct ArgusDSRHeader *dsrs[ARGUSMAXDSRTYPE];
222    struct ArgusCorStruct *correlates;
223    float srate, drate, sload, dload, dur, mean;
224    float pcr, sploss, dploss;
225    long long offset;
226 };
227 
228 struct ArgusRemoteStruct {
229    struct ArgusQueueHeader qhdr;
230    struct ArgusInput *input;
231 
232 #if defined(ARGUS_THREADS)
233    pthread_t tid;
234    pthread_mutex_t lock;
235 #endif
236 };
237 
238 struct ArgusCIDRAddr {
239    u_char type, len, masklen, pad;
240    u_int addr[4], mask[4];
241    char *str;
242 };
243 
244 
245 #define ARGUS_MAXTHREADS	128
246 #define ARGUS_MAXLISTEN		32
247 
248 #define ARGUS_REAL_TIME_PROCESS   	0x0100
249 #define ARGUS_FILE_LIST_PROCESSED	0x1000
250 
251 #define ARGUS_FORCE_LOCAL_SRC		0x02
252 #define ARGUS_FORCE_LOCAL_DST		0x04
253 #define ARGUS_SUGGEST_LOCAL_SRC		0x08
254 #define ARGUS_SUGGEST_LOCAL_DST		0x10
255 
256 #define ARGUS_PORT_SERVICES		0x100
257 #define ARGUS_PORT_WELLKNOWN		0x200
258 #define ARGUS_PORT_REGISTERED		0x400
259 
260 #define ARGUS_ADDR_DIR_MASK (ARGUS_SUGGEST_LOCAL_SRC | ARGUS_SUGGEST_LOCAL_DST | ARGUS_FORCE_LOCAL_SRC | ARGUS_FORCE_LOCAL_DST)
261 #define ARGUS_PORT_DIR_MASK (ARGUS_PORT_SERVICES | ARGUS_PORT_WELLKNOWN | ARGUS_PORT_REGISTERED)
262 
263 #define ARGUS_REPLACE_MODE_TRUE		0x01
264 #define ARGUS_REPLACE_COMPRESSED_GZ	0x02
265 #define ARGUS_REPLACE_COMPRESSED_BZ	0x04
266 #define ARGUS_REPLACE_FILENAME_MODIFIED	0x08
267 
268 #define ARGUS_PRINT_NULL                0x01
269 #define ARGUS_PRINT_EMPTY_STRING        0x02
270 #define ARGUS_OMIT_EMPTY_STRING         0x04
271 
272 
273 struct ArgusParserStruct {
274    int status, RaParseCompleting, RaParseDone;
275    int RaDonePending, RaShutDown, RaSortedInput;
276    int RaTasksToDo, ArgusReliableConnection;
277    int ArgusCorrelateEvents, ArgusPerformCorrection;
278    int ArgusDirectionFunction;
279    int ArgusExitStatus, ArgusPassNum;
280    int ArgusFractionalDate;
281 
282    char *ArgusProgramName, *RaTimeFormat, *RaTimeZone;
283    char *ArgusProgramArgs, *ArgusProgramOptions;
284    char *ArgusSQLStatement, *MySQLDBEngine;
285    char *ArgusSearchString;
286 
287    struct timeval ArgusRealTime, ArgusGlobalTime;
288    struct timeval ArgusStartRealTime, ArgusEndRealTime;
289    struct timeval ArgusTimeDifferential;
290    struct timeval RaClientTimeout, RaClientUpdate;
291    struct timeval RaStartTime, RaEndTime;
292    struct timeval ArgusStartTimeVal;
293    struct timeval ArgusTimeDelta;
294    struct timeval ArgusTimeOffset;
295 
296    double ArgusLastRecordTime, ArgusTimeMultiplier;
297 
298    struct tm RaStartFilter, RaLastFilter;
299    struct tm RaTmStruct;
300 
301    struct ArgusAggregatorStruct *ArgusAggregator;
302    struct ArgusLabelerStruct *ArgusLocalLabeler;
303    struct ArgusLabelerStruct *ArgusColorLabeler;
304    struct ArgusLabelerStruct *ArgusLabeler;
305    struct RaBinProcessStruct *RaBinProcess;
306 
307 #if defined(ARGUS_THREADS)
308    pthread_t thread, remote, output, timer, dns;
309    pthread_mutex_t lock;
310    pthread_cond_t cond;
311 #endif /* ARGUS_THREADS */
312 
313    void *ArgusClientContext;
314 
315    int ArgusTimeoutThread, ArgusSessionId, NonBlockingDNS, RaDNSNameCacheTimeout;
316    int ArgusDSCodePoints;
317    int ArgusColorSupport, RaSeparateAddrPortWithPeriod;
318 
319    char *ArgusPidFile, *ArgusPidPath;
320    char *ArgusColorConfig;
321 
322    struct ArgusRecordStruct *ns;
323 
324    struct ArgusOutputStruct *ArgusOutput;
325    struct ArgusListStruct *ArgusOutputList, *ArgusInputList;
326    struct ArgusListStruct *ArgusNameList;
327 
328    struct ArgusQueueStruct *ArgusRemoteHosts, *ArgusActiveHosts;
329    struct ArgusQueueStruct *ArgusRemoteList;
330 
331    regex_t upreg[ARGUS_MAX_REGEX];
332    regex_t lpreg;
333    regex_t dpreg;
334 
335    int ArgusRegExItems;
336    int ArgusRemotes;
337    int ArgusReplaceMode;
338    int ArgusHostsActive;
339    int ArgusLfd[ARGUS_MAXLISTEN];
340    int ArgusListens;
341    int ArgusAdjustTime;
342    int ArgusConnectTime;
343    int ArgusReverse;
344    int ArgusGenerateManRecords;
345    int ArgusPrintMan, ArgusPrintEvent;
346    int ArgusPrintXml, ArgusAsnFormat;
347    int ArgusPrintJson, ArgusPrintD3;
348    int ArgusPrintJsonEmptyString;
349    char RaOutputStarted;
350    int RaXMLStarted;
351    int ArgusSrvInit;
352    int ArgusGrepSource;
353    int ArgusGrepDestination;
354    int ArgusAutoId;
355 
356    int ArgusStripFields;
357    int ArgusDSRFields[ARGUSMAXDSRTYPE];
358 
359    char *RadiumArchive;
360    char *ArgusMatchLabel;
361 
362    unsigned int ArgusID, ArgusIDType;
363 
364    struct timeval ArgusReportTime;
365    struct timeval ArgusUpdateInterval;
366    struct timeval ArgusMarReportInterval;
367    struct timeval timeout;
368 
369    struct timeval ArgusThisTime;
370    struct timeval ArgusLastTime;
371    struct timeval ArgusCurrentTime;
372    struct timeval ArgusLastRealTime;
373 
374    long long ArgusTotalRecords;
375    long long ArgusTotalMarRecords;
376    long long ArgusTotalEventRecords;
377    long long ArgusTotalFarRecords;
378    long long ArgusTotalPkts, ArgusTotalSrcPkts, ArgusTotalDstPkts;
379    long long ArgusTotalBytes, ArgusTotalSrcBytes, ArgusTotalDstBytes;
380 
381    signed char aflag, Aflag, bflag, cidrflag;
382    signed char cflag, Cflag, dflag, Dflag, eflag, Eflag;
383    signed char fflag, Fflag, gflag, Gflag, Hflag;
384    signed char idflag, jflag, Jflag, lflag, Lflag, mflag, hflag;
385    signed char notNetflag, Oflag, pflag, Pflag, qflag, Qflag;
386    signed char Netflag, nflag, Normflag, Pctflag, pidflag;
387 
388    signed char tflag, uflag, Wflag, vflag, Vflag, iflag;
389    signed char Iflag, rflag, Rflag, Sflag, sflag, xflag;
390    signed char Xflag, XMLflag, yflag, zflag, Zflag, domainonly;
391 
392    char *estr, *Hstr, *Mflag;
393 
394    int  Tflag, debugflag, RaInitialized;
395    double Bflag;
396 
397    char RaFieldDelimiter, RaFieldQuoted;
398    signed int RaFieldWidth, RaWriteOut;
399 
400    int Uflag, sNflag,   eNflag;
401    int        sNoflag, eNoflag;
402    struct timeval startime_t, lasttime_t;
403 
404    float Pauseflag;
405    float ProcessRealTime;
406    float RaFilterTimeout;
407 
408    char RaLabelStr[0x10000], *RaLabel;
409    char RaDBString[0x10000], *RaDBStr;
410    int ArgusRandomSeed;
411    int RaLabelCounter;
412 
413    int RaPrintOptionIndex;
414    char *RaPrintOptionStrings[ARGUS_MAX_S_OPTIONS];
415 
416    int RaSortOptionIndex;
417    char *RaSortOptionStrings[ARGUS_MAX_S_OPTIONS];
418 
419    int ArgusFilterFiledes[2];
420    int ArgusControlFiledes[2];
421 
422    int RaCumulativeMerge;
423    int RaFlowMajorModified;
424    int RaAllocHashTableHeaders;
425    int RaAllocArgusRecord;
426    int RaThisActiveIndex;
427    int RaThisFlowNum;
428    int RaThisModelNum;
429    int RaParseError;
430    int ArgusMinuteUpdate;
431    int ArgusHourlyUpdate;
432 
433    int RaPolicyStatus;
434 
435    int RaHistoMetricSeries;
436    int RaHistoMetricLog;
437    int RaHistoRangeState;
438 
439    double RaHistoLogInterval;
440    double RaHistoBinSize;
441    double RaHistoStart, RaHistoStartLog;
442    double RaHistoEnd, RaHistoEndLog;
443    struct ArgusRecordStruct **RaHistoRecords;
444 
445    unsigned short ArgusSourcePort, ArgusPortNum;
446 
447    int RaHistoBins, RaCloseInputFd;
448 
449    int RaMpcProbeMode;
450    int RaMpcNetMode;
451    int RaCorrelate;
452    int RaPollMode;
453    int RaAgMode;
454    int RaMonMode;
455    int RaUniMode;
456    int RaZeroMode;
457    int RaPrintMode;
458    int RaCursesMode;
459    int RaPrintIndex;
460    int RaExplicitDate;
461    int RaWildCardDate;
462 
463    char *RaFlowModelFile, *exceptfile;
464    char *writeDbstr, *readDbstr;
465    char *dbuserstr, *dbpassstr, *dbportstr, *dbhoststr;
466    char *ais, *ustr, *pstr;
467    char *timearg, *wfile;
468 
469    char *ArgusFlowModelFile;
470    char *ArgusAggregatorFile;
471    char *ArgusDelegatedIPFile;
472    char *ArgusLocalFilter;
473    char *ArgusRemoteFilter;
474    char *ArgusDisplayFilter;
475 
476    char *ArgusBindAddr;
477    char *ArgusEthernetVendorFile;
478    int ArgusPrintEthernetVendors;
479 
480    struct nff_program ArgusFilterCode;
481    struct nff_program ArgusDisplayCode;
482 
483    struct RaFlowModelStruct *RaFlowModel;
484 
485    struct ArgusCIDRAddr ArgusCIDRBuffer, *ArgusCIDRPtr;
486 
487    struct ArgusModeStruct *ArgusModeList;
488    struct ArgusModeStruct *ArgusMaskList;
489    struct ArgusInput *ArgusInputFileList;
490    struct ArgusInput *ArgusRemoteHostList;
491    struct ArgusListStruct *ArgusWfileList;
492 
493    struct ArgusInput *ArgusCurrentInput;
494 
495    struct ArgusPrintFieldStruct *RaPrintAlgorithm;
496    struct ArgusPrintFieldStruct *RaPrintAlgorithmList[ARGUS_MAX_PRINT_ALG];
497 
498    char RaDebugString[MAXSTRLEN];
499    int  RaDebugStatus;
500 
501    struct ArgusRecordStruct argus;
502    struct ArgusCanonRecord canon;
503    struct ArgusRecord ArgusInitCon;
504 
505    char ArgusSrcUserData[0x10000];
506    char ArgusDstUserData[0x10000];
507 
508    char ArgusSrcActDist[256];
509    char ArgusSrcIdleDist[256];
510    char ArgusDstActDist[256];
511    char ArgusDstIdleDist[256];
512 };
513 
514 
515 #ifdef ArgusParse
516 struct ArgusParserStruct *ArgusNewParser(char *);
517 void ArgusInitializeParser(struct ArgusParserStruct *);
518 void ArgusCloseParser(struct ArgusParserStruct *);
519 
520 #else
521 extern struct ArgusParserStruct *ArgusNewParser(char *);
522 extern void ArgusInitializeParser(struct ArgusParserStruct *);
523 extern void ArgusCloseParser(struct ArgusParserStruct *);
524 #endif
525 
526 #ifdef __cplusplus
527 }
528 #endif
529 #endif
530