1 /* 2 ** Copyright (C) 2001-2020 by Carnegie Mellon University. 3 ** 4 ** @OPENSOURCE_LICENSE_START@ 5 ** See license information in ../../LICENSE.txt 6 ** @OPENSOURCE_LICENSE_END@ 7 */ 8 9 /* 10 ** rwcombine.h 11 ** 12 ** Common declarations needed by rwcombine. See rwcombine.c for 13 ** implementation details. 14 */ 15 #ifndef _RWCOMBINE_H 16 #define _RWCOMBINE_H 17 #ifdef __cplusplus 18 extern "C" { 19 #endif 20 21 #include <silk/silk.h> 22 23 RCSIDENTVAR(rcsID_RWCOMBINE_H, "$SiLK: rwcombine.h ef14e54179be 2020-04-14 21:57:45Z mthomas $"); 24 25 #include <silk/rwascii.h> 26 #include <silk/rwrec.h> 27 #include <silk/skipaddr.h> 28 #include <silk/skstream.h> 29 #include <silk/sktempfile.h> 30 #include <silk/utils.h> 31 32 /* use TRACEMSG_LEVEL as our tracing variable */ 33 #define TRACEMSG(msg) TRACEMSG_TO_TRACEMSGLVL(1, msg) 34 #include <silk/sktracemsg.h> 35 36 37 /* LOCAL DEFINES AND TYPEDEFS */ 38 39 /* 40 * The default buffer size to use, unless the user selects a 41 * different value with the --buffer-size switch. 42 * 43 * Support of a buffer of almost 2GB. 44 */ 45 #define DEFAULT_BUFFER_SIZE "1920m" 46 47 /* 48 * We do not allocate the buffer at once, but use realloc() to grow 49 * the buffer linearly to the maximum size. The following is the 50 * number of steps to take to reach the maximum size. The number 51 * of realloc() calls will be once less than this value. 52 * 53 * If the initial allocation fails, the number of chunks is 54 * incremented---making the size of the initial malloc() 55 * smaller---and allocation is attempted again. 56 */ 57 #define NUM_CHUNKS 6 58 59 /* 60 * Do not allocate more than this number of bytes at a time. 61 * 62 * If dividing the buffer size by NUM_CHUNKS gives a chunk size 63 * larger than this; determine the number of chunks by dividing the 64 * buffer size by this value. 65 * 66 * Use a value of 1g 67 */ 68 #define MAX_CHUNK_SIZE ((size_t)(0x40000000)) 69 70 /* 71 * If we cannot allocate a buffer that will hold at least this many 72 * records, give up. 73 */ 74 #define MIN_IN_CORE_RECORDS 1000 75 76 /* 77 * Maximum number of files to attempt to merge-sort at once. 78 */ 79 #define MAX_MERGE_FILES 1024 80 81 /* 82 * Size of a node is constant: the size of a complete rwRec. 83 */ 84 #define NODE_SIZE sizeof(rwRec) 85 86 /* 87 * The maximum buffer size is the maximum size we can allocate. 88 */ 89 #define MAXIMUM_BUFFER_SIZE ((size_t)(SIZE_MAX)) 90 91 /* 92 * The minium buffer size. 93 */ 94 #define MINIMUM_BUFFER_SIZE (NODE_SIZE * MIN_IN_CORE_RECORDS) 95 96 97 /* VARIABLES */ 98 99 /* number of fields to sort over */ 100 extern uint32_t num_fields; 101 102 /* IDs of the fields to sort over; values are from the 103 * rwrec_printable_fields_t enum. */ 104 extern uint32_t sort_fields[RWREC_PRINTABLE_FIELD_COUNT]; 105 106 /* output stream */ 107 extern skstream_t *out_stream; 108 109 /* statistics stream */ 110 extern skstream_t *print_statistics; 111 112 /* temp file context */ 113 extern sk_tempfilectx_t *tmpctx; 114 115 /* maximum amount of RAM to attempt to allocate */ 116 extern size_t buffer_size; 117 118 /* maximum amount of idle time to allow between flows */ 119 extern int64_t max_idle_time; 120 121 122 /* FUNCTIONS */ 123 124 void 125 appExit( 126 int status) 127 NORETURN; 128 void 129 appSetup( 130 int argc, 131 char **argv); 132 int 133 appNextInput( 134 skstream_t **stream); 135 136 137 #ifdef __cplusplus 138 } 139 #endif 140 #endif /* _RWCOMBINE_H */ 141 142 /* 143 ** Local Variables: 144 ** mode:c 145 ** indent-tabs-mode:nil 146 ** c-basic-offset:4 147 ** End: 148 */ 149