1#!/usr/bin/env python3 2# -*- coding: utf-8 -*- 3 4import os 5import sys 6import io 7import datetime 8import bz2 9import shutil 10import operator 11from collections import OrderedDict 12from ipaddress import ip_address 13 14 15# ###### Abort with error ################################################### 16def error(logstring): 17 sys.stderr.write(datetime.datetime.now().isoformat() + \ 18 ' ===== ERROR: ' + logstring + ' =====\n') 19 sys.exit(1) 20 21 22 23# ###### Read input and prepare output ###################################### 24 25# Input types: 26IT_NONE = 0 27IT_PING = 1 28IT_TRACEROUTE = 2 29 30# Output types: 31OT_POSTGRES = 1 32OT_MONGODB = 2 33 34def processInput(inputFile, outputType): 35 inputType = IT_NONE 36 lineNumber = 0 37 output = {} 38 hopCheck = {} 39 for inputLine in inputFile.readlines(): 40 lineNumber = lineNumber + 1 41 tuples = inputLine.rstrip().split(' ') 42 if len(tuples) > 0: 43 # ====== Ping ====================================================== 44 if tuples[0] == '#P': 45 if len(tuples) >= 7: 46 # ------ Handle input ---------------------------------------- 47 if inputType == IT_NONE: 48 inputType = IT_PING 49 elif inputType != IT_PING: 50 raise Exception('Multiple input types in the same file?!') 51 52 sourceIP = ip_address(tuples[1]) 53 destinationIP = ip_address(tuples[2]) 54 timeStamp = int(tuples[3], 16) 55 checksum = int(tuples[4], 16) 56 status = int(tuples[5]) 57 rtt = int(tuples[6]) 58 59 assert ('0x' + tuples[3]) == hex(timeStamp) 60 assert ('0x' + tuples[4]) == hex(checksum) 61 # print('ping', sourceIP, destinationIP, timeStamp, status, rtt) 62 63 # ------ Generate output ------------------------------------- 64 label = str(sourceIP) + '-' + str(destinationIP) + '-' + str(timeStamp) 65 if outputType == OT_POSTGRES: 66 timeStampDT = datetime.datetime(1970, 1, 1, 0, 0, 0, 0) + datetime.timedelta(microseconds = timeStamp) 67 timeStampStr = timeStampDT.strftime("%Y-%m-%dT%H:%M:%S.%f") 68 output[label] = '(' + \ 69 '\'' + timeStampStr + '\',' + \ 70 '\'' + str(sourceIP) + '\',' + \ 71 '\'' + str(destinationIP) + '\',' + \ 72 str(status) + ',' + \ 73 str(rtt) + \ 74 ')' 75 76 elif outputType == OT_MONGODB: 77 output[label] = OrderedDict([ 78 ( 'source', str(sourceIP)), 79 ( 'destination', str(destinationIP)), 80 ( 'timestamp', int(timeStamp)), 81 ( 'checksum', int(checksum)), 82 ( 'status', int(status)), 83 ( 'rtt', int(rtt)) ]) 84 85 else: 86 raise Exception('Bad input for Ping in line ' + str(lineNumber)) 87 88 89 # ====== Traceroute ================================================ 90 elif tuples[0] == '#T': 91 if len(tuples) >= 9: 92 # ------ Handle input ---------------------------------------- 93 if inputType == IT_NONE: 94 inputType = IT_TRACEROUTE 95 elif inputType != IT_TRACEROUTE: 96 raise Exception('Multiple input types in the same file?!') 97 98 sourceIP = ip_address(tuples[1]) 99 destinationIP = ip_address(tuples[2]) 100 timeStamp = int(tuples[3], 16) 101 roundNumber = int(tuples[4]) 102 checksum = int(tuples[5], 16) 103 totalHops = int(tuples[6]) 104 statusFlags = int(tuples[7], 16) 105 pathHashStr = tuples[8] 106 pathHash = int(pathHashStr, 16) 107 108 assert ('0x' + tuples[3]) == hex(timeStamp) 109 assert ('0x' + tuples[5]) == hex(checksum) 110 assert ('0x' + tuples[7]) == hex(statusFlags) 111 assert ('0x' + tuples[8]) == hex(pathHash) 112 # print('traceroute', sourceIP, destinationIP, timeStamp, roundNumber, checksum, totalHops, statusFlags, pathHash) 113 114 if outputType == OT_POSTGRES: 115 timeStampDT = datetime.datetime(1970, 1, 1, 0, 0, 0, 0) + datetime.timedelta(microseconds = timeStamp) 116 timeStampStr = timeStampDT.strftime("%Y%m%dT%H%M%S.%f") 117 118 elif outputType == OT_MONGODB: 119 label = str(sourceIP) + '-' + str(destinationIP) + '-' + str(timeStamp) + '-' + str(roundNumber).zfill(3) 120 # MongoDB only supports signed integers: 121 if pathHash > 0x7FFFFFFFFFFFFFFF: 122 pathHash -= 0x10000000000000000 123 hopCheck[label] = 0 124 output[label] = OrderedDict([ 125 ( 'source', str(sourceIP)), 126 ( 'destination', str(destinationIP)), 127 ( 'timestamp', int(timeStamp)), 128 ( 'round', int(roundNumber)), 129 ( 'checksum', int(checksum)), 130 ( 'totalHops', int(totalHops)), 131 ( 'statusflags', int(statusFlags)), 132 ( 'pathhash', int(pathHash)), 133 ( 'hops', [] ) ]) 134 135 else: 136 raise Exception('Bad input for Traceroute in line ' + str(lineNumber)) 137 138 139 elif ((tuples[0] == '\t') and (inputType == IT_TRACEROUTE)): 140 if len(tuples) >= 4: 141 # ------ Handle input ---------------------------------------- 142 hopNumber = int(tuples[1]) 143 status = int(tuples[2], 16) 144 rtt = int(tuples[3]) 145 hopIP = ip_address(tuples[4]) 146 147 assert hopNumber <= totalHops 148 assert ('0x' + tuples[2]) == hex(status) 149 # print('\t', hopNumber, status, rtt, hopIP) 150 151 # ------ Generate output ------------------------------------- 152 if outputType == OT_POSTGRES: 153 label = str(sourceIP) + '-' + str(destinationIP) + '-' + str(timeStamp) + '-' + str(roundNumber).zfill(3) + str(hopNumber).zfill(3) 154 output[label] = '(' + \ 155 '\'' + timeStampStr + '\',' + \ 156 '\'' + str(sourceIP) + '\',' + \ 157 '\'' + str(destinationIP) + '\',' + \ 158 str(hopNumber) + ',' + \ 159 str(totalHops) + ',' + \ 160 str(status | statusFlags) + ',' + \ 161 str(rtt) + ',' + \ 162 '\'' + str(hopIP) + '\',' + \ 163 'CAST(X\'' + pathHashStr + '\' AS BIGINT),' + \ 164 str(roundNumber) + \ 165 ')' 166 167 elif outputType == OT_MONGODB: 168 label = str(sourceIP) + '-' + str(destinationIP) + '-' + str(timeStamp) + '-' + str(roundNumber).zfill(3) 169 assert(hopCheck[label] + 1 == hopNumber) # Make sure that all hops are in order! 170 hopCheck[label] = hopNumber 171 172 output[label]['hops'].append(OrderedDict([ 173 ( 'status', int(status)), 174 ( 'rtt', int(rtt)), 175 ( 'hop', str(hopIP)) ])) 176 177 else: 178 raise Exception('Bad input for Traceroute in line ' + str(lineNumber)) 179 180 # ====== Error ===================================================== 181 else: 182 raise Exception('Unexpected input in line ' + str(lineNumber)) 183 184 185 # ====== Sort result ===================================================== 186 resultsList = sorted(output.items(), key=operator.itemgetter(0)) 187 188 # ====== Generate output string ========================================== 189 if outputType == OT_POSTGRES: 190 outputString = "" 191 if inputType == IT_PING: 192 outputString = 'INSERT INTO Ping VALUES ' 193 elif inputType == IT_TRACEROUTE: 194 outputString = 'INSERT INTO Traceroute VALUES ' 195 196 firstItem = True 197 for result in resultsList: 198 if firstItem: 199 outputString = outputString + '\n' + result[1] 200 firstItem = False 201 else: 202 outputString = outputString + ',\n' + result[1] 203 204 outputString = outputString + ';' 205 return outputString 206 207 elif outputType == OT_MONGODB: 208 outputList = [] 209 for result in resultsList: 210 outputList.append(result[1]) 211 return [ inputType, outputList ] 212 213 214 215inputFile = bz2.open('/tmp/xy/Ping-172.16.0.206-20180111T124131.681518-000000014.results.bz2', 'rt') 216#inputFile = bz2.open('/tmp/xy/Traceroute-172.16.0.206-20180111T105853.648711-000000001.results.bz2', 'rt') 217 218 219outputString = processInput(inputFile, OT_MONGODB) # OT_POSTGRES OT_MONGODB 220print(outputString) 221