1#!/usr/bin/env python3
2# -*- coding: utf-8 -*-
3
4import os
5import sys
6import io
7import datetime
8import bz2
9import shutil
10import operator
11from collections import OrderedDict
12from ipaddress   import ip_address
13
14
15# ###### Abort with error ###################################################
16def error(logstring):
17   sys.stderr.write(datetime.datetime.now().isoformat() + \
18                    ' ===== ERROR: ' + logstring + ' =====\n')
19   sys.exit(1)
20
21
22
23# ###### Read input and prepare output ######################################
24
25# Input types:
26IT_NONE       = 0
27IT_PING       = 1
28IT_TRACEROUTE = 2
29
30# Output types:
31OT_POSTGRES   = 1
32OT_MONGODB    = 2
33
34def processInput(inputFile, outputType):
35   inputType  = IT_NONE
36   lineNumber = 0
37   output     = {}
38   hopCheck   = {}
39   for inputLine in inputFile.readlines():
40      lineNumber = lineNumber + 1
41      tuples = inputLine.rstrip().split(' ')
42      if len(tuples) > 0:
43         # ====== Ping ======================================================
44         if tuples[0] == '#P':
45            if len(tuples) >= 7:
46               # ------ Handle input ----------------------------------------
47               if inputType == IT_NONE:
48                  inputType = IT_PING
49               elif inputType != IT_PING:
50                  raise Exception('Multiple input types in the same file?!')
51
52               sourceIP      = ip_address(tuples[1])
53               destinationIP = ip_address(tuples[2])
54               timeStamp     = int(tuples[3], 16)
55               checksum      = int(tuples[4], 16)
56               status        = int(tuples[5])
57               rtt           = int(tuples[6])
58
59               assert ('0x' + tuples[3]) == hex(timeStamp)
60               assert ('0x' + tuples[4]) == hex(checksum)
61               # print('ping', sourceIP, destinationIP, timeStamp, status, rtt)
62
63               # ------ Generate output -------------------------------------
64               label = str(sourceIP) + '-' + str(destinationIP) + '-' + str(timeStamp)
65               if outputType == OT_POSTGRES:
66                  timeStampDT  = datetime.datetime(1970, 1, 1, 0, 0, 0, 0) +  datetime.timedelta(microseconds = timeStamp)
67                  timeStampStr = timeStampDT.strftime("%Y-%m-%dT%H:%M:%S.%f")
68                  output[label] = '(' + \
69                     '\'' + timeStampStr       + '\',' + \
70                     '\'' + str(sourceIP)      + '\',' + \
71                     '\'' + str(destinationIP) + '\',' + \
72                     str(status) + ',' + \
73                     str(rtt) + \
74                     ')'
75
76               elif outputType == OT_MONGODB:
77                  output[label] = OrderedDict([
78                                     ( 'source',      str(sourceIP)),
79                                     ( 'destination', str(destinationIP)),
80                                     ( 'timestamp',   int(timeStamp)),
81                                     ( 'checksum',    int(checksum)),
82                                     ( 'status',      int(status)),
83                                     ( 'rtt',         int(rtt)) ])
84
85            else:
86               raise Exception('Bad input for Ping in line ' + str(lineNumber))
87
88
89         # ====== Traceroute ================================================
90         elif tuples[0] == '#T':
91            if len(tuples) >= 9:
92               # ------ Handle input ----------------------------------------
93               if inputType == IT_NONE:
94                  inputType = IT_TRACEROUTE
95               elif inputType != IT_TRACEROUTE:
96                  raise Exception('Multiple input types in the same file?!')
97
98               sourceIP      = ip_address(tuples[1])
99               destinationIP = ip_address(tuples[2])
100               timeStamp     = int(tuples[3], 16)
101               roundNumber   = int(tuples[4])
102               checksum      = int(tuples[5], 16)
103               totalHops     = int(tuples[6])
104               statusFlags   = int(tuples[7], 16)
105               pathHashStr   = tuples[8]
106               pathHash      = int(pathHashStr, 16)
107
108               assert ('0x' + tuples[3]) == hex(timeStamp)
109               assert ('0x' + tuples[5]) == hex(checksum)
110               assert ('0x' + tuples[7]) == hex(statusFlags)
111               assert ('0x' + tuples[8]) == hex(pathHash)
112               # print('traceroute', sourceIP, destinationIP, timeStamp, roundNumber, checksum, totalHops, statusFlags, pathHash)
113
114               if outputType == OT_POSTGRES:
115                  timeStampDT  = datetime.datetime(1970, 1, 1, 0, 0, 0, 0) +  datetime.timedelta(microseconds = timeStamp)
116                  timeStampStr = timeStampDT.strftime("%Y%m%dT%H%M%S.%f")
117
118               elif outputType == OT_MONGODB:
119                  label = str(sourceIP) + '-' + str(destinationIP) + '-' + str(timeStamp) + '-' + str(roundNumber).zfill(3)
120                  # MongoDB only supports signed integers:
121                  if pathHash > 0x7FFFFFFFFFFFFFFF:
122                     pathHash -= 0x10000000000000000
123                  hopCheck[label] = 0
124                  output[label] = OrderedDict([
125                                     ( 'source',                   str(sourceIP)),
126                                     ( 'destination',              str(destinationIP)),
127                                     ( 'timestamp',                int(timeStamp)),
128                                     ( 'round',                    int(roundNumber)),
129                                     ( 'checksum',                 int(checksum)),
130                                     ( 'totalHops',                int(totalHops)),
131                                     ( 'statusflags',              int(statusFlags)),
132                                     ( 'pathhash',                 int(pathHash)),
133                                     ( 'hops',                     [] ) ])
134
135            else:
136               raise Exception('Bad input for Traceroute in line ' + str(lineNumber))
137
138
139         elif ((tuples[0] == '\t') and (inputType == IT_TRACEROUTE)):
140            if len(tuples) >= 4:
141               # ------ Handle input ----------------------------------------
142               hopNumber = int(tuples[1])
143               status    = int(tuples[2], 16)
144               rtt       = int(tuples[3])
145               hopIP     = ip_address(tuples[4])
146
147               assert hopNumber <= totalHops
148               assert ('0x' + tuples[2]) == hex(status)
149               # print('\t', hopNumber, status, rtt, hopIP)
150
151               # ------ Generate output -------------------------------------
152               if outputType == OT_POSTGRES:
153                  label = str(sourceIP) + '-' + str(destinationIP) + '-' + str(timeStamp) + '-' + str(roundNumber).zfill(3) + str(hopNumber).zfill(3)
154                  output[label] = '(' + \
155                     '\'' + timeStampStr       + '\',' + \
156                     '\'' + str(sourceIP)      + '\',' + \
157                     '\'' + str(destinationIP) + '\',' + \
158                     str(hopNumber) + ',' + \
159                     str(totalHops) + ',' + \
160                     str(status | statusFlags) + ',' + \
161                     str(rtt) + ',' + \
162                     '\'' + str(hopIP) + '\',' + \
163                     'CAST(X\'' + pathHashStr + '\' AS BIGINT),' + \
164                     str(roundNumber) + \
165                     ')'
166
167               elif outputType == OT_MONGODB:
168                  label = str(sourceIP) + '-' + str(destinationIP) + '-' + str(timeStamp) + '-' + str(roundNumber).zfill(3)
169                  assert(hopCheck[label] + 1 == hopNumber)   # Make sure that all hops are in order!
170                  hopCheck[label] = hopNumber
171
172                  output[label]['hops'].append(OrderedDict([
173                     ( 'status', int(status)),
174                     ( 'rtt',    int(rtt)),
175                     ( 'hop',    str(hopIP)) ]))
176
177            else:
178               raise Exception('Bad input for Traceroute in line ' + str(lineNumber))
179
180         # ====== Error =====================================================
181         else:
182            raise Exception('Unexpected input in line ' + str(lineNumber))
183
184
185   # ====== Sort result =====================================================
186   resultsList = sorted(output.items(), key=operator.itemgetter(0))
187
188   # ====== Generate output string ==========================================
189   if outputType == OT_POSTGRES:
190      outputString = ""
191      if inputType == IT_PING:
192         outputString = 'INSERT INTO Ping VALUES '
193      elif inputType == IT_TRACEROUTE:
194         outputString = 'INSERT INTO Traceroute VALUES '
195
196      firstItem = True
197      for result in resultsList:
198         if firstItem:
199            outputString = outputString + '\n' + result[1]
200            firstItem = False
201         else:
202            outputString = outputString + ',\n' + result[1]
203
204      outputString = outputString + ';'
205      return outputString
206
207   elif outputType == OT_MONGODB:
208      outputList = []
209      for result in resultsList:
210            outputList.append(result[1])
211      return [ inputType, outputList ]
212
213
214
215inputFile = bz2.open('/tmp/xy/Ping-172.16.0.206-20180111T124131.681518-000000014.results.bz2', 'rt')
216#inputFile = bz2.open('/tmp/xy/Traceroute-172.16.0.206-20180111T105853.648711-000000001.results.bz2', 'rt')
217
218
219outputString = processInput(inputFile, OT_MONGODB)   # OT_POSTGRES OT_MONGODB
220print(outputString)
221