1#   Copyright 2009-2018 Oli Schacher
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14#
15#
16#
17
18import time
19import threading
20import logging
21import os
22
23class StatDelta(object):
24    """Represents the delta to be applied on the total statistics"""
25    def __init__(self, **kwargs):
26        self.total = 0
27        self.spam = 0
28        self.ham = 0
29        self.virus = 0
30        self.blocked = 0
31        self.in_ = 0
32        self.out = 0
33        self.scantime = 0
34
35        for k,v in kwargs.items():
36            setattr(self,k,v)
37
38    def as_message(self):
39        return dict(event_type='statsdelta', total=self.total , spam=self.spam, ham=self.ham, virus=self.virus, blocked=self.blocked, in_=self.in_ , out=self.out, scantime=self.scantime)
40
41
42class Statskeeper(object):
43
44    """Keeps track of a few stats to generate mrtg graphs and stuff"""
45    __shared_state = {}
46
47    def __init__(self):
48        self.__dict__ = self.__shared_state
49        if not hasattr(self, 'totalcount'):
50            self.totalcount = 0
51            self.spamcount = 0
52            self.hamcount = 0
53            self.viruscount = 0
54            self.blockedcount = 0
55            self.incount = 0
56            self.outcount = 0
57            self.scantimes = []
58            self.starttime = time.time()
59            self.lastscan = 0
60            self.stat_listener_callback = []
61
62
63    def uptime(self):
64        """uptime since we started fuglu"""
65        total_seconds = time.time() - self.starttime
66        MINUTE = 60
67        HOUR = MINUTE * 60
68        DAY = HOUR * 24
69        # Get the days, hours, etc:
70        days = int(total_seconds / DAY)
71        hours = int((total_seconds % DAY) / HOUR)
72        minutes = int((total_seconds % HOUR) / MINUTE)
73        seconds = int(total_seconds % MINUTE)
74        # Build up the pretty string (like this: "N days, N hours, N minutes, N
75        # seconds")
76        string = ""
77        if days > 0:
78            string += str(days) + " " + (days == 1 and "day" or "days") + ", "
79        if len(string) > 0 or hours > 0:
80            string += str(hours) + " " + \
81                (hours == 1 and "hour" or "hours") + ", "
82        if len(string) > 0 or minutes > 0:
83            string += str(minutes) + " " + \
84                (minutes == 1 and "minute" or "minutes") + ", "
85        string += str(seconds) + " " + (seconds == 1 and "second" or "seconds")
86        return string
87
88    def numthreads(self):
89        """return the number of threads"""
90        return len(threading.enumerate())
91
92    def increasecounters(self, suspect):
93        """Update local counters after a suspect has passed the system"""
94
95        delta = StatDelta()
96        delta.total = 1
97
98        isspam = suspect.is_spam()
99        isvirus = suspect.is_virus()
100        isblocked = suspect.is_blocked()
101
102        if isspam:
103            delta.spam = 1
104
105        if isvirus:
106            delta.virus = 1
107
108        if isblocked:
109            delta.blocked = 1
110
111        if not (isspam or isvirus): # blocked is currently still counted as ham.
112            delta.ham = 1
113
114        delta.scantime = suspect.get_tag('fuglu.scantime')
115        self.increase_counter_values(delta)
116
117    def increase_counter_values(self, statdelta):
118        self.totalcount += statdelta.total
119        self.spamcount += statdelta.spam
120        self.viruscount += statdelta.virus
121        self.hamcount += statdelta.ham
122        self.blockedcount += statdelta.blocked
123        if statdelta.scantime:
124            self._appendscantime(statdelta.scantime)
125        self.lastscan = time.time()
126        self.incount += statdelta.in_
127        self.outcount += statdelta.out
128        self.fire_stats_changed_event(statdelta)
129
130    def fire_stats_changed_event(self,statdelta):
131        for callback in self.stat_listener_callback:
132            callback(statdelta)
133
134    def scantime(self):
135        """Get the average scantime of the last 100 messages.
136        If last msg is older than five minutes, return 0"""
137        tms = self.scantimes[:]
138        length = len(tms)
139
140        # no entries in scantime list
141        if length == 0:
142            return "0"
143
144        # newest entry is older than five minutes
145        # clear entries
146        if time.time() - self.lastscan > 300:
147            self.scantimes = []
148            return "0"
149
150        avg = sum(tms) / length
151        avgstring = "%.4f" % avg
152        return avgstring
153
154    def _appendscantime(self, scantime):
155        """add new entry to the list of scantimes"""
156        try:
157            f = float(scantime)
158        except:
159            return
160        while len(self.scantimes) > 100:
161            del self.scantimes[0]
162
163        self.scantimes.append(f)
164
165
166class StatsThread(object):
167
168    """Keep Track of statistics and write mrtg data"""
169
170    def __init__(self, config):
171        self.config = config
172        self.stats = Statskeeper()
173        self.logger = logging.getLogger('fuglu.stats')
174        self.writeinterval = 30
175        self.identifier = 'FuGLU'
176        self.stayalive = True
177
178    def writestats(self):
179        mrtgdir = self.config.get('main', 'mrtgdir')
180        if mrtgdir is None or mrtgdir.strip() == "":
181            self.logger.debug(
182                'No mrtg directory defined, disabling stats writer')
183            return
184
185        if not os.path.isdir(mrtgdir):
186            self.logger.error(
187                'MRTG directory %s not found, disabling stats writer' % mrtgdir)
188            return
189
190        self.logger.info('Writing statistics to %s' % mrtgdir)
191
192        while self.stayalive:
193            time.sleep(self.writeinterval)
194            uptime = self.stats.uptime()
195
196            # total messages
197            self.write_mrtg('%s/inout' % mrtgdir, float(self.stats.incount),
198                            float(self.stats.outcount), uptime, self.identifier)
199            # spam ham
200            self.write_mrtg('%s/hamspam' % mrtgdir, float(self.stats.hamcount),
201                            float(self.stats.spamcount), uptime, self.identifier)
202
203            # num threads
204            self.write_mrtg(
205                '%s/threads' % mrtgdir, self.stats.numthreads(), None, uptime, self.identifier)
206
207            # virus
208            self.write_mrtg(
209                '%s/virus' % mrtgdir, float(self.stats.viruscount), None, uptime, self.identifier)
210
211            # scan time
212            self.write_mrtg(
213                '%s/scantime' % mrtgdir, self.stats.scantime(), None, uptime, self.identifier)
214
215    def write_mrtg(self, filename, value1, value2, uptime, identifier):
216        try:
217            with open(filename, 'w') as fp:
218                fp.write("%s\n" % value1)
219                if value2:
220                    fp.write("%s\n" % value2)
221                else:
222                    fp.write("0\n")
223                fp.write("%s\n%s\n" % (uptime, identifier))
224        except Exception as e:
225            self.logger.error(
226                'Could not write mrtg stats file %s : %s)' % (filename, e))
227