1""" profile - moin profiling utilities
2
3This module provides profilers used to profile the memory usage of a
4long running process.
5
6Typical usage:
7
8 1. Create a profiler:
9
10    from MoinMoin.util.profile import Profiler
11    profiler = Profiler('my log')
12
13 2. In the request handler, add each request to the profiler:
14
15    profiler.addRequest()
16
17 3. If you like, you can add extra samples:
18
19    profiler.sample()
20
21You can customize the profiler when you create it:
22
23 * requestsPerSample (default 100):
24
25   How many requests to run between samples. Set higher for live wiki or
26   lower for more accurate results.
27
28 * collect (default 0):
29
30   Use gc.collect to force a memory cleanup each sample. Keeps the
31   memory usage much lower, but your profile data will not reflect the
32   real world memory usage of the application.
33
34Based on code by Oliver Graf
35
36@copyright: 2004 Nir Soffer
37@license: GNU GPL, see COPYING for details.
38"""
39
40import os, time, gc
41
42
43class Profiler:
44    """ Profile memory usage
45
46    Profiler count requests and sample memory usage.
47
48    FIXME: We might want to save the profiler log in the profiled wiki
49    data dir, but the data dir is available only later in request. This
50    should be fixed by loading the config earlier.
51    """
52    def __init__(self, name, requestsPerSample=100, collect=0):
53        """ Initialize a profiler
54
55        @param name: profiler name, used in the log file name
56        @param requestsPerSample: how many request to run between samples
57        @param collect: should call gc.collect() in each sample
58        """
59        logname = '%s--%s.log' % (name, time.strftime('%Y-%m-%d--%H-%M'))
60        self.logfile = file(logname, 'a')
61        self.requestsPerSample = requestsPerSample
62        self.collect = collect
63        self.pid = os.getpid()
64        self.count = 0 # count between somples
65        self.requests = 0 # requests added
66        self.data = {'collect': 'NA'} # Sample data
67
68    def addRequest(self):
69        """ Add a request to the profile
70
71        Call this for each page request.
72
73        WARNING: This is the most important call. if you don't call this
74        for each request - you will not have any profile data.
75
76        Invoke sample when self.count reach self.requestsPerSample.
77        """
78        self.requests += 1
79        self.count += 1
80        if self.count == self.requestsPerSample:
81            # Time for a sample
82            self.count = 0
83            self.sample()
84
85    def sample(self):
86        """ Make a sample of memory usage and log it
87
88        You can call this to make samples between the samples done each
89        requestsPerSample, for example, at startup.
90
91        Invoke common methods for all profilers. Some profilers like
92        TwistedProfiler override this method.
93        """
94        self._setData()
95        self._setMemory()
96        self._log()
97
98    # Private methods ------------------------------------------------------
99
100    def _setData(self):
101        """ Collect sample data into self.data
102
103        Private method used by profilers.
104        """
105        d = self.data
106        d['date'] = time.strftime('%Y-%m-%d %H:%M:%S')
107        d['requests'] = self.requests
108        if self.collect:
109            d['collect'] = str(gc.collect())
110        d['objects'] = len(gc.get_objects())
111        d['garbage'] = len(gc.garbage)
112
113    def _setMemory(self):
114        """ Get process memory usage
115
116        Private method used by profilers.
117
118        Uses ps call, maybe we should use procfs on Linux or maybe
119        getrusage system call (using the ctypes module).
120        """
121        lines = os.popen('/bin/ps -p %s -o rss' % self.pid).readlines()
122        self.data['memory'] = lines[1].strip()
123
124    def _log(self):
125        """ Format sample and write to log
126
127        Private method used by profilers.
128        """
129        line = ('%(date)s req:%(requests)d mem:%(memory)sKB collect:%(collect)s '
130                'objects:%(objects)d garbage:%(garbage)d\n' % self.data)
131        self.logfile.write(line)
132        self.logfile.flush()
133
134
135class TwistedProfiler(Profiler):
136    """ Twisted specific memory profiler
137
138    Customize the way we call ps, to overcome blocking problems on
139    twisted.
140    """
141
142    def __init__(self, name, requestsPerSample=100, collect=0):
143        """ Initialized twisted profiler
144
145        Invoke Profiler.__init__ and import getProcessOuput from
146        twisted.
147        """
148        Profiler.__init__(self, name, requestsPerSample, collect)
149        from twisted.internet.utils import getProcessOutput
150        self._getProcessOutput = getProcessOutput
151
152    def sample(self):
153        """ Make a sample of memory usage and log it
154
155        On twisted we can't just call ps - we have to use deferred,
156        which will call us using a callback when its finished, and then
157        we log.
158
159        Since twisted continue to serve while the deferred fetch the
160        memory, the reading may be late in few requests.
161        """
162        self._setData()
163        # Memory will be available little later
164        deferred = self._getProcessOutput('/bin/ps',
165                                          ('-p', str(self.pid), '-o', 'rss'))
166        deferred.addCallback(self._callback)
167
168    # Private methods ------------------------------------------------------
169
170    def _callback(self, data):
171        """ Called from deferred when ps output is available
172
173        Private method, don't call this.
174        """
175        self.data['memory'] = data.split('\n')[1].strip()
176        self._log()
177
178
179if __name__ == '__main__':
180    # In case someone try to run as a script
181    print __doc__
182
183