1""" profile - moin profiling utilities 2 3This module provides profilers used to profile the memory usage of a 4long running process. 5 6Typical usage: 7 8 1. Create a profiler: 9 10 from MoinMoin.util.profile import Profiler 11 profiler = Profiler('my log') 12 13 2. In the request handler, add each request to the profiler: 14 15 profiler.addRequest() 16 17 3. If you like, you can add extra samples: 18 19 profiler.sample() 20 21You can customize the profiler when you create it: 22 23 * requestsPerSample (default 100): 24 25 How many requests to run between samples. Set higher for live wiki or 26 lower for more accurate results. 27 28 * collect (default 0): 29 30 Use gc.collect to force a memory cleanup each sample. Keeps the 31 memory usage much lower, but your profile data will not reflect the 32 real world memory usage of the application. 33 34Based on code by Oliver Graf 35 36@copyright: 2004 Nir Soffer 37@license: GNU GPL, see COPYING for details. 38""" 39 40import os, time, gc 41 42 43class Profiler: 44 """ Profile memory usage 45 46 Profiler count requests and sample memory usage. 47 48 FIXME: We might want to save the profiler log in the profiled wiki 49 data dir, but the data dir is available only later in request. This 50 should be fixed by loading the config earlier. 51 """ 52 def __init__(self, name, requestsPerSample=100, collect=0): 53 """ Initialize a profiler 54 55 @param name: profiler name, used in the log file name 56 @param requestsPerSample: how many request to run between samples 57 @param collect: should call gc.collect() in each sample 58 """ 59 logname = '%s--%s.log' % (name, time.strftime('%Y-%m-%d--%H-%M')) 60 self.logfile = file(logname, 'a') 61 self.requestsPerSample = requestsPerSample 62 self.collect = collect 63 self.pid = os.getpid() 64 self.count = 0 # count between somples 65 self.requests = 0 # requests added 66 self.data = {'collect': 'NA'} # Sample data 67 68 def addRequest(self): 69 """ Add a request to the profile 70 71 Call this for each page request. 72 73 WARNING: This is the most important call. if you don't call this 74 for each request - you will not have any profile data. 75 76 Invoke sample when self.count reach self.requestsPerSample. 77 """ 78 self.requests += 1 79 self.count += 1 80 if self.count == self.requestsPerSample: 81 # Time for a sample 82 self.count = 0 83 self.sample() 84 85 def sample(self): 86 """ Make a sample of memory usage and log it 87 88 You can call this to make samples between the samples done each 89 requestsPerSample, for example, at startup. 90 91 Invoke common methods for all profilers. Some profilers like 92 TwistedProfiler override this method. 93 """ 94 self._setData() 95 self._setMemory() 96 self._log() 97 98 # Private methods ------------------------------------------------------ 99 100 def _setData(self): 101 """ Collect sample data into self.data 102 103 Private method used by profilers. 104 """ 105 d = self.data 106 d['date'] = time.strftime('%Y-%m-%d %H:%M:%S') 107 d['requests'] = self.requests 108 if self.collect: 109 d['collect'] = str(gc.collect()) 110 d['objects'] = len(gc.get_objects()) 111 d['garbage'] = len(gc.garbage) 112 113 def _setMemory(self): 114 """ Get process memory usage 115 116 Private method used by profilers. 117 118 Uses ps call, maybe we should use procfs on Linux or maybe 119 getrusage system call (using the ctypes module). 120 """ 121 lines = os.popen('/bin/ps -p %s -o rss' % self.pid).readlines() 122 self.data['memory'] = lines[1].strip() 123 124 def _log(self): 125 """ Format sample and write to log 126 127 Private method used by profilers. 128 """ 129 line = ('%(date)s req:%(requests)d mem:%(memory)sKB collect:%(collect)s ' 130 'objects:%(objects)d garbage:%(garbage)d\n' % self.data) 131 self.logfile.write(line) 132 self.logfile.flush() 133 134 135class TwistedProfiler(Profiler): 136 """ Twisted specific memory profiler 137 138 Customize the way we call ps, to overcome blocking problems on 139 twisted. 140 """ 141 142 def __init__(self, name, requestsPerSample=100, collect=0): 143 """ Initialized twisted profiler 144 145 Invoke Profiler.__init__ and import getProcessOuput from 146 twisted. 147 """ 148 Profiler.__init__(self, name, requestsPerSample, collect) 149 from twisted.internet.utils import getProcessOutput 150 self._getProcessOutput = getProcessOutput 151 152 def sample(self): 153 """ Make a sample of memory usage and log it 154 155 On twisted we can't just call ps - we have to use deferred, 156 which will call us using a callback when its finished, and then 157 we log. 158 159 Since twisted continue to serve while the deferred fetch the 160 memory, the reading may be late in few requests. 161 """ 162 self._setData() 163 # Memory will be available little later 164 deferred = self._getProcessOutput('/bin/ps', 165 ('-p', str(self.pid), '-o', 'rss')) 166 deferred.addCallback(self._callback) 167 168 # Private methods ------------------------------------------------------ 169 170 def _callback(self, data): 171 """ Called from deferred when ps output is available 172 173 Private method, don't call this. 174 """ 175 self.data['memory'] = data.split('\n')[1].strip() 176 self._log() 177 178 179if __name__ == '__main__': 180 # In case someone try to run as a script 181 print __doc__ 182 183