1#!/usr/bin/env python
2#
3# Copyright 2010 the V8 project authors. All rights reserved.
4# Redistribution and use in source and binary forms, with or without
5# modification, are permitted provided that the following conditions are
6# met:
7#
8#     * Redistributions of source code must retain the above copyright
9#       notice, this list of conditions and the following disclaimer.
10#     * Redistributions in binary form must reproduce the above
11#       copyright notice, this list of conditions and the following
12#       disclaimer in the documentation and/or other materials provided
13#       with the distribution.
14#     * Neither the name of Google Inc. nor the names of its
15#       contributors may be used to endorse or promote products derived
16#       from this software without specific prior written permission.
17#
18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29#
30
31#
32# This is an utility for plotting charts based on GC traces produced by V8 when
33# run with flags --trace-gc --trace-gc-nvp. Relies on gnuplot for actual
34# plotting.
35#
36# Usage: gc-nvp-trace-processor.py <GC-trace-filename>
37#
38
39
40from __future__ import with_statement
41import sys, types, subprocess, math
42import gc_nvp_common
43
44def flatten(l):
45  flat = []
46  for i in l: flat.extend(i)
47  return flat
48
49def gnuplot(script):
50  gnuplot = subprocess.Popen(["gnuplot"], stdin=subprocess.PIPE)
51  gnuplot.stdin.write(script)
52  gnuplot.stdin.close()
53  gnuplot.wait()
54
55x1y1 = 'x1y1'
56x1y2 = 'x1y2'
57x2y1 = 'x2y1'
58x2y2 = 'x2y2'
59
60class Item(object):
61  def __init__(self, title, field, axis = x1y1, **keywords):
62    self.title = title
63    self.axis = axis
64    self.props = keywords
65    if type(field) is types.ListType:
66      self.field = field
67    else:
68      self.field = [field]
69
70  def fieldrefs(self):
71    return self.field
72
73  def to_gnuplot(self, context):
74    args = ['"%s"' % context.datafile,
75            'using %s' % context.format_fieldref(self.field),
76            'title "%s"' % self.title,
77            'axis %s' % self.axis]
78    if 'style' in self.props:
79      args.append('with %s' % self.props['style'])
80    if 'lc' in self.props:
81      args.append('lc rgb "%s"' % self.props['lc'])
82    if 'fs' in self.props:
83      args.append('fs %s' % self.props['fs'])
84    return ' '.join(args)
85
86class Plot(object):
87  def __init__(self, *items):
88    self.items = items
89
90  def fieldrefs(self):
91    return flatten([item.fieldrefs() for item in self.items])
92
93  def to_gnuplot(self, ctx):
94    return 'plot ' + ', '.join([item.to_gnuplot(ctx) for item in self.items])
95
96class Set(object):
97  def __init__(self, value):
98    self.value = value
99
100  def to_gnuplot(self, ctx):
101    return 'set ' + self.value
102
103  def fieldrefs(self):
104    return []
105
106class Context(object):
107  def __init__(self, datafile, field_to_index):
108    self.datafile = datafile
109    self.field_to_index = field_to_index
110
111  def format_fieldref(self, fieldref):
112    return ':'.join([str(self.field_to_index[field]) for field in fieldref])
113
114def collect_fields(plot):
115  field_to_index = {}
116  fields = []
117
118  def add_field(field):
119    if field not in field_to_index:
120      fields.append(field)
121      field_to_index[field] = len(fields)
122
123  for field in flatten([item.fieldrefs() for item in plot]):
124    add_field(field)
125
126  return (fields, field_to_index)
127
128def is_y2_used(plot):
129  for subplot in plot:
130    if isinstance(subplot, Plot):
131      for item in subplot.items:
132        if item.axis == x1y2 or item.axis == x2y2:
133          return True
134  return False
135
136def get_field(trace_line, field):
137  t = type(field)
138  if t is types.StringType:
139    return trace_line[field]
140  elif t is types.FunctionType:
141    return field(trace_line)
142
143def generate_datafile(datafile_name, trace, fields):
144  with open(datafile_name, 'w') as datafile:
145    for line in trace:
146      data_line = [str(get_field(line, field)) for field in fields]
147      datafile.write('\t'.join(data_line))
148      datafile.write('\n')
149
150def generate_script_and_datafile(plot, trace, datafile, output):
151  (fields, field_to_index) = collect_fields(plot)
152  generate_datafile(datafile, trace, fields)
153  script = [
154      'set terminal png',
155      'set output "%s"' % output,
156      'set autoscale',
157      'set ytics nomirror',
158      'set xtics nomirror',
159      'set key below'
160  ]
161
162  if is_y2_used(plot):
163    script.append('set autoscale y2')
164    script.append('set y2tics')
165
166  context = Context(datafile, field_to_index)
167
168  for item in plot:
169    script.append(item.to_gnuplot(context))
170
171  return '\n'.join(script)
172
173def plot_all(plots, trace, prefix):
174  charts = []
175
176  for plot in plots:
177    outfilename = "%s_%d.png" % (prefix, len(charts))
178    charts.append(outfilename)
179    script = generate_script_and_datafile(plot, trace, '~datafile', outfilename)
180    print 'Plotting %s...' % outfilename
181    gnuplot(script)
182
183  return charts
184
185def reclaimed_bytes(row):
186  return row['total_size_before'] - row['total_size_after']
187
188def other_scope(r):
189  if r['gc'] == 's':
190    # there is no 'other' scope for scavenging collections.
191    return 0
192  return r['pause'] - r['mark'] - r['sweep'] - r['external']
193
194def scavenge_scope(r):
195  if r['gc'] == 's':
196    return r['pause'] - r['external']
197  return 0
198
199
200def real_mutator(r):
201  return r['mutator'] - r['steps_took']
202
203plots = [
204  [
205    Set('style fill solid 0.5 noborder'),
206    Set('style histogram rowstacked'),
207    Set('style data histograms'),
208    Plot(Item('Scavenge', scavenge_scope, lc = 'green'),
209         Item('Marking', 'mark', lc = 'purple'),
210         Item('Sweep', 'sweep', lc = 'blue'),
211         Item('External', 'external', lc = '#489D43'),
212         Item('Other', other_scope, lc = 'grey'),
213         Item('IGC Steps', 'steps_took', lc = '#FF6347'))
214  ],
215  [
216    Set('style fill solid 0.5 noborder'),
217    Set('style histogram rowstacked'),
218    Set('style data histograms'),
219    Plot(Item('Scavenge', scavenge_scope, lc = 'green'),
220         Item('Marking', 'mark', lc = 'purple'),
221         Item('Sweep', 'sweep', lc = 'blue'),
222         Item('External', 'external', lc = '#489D43'),
223         Item('Other', other_scope, lc = '#ADD8E6'),
224         Item('External', 'external', lc = '#D3D3D3'))
225  ],
226
227  [
228    Plot(Item('Mutator', real_mutator, lc = 'black', style = 'lines'))
229  ],
230  [
231    Set('style histogram rowstacked'),
232    Set('style data histograms'),
233    Plot(Item('Heap Size (before GC)', 'total_size_before', x1y2,
234              fs = 'solid 0.4 noborder',
235              lc = 'green'),
236         Item('Total holes (after GC)', 'holes_size_before', x1y2,
237              fs = 'solid 0.4 noborder',
238              lc = 'red'),
239         Item('GC Time', ['i', 'pause'], style = 'lines', lc = 'red'))
240  ],
241  [
242    Set('style histogram rowstacked'),
243    Set('style data histograms'),
244    Plot(Item('Heap Size (after GC)', 'total_size_after', x1y2,
245              fs = 'solid 0.4 noborder',
246              lc = 'green'),
247         Item('Total holes (after GC)', 'holes_size_after', x1y2,
248              fs = 'solid 0.4 noborder',
249              lc = 'red'),
250         Item('GC Time', ['i', 'pause'],
251              style = 'lines',
252              lc = 'red'))
253  ],
254  [
255    Set('style fill solid 0.5 noborder'),
256    Set('style data histograms'),
257    Plot(Item('Allocated', 'allocated'),
258         Item('Reclaimed', reclaimed_bytes),
259         Item('Promoted', 'promoted', style = 'lines', lc = 'black'))
260  ],
261]
262
263def freduce(f, field, trace, init):
264  return reduce(lambda t,r: f(t, r[field]), trace, init)
265
266def calc_total(trace, field):
267  return freduce(lambda t,v: t + long(v), field, trace, long(0))
268
269def calc_max(trace, field):
270  return freduce(lambda t,r: max(t, r), field, trace, 0)
271
272def count_nonzero(trace, field):
273  return freduce(lambda t,r: t if r == 0 else t + 1, field, trace, 0)
274
275
276def process_trace(filename):
277  trace = gc_nvp_common.parse_gc_trace(filename)
278
279  marksweeps = filter(lambda r: r['gc'] == 'ms', trace)
280  scavenges = filter(lambda r: r['gc'] == 's', trace)
281  globalgcs = filter(lambda r: r['gc'] != 's', trace)
282
283
284  charts = plot_all(plots, trace, filename)
285
286  def stats(out, prefix, trace, field):
287    n = len(trace)
288    total = calc_total(trace, field)
289    max = calc_max(trace, field)
290    if n > 0:
291      avg = total / n
292    else:
293      avg = 0
294    if n > 1:
295      dev = math.sqrt(freduce(lambda t,r: t + (r - avg) ** 2, field, trace, 0) /
296                      (n - 1))
297    else:
298      dev = 0
299
300    out.write('<tr><td>%s</td><td>%d</td><td>%d</td>'
301              '<td>%d</td><td>%d [dev %f]</td></tr>' %
302              (prefix, n, total, max, avg, dev))
303
304  def HumanReadable(size):
305    suffixes = ['bytes', 'kB', 'MB', 'GB']
306    power = 1
307    for i in range(len(suffixes)):
308      if size < power*1024:
309        return "%.1f" % (float(size) / power) + " " + suffixes[i]
310      power *= 1024
311
312  def throughput(name, trace):
313    total_live_after = calc_total(trace, 'total_size_after')
314    total_live_before = calc_total(trace, 'total_size_before')
315    total_gc = calc_total(trace, 'pause')
316    if total_gc == 0:
317      return
318    out.write('GC %s Throughput (after): %s / %s ms = %s/ms<br/>' %
319              (name,
320               HumanReadable(total_live_after),
321               total_gc,
322               HumanReadable(total_live_after / total_gc)))
323    out.write('GC %s Throughput (before): %s / %s ms = %s/ms<br/>' %
324              (name,
325               HumanReadable(total_live_before),
326               total_gc,
327               HumanReadable(total_live_before / total_gc)))
328
329
330  with open(filename + '.html', 'w') as out:
331    out.write('<html><body>')
332    out.write('<table>')
333    out.write('<tr><td>Phase</td><td>Count</td><td>Time (ms)</td>')
334    out.write('<td>Max</td><td>Avg</td></tr>')
335    stats(out, 'Total in GC', trace, 'pause')
336    stats(out, 'Scavenge', scavenges, 'pause')
337    stats(out, 'MarkSweep', marksweeps, 'pause')
338    stats(out, 'Mark', filter(lambda r: r['mark'] != 0, trace), 'mark')
339    stats(out, 'Sweep', filter(lambda r: r['sweep'] != 0, trace), 'sweep')
340    stats(out,
341          'External',
342          filter(lambda r: r['external'] != 0, trace),
343          'external')
344    out.write('</table>')
345    throughput('TOTAL', trace)
346    throughput('MS', marksweeps)
347    throughput('OLDSPACE', globalgcs)
348    out.write('<br/>')
349    for chart in charts:
350      out.write('<img src="%s">' % chart)
351      out.write('</body></html>')
352
353  print "%s generated." % (filename + '.html')
354
355if len(sys.argv) != 2:
356  print "Usage: %s <GC-trace-filename>" % sys.argv[0]
357  sys.exit(1)
358
359process_trace(sys.argv[1])
360