1#!/usr/bin/env python
2#
3# Copyright 2010 the V8 project authors. All rights reserved.
4# Redistribution and use in source and binary forms, with or without
5# modification, are permitted provided that the following conditions are
6# met:
7#
8#     * Redistributions of source code must retain the above copyright
9#       notice, this list of conditions and the following disclaimer.
10#     * Redistributions in binary form must reproduce the above
11#       copyright notice, this list of conditions and the following
12#       disclaimer in the documentation and/or other materials provided
13#       with the distribution.
14#     * Neither the name of Google Inc. nor the names of its
15#       contributors may be used to endorse or promote products derived
16#       from this software without specific prior written permission.
17#
18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29#
30
31#
32# This is an utility for plotting charts based on GC traces produced by V8 when
33# run with flags --trace-gc --trace-gc-nvp. Relies on gnuplot for actual
34# plotting.
35#
36# Usage: gc-nvp-trace-processor.py <GC-trace-filename>
37#
38
39
40# for py2/py3 compatibility
41from __future__ import with_statement
42from __future__ import print_function
43from functools import reduce
44
45import sys, types, subprocess, math
46import gc_nvp_common
47
48
49try:
50  long        # Python 2
51except NameError:
52  long = int  # Python 3
53
54
55def flatten(l):
56  flat = []
57  for i in l: flat.extend(i)
58  return flat
59
60def gnuplot(script):
61  gnuplot = subprocess.Popen(["gnuplot"], stdin=subprocess.PIPE)
62  gnuplot.stdin.write(script)
63  gnuplot.stdin.close()
64  gnuplot.wait()
65
66x1y1 = 'x1y1'
67x1y2 = 'x1y2'
68x2y1 = 'x2y1'
69x2y2 = 'x2y2'
70
71class Item(object):
72  def __init__(self, title, field, axis = x1y1, **keywords):
73    self.title = title
74    self.axis = axis
75    self.props = keywords
76    if type(field) is list:
77      self.field = field
78    else:
79      self.field = [field]
80
81  def fieldrefs(self):
82    return self.field
83
84  def to_gnuplot(self, context):
85    args = ['"%s"' % context.datafile,
86            'using %s' % context.format_fieldref(self.field),
87            'title "%s"' % self.title,
88            'axis %s' % self.axis]
89    if 'style' in self.props:
90      args.append('with %s' % self.props['style'])
91    if 'lc' in self.props:
92      args.append('lc rgb "%s"' % self.props['lc'])
93    if 'fs' in self.props:
94      args.append('fs %s' % self.props['fs'])
95    return ' '.join(args)
96
97class Plot(object):
98  def __init__(self, *items):
99    self.items = items
100
101  def fieldrefs(self):
102    return flatten([item.fieldrefs() for item in self.items])
103
104  def to_gnuplot(self, ctx):
105    return 'plot ' + ', '.join([item.to_gnuplot(ctx) for item in self.items])
106
107class Set(object):
108  def __init__(self, value):
109    self.value = value
110
111  def to_gnuplot(self, ctx):
112    return 'set ' + self.value
113
114  def fieldrefs(self):
115    return []
116
117class Context(object):
118  def __init__(self, datafile, field_to_index):
119    self.datafile = datafile
120    self.field_to_index = field_to_index
121
122  def format_fieldref(self, fieldref):
123    return ':'.join([str(self.field_to_index[field]) for field in fieldref])
124
125def collect_fields(plot):
126  field_to_index = {}
127  fields = []
128
129  def add_field(field):
130    if field not in field_to_index:
131      fields.append(field)
132      field_to_index[field] = len(fields)
133
134  for field in flatten([item.fieldrefs() for item in plot]):
135    add_field(field)
136
137  return (fields, field_to_index)
138
139def is_y2_used(plot):
140  for subplot in plot:
141    if isinstance(subplot, Plot):
142      for item in subplot.items:
143        if item.axis == x1y2 or item.axis == x2y2:
144          return True
145  return False
146
147def get_field(trace_line, field):
148  t = type(field)
149  if t is bytes:
150    return trace_line[field]
151  elif t is types.FunctionType:
152    return field(trace_line)
153
154def generate_datafile(datafile_name, trace, fields):
155  with open(datafile_name, 'w') as datafile:
156    for line in trace:
157      data_line = [str(get_field(line, field)) for field in fields]
158      datafile.write('\t'.join(data_line))
159      datafile.write('\n')
160
161def generate_script_and_datafile(plot, trace, datafile, output):
162  (fields, field_to_index) = collect_fields(plot)
163  generate_datafile(datafile, trace, fields)
164  script = [
165      'set terminal png',
166      'set output "%s"' % output,
167      'set autoscale',
168      'set ytics nomirror',
169      'set xtics nomirror',
170      'set key below'
171  ]
172
173  if is_y2_used(plot):
174    script.append('set autoscale y2')
175    script.append('set y2tics')
176
177  context = Context(datafile, field_to_index)
178
179  for item in plot:
180    script.append(item.to_gnuplot(context))
181
182  return '\n'.join(script)
183
184def plot_all(plots, trace, prefix):
185  charts = []
186
187  for plot in plots:
188    outfilename = "%s_%d.png" % (prefix, len(charts))
189    charts.append(outfilename)
190    script = generate_script_and_datafile(plot, trace, '~datafile', outfilename)
191    print('Plotting %s...' % outfilename)
192    gnuplot(script)
193
194  return charts
195
196def reclaimed_bytes(row):
197  return row['total_size_before'] - row['total_size_after']
198
199def other_scope(r):
200  if r['gc'] == 's':
201    # there is no 'other' scope for scavenging collections.
202    return 0
203  return r['pause'] - r['mark'] - r['sweep'] - r['external']
204
205def scavenge_scope(r):
206  if r['gc'] == 's':
207    return r['pause'] - r['external']
208  return 0
209
210
211def real_mutator(r):
212  return r['mutator'] - r['steps_took']
213
214plots = [
215  [
216    Set('style fill solid 0.5 noborder'),
217    Set('style histogram rowstacked'),
218    Set('style data histograms'),
219    Plot(Item('Scavenge', scavenge_scope, lc = 'green'),
220         Item('Marking', 'mark', lc = 'purple'),
221         Item('Sweep', 'sweep', lc = 'blue'),
222         Item('External', 'external', lc = '#489D43'),
223         Item('Other', other_scope, lc = 'grey'),
224         Item('IGC Steps', 'steps_took', lc = '#FF6347'))
225  ],
226  [
227    Set('style fill solid 0.5 noborder'),
228    Set('style histogram rowstacked'),
229    Set('style data histograms'),
230    Plot(Item('Scavenge', scavenge_scope, lc = 'green'),
231         Item('Marking', 'mark', lc = 'purple'),
232         Item('Sweep', 'sweep', lc = 'blue'),
233         Item('External', 'external', lc = '#489D43'),
234         Item('Other', other_scope, lc = '#ADD8E6'),
235         Item('External', 'external', lc = '#D3D3D3'))
236  ],
237
238  [
239    Plot(Item('Mutator', real_mutator, lc = 'black', style = 'lines'))
240  ],
241  [
242    Set('style histogram rowstacked'),
243    Set('style data histograms'),
244    Plot(Item('Heap Size (before GC)', 'total_size_before', x1y2,
245              fs = 'solid 0.4 noborder',
246              lc = 'green'),
247         Item('Total holes (after GC)', 'holes_size_before', x1y2,
248              fs = 'solid 0.4 noborder',
249              lc = 'red'),
250         Item('GC Time', ['i', 'pause'], style = 'lines', lc = 'red'))
251  ],
252  [
253    Set('style histogram rowstacked'),
254    Set('style data histograms'),
255    Plot(Item('Heap Size (after GC)', 'total_size_after', x1y2,
256              fs = 'solid 0.4 noborder',
257              lc = 'green'),
258         Item('Total holes (after GC)', 'holes_size_after', x1y2,
259              fs = 'solid 0.4 noborder',
260              lc = 'red'),
261         Item('GC Time', ['i', 'pause'],
262              style = 'lines',
263              lc = 'red'))
264  ],
265  [
266    Set('style fill solid 0.5 noborder'),
267    Set('style data histograms'),
268    Plot(Item('Allocated', 'allocated'),
269         Item('Reclaimed', reclaimed_bytes),
270         Item('Promoted', 'promoted', style = 'lines', lc = 'black'))
271  ],
272]
273
274def freduce(f, field, trace, init):
275  return reduce(lambda t,r: f(t, r[field]), trace, init)
276
277def calc_total(trace, field):
278  return freduce(lambda t,v: t + long(v), field, trace, long(0))
279
280def calc_max(trace, field):
281  return freduce(lambda t,r: max(t, r), field, trace, 0)
282
283def count_nonzero(trace, field):
284  return freduce(lambda t,r: t if r == 0 else t + 1, field, trace, 0)
285
286
287def process_trace(filename):
288  trace = gc_nvp_common.parse_gc_trace(filename)
289
290  marksweeps = filter(lambda r: r['gc'] == 'ms', trace)
291  scavenges = filter(lambda r: r['gc'] == 's', trace)
292  globalgcs = filter(lambda r: r['gc'] != 's', trace)
293
294
295  charts = plot_all(plots, trace, filename)
296
297  def stats(out, prefix, trace, field):
298    n = len(trace)
299    total = calc_total(trace, field)
300    max = calc_max(trace, field)
301    if n > 0:
302      avg = total / n
303    else:
304      avg = 0
305    if n > 1:
306      dev = math.sqrt(freduce(lambda t,r: t + (r - avg) ** 2, field, trace, 0) /
307                      (n - 1))
308    else:
309      dev = 0
310
311    out.write('<tr><td>%s</td><td>%d</td><td>%d</td>'
312              '<td>%d</td><td>%d [dev %f]</td></tr>' %
313              (prefix, n, total, max, avg, dev))
314
315  def HumanReadable(size):
316    suffixes = ['bytes', 'kB', 'MB', 'GB']
317    power = 1
318    for i in range(len(suffixes)):
319      if size < power*1024:
320        return "%.1f" % (float(size) / power) + " " + suffixes[i]
321      power *= 1024
322
323  def throughput(name, trace):
324    total_live_after = calc_total(trace, 'total_size_after')
325    total_live_before = calc_total(trace, 'total_size_before')
326    total_gc = calc_total(trace, 'pause')
327    if total_gc == 0:
328      return
329    out.write('GC %s Throughput (after): %s / %s ms = %s/ms<br/>' %
330              (name,
331               HumanReadable(total_live_after),
332               total_gc,
333               HumanReadable(total_live_after / total_gc)))
334    out.write('GC %s Throughput (before): %s / %s ms = %s/ms<br/>' %
335              (name,
336               HumanReadable(total_live_before),
337               total_gc,
338               HumanReadable(total_live_before / total_gc)))
339
340
341  with open(filename + '.html', 'w') as out:
342    out.write('<html><body>')
343    out.write('<table>')
344    out.write('<tr><td>Phase</td><td>Count</td><td>Time (ms)</td>')
345    out.write('<td>Max</td><td>Avg</td></tr>')
346    stats(out, 'Total in GC', trace, 'pause')
347    stats(out, 'Scavenge', scavenges, 'pause')
348    stats(out, 'MarkSweep', marksweeps, 'pause')
349    stats(out, 'Mark', filter(lambda r: r['mark'] != 0, trace), 'mark')
350    stats(out, 'Sweep', filter(lambda r: r['sweep'] != 0, trace), 'sweep')
351    stats(out,
352          'External',
353          filter(lambda r: r['external'] != 0, trace),
354          'external')
355    out.write('</table>')
356    throughput('TOTAL', trace)
357    throughput('MS', marksweeps)
358    throughput('OLDSPACE', globalgcs)
359    out.write('<br/>')
360    for chart in charts:
361      out.write('<img src="%s">' % chart)
362      out.write('</body></html>')
363
364  print("%s generated." % (filename + '.html'))
365
366if len(sys.argv) != 2:
367  print("Usage: %s <GC-trace-filename>" % sys.argv[0])
368  sys.exit(1)
369
370process_trace(sys.argv[1])
371