1#!/usr/bin/env python
2
3from __future__ import print_function
4
5import argparse
6import errno
7import functools
8import html
9import io
10from multiprocessing import cpu_count
11import os.path
12import re
13import shutil
14import sys
15
16from pygments import highlight
17from pygments.lexers.c_cpp import CppLexer
18from pygments.formatters import HtmlFormatter
19
20import optpmap
21import optrecord
22
23
24desc = '''Generate HTML output to visualize optimization records from the YAML files
25generated with -fsave-optimization-record and -fdiagnostics-show-hotness.
26
27The tools requires PyYAML and Pygments Python packages.'''
28
29
30# This allows passing the global context to the child processes.
31class Context:
32    def __init__(self, caller_loc = dict()):
33       # Map function names to their source location for function where inlining happened
34       self.caller_loc = caller_loc
35
36context = Context()
37
38def suppress(remark):
39    if remark.Name == 'sil.Specialized':
40        return remark.getArgDict()['Function'][0].startswith('\"Swift.')
41    elif remark.Name == 'sil.Inlined':
42        return remark.getArgDict()['Callee'][0].startswith(('\"Swift.', '\"specialized Swift.'))
43    return False
44
45class SourceFileRenderer:
46    def __init__(self, source_dir, output_dir, filename, no_highlight):
47        self.filename = filename
48        existing_filename = None
49        if os.path.exists(filename):
50            existing_filename = filename
51        else:
52            fn = os.path.join(source_dir, filename)
53            if os.path.exists(fn):
54                existing_filename = fn
55
56        self.no_highlight = no_highlight
57        self.stream = io.open(os.path.join(output_dir, optrecord.html_file_name(filename)), 'w', encoding='utf-8')
58        if existing_filename:
59            self.source_stream = io.open(existing_filename, encoding='utf-8')
60        else:
61            self.source_stream = None
62            print(u'''
63<html>
64<h1>Unable to locate file {}</h1>
65</html>
66            '''.format(filename), file=self.stream)
67
68        self.html_formatter = HtmlFormatter(encoding='utf-8')
69        self.cpp_lexer = CppLexer(stripnl=False)
70
71    def render_source_lines(self, stream, line_remarks):
72        file_text = stream.read()
73
74        if self.no_highlight:
75            html_highlighted = file_text
76        else:
77            html_highlighted = highlight(
78            file_text,
79                self.cpp_lexer,
80                self.html_formatter)
81
82            # Note that the API is different between Python 2 and 3.  On
83            # Python 3, pygments.highlight() returns a bytes object, so we
84            # have to decode.  On Python 2, the output is str but since we
85            # support unicode characters and the output streams is unicode we
86            # decode too.
87            html_highlighted = html_highlighted.decode('utf-8')
88
89            # Take off the header and footer, these must be
90            #   reapplied line-wise, within the page structure
91            html_highlighted = html_highlighted.replace('<div class="highlight"><pre>', '')
92            html_highlighted = html_highlighted.replace('</pre></div>', '')
93
94        for (linenum, html_line) in enumerate(html_highlighted.split('\n'), start=1):
95            print(u'''
96<tr>
97<td><a name=\"L{linenum}\">{linenum}</a></td>
98<td></td>
99<td></td>
100<td><div class="highlight"><pre>{html_line}</pre></div></td>
101</tr>'''.format(**locals()), file=self.stream)
102
103            for remark in line_remarks.get(linenum, []):
104                if not suppress(remark):
105                    self.render_inline_remarks(remark, html_line)
106
107    def render_inline_remarks(self, r, line):
108        inlining_context = r.DemangledFunctionName
109        dl = context.caller_loc.get(r.Function)
110        if dl:
111            dl_dict = dict(list(dl))
112            link = optrecord.make_link(dl_dict['File'], dl_dict['Line'] - 2)
113            inlining_context = "<a href={link}>{r.DemangledFunctionName}</a>".format(**locals())
114
115        # Column is the number of characters *including* tabs, keep those and
116        # replace everything else with spaces.
117        indent = line[:max(r.Column, 1) - 1]
118        indent = re.sub('\S', ' ', indent)
119
120        # Create expanded message and link if we have a multiline message.
121        lines = r.message.split('\n')
122        if len(lines) > 1:
123            expand_link = '<a style="text-decoration: none;" href="" onclick="toggleExpandedMessage(this); return false;">+</a>'
124            message = lines[0]
125            expand_message = u'''
126<div class="full-info" style="display:none;">
127  <div class="col-left"><pre style="display:inline">{}</pre></div>
128  <div class="expanded col-left"><pre>{}</pre></div>
129</div>'''.format(indent, '\n'.join(lines[1:]))
130        else:
131            expand_link = ''
132            expand_message = ''
133            message = r.message
134        print(u'''
135<tr>
136<td></td>
137<td>{r.RelativeHotness}</td>
138<td class=\"column-entry-{r.color}\">{r.PassWithDiffPrefix}</td>
139<td><pre style="display:inline">{indent}</pre><span class=\"column-entry-yellow\">{expand_link} {message}&nbsp;</span>{expand_message}</td>
140<td class=\"column-entry-yellow\">{inlining_context}</td>
141</tr>'''.format(**locals()), file=self.stream)
142
143    def render(self, line_remarks):
144        if not self.source_stream:
145            return
146
147        print(u'''
148<html>
149<title>{}</title>
150<meta charset="utf-8" />
151<head>
152<link rel='stylesheet' type='text/css' href='style.css'>
153<script type="text/javascript">
154/* Simple helper to show/hide the expanded message of a remark. */
155function toggleExpandedMessage(e) {{
156  var FullTextElems = e.parentElement.parentElement.getElementsByClassName("full-info");
157  if (!FullTextElems || FullTextElems.length < 1) {{
158      return false;
159  }}
160  var FullText = FullTextElems[0];
161  if (FullText.style.display == 'none') {{
162    e.innerHTML = '-';
163    FullText.style.display = 'block';
164  }} else {{
165    e.innerHTML = '+';
166    FullText.style.display = 'none';
167  }}
168}}
169</script>
170</head>
171<body>
172<div class="centered">
173<table class="source">
174<thead>
175<tr>
176<th style="width: 2%">Line</td>
177<th style="width: 3%">Hotness</td>
178<th style="width: 10%">Optimization</td>
179<th style="width: 70%">Source</td>
180<th style="width: 15%">Inline Context</td>
181</tr>
182</thead>
183<tbody>'''.format(os.path.basename(self.filename)), file=self.stream)
184        self.render_source_lines(self.source_stream, line_remarks)
185
186        print(u'''
187</tbody>
188</table>
189</body>
190</html>''', file=self.stream)
191
192
193class IndexRenderer:
194    def __init__(self, output_dir, should_display_hotness, max_hottest_remarks_on_index):
195        self.stream = io.open(os.path.join(output_dir, 'index.html'), 'w', encoding='utf-8')
196        self.should_display_hotness = should_display_hotness
197        self.max_hottest_remarks_on_index = max_hottest_remarks_on_index
198
199    def render_entry(self, r, odd):
200        escaped_name = html.escape(r.DemangledFunctionName)
201        print(u'''
202<tr>
203<td class=\"column-entry-{odd}\"><a href={r.Link}>{r.DebugLocString}</a></td>
204<td class=\"column-entry-{odd}\">{r.RelativeHotness}</td>
205<td class=\"column-entry-{odd}\">{escaped_name}</td>
206<td class=\"column-entry-{r.color}\">{r.PassWithDiffPrefix}</td>
207</tr>'''.format(**locals()), file=self.stream)
208
209    def render(self, all_remarks):
210        print(u'''
211<html>
212<meta charset="utf-8" />
213<head>
214<link rel='stylesheet' type='text/css' href='style.css'>
215</head>
216<body>
217<div class="centered">
218<table>
219<tr>
220<td>Source Location</td>
221<td>Hotness</td>
222<td>Function</td>
223<td>Pass</td>
224</tr>''', file=self.stream)
225
226        max_entries = None
227        if self.should_display_hotness:
228            max_entries = self.max_hottest_remarks_on_index
229
230        for i, remark in enumerate(all_remarks[:max_entries]):
231            if not suppress(remark):
232                self.render_entry(remark, i % 2)
233        print(u'''
234</table>
235</body>
236</html>''', file=self.stream)
237
238
239def _render_file(source_dir, output_dir, ctx, no_highlight, entry, filter_):
240    global context
241    context = ctx
242    filename, remarks = entry
243    SourceFileRenderer(source_dir, output_dir, filename, no_highlight).render(remarks)
244
245
246def map_remarks(all_remarks):
247    # Set up a map between function names and their source location for
248    # function where inlining happened
249    for remark in optrecord.itervalues(all_remarks):
250        if isinstance(remark, optrecord.Passed) and remark.Pass == "inline" and remark.Name == "Inlined":
251            for arg in remark.Args:
252                arg_dict = dict(list(arg))
253                caller = arg_dict.get('Caller')
254                if caller:
255                    try:
256                        context.caller_loc[caller] = arg_dict['DebugLoc']
257                    except KeyError:
258                        pass
259
260
261def generate_report(all_remarks,
262                    file_remarks,
263                    source_dir,
264                    output_dir,
265                    no_highlight,
266                    should_display_hotness,
267                    max_hottest_remarks_on_index,
268                    num_jobs,
269                    should_print_progress):
270    try:
271        os.makedirs(output_dir)
272    except OSError as e:
273        if e.errno == errno.EEXIST and os.path.isdir(output_dir):
274            pass
275        else:
276            raise
277
278    if should_print_progress:
279        print('Rendering index page...')
280    if should_display_hotness:
281        sorted_remarks = sorted(optrecord.itervalues(all_remarks), key=lambda r: (r.Hotness, r.File, r.Line, r.Column, r.PassWithDiffPrefix, r.yaml_tag, r.Function), reverse=True)
282    else:
283        sorted_remarks = sorted(optrecord.itervalues(all_remarks), key=lambda r: (r.File, r.Line, r.Column, r.PassWithDiffPrefix, r.yaml_tag, r.Function))
284    IndexRenderer(output_dir, should_display_hotness, max_hottest_remarks_on_index).render(sorted_remarks)
285
286    shutil.copy(os.path.join(os.path.dirname(os.path.realpath(__file__)),
287            "style.css"), output_dir)
288
289    _render_file_bound = functools.partial(_render_file, source_dir, output_dir, context, no_highlight)
290    if should_print_progress:
291        print('Rendering HTML files...')
292    optpmap.pmap(_render_file_bound,
293                 file_remarks.items(),
294                 num_jobs,
295                 should_print_progress)
296
297
298def main():
299    parser = argparse.ArgumentParser(description=desc)
300    parser.add_argument(
301        'yaml_dirs_or_files',
302        nargs='+',
303        help='List of optimization record files or directories searched '
304             'for optimization record files.')
305    parser.add_argument(
306        '--output-dir',
307        '-o',
308        default='html',
309        help='Path to a directory where generated HTML files will be output. '
310             'If the directory does not already exist, it will be created. '
311             '"%(default)s" by default.')
312    parser.add_argument(
313        '--jobs',
314        '-j',
315        default=None,
316        type=int,
317        help='Max job count (defaults to %(default)s, the current CPU count)')
318    parser.add_argument(
319        '--source-dir',
320        '-s',
321        default='',
322        help='set source directory')
323    parser.add_argument(
324        '--no-progress-indicator',
325        '-n',
326        action='store_true',
327        default=False,
328        help='Do not display any indicator of how many YAML files were read '
329             'or rendered into HTML.')
330    parser.add_argument(
331        '--max-hottest-remarks-on-index',
332        default=1000,
333        type=int,
334        help='Maximum number of the hottest remarks to appear on the index page')
335    parser.add_argument(
336        '--no-highlight',
337        action='store_true',
338        default=False,
339        help='Do not use a syntax highlighter when rendering the source code')
340    parser.add_argument(
341        '--demangler',
342        help='Set the demangler to be used (defaults to %s)' % optrecord.Remark.default_demangler)
343
344    parser.add_argument(
345        '--filter',
346        default='',
347        help='Only display remarks from passes matching filter expression')
348
349    # Do not make this a global variable.  Values needed to be propagated through
350    # to individual classes and functions to be portable with multiprocessing across
351    # Windows and non-Windows.
352    args = parser.parse_args()
353
354    print_progress = not args.no_progress_indicator
355    if args.demangler:
356        optrecord.Remark.set_demangler(args.demangler)
357
358    files = optrecord.find_opt_files(*args.yaml_dirs_or_files)
359    if not files:
360        parser.error("No *.opt.yaml files found")
361        sys.exit(1)
362
363    all_remarks, file_remarks, should_display_hotness = \
364        optrecord.gather_results(files, args.jobs, print_progress, args.filter)
365
366    map_remarks(all_remarks)
367
368    generate_report(all_remarks,
369                    file_remarks,
370                    args.source_dir,
371                    args.output_dir,
372                    args.no_highlight,
373                    should_display_hotness,
374                    args.max_hottest_remarks_on_index,
375                    args.jobs,
376                    print_progress)
377
378if __name__ == '__main__':
379    main()
380