1# Note: Work in progress
2
3from __future__ import absolute_import
4
5import os
6import os.path
7import re
8import codecs
9import textwrap
10from datetime import datetime
11from functools import partial
12from collections import defaultdict
13from xml.sax.saxutils import escape as html_escape
14try:
15    from StringIO import StringIO
16except ImportError:
17    from io import StringIO  # does not support writing 'str' in Py2
18
19from . import Version
20from .Code import CCodeWriter
21from .. import Utils
22
23
24class AnnotationCCodeWriter(CCodeWriter):
25
26    def __init__(self, create_from=None, buffer=None, copy_formatting=True):
27        CCodeWriter.__init__(self, create_from, buffer, copy_formatting=copy_formatting)
28        if create_from is None:
29            self.annotation_buffer = StringIO()
30            self.last_annotated_pos = None
31            # annotations[filename][line] -> [(column, AnnotationItem)*]
32            self.annotations = defaultdict(partial(defaultdict, list))
33            # code[filename][line] -> str
34            self.code = defaultdict(partial(defaultdict, str))
35            # scopes[filename][line] -> set(scopes)
36            self.scopes = defaultdict(partial(defaultdict, set))
37        else:
38            # When creating an insertion point, keep references to the same database
39            self.annotation_buffer = create_from.annotation_buffer
40            self.annotations = create_from.annotations
41            self.code = create_from.code
42            self.scopes = create_from.scopes
43            self.last_annotated_pos = create_from.last_annotated_pos
44
45    def create_new(self, create_from, buffer, copy_formatting):
46        return AnnotationCCodeWriter(create_from, buffer, copy_formatting)
47
48    def write(self, s):
49        CCodeWriter.write(self, s)
50        self.annotation_buffer.write(s)
51
52    def mark_pos(self, pos, trace=True):
53        if pos is not None:
54            CCodeWriter.mark_pos(self, pos, trace)
55            if self.funcstate and self.funcstate.scope:
56                # lambdas and genexprs can result in multiple scopes per line => keep them in a set
57                self.scopes[pos[0].filename][pos[1]].add(self.funcstate.scope)
58        if self.last_annotated_pos:
59            source_desc, line, _ = self.last_annotated_pos
60            pos_code = self.code[source_desc.filename]
61            pos_code[line] += self.annotation_buffer.getvalue()
62        self.annotation_buffer = StringIO()
63        self.last_annotated_pos = pos
64
65    def annotate(self, pos, item):
66        self.annotations[pos[0].filename][pos[1]].append((pos[2], item))
67
68    def _css(self):
69        """css template will later allow to choose a colormap"""
70        css = [self._css_template]
71        for i in range(255):
72            color = u"FFFF%02x" % int(255/(1+i/10.0))
73            css.append('.cython.score-%d {background-color: #%s;}' % (i, color))
74        try:
75            from pygments.formatters import HtmlFormatter
76        except ImportError:
77            pass
78        else:
79            css.append(HtmlFormatter().get_style_defs('.cython'))
80        return '\n'.join(css)
81
82    _css_template = textwrap.dedent("""
83        body.cython { font-family: courier; font-size: 12; }
84
85        .cython.tag  {  }
86        .cython.line { margin: 0em }
87        .cython.code { font-size: 9; color: #444444; display: none; margin: 0px 0px 0px 8px; border-left: 8px none; }
88
89        .cython.line .run { background-color: #B0FFB0; }
90        .cython.line .mis { background-color: #FFB0B0; }
91        .cython.code.run  { border-left: 8px solid #B0FFB0; }
92        .cython.code.mis  { border-left: 8px solid #FFB0B0; }
93
94        .cython.code .py_c_api  { color: red; }
95        .cython.code .py_macro_api  { color: #FF7000; }
96        .cython.code .pyx_c_api  { color: #FF3000; }
97        .cython.code .pyx_macro_api  { color: #FF7000; }
98        .cython.code .refnanny  { color: #FFA000; }
99        .cython.code .trace  { color: #FFA000; }
100        .cython.code .error_goto  { color: #FFA000; }
101
102        .cython.code .coerce  { color: #008000; border: 1px dotted #008000 }
103        .cython.code .py_attr { color: #FF0000; font-weight: bold; }
104        .cython.code .c_attr  { color: #0000FF; }
105        .cython.code .py_call { color: #FF0000; font-weight: bold; }
106        .cython.code .c_call  { color: #0000FF; }
107    """)
108
109    # on-click toggle function to show/hide C source code
110    _onclick_attr = ' onclick="{0}"'.format((
111        "(function(s){"
112        "    s.display =  s.display === 'block' ? 'none' : 'block'"
113        "})(this.nextElementSibling.style)"
114        ).replace(' ', '')  # poor dev's JS minification
115    )
116
117    def save_annotation(self, source_filename, target_filename, coverage_xml=None):
118        with Utils.open_source_file(source_filename) as f:
119            code = f.read()
120        generated_code = self.code.get(source_filename, {})
121        c_file = Utils.decode_filename(os.path.basename(target_filename))
122        html_filename = os.path.splitext(target_filename)[0] + ".html"
123
124        with codecs.open(html_filename, "w", encoding="UTF-8") as out_buffer:
125            out_buffer.write(self._save_annotation(code, generated_code, c_file, source_filename, coverage_xml))
126
127    def _save_annotation_header(self, c_file, source_filename, coverage_timestamp=None):
128        coverage_info = ''
129        if coverage_timestamp:
130            coverage_info = u' with coverage data from {timestamp}'.format(
131                timestamp=datetime.fromtimestamp(int(coverage_timestamp) // 1000))
132
133        outlist = [
134            textwrap.dedent(u'''\
135            <!DOCTYPE html>
136            <!-- Generated by Cython {watermark} -->
137            <html>
138            <head>
139                <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
140                <title>Cython: {filename}</title>
141                <style type="text/css">
142                {css}
143                </style>
144            </head>
145            <body class="cython">
146            <p><span style="border-bottom: solid 1px grey;">Generated by Cython {watermark}</span>{more_info}</p>
147            <p>
148                <span style="background-color: #FFFF00">Yellow lines</span> hint at Python interaction.<br />
149                Click on a line that starts with a "<code>+</code>" to see the C code that Cython generated for it.
150            </p>
151            ''').format(css=self._css(), watermark=Version.watermark,
152                        filename=os.path.basename(source_filename) if source_filename else '',
153                        more_info=coverage_info)
154        ]
155        if c_file:
156            outlist.append(u'<p>Raw output: <a href="%s">%s</a></p>\n' % (c_file, c_file))
157        return outlist
158
159    def _save_annotation_footer(self):
160        return (u'</body></html>\n',)
161
162    def _save_annotation(self, code, generated_code, c_file=None, source_filename=None, coverage_xml=None):
163        """
164        lines : original cython source code split by lines
165        generated_code : generated c code keyed by line number in original file
166        target filename : name of the file in which to store the generated html
167        c_file : filename in which the c_code has been written
168        """
169        if coverage_xml is not None and source_filename:
170            coverage_timestamp = coverage_xml.get('timestamp', '').strip()
171            covered_lines = self._get_line_coverage(coverage_xml, source_filename)
172        else:
173            coverage_timestamp = covered_lines = None
174        annotation_items = dict(self.annotations[source_filename])
175        scopes = dict(self.scopes[source_filename])
176
177        outlist = []
178        outlist.extend(self._save_annotation_header(c_file, source_filename, coverage_timestamp))
179        outlist.extend(self._save_annotation_body(code, generated_code, annotation_items, scopes, covered_lines))
180        outlist.extend(self._save_annotation_footer())
181        return ''.join(outlist)
182
183    def _get_line_coverage(self, coverage_xml, source_filename):
184        coverage_data = None
185        for entry in coverage_xml.iterfind('.//class'):
186            if not entry.get('filename'):
187                continue
188            if (entry.get('filename') == source_filename or
189                    os.path.abspath(entry.get('filename')) == source_filename):
190                coverage_data = entry
191                break
192            elif source_filename.endswith(entry.get('filename')):
193                coverage_data = entry  # but we might still find a better match...
194        if coverage_data is None:
195            return None
196        return dict(
197            (int(line.get('number')), int(line.get('hits')))
198            for line in coverage_data.iterfind('lines/line')
199        )
200
201    def _htmlify_code(self, code):
202        try:
203            from pygments import highlight
204            from pygments.lexers import CythonLexer
205            from pygments.formatters import HtmlFormatter
206        except ImportError:
207            # no Pygments, just escape the code
208            return html_escape(code)
209
210        html_code = highlight(
211            code, CythonLexer(stripnl=False, stripall=False),
212            HtmlFormatter(nowrap=True))
213        return html_code
214
215    def _save_annotation_body(self, cython_code, generated_code, annotation_items, scopes, covered_lines=None):
216        outlist = [u'<div class="cython">']
217        pos_comment_marker = u'/* \N{HORIZONTAL ELLIPSIS} */\n'
218        new_calls_map = dict(
219            (name, 0) for name in
220            'refnanny trace py_macro_api py_c_api pyx_macro_api pyx_c_api error_goto'.split()
221        ).copy
222
223        self.mark_pos(None)
224
225        def annotate(match):
226            group_name = match.lastgroup
227            calls[group_name] += 1
228            return u"<span class='%s'>%s</span>" % (
229                group_name, match.group(group_name))
230
231        lines = self._htmlify_code(cython_code).splitlines()
232        lineno_width = len(str(len(lines)))
233        if not covered_lines:
234            covered_lines = None
235
236        for k, line in enumerate(lines, 1):
237            try:
238                c_code = generated_code[k]
239            except KeyError:
240                c_code = ''
241            else:
242                c_code = _replace_pos_comment(pos_comment_marker, c_code)
243                if c_code.startswith(pos_comment_marker):
244                    c_code = c_code[len(pos_comment_marker):]
245                c_code = html_escape(c_code)
246
247            calls = new_calls_map()
248            c_code = _parse_code(annotate, c_code)
249            score = (5 * calls['py_c_api'] + 2 * calls['pyx_c_api'] +
250                     calls['py_macro_api'] + calls['pyx_macro_api'])
251
252            if c_code:
253                onclick = self._onclick_attr
254                expandsymbol = '+'
255            else:
256                onclick = ''
257                expandsymbol = '&#xA0;'
258
259            covered = ''
260            if covered_lines is not None and k in covered_lines:
261                hits = covered_lines[k]
262                if hits is not None:
263                    covered = 'run' if hits else 'mis'
264
265            outlist.append(
266                u'<pre class="cython line score-{score}"{onclick}>'
267                # generate line number with expand symbol in front,
268                # and the right  number of digit
269                u'{expandsymbol}<span class="{covered}">{line:0{lineno_width}d}</span>: {code}</pre>\n'.format(
270                    score=score,
271                    expandsymbol=expandsymbol,
272                    covered=covered,
273                    lineno_width=lineno_width,
274                    line=k,
275                    code=line.rstrip(),
276                    onclick=onclick,
277                ))
278            if c_code:
279                outlist.append(u"<pre class='cython code score-{score} {covered}'>{code}</pre>".format(
280                    score=score, covered=covered, code=c_code))
281        outlist.append(u"</div>")
282        return outlist
283
284
285_parse_code = re.compile((
286    br'(?P<refnanny>__Pyx_X?(?:GOT|GIVE)REF|__Pyx_RefNanny[A-Za-z]+)|'
287    br'(?P<trace>__Pyx_Trace[A-Za-z]+)|'
288    br'(?:'
289    br'(?P<pyx_macro_api>__Pyx_[A-Z][A-Z_]+)|'
290    br'(?P<pyx_c_api>(?:__Pyx_[A-Z][a-z_][A-Za-z_]*)|__pyx_convert_[A-Za-z_]*)|'
291    br'(?P<py_macro_api>Py[A-Z][a-z]+_[A-Z][A-Z_]+)|'
292    br'(?P<py_c_api>Py[A-Z][a-z]+_[A-Z][a-z][A-Za-z_]*)'
293    br')(?=\()|'       # look-ahead to exclude subsequent '(' from replacement
294    br'(?P<error_goto>(?:(?<=;) *if [^;]* +)?__PYX_ERR\([^)]+\))'
295).decode('ascii')).sub
296
297
298_replace_pos_comment = re.compile(
299    # this matches what Cython generates as code line marker comment
300    br'^\s*/\*(?:(?:[^*]|\*[^/])*\n)+\s*\*/\s*\n'.decode('ascii'),
301    re.M
302).sub
303
304
305class AnnotationItem(object):
306
307    def __init__(self, style, text, tag="", size=0):
308        self.style = style
309        self.text = text
310        self.tag = tag
311        self.size = size
312
313    def start(self):
314        return u"<span class='cython tag %s' title='%s'>%s" % (self.style, self.text, self.tag)
315
316    def end(self):
317        return self.size, u"</span>"
318