1import re
2import os
3import subprocess
4import traceback
5import shutil
6from collections import defaultdict
7import unidecode
8
9import yaml
10from sphinx.util.osutil import ensuredir
11from sphinx.util.console import darkgreen, bold
12import sphinx.util.logging
13from sphinx.errors import ExtensionError
14
15from .base import PythonMapperBase, SphinxMapperBase
16
17LOGGER = sphinx.util.logging.getLogger(__name__)
18
19# Doc comment patterns
20DOC_COMMENT_PATTERN = r"""
21    \<%(tag)s
22    \s+%(attr)s="(?P<attr_value>[^"]*?)"
23    \s*?
24    (?:
25        \/\>|
26        \>(?P<inner>[^\<]*?)\<\/%(tag)s\>
27    )
28"""
29DOC_COMMENT_SEE_PATTERN = re.compile(
30    DOC_COMMENT_PATTERN % {"tag": "(?:see|seealso)", "attr": "cref"}, re.X
31)
32DOC_COMMENT_PARAM_PATTERN = re.compile(
33    DOC_COMMENT_PATTERN % {"tag": "(?:paramref|typeparamref)", "attr": "name"}, re.X
34)
35
36# Comment member identities
37# From: https://msdn.microsoft.com/en-us/library/vstudio/fsbx0t7x(v=VS.100).aspx
38DOC_COMMENT_IDENTITIES = {
39    "N": "dn:ns",
40    "T": "any",  # can be any type (class, delegate, enum, etc), so use any
41    "F": "dn:field",
42    "P": "dn:prop",
43    "M": "dn:meth",
44    "E": "dn:event",
45}
46
47
48class DotNetSphinxMapper(SphinxMapperBase):
49
50    """Auto API domain handler for .NET
51
52    Searches for YAML files, and soon to be JSON files as well, for auto API
53    sources. If no pattern configuration was explicitly specified, then default
54    to looking up a ``docfx.json`` file.
55
56    :param app: Sphinx application passed in as part of the extension
57    """
58
59    top_namespaces = {}
60
61    DOCFX_OUTPUT_PATH = "_api"
62
63    # pylint: disable=arguments-differ
64    def load(self, patterns, dirs, ignore=None):
65        """Load objects from the filesystem into the ``paths`` dictionary.
66
67        If the setting ``autoapi_patterns`` was not specified, look for a
68        ``docfx.json`` file by default.  A ``docfx.json`` should be treated as
69        the canonical source before the default patterns.  Fallback to default
70        pattern matches if no ``docfx.json`` files are found.
71        """
72        LOGGER.info(bold("[AutoAPI] ") + darkgreen("Loading Data"))
73        all_files = set()
74        if not self.app.config.autoapi_file_patterns:
75            all_files = set(
76                self.find_files(patterns=["docfx.json"], dirs=dirs, ignore=ignore)
77            )
78        if not all_files:
79            all_files = set(
80                self.find_files(patterns=patterns, dirs=dirs, ignore=ignore)
81            )
82
83        if all_files:
84            command = ["docfx", "metadata", "--raw", "--force"]
85            command.extend(all_files)
86            proc = subprocess.Popen(
87                " ".join(command),
88                stdout=subprocess.PIPE,
89                stderr=subprocess.PIPE,
90                shell=True,
91                env=dict(
92                    (key, os.environ[key])
93                    for key in [
94                        "PATH",
95                        "HOME",
96                        "SYSTEMROOT",
97                        "USERPROFILE",
98                        "WINDIR",
99                    ]
100                    if key in os.environ
101                ),
102            )
103            _, error_output = proc.communicate()
104            if error_output:
105                LOGGER.warning(error_output, type="autoapi", subtype="not_readable")
106        # We now have yaml files
107        for xdoc_path in self.find_files(
108            patterns=["*.yml"], dirs=[self.DOCFX_OUTPUT_PATH], ignore=ignore
109        ):
110            data = self.read_file(path=xdoc_path)
111            if data:
112                self.paths[xdoc_path] = data
113
114        return True
115
116    def read_file(self, path, **kwargs):
117        """Read file input into memory, returning deserialized objects
118
119        :param path: Path of file to read
120        """
121        try:
122            with open(path, "r") as handle:
123                parsed_data = yaml.safe_load(handle)
124                return parsed_data
125        except IOError:
126            LOGGER.warning(
127                "Error reading file: {0}".format(path),
128                type="autoapi",
129                subtype="not_readable",
130            )
131        except TypeError:
132            LOGGER.warning(
133                "Error reading file: {0}".format(path),
134                type="autoapi",
135                subtype="not_readable",
136            )
137        return None
138
139    # Subclassed to iterate over items
140    def map(self, options=None):
141        """Trigger find of serialized sources and build objects"""
142        for _, data in sphinx.util.status_iterator(
143            self.paths.items(),
144            bold("[AutoAPI] ") + "Mapping Data... ",
145            length=len(self.paths),
146            stringify_func=(lambda x: x[0]),
147        ):
148            references = data.get("references", [])
149            for item in data["items"]:
150                for obj in self.create_class(item, options, references=references):
151                    self.add_object(obj)
152
153        self.organize_objects()
154
155    def create_class(self, data, options=None, **kwargs):
156        """
157        Return instance of class based on Roslyn type property
158
159        Data keys handled here:
160
161            type
162                Set the object class
163
164            items
165                Recurse into :py:meth:`create_class` to create child object
166                instances
167
168        :param data: dictionary data from Roslyn output artifact
169        """
170        kwargs.pop("path", None)
171        obj_map = {cls.type: cls for cls in ALL_CLASSES}
172        try:
173            cls = obj_map[data["type"].lower()]
174        except KeyError:
175            # this warning intentionally has no (sub-)type
176            LOGGER.warning("Unknown type: %s" % data)
177        else:
178            obj = cls(
179                data, jinja_env=self.jinja_env, app=self.app, options=options, **kwargs
180            )
181            obj.url_root = self.url_root
182
183            # Append child objects
184            # TODO this should recurse in the case we're getting back more
185            # complex argument listings
186
187            yield obj
188
189    def add_object(self, obj):
190        """Add object to local and app environment storage
191
192        :param obj: Instance of a .NET object
193        """
194        if obj.top_level_object:
195            if isinstance(obj, DotNetNamespace):
196                self.namespaces[obj.name] = obj
197        self.objects[obj.id] = obj
198
199    def organize_objects(self):
200        """Organize objects and namespaces"""
201
202        def _render_children(obj):
203            for child in obj.children_strings:
204                child_object = self.objects.get(child)
205                if child_object:
206                    obj.item_map[child_object.plural].append(child_object)
207                    obj.children.append(child_object)
208
209            for key in obj.item_map:
210                obj.item_map[key].sort()
211
212        def _recurse_ns(obj):
213            if not obj:
214                return
215            namespace = obj.top_namespace
216            if namespace is not None:
217                ns_obj = self.top_namespaces.get(namespace)
218                if ns_obj is None or not isinstance(ns_obj, DotNetNamespace):
219                    for ns_obj in self.create_class(
220                        {"uid": namespace, "type": "namespace"}
221                    ):
222                        self.top_namespaces[ns_obj.id] = ns_obj
223                if obj not in ns_obj.children and namespace != obj.id:
224                    ns_obj.children.append(obj)
225
226        for obj in self.objects.values():
227            _render_children(obj)
228            _recurse_ns(obj)
229
230        # Clean out dead namespaces
231        for key, namespace in self.top_namespaces.copy().items():
232            if not namespace.children:
233                del self.top_namespaces[key]
234
235        for key, namespace in self.namespaces.copy().items():
236            if not namespace.children:
237                del self.namespaces[key]
238
239    def output_rst(self, root, source_suffix):
240        if not self.objects:
241            raise ExtensionError("No API objects exist. Can't continue")
242
243        for _, obj in sphinx.util.status_iterator(
244            self.objects.items(),
245            bold("[AutoAPI] ") + "Rendering Data... ",
246            length=len(self.objects),
247            stringify_func=(lambda x: x[0]),
248        ):
249            if not obj or not obj.top_level_object:
250                continue
251
252            rst = obj.render()
253            if not rst:
254                continue
255
256            detail_dir = os.path.join(root, obj.pathname)
257            ensuredir(detail_dir)
258            path = os.path.join(detail_dir, "%s%s" % ("index", source_suffix))
259            with open(path, "wb") as detail_file:
260                detail_file.write(rst.encode("utf-8"))
261
262        # Render Top Index
263        top_level_index = os.path.join(root, "index.rst")
264        with open(top_level_index, "wb") as top_level_file:
265            content = self.jinja_env.get_template("index.rst")
266            top_level_file.write(
267                content.render(pages=self.namespaces.values()).encode("utf-8")
268            )
269
270    @staticmethod
271    def build_finished(app, _):
272        if app.verbosity > 1:
273            LOGGER.info(bold("[AutoAPI] ") + darkgreen("Cleaning generated .yml files"))
274        if os.path.exists(DotNetSphinxMapper.DOCFX_OUTPUT_PATH):
275            shutil.rmtree(DotNetSphinxMapper.DOCFX_OUTPUT_PATH)
276
277
278class DotNetPythonMapper(PythonMapperBase):
279
280    """Base .NET object representation
281
282    :param references: object reference list from docfx
283    :type references: list of dict objects
284    """
285
286    language = "dotnet"
287
288    def __init__(self, obj, **kwargs):
289        self.references = dict(
290            (obj.get("uid"), obj)
291            for obj in kwargs.pop("references", [])
292            if "uid" in obj
293        )
294        super(DotNetPythonMapper, self).__init__(obj, **kwargs)
295
296        # Always exist
297        self.id = obj.get("uid", obj.get("id"))
298        self.definition = obj.get("definition", self.id)
299        self.name = obj.get("fullName", self.definition)
300
301        # Optional
302        self.fullname = obj.get("fullName")
303        self.summary = self.transform_doc_comments(obj.get("summary", ""))
304        self.parameters = []
305        self.items = obj.get("items", [])
306        self.children_strings = obj.get("children", [])
307        self.children = []
308        self.item_map = defaultdict(list)
309        self.inheritance = []
310        self.assemblies = obj.get("assemblies", [])
311
312        # Syntax example and parameter list
313        syntax = obj.get("syntax", None)
314        self.example = ""
315        if syntax is not None:
316            # Code example
317            try:
318                self.example = syntax["content"]
319            except (KeyError, TypeError):
320                traceback.print_exc()
321
322            self.parameters = []
323            for param in syntax.get("parameters", []):
324                if "id" in param:
325                    self.parameters.append(
326                        {
327                            "name": param.get("id"),
328                            "type": self.resolve_spec_identifier(param.get("type")),
329                            "desc": self.transform_doc_comments(
330                                param.get("description", "")
331                            ),
332                        }
333                    )
334
335            self.returns = {}
336            self.returns["type"] = self.resolve_spec_identifier(
337                syntax.get("return", {}).get("type")
338            )
339            self.returns["description"] = self.transform_doc_comments(
340                syntax.get("return", {}).get("description")
341            )
342
343        # Inheritance
344        # TODO Support more than just a class type here, should support enum/etc
345        self.inheritance = [
346            DotNetClass(
347                {"uid": name, "name": name}, jinja_env=self.jinja_env, app=self.app
348            )
349            for name in obj.get("inheritance", [])
350        ]
351
352    def __str__(self):
353        return "<{cls} {id}>".format(cls=self.__class__.__name__, id=self.id)
354
355    @property
356    def pathname(self):
357        """Sluggified path for filenames
358
359        Slugs to a filename using the follow steps
360
361        * Decode unicode to approximate ascii
362        * Remove existing hypens
363        * Substitute hyphens for non-word characters
364        * Break up the string as paths
365        """
366        slug = self.name
367        try:
368            slug = self.name.split("(")[0]
369        except IndexError:
370            pass
371        slug = unidecode.unidecode(slug)
372        slug = slug.replace("-", "")
373        slug = re.sub(r"[^\w\.]+", "-", slug).strip("-")
374        return os.path.join(*slug.split("."))
375
376    @property
377    def short_name(self):
378        """Shorten name property"""
379        return self.name.split(".")[-1]
380
381    @property
382    def edit_link(self):
383        try:
384            repo = self.source["remote"]["repo"].replace(".git", "")
385            path = self.path
386            return "{repo}/blob/master/{path}".format(repo=repo, path=path)
387        except KeyError:
388            return ""
389
390    @property
391    def source(self):
392        return self.obj.get("source")
393
394    @property
395    def path(self):
396        return self.source["path"]
397
398    @property
399    def namespace(self):
400        pieces = self.id.split(".")[:-1]
401        if pieces:
402            return ".".join(pieces)
403        return None
404
405    @property
406    def top_namespace(self):
407        pieces = self.id.split(".")[:2]
408        if pieces:
409            return ".".join(pieces)
410        return None
411
412    @property
413    def ref_type(self):
414        return self.type
415
416    @property
417    def ref_directive(self):
418        return self.type
419
420    @property
421    def ref_name(self):
422        """Return object name suitable for use in references
423
424        Escapes several known strings that cause problems, including the
425        following reference syntax::
426
427            :dotnet:cls:`Foo.Bar<T>`
428
429        As the `<T>` notation is also special syntax in references, indicating
430        the reference to Foo.Bar should be named T.
431
432        See: http://sphinx-doc.org/domains.html#role-cpp:any
433        """
434        return self.name.replace("<", r"\<").replace("`", r"\`")
435
436    @property
437    def ref_short_name(self):
438        """Same as above, return the truncated name instead"""
439        return self.ref_name.split(".")[-1]
440
441    @staticmethod
442    def transform_doc_comments(text):
443        """
444        Parse XML content for references and other syntax.
445
446        This avoids an LXML dependency, we only need to parse out a small subset
447        of elements here. Iterate over string to reduce regex pattern complexity
448        and make substitutions easier
449
450        .. seealso::
451
452            `Doc comment reference <https://msdn.microsoft.com/en-us/library/5ast78ax.aspx>`
453                Reference on XML documentation comment syntax
454        """
455        try:
456            while True:
457                found = DOC_COMMENT_SEE_PATTERN.search(text)
458                if found is None:
459                    break
460                ref = found.group("attr_value").replace("<", r"\<").replace("`", r"\`")
461
462                reftype = "any"
463                replacement = ""
464                # Given the pattern of `\w:\w+`, inspect first letter of
465                # reference for identity type
466                if ref[1] == ":" and ref[0] in DOC_COMMENT_IDENTITIES:
467                    reftype = DOC_COMMENT_IDENTITIES[ref[:1]]
468                    ref = ref[2:]
469                    replacement = ":{reftype}:`{ref}`".format(reftype=reftype, ref=ref)
470                elif ref[:2] == "!:":
471                    replacement = ref[2:]
472                else:
473                    replacement = ":any:`{ref}`".format(ref=ref)
474
475                # Escape following text
476                text_end = text[found.end() :]
477                text_start = text[: found.start()]
478                text_end = re.sub(r"^(\S)", r"\\\1", text_end)
479                text_start = re.sub(r"(\S)$", r"\1 ", text_start)
480
481                text = "".join([text_start, replacement, text_end])
482            while True:
483                found = DOC_COMMENT_PARAM_PATTERN.search(text)
484                if found is None:
485                    break
486
487                # Escape following text
488                text_end = text[found.end() :]
489                text_start = text[: found.start()]
490                text_end = re.sub(r"^(\S)", r"\\\1", text_end)
491                text_start = re.sub(r"(\S)$", r"\1 ", text_start)
492
493                text = "".join(
494                    [text_start, "``", found.group("attr_value"), "``", text_end]
495                )
496        except TypeError:
497            pass
498        return text
499
500    def resolve_spec_identifier(self, obj_name):
501        """Find reference name based on spec identifier
502
503        Spec identifiers are used in parameter and return type definitions, but
504        should be a user-friendly version instead. Use docfx ``references``
505        lookup mapping for resolution.
506
507        If the spec identifier reference has a ``spec.csharp`` key, this implies
508        a compound reference that should be linked in a special way. Resolve to
509        a nested reference, with the corrected nodes.
510
511        .. note::
512            This uses a special format that is interpreted by the domain for
513            parameter type and return type fields.
514
515        :param obj_name: spec identifier to resolve to a correct reference
516        :returns: resolved string with one or more references
517        :rtype: str
518        """
519        ref = self.references.get(obj_name)
520        if ref is None:
521            return obj_name
522
523        resolved = ref.get("fullName", obj_name)
524        spec = ref.get("spec.csharp", [])
525        parts = []
526        for part in spec:
527            if part.get("name") == "<":
528                parts.append("{")
529            elif part.get("name") == ">":
530                parts.append("}")
531            elif "fullName" in part and "uid" in part:
532                parts.append("{fullName}<{uid}>".format(**part))
533            elif "uid" in part:
534                parts.append(part["uid"])
535            elif "fullName" in part:
536                parts.append(part["fullName"])
537        if parts:
538            resolved = "".join(parts)
539        return resolved
540
541
542class DotNetNamespace(DotNetPythonMapper):
543    type = "namespace"
544    ref_directive = "ns"
545    plural = "namespaces"
546    top_level_object = True
547
548
549class DotNetMethod(DotNetPythonMapper):
550    type = "method"
551    ref_directive = "meth"
552    plural = "methods"
553
554
555class DotNetOperator(DotNetPythonMapper):
556    type = "operator"
557    ref_directive = "op"
558    plural = "operators"
559
560
561class DotNetProperty(DotNetPythonMapper):
562    type = "property"
563    ref_directive = "prop"
564    plural = "properties"
565
566
567class DotNetEnum(DotNetPythonMapper):
568    type = "enum"
569    ref_type = "enumeration"
570    ref_directive = "enum"
571    plural = "enumerations"
572    top_level_object = True
573
574
575class DotNetStruct(DotNetPythonMapper):
576    type = "struct"
577    ref_type = "structure"
578    ref_directive = "struct"
579    plural = "structures"
580    top_level_object = True
581
582
583class DotNetConstructor(DotNetPythonMapper):
584    type = "constructor"
585    ref_directive = "ctor"
586    plural = "constructors"
587
588
589class DotNetInterface(DotNetPythonMapper):
590    type = "interface"
591    ref_directive = "iface"
592    plural = "interfaces"
593    top_level_object = True
594
595
596class DotNetDelegate(DotNetPythonMapper):
597    type = "delegate"
598    ref_directive = "del"
599    plural = "delegates"
600    top_level_object = True
601
602
603class DotNetClass(DotNetPythonMapper):
604    type = "class"
605    ref_directive = "cls"
606    plural = "classes"
607    top_level_object = True
608
609
610class DotNetField(DotNetPythonMapper):
611    type = "field"
612    plural = "fields"
613
614
615class DotNetEvent(DotNetPythonMapper):
616    type = "event"
617    plural = "events"
618
619
620ALL_CLASSES = [
621    DotNetNamespace,
622    DotNetClass,
623    DotNetEnum,
624    DotNetStruct,
625    DotNetInterface,
626    DotNetDelegate,
627    DotNetOperator,
628    DotNetProperty,
629    DotNetMethod,
630    DotNetConstructor,
631    DotNetField,
632    DotNetEvent,
633]
634