1"""
2Generate documentation for pyboard API from C files.
3"""
4
5import os
6import argparse
7import re
8import markdown
9
10# given a list of (name,regex) pairs, find the first one that matches the given line
11def re_match_first(regexs, line):
12    for name, regex in regexs:
13        match = re.match(regex, line)
14        if match:
15            return name, match
16    return None, None
17
18
19def makedirs(d):
20    if not os.path.isdir(d):
21        os.makedirs(d)
22
23
24class Lexer:
25    class LexerError(Exception):
26        pass
27
28    class EOF(Exception):
29        pass
30
31    class Break(Exception):
32        pass
33
34    def __init__(self, file):
35        self.filename = file
36        with open(file, "rt") as f:
37            line_num = 0
38            lines = []
39            for line in f:
40                line_num += 1
41                line = line.strip()
42                if line == "///":
43                    lines.append((line_num, ""))
44                elif line.startswith("/// "):
45                    lines.append((line_num, line[4:]))
46                elif len(lines) > 0 and lines[-1][1] is not None:
47                    lines.append((line_num, None))
48        if len(lines) > 0 and lines[-1][1] is not None:
49            lines.append((line_num, None))
50        self.cur_line = 0
51        self.lines = lines
52
53    def opt_break(self):
54        if len(self.lines) > 0 and self.lines[0][1] is None:
55            self.lines.pop(0)
56
57    def next(self):
58        if len(self.lines) == 0:
59            raise Lexer.EOF
60        else:
61            l = self.lines.pop(0)
62            self.cur_line = l[0]
63            if l[1] is None:
64                raise Lexer.Break
65            else:
66                return l[1]
67
68    def error(self, msg):
69        print("({}:{}) {}".format(self.filename, self.cur_line, msg))
70        raise Lexer.LexerError
71
72
73class MarkdownWriter:
74    def __init__(self):
75        pass
76
77    def start(self):
78        self.lines = []
79
80    def end(self):
81        return "\n".join(self.lines)
82
83    def heading(self, level, text):
84        if len(self.lines) > 0:
85            self.lines.append("")
86        self.lines.append(level * "#" + " " + text)
87        self.lines.append("")
88
89    def para(self, text):
90        if len(self.lines) > 0 and self.lines[-1] != "":
91            self.lines.append("")
92        if isinstance(text, list):
93            self.lines.extend(text)
94        elif isinstance(text, str):
95            self.lines.append(text)
96        else:
97            assert False
98        self.lines.append("")
99
100    def single_line(self, text):
101        self.lines.append(text)
102
103    def module(self, name, short_descr, descr):
104        self.heading(1, "module {}".format(name))
105        self.para(descr)
106
107    def function(self, ctx, name, args, descr):
108        proto = "{}.{}{}".format(ctx, self.name, self.args)
109        self.heading(3, "`" + proto + "`")
110        self.para(descr)
111
112    def method(self, ctx, name, args, descr):
113        if name == "\\constructor":
114            proto = "{}{}".format(ctx, args)
115        elif name == "\\call":
116            proto = "{}{}".format(ctx, args)
117        else:
118            proto = "{}.{}{}".format(ctx, name, args)
119        self.heading(3, "`" + proto + "`")
120        self.para(descr)
121
122    def constant(self, ctx, name, descr):
123        self.single_line("`{}.{}` - {}".format(ctx, name, descr))
124
125
126class ReStructuredTextWriter:
127    head_chars = {1: "=", 2: "-", 3: "."}
128
129    def __init__(self):
130        pass
131
132    def start(self):
133        self.lines = []
134
135    def end(self):
136        return "\n".join(self.lines)
137
138    def _convert(self, text):
139        return text.replace("`", "``").replace("*", "\\*")
140
141    def heading(self, level, text, convert=True):
142        if len(self.lines) > 0:
143            self.lines.append("")
144        if convert:
145            text = self._convert(text)
146        self.lines.append(text)
147        self.lines.append(len(text) * self.head_chars[level])
148        self.lines.append("")
149
150    def para(self, text, indent=""):
151        if len(self.lines) > 0 and self.lines[-1] != "":
152            self.lines.append("")
153        if isinstance(text, list):
154            for t in text:
155                self.lines.append(indent + self._convert(t))
156        elif isinstance(text, str):
157            self.lines.append(indent + self._convert(text))
158        else:
159            assert False
160        self.lines.append("")
161
162    def single_line(self, text):
163        self.lines.append(self._convert(text))
164
165    def module(self, name, short_descr, descr):
166        self.heading(1, ":mod:`{}` --- {}".format(name, self._convert(short_descr)), convert=False)
167        self.lines.append(".. module:: {}".format(name))
168        self.lines.append("   :synopsis: {}".format(short_descr))
169        self.para(descr)
170
171    def function(self, ctx, name, args, descr):
172        args = self._convert(args)
173        self.lines.append(".. function:: " + name + args)
174        self.para(descr, indent="   ")
175
176    def method(self, ctx, name, args, descr):
177        args = self._convert(args)
178        if name == "\\constructor":
179            self.lines.append(".. class:: " + ctx + args)
180        elif name == "\\call":
181            self.lines.append(".. method:: " + ctx + args)
182        else:
183            self.lines.append(".. method:: " + ctx + "." + name + args)
184        self.para(descr, indent="   ")
185
186    def constant(self, ctx, name, descr):
187        self.lines.append(".. data:: " + name)
188        self.para(descr, indent="   ")
189
190
191class DocValidateError(Exception):
192    pass
193
194
195class DocItem:
196    def __init__(self):
197        self.doc = []
198
199    def add_doc(self, lex):
200        try:
201            while True:
202                line = lex.next()
203                if len(line) > 0 or len(self.doc) > 0:
204                    self.doc.append(line)
205        except Lexer.Break:
206            pass
207
208    def dump(self, writer):
209        writer.para(self.doc)
210
211
212class DocConstant(DocItem):
213    def __init__(self, name, descr):
214        super().__init__()
215        self.name = name
216        self.descr = descr
217
218    def dump(self, ctx, writer):
219        writer.constant(ctx, self.name, self.descr)
220
221
222class DocFunction(DocItem):
223    def __init__(self, name, args):
224        super().__init__()
225        self.name = name
226        self.args = args
227
228    def dump(self, ctx, writer):
229        writer.function(ctx, self.name, self.args, self.doc)
230
231
232class DocMethod(DocItem):
233    def __init__(self, name, args):
234        super().__init__()
235        self.name = name
236        self.args = args
237
238    def dump(self, ctx, writer):
239        writer.method(ctx, self.name, self.args, self.doc)
240
241
242class DocClass(DocItem):
243    def __init__(self, name, descr):
244        super().__init__()
245        self.name = name
246        self.descr = descr
247        self.constructors = {}
248        self.classmethods = {}
249        self.methods = {}
250        self.constants = {}
251
252    def process_classmethod(self, lex, d):
253        name = d["id"]
254        if name == "\\constructor":
255            dict_ = self.constructors
256        else:
257            dict_ = self.classmethods
258        if name in dict_:
259            lex.error("multiple definition of method '{}'".format(name))
260        method = dict_[name] = DocMethod(name, d["args"])
261        method.add_doc(lex)
262
263    def process_method(self, lex, d):
264        name = d["id"]
265        dict_ = self.methods
266        if name in dict_:
267            lex.error("multiple definition of method '{}'".format(name))
268        method = dict_[name] = DocMethod(name, d["args"])
269        method.add_doc(lex)
270
271    def process_constant(self, lex, d):
272        name = d["id"]
273        if name in self.constants:
274            lex.error("multiple definition of constant '{}'".format(name))
275        self.constants[name] = DocConstant(name, d["descr"])
276        lex.opt_break()
277
278    def dump(self, writer):
279        writer.heading(1, "class {}".format(self.name))
280        super().dump(writer)
281        if len(self.constructors) > 0:
282            writer.heading(2, "Constructors")
283            for f in sorted(self.constructors.values(), key=lambda x: x.name):
284                f.dump(self.name, writer)
285        if len(self.classmethods) > 0:
286            writer.heading(2, "Class methods")
287            for f in sorted(self.classmethods.values(), key=lambda x: x.name):
288                f.dump(self.name, writer)
289        if len(self.methods) > 0:
290            writer.heading(2, "Methods")
291            for f in sorted(self.methods.values(), key=lambda x: x.name):
292                f.dump(self.name.lower(), writer)
293        if len(self.constants) > 0:
294            writer.heading(2, "Constants")
295            for c in sorted(self.constants.values(), key=lambda x: x.name):
296                c.dump(self.name, writer)
297
298
299class DocModule(DocItem):
300    def __init__(self, name, descr):
301        super().__init__()
302        self.name = name
303        self.descr = descr
304        self.functions = {}
305        self.constants = {}
306        self.classes = {}
307        self.cur_class = None
308
309    def new_file(self):
310        self.cur_class = None
311
312    def process_function(self, lex, d):
313        name = d["id"]
314        if name in self.functions:
315            lex.error("multiple definition of function '{}'".format(name))
316        function = self.functions[name] = DocFunction(name, d["args"])
317        function.add_doc(lex)
318
319    # def process_classref(self, lex, d):
320    #    name = d['id']
321    #    self.classes[name] = name
322    #    lex.opt_break()
323
324    def process_class(self, lex, d):
325        name = d["id"]
326        if name in self.classes:
327            lex.error("multiple definition of class '{}'".format(name))
328        self.cur_class = self.classes[name] = DocClass(name, d["descr"])
329        self.cur_class.add_doc(lex)
330
331    def process_classmethod(self, lex, d):
332        self.cur_class.process_classmethod(lex, d)
333
334    def process_method(self, lex, d):
335        self.cur_class.process_method(lex, d)
336
337    def process_constant(self, lex, d):
338        if self.cur_class is None:
339            # a module-level constant
340            name = d["id"]
341            if name in self.constants:
342                lex.error("multiple definition of constant '{}'".format(name))
343            self.constants[name] = DocConstant(name, d["descr"])
344            lex.opt_break()
345        else:
346            # a class-level constant
347            self.cur_class.process_constant(lex, d)
348
349    def validate(self):
350        if self.descr is None:
351            raise DocValidateError("module {} referenced but never defined".format(self.name))
352
353    def dump(self, writer):
354        writer.module(self.name, self.descr, self.doc)
355        if self.functions:
356            writer.heading(2, "Functions")
357            for f in sorted(self.functions.values(), key=lambda x: x.name):
358                f.dump(self.name, writer)
359        if self.constants:
360            writer.heading(2, "Constants")
361            for c in sorted(self.constants.values(), key=lambda x: x.name):
362                c.dump(self.name, writer)
363        if self.classes:
364            writer.heading(2, "Classes")
365            for c in sorted(self.classes.values(), key=lambda x: x.name):
366                writer.para("[`{}.{}`]({}) - {}".format(self.name, c.name, c.name, c.descr))
367
368    def write_html(self, dir):
369        md_writer = MarkdownWriter()
370        md_writer.start()
371        self.dump(md_writer)
372        with open(os.path.join(dir, "index.html"), "wt") as f:
373            f.write(markdown.markdown(md_writer.end()))
374        for c in self.classes.values():
375            class_dir = os.path.join(dir, c.name)
376            makedirs(class_dir)
377            md_writer.start()
378            md_writer.para("part of the [{} module](./)".format(self.name))
379            c.dump(md_writer)
380            with open(os.path.join(class_dir, "index.html"), "wt") as f:
381                f.write(markdown.markdown(md_writer.end()))
382
383    def write_rst(self, dir):
384        rst_writer = ReStructuredTextWriter()
385        rst_writer.start()
386        self.dump(rst_writer)
387        with open(dir + "/" + self.name + ".rst", "wt") as f:
388            f.write(rst_writer.end())
389        for c in self.classes.values():
390            rst_writer.start()
391            c.dump(rst_writer)
392            with open(dir + "/" + self.name + "." + c.name + ".rst", "wt") as f:
393                f.write(rst_writer.end())
394
395
396class Doc:
397    def __init__(self):
398        self.modules = {}
399        self.cur_module = None
400
401    def new_file(self):
402        self.cur_module = None
403        for m in self.modules.values():
404            m.new_file()
405
406    def check_module(self, lex):
407        if self.cur_module is None:
408            lex.error("module not defined")
409
410    def process_module(self, lex, d):
411        name = d["id"]
412        if name not in self.modules:
413            self.modules[name] = DocModule(name, None)
414        self.cur_module = self.modules[name]
415        if self.cur_module.descr is not None:
416            lex.error("multiple definition of module '{}'".format(name))
417        self.cur_module.descr = d["descr"]
418        self.cur_module.add_doc(lex)
419
420    def process_moduleref(self, lex, d):
421        name = d["id"]
422        if name not in self.modules:
423            self.modules[name] = DocModule(name, None)
424        self.cur_module = self.modules[name]
425        lex.opt_break()
426
427    def process_class(self, lex, d):
428        self.check_module(lex)
429        self.cur_module.process_class(lex, d)
430
431    def process_function(self, lex, d):
432        self.check_module(lex)
433        self.cur_module.process_function(lex, d)
434
435    def process_classmethod(self, lex, d):
436        self.check_module(lex)
437        self.cur_module.process_classmethod(lex, d)
438
439    def process_method(self, lex, d):
440        self.check_module(lex)
441        self.cur_module.process_method(lex, d)
442
443    def process_constant(self, lex, d):
444        self.check_module(lex)
445        self.cur_module.process_constant(lex, d)
446
447    def validate(self):
448        for m in self.modules.values():
449            m.validate()
450
451    def dump(self, writer):
452        writer.heading(1, "Modules")
453        writer.para("These are the Python modules that are implemented.")
454        for m in sorted(self.modules.values(), key=lambda x: x.name):
455            writer.para("[`{}`]({}/) - {}".format(m.name, m.name, m.descr))
456
457    def write_html(self, dir):
458        md_writer = MarkdownWriter()
459        with open(os.path.join(dir, "module", "index.html"), "wt") as f:
460            md_writer.start()
461            self.dump(md_writer)
462            f.write(markdown.markdown(md_writer.end()))
463        for m in self.modules.values():
464            mod_dir = os.path.join(dir, "module", m.name)
465            makedirs(mod_dir)
466            m.write_html(mod_dir)
467
468    def write_rst(self, dir):
469        # with open(os.path.join(dir, 'module', 'index.html'), 'wt') as f:
470        #    f.write(markdown.markdown(self.dump()))
471        for m in self.modules.values():
472            m.write_rst(dir)
473
474
475regex_descr = r"(?P<descr>.*)"
476
477doc_regexs = (
478    (Doc.process_module, re.compile(r"\\module (?P<id>[a-z][a-z0-9]*) - " + regex_descr + r"$")),
479    (Doc.process_moduleref, re.compile(r"\\moduleref (?P<id>[a-z]+)$")),
480    (Doc.process_function, re.compile(r"\\function (?P<id>[a-z0-9_]+)(?P<args>\(.*\))$")),
481    (Doc.process_classmethod, re.compile(r"\\classmethod (?P<id>\\?[a-z0-9_]+)(?P<args>\(.*\))$")),
482    (Doc.process_method, re.compile(r"\\method (?P<id>\\?[a-z0-9_]+)(?P<args>\(.*\))$")),
483    (
484        Doc.process_constant,
485        re.compile(r"\\constant (?P<id>[A-Za-z0-9_]+) - " + regex_descr + r"$"),
486    ),
487    # (Doc.process_classref, re.compile(r'\\classref (?P<id>[A-Za-z0-9_]+)$')),
488    (Doc.process_class, re.compile(r"\\class (?P<id>[A-Za-z0-9_]+) - " + regex_descr + r"$")),
489)
490
491
492def process_file(file, doc):
493    lex = Lexer(file)
494    doc.new_file()
495    try:
496        try:
497            while True:
498                line = lex.next()
499                fun, match = re_match_first(doc_regexs, line)
500                if fun == None:
501                    lex.error("unknown line format: {}".format(line))
502                fun(doc, lex, match.groupdict())
503
504        except Lexer.Break:
505            lex.error("unexpected break")
506
507        except Lexer.EOF:
508            pass
509
510    except Lexer.LexerError:
511        return False
512
513    return True
514
515
516def main():
517    cmd_parser = argparse.ArgumentParser(
518        description="Generate documentation for pyboard API from C files."
519    )
520    cmd_parser.add_argument(
521        "--outdir", metavar="<output dir>", default="gendoc-out", help="ouput directory"
522    )
523    cmd_parser.add_argument("--format", default="html", help="output format: html or rst")
524    cmd_parser.add_argument("files", nargs="+", help="input files")
525    args = cmd_parser.parse_args()
526
527    doc = Doc()
528    for file in args.files:
529        print("processing", file)
530        if not process_file(file, doc):
531            return
532    try:
533        doc.validate()
534    except DocValidateError as e:
535        print(e)
536
537    makedirs(args.outdir)
538
539    if args.format == "html":
540        doc.write_html(args.outdir)
541    elif args.format == "rst":
542        doc.write_rst(args.outdir)
543    else:
544        print("unknown format:", args.format)
545        return
546
547    print("written to", args.outdir)
548
549
550if __name__ == "__main__":
551    main()
552