1""" 2Generate documentation for pyboard API from C files. 3""" 4 5import os 6import argparse 7import re 8import markdown 9 10# given a list of (name,regex) pairs, find the first one that matches the given line 11def re_match_first(regexs, line): 12 for name, regex in regexs: 13 match = re.match(regex, line) 14 if match: 15 return name, match 16 return None, None 17 18 19def makedirs(d): 20 if not os.path.isdir(d): 21 os.makedirs(d) 22 23 24class Lexer: 25 class LexerError(Exception): 26 pass 27 28 class EOF(Exception): 29 pass 30 31 class Break(Exception): 32 pass 33 34 def __init__(self, file): 35 self.filename = file 36 with open(file, "rt") as f: 37 line_num = 0 38 lines = [] 39 for line in f: 40 line_num += 1 41 line = line.strip() 42 if line == "///": 43 lines.append((line_num, "")) 44 elif line.startswith("/// "): 45 lines.append((line_num, line[4:])) 46 elif len(lines) > 0 and lines[-1][1] is not None: 47 lines.append((line_num, None)) 48 if len(lines) > 0 and lines[-1][1] is not None: 49 lines.append((line_num, None)) 50 self.cur_line = 0 51 self.lines = lines 52 53 def opt_break(self): 54 if len(self.lines) > 0 and self.lines[0][1] is None: 55 self.lines.pop(0) 56 57 def next(self): 58 if len(self.lines) == 0: 59 raise Lexer.EOF 60 else: 61 l = self.lines.pop(0) 62 self.cur_line = l[0] 63 if l[1] is None: 64 raise Lexer.Break 65 else: 66 return l[1] 67 68 def error(self, msg): 69 print("({}:{}) {}".format(self.filename, self.cur_line, msg)) 70 raise Lexer.LexerError 71 72 73class MarkdownWriter: 74 def __init__(self): 75 pass 76 77 def start(self): 78 self.lines = [] 79 80 def end(self): 81 return "\n".join(self.lines) 82 83 def heading(self, level, text): 84 if len(self.lines) > 0: 85 self.lines.append("") 86 self.lines.append(level * "#" + " " + text) 87 self.lines.append("") 88 89 def para(self, text): 90 if len(self.lines) > 0 and self.lines[-1] != "": 91 self.lines.append("") 92 if isinstance(text, list): 93 self.lines.extend(text) 94 elif isinstance(text, str): 95 self.lines.append(text) 96 else: 97 assert False 98 self.lines.append("") 99 100 def single_line(self, text): 101 self.lines.append(text) 102 103 def module(self, name, short_descr, descr): 104 self.heading(1, "module {}".format(name)) 105 self.para(descr) 106 107 def function(self, ctx, name, args, descr): 108 proto = "{}.{}{}".format(ctx, self.name, self.args) 109 self.heading(3, "`" + proto + "`") 110 self.para(descr) 111 112 def method(self, ctx, name, args, descr): 113 if name == "\\constructor": 114 proto = "{}{}".format(ctx, args) 115 elif name == "\\call": 116 proto = "{}{}".format(ctx, args) 117 else: 118 proto = "{}.{}{}".format(ctx, name, args) 119 self.heading(3, "`" + proto + "`") 120 self.para(descr) 121 122 def constant(self, ctx, name, descr): 123 self.single_line("`{}.{}` - {}".format(ctx, name, descr)) 124 125 126class ReStructuredTextWriter: 127 head_chars = {1: "=", 2: "-", 3: "."} 128 129 def __init__(self): 130 pass 131 132 def start(self): 133 self.lines = [] 134 135 def end(self): 136 return "\n".join(self.lines) 137 138 def _convert(self, text): 139 return text.replace("`", "``").replace("*", "\\*") 140 141 def heading(self, level, text, convert=True): 142 if len(self.lines) > 0: 143 self.lines.append("") 144 if convert: 145 text = self._convert(text) 146 self.lines.append(text) 147 self.lines.append(len(text) * self.head_chars[level]) 148 self.lines.append("") 149 150 def para(self, text, indent=""): 151 if len(self.lines) > 0 and self.lines[-1] != "": 152 self.lines.append("") 153 if isinstance(text, list): 154 for t in text: 155 self.lines.append(indent + self._convert(t)) 156 elif isinstance(text, str): 157 self.lines.append(indent + self._convert(text)) 158 else: 159 assert False 160 self.lines.append("") 161 162 def single_line(self, text): 163 self.lines.append(self._convert(text)) 164 165 def module(self, name, short_descr, descr): 166 self.heading(1, ":mod:`{}` --- {}".format(name, self._convert(short_descr)), convert=False) 167 self.lines.append(".. module:: {}".format(name)) 168 self.lines.append(" :synopsis: {}".format(short_descr)) 169 self.para(descr) 170 171 def function(self, ctx, name, args, descr): 172 args = self._convert(args) 173 self.lines.append(".. function:: " + name + args) 174 self.para(descr, indent=" ") 175 176 def method(self, ctx, name, args, descr): 177 args = self._convert(args) 178 if name == "\\constructor": 179 self.lines.append(".. class:: " + ctx + args) 180 elif name == "\\call": 181 self.lines.append(".. method:: " + ctx + args) 182 else: 183 self.lines.append(".. method:: " + ctx + "." + name + args) 184 self.para(descr, indent=" ") 185 186 def constant(self, ctx, name, descr): 187 self.lines.append(".. data:: " + name) 188 self.para(descr, indent=" ") 189 190 191class DocValidateError(Exception): 192 pass 193 194 195class DocItem: 196 def __init__(self): 197 self.doc = [] 198 199 def add_doc(self, lex): 200 try: 201 while True: 202 line = lex.next() 203 if len(line) > 0 or len(self.doc) > 0: 204 self.doc.append(line) 205 except Lexer.Break: 206 pass 207 208 def dump(self, writer): 209 writer.para(self.doc) 210 211 212class DocConstant(DocItem): 213 def __init__(self, name, descr): 214 super().__init__() 215 self.name = name 216 self.descr = descr 217 218 def dump(self, ctx, writer): 219 writer.constant(ctx, self.name, self.descr) 220 221 222class DocFunction(DocItem): 223 def __init__(self, name, args): 224 super().__init__() 225 self.name = name 226 self.args = args 227 228 def dump(self, ctx, writer): 229 writer.function(ctx, self.name, self.args, self.doc) 230 231 232class DocMethod(DocItem): 233 def __init__(self, name, args): 234 super().__init__() 235 self.name = name 236 self.args = args 237 238 def dump(self, ctx, writer): 239 writer.method(ctx, self.name, self.args, self.doc) 240 241 242class DocClass(DocItem): 243 def __init__(self, name, descr): 244 super().__init__() 245 self.name = name 246 self.descr = descr 247 self.constructors = {} 248 self.classmethods = {} 249 self.methods = {} 250 self.constants = {} 251 252 def process_classmethod(self, lex, d): 253 name = d["id"] 254 if name == "\\constructor": 255 dict_ = self.constructors 256 else: 257 dict_ = self.classmethods 258 if name in dict_: 259 lex.error("multiple definition of method '{}'".format(name)) 260 method = dict_[name] = DocMethod(name, d["args"]) 261 method.add_doc(lex) 262 263 def process_method(self, lex, d): 264 name = d["id"] 265 dict_ = self.methods 266 if name in dict_: 267 lex.error("multiple definition of method '{}'".format(name)) 268 method = dict_[name] = DocMethod(name, d["args"]) 269 method.add_doc(lex) 270 271 def process_constant(self, lex, d): 272 name = d["id"] 273 if name in self.constants: 274 lex.error("multiple definition of constant '{}'".format(name)) 275 self.constants[name] = DocConstant(name, d["descr"]) 276 lex.opt_break() 277 278 def dump(self, writer): 279 writer.heading(1, "class {}".format(self.name)) 280 super().dump(writer) 281 if len(self.constructors) > 0: 282 writer.heading(2, "Constructors") 283 for f in sorted(self.constructors.values(), key=lambda x: x.name): 284 f.dump(self.name, writer) 285 if len(self.classmethods) > 0: 286 writer.heading(2, "Class methods") 287 for f in sorted(self.classmethods.values(), key=lambda x: x.name): 288 f.dump(self.name, writer) 289 if len(self.methods) > 0: 290 writer.heading(2, "Methods") 291 for f in sorted(self.methods.values(), key=lambda x: x.name): 292 f.dump(self.name.lower(), writer) 293 if len(self.constants) > 0: 294 writer.heading(2, "Constants") 295 for c in sorted(self.constants.values(), key=lambda x: x.name): 296 c.dump(self.name, writer) 297 298 299class DocModule(DocItem): 300 def __init__(self, name, descr): 301 super().__init__() 302 self.name = name 303 self.descr = descr 304 self.functions = {} 305 self.constants = {} 306 self.classes = {} 307 self.cur_class = None 308 309 def new_file(self): 310 self.cur_class = None 311 312 def process_function(self, lex, d): 313 name = d["id"] 314 if name in self.functions: 315 lex.error("multiple definition of function '{}'".format(name)) 316 function = self.functions[name] = DocFunction(name, d["args"]) 317 function.add_doc(lex) 318 319 # def process_classref(self, lex, d): 320 # name = d['id'] 321 # self.classes[name] = name 322 # lex.opt_break() 323 324 def process_class(self, lex, d): 325 name = d["id"] 326 if name in self.classes: 327 lex.error("multiple definition of class '{}'".format(name)) 328 self.cur_class = self.classes[name] = DocClass(name, d["descr"]) 329 self.cur_class.add_doc(lex) 330 331 def process_classmethod(self, lex, d): 332 self.cur_class.process_classmethod(lex, d) 333 334 def process_method(self, lex, d): 335 self.cur_class.process_method(lex, d) 336 337 def process_constant(self, lex, d): 338 if self.cur_class is None: 339 # a module-level constant 340 name = d["id"] 341 if name in self.constants: 342 lex.error("multiple definition of constant '{}'".format(name)) 343 self.constants[name] = DocConstant(name, d["descr"]) 344 lex.opt_break() 345 else: 346 # a class-level constant 347 self.cur_class.process_constant(lex, d) 348 349 def validate(self): 350 if self.descr is None: 351 raise DocValidateError("module {} referenced but never defined".format(self.name)) 352 353 def dump(self, writer): 354 writer.module(self.name, self.descr, self.doc) 355 if self.functions: 356 writer.heading(2, "Functions") 357 for f in sorted(self.functions.values(), key=lambda x: x.name): 358 f.dump(self.name, writer) 359 if self.constants: 360 writer.heading(2, "Constants") 361 for c in sorted(self.constants.values(), key=lambda x: x.name): 362 c.dump(self.name, writer) 363 if self.classes: 364 writer.heading(2, "Classes") 365 for c in sorted(self.classes.values(), key=lambda x: x.name): 366 writer.para("[`{}.{}`]({}) - {}".format(self.name, c.name, c.name, c.descr)) 367 368 def write_html(self, dir): 369 md_writer = MarkdownWriter() 370 md_writer.start() 371 self.dump(md_writer) 372 with open(os.path.join(dir, "index.html"), "wt") as f: 373 f.write(markdown.markdown(md_writer.end())) 374 for c in self.classes.values(): 375 class_dir = os.path.join(dir, c.name) 376 makedirs(class_dir) 377 md_writer.start() 378 md_writer.para("part of the [{} module](./)".format(self.name)) 379 c.dump(md_writer) 380 with open(os.path.join(class_dir, "index.html"), "wt") as f: 381 f.write(markdown.markdown(md_writer.end())) 382 383 def write_rst(self, dir): 384 rst_writer = ReStructuredTextWriter() 385 rst_writer.start() 386 self.dump(rst_writer) 387 with open(dir + "/" + self.name + ".rst", "wt") as f: 388 f.write(rst_writer.end()) 389 for c in self.classes.values(): 390 rst_writer.start() 391 c.dump(rst_writer) 392 with open(dir + "/" + self.name + "." + c.name + ".rst", "wt") as f: 393 f.write(rst_writer.end()) 394 395 396class Doc: 397 def __init__(self): 398 self.modules = {} 399 self.cur_module = None 400 401 def new_file(self): 402 self.cur_module = None 403 for m in self.modules.values(): 404 m.new_file() 405 406 def check_module(self, lex): 407 if self.cur_module is None: 408 lex.error("module not defined") 409 410 def process_module(self, lex, d): 411 name = d["id"] 412 if name not in self.modules: 413 self.modules[name] = DocModule(name, None) 414 self.cur_module = self.modules[name] 415 if self.cur_module.descr is not None: 416 lex.error("multiple definition of module '{}'".format(name)) 417 self.cur_module.descr = d["descr"] 418 self.cur_module.add_doc(lex) 419 420 def process_moduleref(self, lex, d): 421 name = d["id"] 422 if name not in self.modules: 423 self.modules[name] = DocModule(name, None) 424 self.cur_module = self.modules[name] 425 lex.opt_break() 426 427 def process_class(self, lex, d): 428 self.check_module(lex) 429 self.cur_module.process_class(lex, d) 430 431 def process_function(self, lex, d): 432 self.check_module(lex) 433 self.cur_module.process_function(lex, d) 434 435 def process_classmethod(self, lex, d): 436 self.check_module(lex) 437 self.cur_module.process_classmethod(lex, d) 438 439 def process_method(self, lex, d): 440 self.check_module(lex) 441 self.cur_module.process_method(lex, d) 442 443 def process_constant(self, lex, d): 444 self.check_module(lex) 445 self.cur_module.process_constant(lex, d) 446 447 def validate(self): 448 for m in self.modules.values(): 449 m.validate() 450 451 def dump(self, writer): 452 writer.heading(1, "Modules") 453 writer.para("These are the Python modules that are implemented.") 454 for m in sorted(self.modules.values(), key=lambda x: x.name): 455 writer.para("[`{}`]({}/) - {}".format(m.name, m.name, m.descr)) 456 457 def write_html(self, dir): 458 md_writer = MarkdownWriter() 459 with open(os.path.join(dir, "module", "index.html"), "wt") as f: 460 md_writer.start() 461 self.dump(md_writer) 462 f.write(markdown.markdown(md_writer.end())) 463 for m in self.modules.values(): 464 mod_dir = os.path.join(dir, "module", m.name) 465 makedirs(mod_dir) 466 m.write_html(mod_dir) 467 468 def write_rst(self, dir): 469 # with open(os.path.join(dir, 'module', 'index.html'), 'wt') as f: 470 # f.write(markdown.markdown(self.dump())) 471 for m in self.modules.values(): 472 m.write_rst(dir) 473 474 475regex_descr = r"(?P<descr>.*)" 476 477doc_regexs = ( 478 (Doc.process_module, re.compile(r"\\module (?P<id>[a-z][a-z0-9]*) - " + regex_descr + r"$")), 479 (Doc.process_moduleref, re.compile(r"\\moduleref (?P<id>[a-z]+)$")), 480 (Doc.process_function, re.compile(r"\\function (?P<id>[a-z0-9_]+)(?P<args>\(.*\))$")), 481 (Doc.process_classmethod, re.compile(r"\\classmethod (?P<id>\\?[a-z0-9_]+)(?P<args>\(.*\))$")), 482 (Doc.process_method, re.compile(r"\\method (?P<id>\\?[a-z0-9_]+)(?P<args>\(.*\))$")), 483 ( 484 Doc.process_constant, 485 re.compile(r"\\constant (?P<id>[A-Za-z0-9_]+) - " + regex_descr + r"$"), 486 ), 487 # (Doc.process_classref, re.compile(r'\\classref (?P<id>[A-Za-z0-9_]+)$')), 488 (Doc.process_class, re.compile(r"\\class (?P<id>[A-Za-z0-9_]+) - " + regex_descr + r"$")), 489) 490 491 492def process_file(file, doc): 493 lex = Lexer(file) 494 doc.new_file() 495 try: 496 try: 497 while True: 498 line = lex.next() 499 fun, match = re_match_first(doc_regexs, line) 500 if fun == None: 501 lex.error("unknown line format: {}".format(line)) 502 fun(doc, lex, match.groupdict()) 503 504 except Lexer.Break: 505 lex.error("unexpected break") 506 507 except Lexer.EOF: 508 pass 509 510 except Lexer.LexerError: 511 return False 512 513 return True 514 515 516def main(): 517 cmd_parser = argparse.ArgumentParser( 518 description="Generate documentation for pyboard API from C files." 519 ) 520 cmd_parser.add_argument( 521 "--outdir", metavar="<output dir>", default="gendoc-out", help="ouput directory" 522 ) 523 cmd_parser.add_argument("--format", default="html", help="output format: html or rst") 524 cmd_parser.add_argument("files", nargs="+", help="input files") 525 args = cmd_parser.parse_args() 526 527 doc = Doc() 528 for file in args.files: 529 print("processing", file) 530 if not process_file(file, doc): 531 return 532 try: 533 doc.validate() 534 except DocValidateError as e: 535 print(e) 536 537 makedirs(args.outdir) 538 539 if args.format == "html": 540 doc.write_html(args.outdir) 541 elif args.format == "rst": 542 doc.write_rst(args.outdir) 543 else: 544 print("unknown format:", args.format) 545 return 546 547 print("written to", args.outdir) 548 549 550if __name__ == "__main__": 551 main() 552