1# -*- coding: utf-8 -*- 2 3import re 4from os import getenv, listdir, path 5import pickle 6import importlib 7 8from django.utils.html import escape, linebreaks 9from django.utils.safestring import mark_safe 10 11from mathics import settings 12 13from mathics import builtin 14from mathics.builtin import get_module_doc 15from mathics.core.evaluation import Message, Print 16from mathics_django.doc.utils import slugify 17 18CHAPTER_RE = re.compile('(?s)<chapter title="(.*?)">(.*?)</chapter>') 19SECTION_RE = re.compile('(?s)(.*?)<section title="(.*?)">(.*?)</section>') 20SUBSECTION_RE = re.compile('(?s)<subsection title="(.*?)">') 21SUBSECTION_END_RE = re.compile("</subsection>") 22 23TESTCASE_RE = re.compile( 24 r"""(?mx)^ 25 ((?:.|\n)*?) 26 ^\s*([>#SX])>[ ](.*) 27 ((?:\n\s*(?:[:|=.][ ]|\.).*)*) 28""" 29) 30TESTCASE_OUT_RE = re.compile(r"^\s*([:|=])(.*)$") 31 32MATHICS_RE = re.compile(r"(?<!\\)\'(.*?)(?<!\\)\'") 33 34# preserve space before and after inline code variables 35LATEX_RE = re.compile(r"(\s?)\$(\w+?)\$(\s?)") 36 37DL_RE = re.compile(r"(?s)<dl>(.*?)</dl>") 38DL_ITEM_RE = re.compile( 39 r"(?s)<(?P<tag>d[td])>(?P<content>.*?)(?:</(?P=tag)>|)\s*(?:(?=<d[td]>)|$)" 40) 41LIST_RE = re.compile(r"(?s)<(?P<tag>ul|ol)>(?P<content>.*?)</(?P=tag)>") 42LIST_ITEM_RE = re.compile(r"(?s)<li>(.*?)(?:</li>|(?=<li>)|$)") 43CONSOLE_RE = re.compile(r"(?s)<(?P<tag>con|console)>(?P<content>.*?)</(?P=tag)>") 44ITALIC_RE = re.compile(r"(?s)<(?P<tag>i)>(?P<content>.*?)</(?P=tag)>") 45IMG_RE = re.compile( 46 r'<img src="(?P<src>.*?)" title="(?P<title>.*?)" label="(?P<label>.*?)">' 47) 48IMG_PNG_RE = re.compile( 49 r'<imgpng src="(?P<src>.*?)" title="(?P<title>.*?)" label="(?P<label>.*?)">' 50) 51REF_RE = re.compile(r'<ref label="(?P<label>.*?)">') 52PYTHON_RE = re.compile(r"(?s)<python>(.*?)</python>") 53LATEX_CHAR_RE = re.compile(r"(?<!\\)(\^)") 54 55QUOTATIONS_RE = re.compile(r"\"([\w\s,]*?)\"") 56HYPERTEXT_RE = re.compile(r"(?s)<(?P<tag>em|url)>(?P<content>.*?)</(?P=tag)>") 57 58OUTSIDE_ASY_RE = re.compile(r"(?s)((?:^|\\end\{asy\}).*?(?:$|\\begin\{asy\}))") 59LATEX_TEXT_RE = re.compile( 60 r"(?s)\\text\{([^{}]*?(?:[^{}]*?\{[^{}]*?(?:[^{}]*?\{[^{}]*?\}[^{}]*?)*?" 61 r"[^{}]*?\}[^{}]*?)*?[^{}]*?)\}" 62) 63LATEX_TESTOUT_RE = re.compile( 64 r"(?s)\\begin\{(?P<tag>testmessage|testprint|testresult)\}" 65 r"(?P<content>.*?)\\end\{(?P=tag)\}" 66) 67LATEX_TESTOUT_DELIM_RE = re.compile(r",") 68NUMBER_RE = re.compile(r"(\d*(?<!\.)\.\d+|\d+\.(?!\.)\d*|\d+)") 69LATEX_ARRAY_RE = re.compile( 70 r"(?s)\\begin\{testresult\}\\begin\{array\}\{l\}(.*?)" 71 r"\\end\{array\}\\end\{testresult\}" 72) 73LATEX_INLINE_END_RE = re.compile(r"(?s)(?P<all>\\lstinline'[^']*?'\}?[.,;:])") 74LATEX_CONSOLE_RE = re.compile(r"\\console\{(.*?)\}") 75 76ALLOWED_TAGS = ( 77 "dl", 78 "dd", 79 "dt", 80 "em", 81 "url", 82 "ul", 83 "i", 84 "ol", 85 "li", 86 "con", 87 "console", 88 "img", 89 "imgpng", 90 "ref", 91 "subsection", 92) 93ALLOWED_TAGS_RE = dict( 94 (allowed, re.compile("<(%s.*?)>" % allowed)) for allowed in ALLOWED_TAGS 95) 96 97SPECIAL_COMMANDS = { 98 "LaTeX": (r"<em>LaTeX</em>", r"\LaTeX{}"), 99 "Mathematica": ( 100 r"<em>Mathematica</em>®", 101 r"\emph{Mathematica}\textregistered{}", 102 ), 103 "Mathics": (r"<em>Mathics</em>", r"\emph{Mathics}"), 104 "Sage": (r"<em>Sage</em>", r"\emph{Sage}"), 105 "Wolfram": (r"<em>Wolfram</em>", r"\emph{Wolfram}"), 106 "skip": (r"<br /><br />", r"\bigskip"), 107} 108 109try: 110 with open(settings.DOC_XML_DATA, "rb") as xml_data_file: 111 xml_data = pickle.load(xml_data_file) 112except IOError: 113 xml_data = {} 114 115 116def filter_comments(doc): 117 return "\n".join( 118 line for line in doc.splitlines() if not line.lstrip().startswith("##") 119 ) 120 121 122def strip_system_prefix(name): 123 if name.startswith("System`"): 124 stripped_name = name[len("System`") :] 125 # don't return Private`sym for System`Private`sym 126 if "`" not in stripped_name: 127 return stripped_name 128 return name 129 130 131def get_latex_escape_char(text): 132 for escape_char in ("'", "~", "@"): 133 if escape_char not in text: 134 return escape_char 135 raise ValueError 136 137 138def _replace_all(text, pairs): 139 for (i, j) in pairs: 140 text = text.replace(i, j) 141 return text 142 143 144def escape_latex_output(text): 145 " Escape Mathics output " 146 147 text = _replace_all( 148 text, 149 [ 150 ("\\", "\\\\"), 151 ("{", "\\{"), 152 ("}", "\\}"), 153 ("~", "\\~"), 154 ("&", "\\&"), 155 ("%", "\\%"), 156 ("$", r"\$"), 157 ("_", "\\_"), 158 ], 159 ) 160 return text 161 162 163def escape_latex_code(text): 164 " Escape verbatim Mathics input " 165 166 text = escape_latex_output(text) 167 escape_char = get_latex_escape_char(text) 168 return "\\lstinline%s%s%s" % (escape_char, text, escape_char) 169 170 171def escape_latex(text): 172 " Escape documentation text " 173 174 def repl_python(match): 175 return ( 176 r"""\begin{lstlisting}[style=python] 177%s 178\end{lstlisting}""" 179 % match.group(1).strip() 180 ) 181 182 text, post_substitutions = pre_sub(PYTHON_RE, text, repl_python) 183 184 text = _replace_all( 185 text, 186 [ 187 ("\\", "\\\\"), 188 ("{", "\\{"), 189 ("}", "\\}"), 190 ("~", "\\~{ }"), 191 ("&", "\\&"), 192 ("%", "\\%"), 193 ("#", "\\#"), 194 ], 195 ) 196 197 def repl(match): 198 text = match.group(1) 199 if text: 200 text = _replace_all(text, [("\\'", "'"), ("^", "\\^")]) 201 escape_char = get_latex_escape_char(text) 202 text = LATEX_RE.sub( 203 lambda m: "%s%s\\codevar{\\textit{%s}}%s\\lstinline%s" 204 % (escape_char, m.group(1), m.group(2), m.group(3), escape_char), 205 text, 206 ) 207 if text.startswith(" "): 208 text = r"\ " + text[1:] 209 if text.endswith(" "): 210 text = text[:-1] + r"\ " 211 return "\\code{\\lstinline%s%s%s}" % (escape_char, text, escape_char) 212 else: 213 # treat double '' literaly 214 return "''" 215 216 text = MATHICS_RE.sub(repl, text) 217 218 text = LATEX_RE.sub( 219 lambda m: "%s\\textit{%s}%s" % (m.group(1), m.group(2), m.group(3)), text 220 ) 221 222 text = text.replace("\\\\'", "'") 223 224 def repl_dl(match): 225 text = match.group(1) 226 text = DL_ITEM_RE.sub( 227 lambda m: "\\%(tag)s{%(content)s}\n" % m.groupdict(), text 228 ) 229 return "\\begin{definitions}%s\\end{definitions}" % text 230 231 text = DL_RE.sub(repl_dl, text) 232 233 def repl_list(match): 234 tag = match.group("tag") 235 content = match.group("content") 236 content = LIST_ITEM_RE.sub(lambda m: "\\item %s\n" % m.group(1), content) 237 env = "itemize" if tag == "ul" else "enumerate" 238 return "\\begin{%s}%s\\end{%s}" % (env, content, env) 239 240 text = LIST_RE.sub(repl_list, text) 241 242 text = _replace_all( 243 text, 244 [ 245 ("$", r"\$"), 246 ("\u03c0", r"$\pi$"), 247 (" ", r"$\ge$"), 248 (" ", r"$\le$"), 249 (" ", r"$\ne$"), 250 ("\u00E7", r"\c{c}"), 251 ("\u00e9", r"\'e"), 252 ("\u00ea", r"\^e"), 253 ("\u00f1", r"\~n"), 254 (r" ", r"\int"), 255 ("\uF74C", r"d"), 256 ], 257 ) 258 259 def repl_char(match): 260 char = match.group(1) 261 return { 262 "^": "$^\wedge$", 263 }[char] 264 265 text = LATEX_CHAR_RE.sub(repl_char, text) 266 267 def repl_img(match): 268 src = match.group("src") 269 title = match.group("title") 270 label = match.group("label") 271 return r"""\begin{figure*}[htp] 272\centering 273\includegraphics[width=\textwidth]{images/%(src)s} 274\caption{%(title)s} 275\label{%(label)s} 276\end{figure*}""" % { 277 "src": src, 278 "title": title, 279 "label": label, 280 } 281 282 text = IMG_RE.sub(repl_img, text) 283 284 def repl_imgpng(match): 285 src = match.group("src") 286 return r"\includegraphics[scale=1.0]{images/%(src)s}" % {"src": src} 287 288 text = IMG_PNG_RE.sub(repl_imgpng, text) 289 290 def repl_ref(match): 291 return r"figure \ref{%s}" % match.group("label") 292 293 text = REF_RE.sub(repl_ref, text) 294 295 def repl_quotation(match): 296 return r"``%s''" % match.group(1) 297 298 def repl_hypertext(match): 299 tag = match.group("tag") 300 content = match.group("content") 301 if tag == "em": 302 return r"\emph{%s}" % content 303 elif tag == "url": 304 return "\\url{%s}" % content 305 306 text = QUOTATIONS_RE.sub(repl_quotation, text) 307 text = HYPERTEXT_RE.sub(repl_hypertext, text) 308 309 def repl_console(match): 310 tag = match.group("tag") 311 content = match.group("content") 312 content = content.strip() 313 content = content.replace(r"\$", "$") 314 if tag == "con": 315 return "\\console{%s}" % content 316 else: 317 return "\\begin{lstlisting}\n%s\n\\end{lstlisting}" % content 318 319 text = CONSOLE_RE.sub(repl_console, text) 320 321 def repl_italic(match): 322 content = match.group("content") 323 return "\\emph{%s}" % content 324 325 text = ITALIC_RE.sub(repl_italic, text) 326 327 '''def repl_asy(match): 328 """ 329 Ensure \begin{asy} and \end{asy} are on their own line, 330 but there shall be no extra empty lines 331 """ 332 #tag = match.group(1) 333 #return '\n%s\n' % tag 334 #print "replace" 335 return '\\end{asy}\n\\begin{asy}' 336 text = LATEX_BETWEEN_ASY_RE.sub(repl_asy, text)''' 337 338 def repl_subsection(match): 339 return "\n\\subsection*{%s}\n" % match.group(1) 340 341 text = SUBSECTION_RE.sub(repl_subsection, text) 342 text = SUBSECTION_END_RE.sub("", text) 343 344 for key, (xml, tex) in SPECIAL_COMMANDS.items(): 345 # "\" has been escaped already => 2 \ 346 text = text.replace("\\\\" + key, tex) 347 348 text = post_sub(text, post_substitutions) 349 350 return text 351 352 353def post_process_latex(result): 354 """ 355 Some post-processing hacks of generated LaTeX code to handle linebreaks 356 """ 357 358 WORD_SPLIT_RE = re.compile(r"(\s+|\\newline\s*)") 359 360 def wrap_word(word): 361 if word.strip() == r"\newline": 362 return word 363 return r"\text{%s}" % word 364 365 def repl_text(match): 366 text = match.group(1) 367 if not text: 368 return r"\text{}" 369 words = WORD_SPLIT_RE.split(text) 370 assert len(words) >= 1 371 if len(words) > 1: 372 text = "" 373 index = 0 374 while index < len(words) - 1: 375 text += "%s%s\\allowbreak{}" % ( 376 wrap_word(words[index]), 377 wrap_word(words[index + 1]), 378 ) 379 index += 2 380 text += wrap_word(words[-1]) 381 else: 382 text = r"\text{%s}" % words[0] 383 if not text: 384 return r"\text{}" 385 text = text.replace("><", r">}\allowbreak\text{<") 386 return text 387 388 def repl_out_delim(match): 389 return ",\\allowbreak{}" 390 391 def repl_number(match): 392 guard = r"\allowbreak{}" 393 inter_groups_pre = r"\,\discretionary{\~{}}{\~{}}{}" 394 inter_groups_post = r"\discretionary{\~{}}{\~{}}{}" 395 number = match.group(1) 396 parts = number.split(".") 397 if len(number) <= 3: 398 return number 399 assert 1 <= len(parts) <= 2 400 pre_dec = parts[0] 401 groups = [] 402 while pre_dec: 403 groups.append(pre_dec[-3:]) 404 pre_dec = pre_dec[:-3] 405 pre_dec = inter_groups_pre.join(reversed(groups)) 406 if len(parts) == 2: 407 post_dec = parts[1] 408 groups = [] 409 while post_dec: 410 groups.append(post_dec[:3]) 411 post_dec = post_dec[3:] 412 post_dec = inter_groups_post.join(groups) 413 result = pre_dec + "." + post_dec 414 else: 415 result = pre_dec 416 return guard + result + guard 417 418 def repl_array(match): 419 content = match.group(1) 420 lines = content.split("\\\\") 421 content = "".join( 422 r"\begin{dmath*}%s\end{dmath*}" % line for line in lines if line.strip() 423 ) 424 return r"\begin{testresultlist}%s\end{testresultlist}" % content 425 426 def repl_out(match): 427 tag = match.group("tag") 428 content = match.group("content") 429 content = LATEX_TESTOUT_DELIM_RE.sub(repl_out_delim, content) 430 content = NUMBER_RE.sub(repl_number, content) 431 content = content.replace(r"\left[", r"\left[\allowbreak{}") 432 return "\\begin{%s}%s\\end{%s}" % (tag, content, tag) 433 434 def repl_inline_end(match): 435 " Prevent linebreaks between inline code and sentence delimeters " 436 437 code = match.group("all") 438 if code[-2] == "}": 439 code = code[:-2] + code[-1] + code[-2] 440 return r"\mbox{%s}" % code 441 442 def repl_console(match): 443 code = match.group(1) 444 code = code.replace("/", r"/\allowbreak{}") 445 return r"\console{%s}" % code 446 447 def repl_nonasy(match): 448 result = match.group(1) 449 result = LATEX_TEXT_RE.sub(repl_text, result) 450 result = LATEX_TESTOUT_RE.sub(repl_out, result) 451 result = LATEX_ARRAY_RE.sub(repl_array, result) 452 result = LATEX_INLINE_END_RE.sub(repl_inline_end, result) 453 result = LATEX_CONSOLE_RE.sub(repl_console, result) 454 return result 455 456 return OUTSIDE_ASY_RE.sub(repl_nonasy, result) 457 458 459POST_SUBSTITUTION_TAG = "_POST_SUBSTITUTION%d_" 460 461 462def pre_sub(re, text, repl_func): 463 post_substitutions = [] 464 465 def repl_pre(match): 466 repl = repl_func(match) 467 index = len(post_substitutions) 468 post_substitutions.append(repl) 469 return POST_SUBSTITUTION_TAG % index 470 471 text = re.sub(repl_pre, text) 472 473 return text, post_substitutions 474 475 476def post_sub(text, post_substitutions): 477 for index, sub in enumerate(post_substitutions): 478 text = text.replace(POST_SUBSTITUTION_TAG % index, sub) 479 return text 480 481 482def escape_html(text, verbatim_mode=False, counters=None, single_line=False): 483 def repl_python(match): 484 return ( 485 r"""<pre><![CDATA[ 486%s 487]]></pre>""" 488 % match.group(1).strip() 489 ) 490 491 text, post_substitutions = pre_sub(PYTHON_RE, text, repl_python) 492 493 text = text.replace("&", "&").replace("<", "<").replace(">", ">") 494 495 if not verbatim_mode: 496 497 def repl_quotation(match): 498 return r"“%s”" % match.group(1) 499 500 text = QUOTATIONS_RE.sub(repl_quotation, text) 501 502 if counters is None: 503 counters = {} 504 505 text = text.replace('"', """) 506 if not verbatim_mode: 507 508 def repl_latex(match): 509 return "%s<var>%s</var>%s" % ( 510 match.group(1), 511 match.group(2), 512 match.group(3), 513 ) 514 515 text = LATEX_RE.sub(repl_latex, text) 516 517 def repl_mathics(match): 518 text = match.group(1) 519 text = text.replace("\\'", "'") 520 text = text.replace(" ", " ") 521 if text: 522 return "<code>%s</code>" % text 523 else: 524 return "'" 525 526 def repl_allowed(match): 527 content = _replace_all( 528 match.group(1), [("“", '"'), ("”", '"'), (""", '"')] 529 ) 530 return "<%s>" % content 531 532 text = MATHICS_RE.sub(repl_mathics, text) 533 for allowed in ALLOWED_TAGS: 534 text = ALLOWED_TAGS_RE[allowed].sub(repl_allowed, text) 535 text = text.replace("</%s>" % allowed, "</%s>" % allowed) 536 537 def repl_dl(match): 538 text = match.group(1) 539 text = DL_ITEM_RE.sub( 540 lambda m: "<%(tag)s>%(content)s</%(tag)s>\n" % m.groupdict(), text 541 ) 542 return "<dl>%s</dl>" % text 543 544 text = DL_RE.sub(repl_dl, text) 545 546 def repl_list(match): 547 tag = match.group("tag") 548 content = match.group("content") 549 content = LIST_ITEM_RE.sub(lambda m: "<li>%s</li>" % m.group(1), content) 550 return "<%s>%s</%s>" % (tag, content, tag) 551 552 text = LIST_RE.sub(repl_list, text) 553 554 def repl_hypertext(match): 555 tag = match.group("tag") 556 content = match.group("content") 557 if tag == "em": 558 return r"<em>%s</em>" % content 559 elif tag == "url": 560 return r'<a href="%s">%s</a>' % (content, content) 561 562 text = HYPERTEXT_RE.sub(repl_hypertext, text) 563 564 def repl_console(match): 565 tag = match.group("tag") 566 content = match.group("content") 567 tag = "div" if tag == "console" else "span" 568 content = content.strip() 569 pre = post = "" 570 571 # gets replaced for <br /> later by DocText.html() 572 content = content.replace("\n", "<br>") 573 574 return r'<%s class="console">%s%s%s</%s>' % (tag, pre, content, post, tag) 575 576 text = CONSOLE_RE.sub(repl_console, text) 577 578 def repl_img(match): 579 src = match.group("src") 580 title = match.group("title") 581 return ( 582 r'<a href="/media/doc/%(src)s.pdf">' 583 r'<img src="/media/doc/%(src)s.png" title="%(title)s" />' 584 r"</a>" 585 ) % {"src": src, "title": title} 586 587 text = IMG_RE.sub(repl_img, text) 588 589 def repl_imgpng(match): 590 src = match.group("src") 591 title = match.group("title") 592 return (r'<img src="/media/doc/%(src)s" title="%(title)s" />') % { 593 "src": src, 594 "title": title, 595 } 596 597 text = IMG_PNG_RE.sub(repl_imgpng, text) 598 599 def repl_ref(match): 600 # TODO: this is not an optimal solution - maybe we need figure 601 # numbers in the XML doc as well? 602 return r"the following figure" 603 604 text = REF_RE.sub(repl_ref, text) 605 606 def repl_subsection(match): 607 return '\n<h2 label="%s">%s</h2>\n' % (match.group(1), match.group(1)) 608 609 text = SUBSECTION_RE.sub(repl_subsection, text) 610 text = SUBSECTION_END_RE.sub("", text) 611 612 text = text.replace("\\'", "'") 613 else: 614 text = text.replace(" ", " ") 615 text = "<code>%s</code>" % text 616 text = text.replace("'", "'") 617 text = text.replace("---", "—") 618 for key, (xml, tex) in SPECIAL_COMMANDS.items(): 619 text = text.replace("\\" + key, xml) 620 621 if not single_line: 622 text = linebreaks(text) 623 text = text.replace("<br />", "\n").replace("<br>", "<br />") 624 625 text = post_sub(text, post_substitutions) 626 627 text = text.replace("<p><pre>", "<pre>").replace("</pre></p>", "</pre>") 628 629 return text 630 631 632class Tests(object): 633 def __init__(self, part, chapter, section, tests): 634 self.part, self.chapter = part, chapter 635 self.section, self.tests = section, tests 636 637 638class DocElement(object): 639 def href(self, ajax=False): 640 if ajax: 641 return "javascript:loadDoc('%s')" % self.get_url() 642 else: 643 return "/doc%s" % self.get_url() 644 645 def get_prev(self): 646 return self.get_prev_next()[0] 647 648 def get_next(self): 649 return self.get_prev_next()[1] 650 651 def get_collection(self): 652 return [] 653 654 def get_prev_next(self): 655 collection = self.get_collection() 656 index = collection.index(self) 657 prev = collection[index - 1] if index > 0 else None 658 next = collection[index + 1] if index < len(collection) - 1 else None 659 return prev, next 660 661 def get_title_html(self): 662 return mark_safe(escape_html(self.title, single_line=True)) 663 664 665class Documentation(DocElement): 666 def __str__(self): 667 return "\n\n\n".join(str(part) for part in self.parts) 668 669 def get_tests(self): 670 for part in self.parts: 671 for chapter in part.chapters: 672 tests = chapter.doc.get_tests() 673 if tests: 674 yield Tests(part.title, chapter.title, "", tests) 675 for section in chapter.sections: 676 if section.installed: 677 tests = section.doc.get_tests() 678 if tests: 679 yield Tests(part.title, chapter.title, section.title, tests) 680 681 def get_part(self, part_slug): 682 return self.parts_by_slug.get(part_slug) 683 684 def get_chapter(self, part_slug, chapter_slug): 685 part = self.parts_by_slug.get(part_slug) 686 if part: 687 return part.chapters_by_slug.get(chapter_slug) 688 return None 689 """for part in self.parts: 690 if part.slug == part_slug: 691 for chapter in self: 692 pass""" 693 694 def get_section(self, part_slug, chapter_slug, section_slug): 695 part = self.parts_by_slug.get(part_slug) 696 if part: 697 chapter = part.chapters_by_slug.get(chapter_slug) 698 if chapter: 699 return chapter.sections_by_slug.get(section_slug) 700 return None 701 702 def latex(self, output): 703 parts = [] 704 appendix = False 705 for part in self.parts: 706 text = part.latex(output) 707 if part.is_appendix and not appendix: 708 appendix = True 709 text = "\n\\appendix\n" + text 710 parts.append(text) 711 result = "\n\n".join(parts) 712 result = post_process_latex(result) 713 return result 714 715 def get_url(self): 716 return "/" 717 718 def search(self, query): 719 query = query.strip() 720 query_parts = [q.strip().lower() for q in query.split()] 721 722 def matches(text): 723 text = text.lower() 724 return all(q in text for q in query_parts) 725 726 result = [] 727 for part in self.parts: 728 if matches(part.title): 729 result.append((False, part)) 730 for chapter in part.chapters: 731 if matches(chapter.title): 732 result.append((False, chapter)) 733 for section in chapter.sections: 734 if matches(section.title): 735 result.append((section.title == query, section)) 736 elif query == section.operator: 737 result.append((True, section)) 738 return result 739 740 741class MathicsMainDocumentation(Documentation): 742 def __init__(self): 743 self.title = "Overview" 744 self.parts = [] 745 self.parts_by_slug = {} 746 self.doc_dir = settings.DOC_DIR 747 self.xml_data_file = settings.DOC_XML_DATA 748 self.tex_data_file = settings.DOC_TEX_DATA 749 self.latex_file = settings.DOC_LATEX_FILE 750 self.pymathics_doc_loaded = False 751 files = listdir(self.doc_dir) 752 files.sort() 753 appendix = [] 754 755 for file in files: 756 part_title = file[2:] 757 if part_title.endswith(".mdoc"): 758 part_title = part_title[: -len(".mdoc")] 759 part = DocPart(self, part_title) 760 text = open(self.doc_dir + file, "rb").read().decode("utf8") 761 text = filter_comments(text) 762 chapters = CHAPTER_RE.findall(text) 763 for title, text in chapters: 764 chapter = DocChapter(part, title) 765 text += '<section title=""></section>' 766 sections = SECTION_RE.findall(text) 767 for pre_text, title, text in sections: 768 if not chapter.doc: 769 chapter.doc = Doc(pre_text) 770 if title: 771 section = DocSection(chapter, title, text) 772 chapter.sections.append(section) 773 part.chapters.append(chapter) 774 if file[0].isdigit(): 775 self.parts.append(part) 776 else: 777 part.is_appendix = True 778 appendix.append(part) 779 780 for title, modules, builtins_by_module, start in [ 781 ( 782 "Reference of Built-in Symbols", 783 builtin.modules, 784 builtin.builtins_by_module, 785 True, 786 ) 787 ]: # nopep8 788 # ("Reference of optional symbols", optional.modules, 789 # optional.optional_builtins_by_module, False)]: 790 791 builtin_part = DocPart(self, title, is_reference=start) 792 for module in modules: 793 title, text = get_module_doc(module) 794 chapter = DocChapter(builtin_part, title, Doc(text)) 795 builtins = builtins_by_module[module.__name__] 796 for instance in builtins: 797 installed = True 798 for package in getattr(instance, "requires", []): 799 try: 800 importlib.import_module(package) 801 except ImportError: 802 installed = False 803 break 804 section = DocSection( 805 chapter, 806 strip_system_prefix(instance.get_name()), 807 instance.__doc__ or "", 808 operator=instance.get_operator(), 809 installed=installed, 810 ) 811 chapter.sections.append(section) 812 builtin_part.chapters.append(chapter) 813 self.parts.append(builtin_part) 814 815 for part in appendix: 816 self.parts.append(part) 817 818 # set keys of tests 819 for tests in self.get_tests(): 820 for test in tests.tests: 821 test.key = (tests.part, tests.chapter, tests.section, test.index) 822 823 def load_pymathics_doc(self): 824 if self.pymathics_doc_loaded: 825 return 826 from mathics.settings import default_pymathics_modules 827 828 pymathicspart = None 829 # Look the "Pymathics Modules" part, and if it does not exist, create it. 830 for part in self.parts: 831 if part.title == "Pymathics Modules": 832 pymathicspart = part 833 if pymathicspart is None: 834 pymathicspart = DocPart(self, "Pymathics Modules", is_reference=True) 835 self.parts.append(pymathicspart) 836 837 # For each module, create the documentation object and load the chapters in the pymathics part. 838 for pymmodule in default_pymathics_modules: 839 pymathicsdoc = PyMathicsDocumentation(pymmodule) 840 for part in pymathicsdoc.parts: 841 for ch in part.chapters: 842 ch.title = f"{pymmodule} {part.title} {ch.title}" 843 ch.part = pymathicspart 844 pymathicspart.chapters_by_slug[ch.slug] = ch 845 pymathicspart.chapters.append(ch) 846 847 self.pymathics_doc_loaded = True 848 849 850class PyMathicsDocumentation(Documentation): 851 def __init__(self, module=None): 852 self.title = "Overview" 853 self.parts = [] 854 self.parts_by_slug = {} 855 self.doc_dir = None 856 self.xml_data_file = None 857 self.tex_data_file = None 858 self.latex_file = None 859 self.symbols = {} 860 if module is None: 861 return 862 863 import importlib 864 865 # Load the module and verifies it is a pymathics module 866 try: 867 self.pymathicsmodule = importlib.import_module(module) 868 except ImportError: 869 print("Module does not exist") 870 mainfolder = "" 871 self.pymathicsmodule = None 872 self.parts = [] 873 return 874 875 try: 876 mainfolder = self.pymathicsmodule.__path__[0] 877 if "name" in self.pymathicsmodule.pymathics_version_data: 878 self.name = self.version = self.pymathicsmodule.pymathics_version_data[ 879 "name" 880 ] 881 else: 882 self.name = (self.pymathicsmodule.__package__)[10:] 883 self.version = self.pymathicsmodule.pymathics_version_data["version"] 884 self.author = self.pymathicsmodule.pymathics_version_data["author"] 885 except (AttributeError, KeyError, IndexError): 886 print(module + " is not a pymathics module.") 887 mainfolder = "" 888 self.pymathicsmodule = None 889 self.parts = [] 890 return 891 892 # Paths 893 self.doc_dir = self.pymathicsmodule.__path__[0] + "/doc/" 894 self.xml_data_file = self.doc_dir + "xml/data" 895 self.tex_data_file = self.doc_dir + "tex/data" 896 self.latex_file = self.doc_dir + "tex/documentation.tex" 897 898 # Load the dictionary of mathics symbols defined in the module 899 self.symbols = {} 900 from mathics.builtin import is_builtin, Builtin 901 902 print("loading symbols") 903 for name in dir(self.pymathicsmodule): 904 var = getattr(self.pymathicsmodule, name) 905 if ( 906 hasattr(var, "__module__") 907 and var.__module__ != "mathics.builtin.base" 908 and is_builtin(var) 909 and not name.startswith("_") 910 and var.__module__[: len(self.pymathicsmodule.__name__)] 911 == self.pymathicsmodule.__name__ 912 ): # nopep8 913 instance = var(expression=False) 914 if isinstance(instance, Builtin): 915 self.symbols[instance.get_name()] = instance 916 # Defines de default first part, in case we are building an independent documentation module. 917 self.title = "Overview" 918 self.parts = [] 919 self.parts_by_slug = {} 920 try: 921 files = listdir(self.doc_dir) 922 files.sort() 923 except FileNotFoundError: 924 self.doc_dir = "" 925 self.xml_data_file = "" 926 self.tex_data_file = "" 927 self.latex_file = "" 928 files = [] 929 appendix = [] 930 for file in files: 931 part_title = file[2:] 932 if part_title.endswith(".mdoc"): 933 part_title = part_title[: -len(".mdoc")] 934 part = DocPart(self, part_title) 935 text = open(self.doc_dir + file, "rb").read().decode("utf8") 936 text = filter_comments(text) 937 chapters = CHAPTER_RE.findall(text) 938 for title, text in chapters: 939 chapter = DocChapter(part, title) 940 text += '<section title=""></section>' 941 sections = SECTION_RE.findall(text) 942 for pre_text, title, text in sections: 943 if not chapter.doc: 944 chapter.doc = Doc(pre_text) 945 if title: 946 section = DocSection(chapter, title, text) 947 chapter.sections.append(section) 948 part.chapters.append(chapter) 949 if file[0].isdigit(): 950 self.parts.append(part) 951 else: 952 part.is_appendix = True 953 appendix.append(part) 954 955 # Builds the automatic documentation 956 builtin_part = DocPart(self, "Pymathics Modules", is_reference=True) 957 title, text = get_module_doc(self.pymathicsmodule) 958 chapter = DocChapter(builtin_part, title, Doc(text)) 959 for name in self.symbols: 960 instance = self.symbols[name] 961 installed = True 962 for package in getattr(instance, "requires", []): 963 try: 964 importlib.import_module(package) 965 except ImportError: 966 installed = False 967 break 968 section = DocSection( 969 chapter, 970 strip_system_prefix(name), 971 instance.__doc__ or "", 972 operator=instance.get_operator(), 973 installed=installed, 974 ) 975 chapter.sections.append(section) 976 builtin_part.chapters.append(chapter) 977 self.parts.append(builtin_part) 978 # Adds possible appendices 979 for part in appendix: 980 self.parts.append(part) 981 982 # set keys of tests 983 for tests in self.get_tests(): 984 for test in tests.tests: 985 test.key = (tests.part, tests.chapter, tests.section, test.index) 986 987 988class DocPart(DocElement): 989 def __init__(self, doc, title, is_reference=False): 990 self.doc = doc 991 self.title = title 992 self.slug = slugify(title) 993 self.chapters = [] 994 self.chapters_by_slug = {} 995 self.is_reference = is_reference 996 self.is_appendix = False 997 doc.parts_by_slug[self.slug] = self 998 999 def __str__(self): 1000 return "%s\n\n%s" % ( 1001 self.title, 1002 "\n".join(str(chapter) for chapter in self.chapters), 1003 ) 1004 1005 def latex(self, output): 1006 result = "\n\n\\part{%s}\n\n" % escape_latex(self.title) + ( 1007 "\n\n".join(chapter.latex(output) for chapter in self.chapters) 1008 ) 1009 if self.is_reference: 1010 result = "\n\n\\referencestart" + result 1011 return result 1012 1013 def get_url(self): 1014 return f"/{self.slug}/" 1015 1016 def get_collection(self): 1017 return self.doc.parts 1018 1019 1020class DocChapter(DocElement): 1021 def __init__(self, part, title, doc=None): 1022 self.part = part 1023 self.title = title 1024 self.slug = slugify(title) 1025 self.doc = doc 1026 self.sections = [] 1027 self.sections_by_slug = {} 1028 part.chapters_by_slug[self.slug] = self 1029 1030 def __str__(self): 1031 sections = "\n".join(str(section) for section in self.sections) 1032 return f"= {self.title} =\n\n{sections}" 1033 1034 def latex(self, output): 1035 intro = self.doc.latex(output).strip() 1036 if intro: 1037 short = "short" if len(intro) < 300 else "" 1038 intro = "\\begin{chapterintro%s}\n%s\n\n\\end{chapterintro%s}" % ( 1039 short, 1040 intro, 1041 short, 1042 ) 1043 return "".join( 1044 [ 1045 "\n\n\\chapter{%(title)s}\n\\chapterstart\n\n%(intro)s" 1046 % {"title": escape_latex(self.title), "intro": intro}, 1047 "\\chaptersections\n", 1048 "\n\n".join(section.latex(output) for section in self.sections), 1049 "\n\\chapterend\n", 1050 ] 1051 ) 1052 1053 def get_url(self): 1054 return f"/{self.part.slug}/{self.slug}/" 1055 1056 def get_collection(self): 1057 return self.part.chapters 1058 1059 1060class DocSection(DocElement): 1061 def __init__(self, chapter, title, text, operator=None, installed=True): 1062 self.chapter = chapter 1063 self.title = title 1064 self.slug = slugify(title) 1065 if text.count("<dl>") != text.count("</dl>"): 1066 raise ValueError( 1067 "Missing opening or closing <dl> tag in " 1068 "{} documentation".format(title) 1069 ) 1070 self.doc = Doc(text) 1071 self.operator = operator 1072 self.installed = installed 1073 chapter.sections_by_slug[self.slug] = self 1074 1075 def __str__(self): 1076 return f"== {self.title} ==\n{self.doc}" 1077 1078 def latex(self, output): 1079 title = escape_latex(self.title) 1080 if self.operator: 1081 title += " (\\code{%s})" % escape_latex_code(self.operator) 1082 index = ( 1083 "\index{%s}" % escape_latex(self.title) 1084 if self.chapter.part.is_reference 1085 else "" 1086 ) 1087 return ( 1088 "\n\n\\section*{%(title)s}%(index)s\n" 1089 "\\sectionstart\n\n%(content)s\\sectionend" 1090 "\\addcontentsline{toc}{section}{%(title)s}" 1091 ) % {"title": title, "index": index, "content": self.doc.latex(output)} 1092 1093 def get_url(self): 1094 return f"/{self.chapter.part.slug}/{self.chapter.slug}/{self.slug}/" 1095 1096 def get_collection(self): 1097 return self.chapter.sections 1098 1099 def html_data(self): 1100 indices = set() 1101 for test in self.doc.items: 1102 indices.update(test.test_indices()) 1103 result = {} 1104 for index in indices: 1105 result[index] = xml_data.get( 1106 (self.chapter.part.title, self.chapter.title, self.title, index) 1107 ) 1108 return result 1109 1110 1111class Doc(object): 1112 def __init__(self, doc): 1113 self.items = [] 1114 # remove commented lines 1115 doc = filter_comments(doc) 1116 # pre-substitute Python code because it might contain tests 1117 doc, post_substitutions = pre_sub( 1118 PYTHON_RE, doc, lambda m: "<python>%s</python>" % m.group(1) 1119 ) 1120 # HACK: Artificially construct a last testcase to get the "intertext" 1121 # after the last (real) testcase. Ignore the test, of course. 1122 doc += "\n>> test\n = test" 1123 testcases = TESTCASE_RE.findall(doc) 1124 tests = None 1125 for index in range(len(testcases)): 1126 testcase = list(testcases[index]) 1127 text = testcase.pop(0).strip() 1128 if text: 1129 if tests is not None: 1130 self.items.append(tests) 1131 tests = None 1132 text = post_sub(text, post_substitutions) 1133 self.items.append(DocText(text)) 1134 tests = None 1135 if index < len(testcases) - 1: 1136 test = DocTest(index, testcase) 1137 if tests is None: 1138 tests = DocTests() 1139 tests.tests.append(test) 1140 if tests is not None: 1141 self.items.append(tests) 1142 tests = None 1143 1144 def __str__(self): 1145 return "\n".join(str(item) for item in self.items) 1146 1147 def text(self, detail_level): 1148 # used for introspection 1149 # TODO parse XML and pretty print 1150 # HACK 1151 item = str(self.items[0]) 1152 item = "\n".join(line.strip() for line in item.split("\n")) 1153 item = item.replace("<dl>", "") 1154 item = item.replace("</dl>", "") 1155 item = item.replace("<dt>", " ") 1156 item = item.replace("</dt>", "") 1157 item = item.replace("<dd>", " ") 1158 item = item.replace("</dd>", "") 1159 item = "\n".join(line for line in item.split("\n") if not line.isspace()) 1160 return item 1161 1162 def get_tests(self): 1163 tests = [] 1164 for item in self.items: 1165 tests.extend(item.get_tests()) 1166 return tests 1167 1168 def latex(self, output): 1169 return "\n".join( 1170 item.latex(output) for item in self.items if not item.is_private() 1171 ) 1172 1173 def html(self): 1174 counters = {} 1175 return mark_safe( 1176 "\n".join( 1177 item.html(counters) for item in self.items if not item.is_private() 1178 ) 1179 ) 1180 1181 1182class DocText(object): 1183 def __init__(self, text): 1184 self.text = text 1185 1186 def get_tests(self): 1187 return [] 1188 1189 def is_private(self): 1190 return False 1191 1192 def __str__(self): 1193 return self.text 1194 1195 def latex(self, output): 1196 return escape_latex(self.text) 1197 1198 def html(self, counters=None): 1199 result = escape_html(self.text, counters=counters) 1200 return result 1201 1202 def test_indices(self): 1203 return [] 1204 1205 1206class DocTests(object): 1207 def __init__(self): 1208 self.tests = [] 1209 1210 def get_tests(self): 1211 return self.tests 1212 1213 def is_private(self): 1214 return all(test.private for test in self.tests) 1215 1216 def __str__(self): 1217 return "\n".join(str(test) for test in self.tests) 1218 1219 def latex(self, output): 1220 if len(self.tests) == 0: 1221 return "\n" 1222 1223 testLatexStrings = [ 1224 test.latex(output) for test in self.tests if not test.private 1225 ] 1226 testLatexStrings = [t for t in testLatexStrings if len(t) > 1] 1227 if len(testLatexStrings) == 0: 1228 return "\n" 1229 1230 return "\\begin{tests}%%\n%s%%\n\\end{tests}" % ("%\n".join(testLatexStrings)) 1231 1232 def html(self, counters=None): 1233 if len(self.tests) == 0: 1234 return "\n" 1235 return '<ul class="tests">%s</ul>' % ( 1236 "\n".join( 1237 "<li>%s</li>" % test.html() for test in self.tests if not test.private 1238 ) 1239 ) 1240 1241 def test_indices(self): 1242 return [test.index for test in self.tests] 1243 1244 1245# This string is used so we can indicate a trailing blank at the end of a line by 1246# adding this string to the end of the line which gets stripped off. 1247# Some editors and formatters like to strip off trailing blanks at the ends of lines. 1248END_LINE_SENTINAL = "#<--#" 1249 1250 1251class DocTest(object): 1252 """ 1253 DocTest formatting rules: 1254 1255 * `>>` Marks test case; it will also appear as part of 1256 the documentation. 1257 * `#>` Marks test private or one that does not appear as part of 1258 the documentation. 1259 * `X>` Shows the example in the docs, but disables testing the example. 1260 * `S>` Shows the example in the docs, but disables testing if environment 1261 variable SANDBOX is set. 1262 * `=` Compares the result text. 1263 * `:` Compares an (error) message. 1264 `|` Prints output. 1265 """ 1266 1267 def __init__(self, index, testcase): 1268 def strip_sentinal(line): 1269 """Remove END_LINE_SENTINAL from the end of a line if it appears. 1270 1271 Some editors like to strip blanks at the end of a line. 1272 Since the line ends in END_LINE_SENTINAL which isn't blank, 1273 any blanks that appear before will be preserved. 1274 1275 Some tests require some lines to be blank or entry because 1276 Mathics output can be that way 1277 """ 1278 if line.endswith(END_LINE_SENTINAL): 1279 line = line[: -len(END_LINE_SENTINAL)] 1280 1281 # Also remove any remaining trailing blanks since that 1282 # seems *also* what we want to do. 1283 return line.strip() 1284 1285 self.index = index 1286 self.result = None 1287 self.outs = [] 1288 1289 # Private test cases are executed, but NOT shown as part of the docs 1290 self.private = testcase[0] == "#" 1291 1292 # Ignored test cases are NOT executed, but shown as part of the docs 1293 # Sandboxed test cases are NOT executed if environtment SANDBOX is set 1294 if testcase[0] == "X" or (testcase[0] == "S" and getenv("SANDBOX", False)): 1295 self.ignore = True 1296 # substitute '>' again so we get the correct formatting 1297 testcase[0] = ">" 1298 else: 1299 self.ignore = False 1300 1301 self.test = strip_sentinal(testcase[1]) 1302 1303 self.key = None 1304 outs = testcase[2].splitlines() 1305 for line in outs: 1306 line = strip_sentinal(line) 1307 if line: 1308 if line.startswith("."): 1309 text = line[1:] 1310 if text.startswith(" "): 1311 text = text[1:] 1312 text = "\n" + text 1313 if self.result is not None: 1314 self.result += text 1315 elif self.outs: 1316 self.outs[-1].text += text 1317 continue 1318 1319 match = TESTCASE_OUT_RE.match(line) 1320 symbol, text = match.group(1), match.group(2) 1321 text = text.strip() 1322 if symbol == "=": 1323 self.result = text 1324 elif symbol == ":": 1325 out = Message("", "", text) 1326 self.outs.append(out) 1327 elif symbol == "|": 1328 out = Print(text) 1329 self.outs.append(out) 1330 1331 def __str__(self): 1332 return self.test 1333 1334 def latex(self, output): 1335 text = "" 1336 text += "\\begin{testcase}\n" 1337 text += "\\test{%s}\n" % escape_latex_code(self.test) 1338 if self.key is None: 1339 return "" 1340 results = output[self.key]["results"] 1341 for result in results: 1342 for out in result["out"]: 1343 kind = "message" if out["message"] else "print" 1344 text += "\\begin{test%s}%s\\end{test%s}" % ( 1345 kind, 1346 escape_latex_output(out["text"]), 1347 kind, 1348 ) 1349 if result["result"]: # is not None and result['result'].strip(): 1350 text += "\\begin{testresult}%s\\end{testresult}" % result["result"] 1351 text += "\\end{testcase}" 1352 return text 1353 1354 def html(self): 1355 result = '<div class="test"><span class="move"></span>' 1356 result += '<ul class="test" id="test_%d">' % self.index 1357 result += '<li class="test">%s</li>' % escape_html(self.test, True) 1358 result += "</ul>" 1359 result += "</div>" 1360 return result 1361