1# -*- coding: utf-8 -*- 2 3from html import escape as html_escape 4import re 5from os import getenv, listdir 6import pickle 7import importlib 8import pkgutil 9 10from mathics import settings 11 12from mathics import builtin 13from mathics.builtin import get_module_doc 14from mathics.core.evaluation import Message, Print 15from mathics.doc.utils import slugify 16 17CHAPTER_RE = re.compile('(?s)<chapter title="(.*?)">(.*?)</chapter>') 18SECTION_RE = re.compile('(?s)(.*?)<section title="(.*?)">(.*?)</section>') 19SUBSECTION_RE = re.compile('(?s)<subsection title="(.*?)">') 20SUBSECTION_END_RE = re.compile("</subsection>") 21 22TESTCASE_RE = re.compile( 23 r"""(?mx)^ 24 ((?:.|\n)*?) 25 ^\s*([>#SX])>[ ](.*) 26 ((?:\n\s*(?:[:|=.][ ]|\.).*)*) 27""" 28) 29TESTCASE_OUT_RE = re.compile(r"^\s*([:|=])(.*)$") 30 31MATHICS_RE = re.compile(r"(?<!\\)\'(.*?)(?<!\\)\'") 32 33# preserve space before and after inline code variables 34LATEX_RE = re.compile(r"(\s?)\$(\w+?)\$(\s?)") 35 36DL_RE = re.compile(r"(?s)<dl>(.*?)</dl>") 37DL_ITEM_RE = re.compile( 38 r"(?s)<(?P<tag>d[td])>(?P<content>.*?)(?:</(?P=tag)>|)\s*(?:(?=<d[td]>)|$)" 39) 40LIST_RE = re.compile(r"(?s)<(?P<tag>ul|ol)>(?P<content>.*?)</(?P=tag)>") 41LIST_ITEM_RE = re.compile(r"(?s)<li>(.*?)(?:</li>|(?=<li>)|$)") 42CONSOLE_RE = re.compile(r"(?s)<(?P<tag>con|console)>(?P<content>.*?)</(?P=tag)>") 43ITALIC_RE = re.compile(r"(?s)<(?P<tag>i)>(?P<content>.*?)</(?P=tag)>") 44IMG_RE = re.compile( 45 r'<img src="(?P<src>.*?)" title="(?P<title>.*?)" label="(?P<label>.*?)">' 46) 47IMG_PNG_RE = re.compile( 48 r'<imgpng src="(?P<src>.*?)" title="(?P<title>.*?)" label="(?P<label>.*?)">' 49) 50REF_RE = re.compile(r'<ref label="(?P<label>.*?)">') 51PYTHON_RE = re.compile(r"(?s)<python>(.*?)</python>") 52LATEX_CHAR_RE = re.compile(r"(?<!\\)(\^)") 53 54QUOTATIONS_RE = re.compile(r"\"([\w\s,]*?)\"") 55HYPERTEXT_RE = re.compile(r"(?s)<(?P<tag>em|url)>(?P<content>.*?)</(?P=tag)>") 56 57OUTSIDE_ASY_RE = re.compile(r"(?s)((?:^|\\end\{asy\}).*?(?:$|\\begin\{asy\}))") 58LATEX_TEXT_RE = re.compile( 59 r"(?s)\\text\{([^{}]*?(?:[^{}]*?\{[^{}]*?(?:[^{}]*?\{[^{}]*?\}[^{}]*?)*?" 60 r"[^{}]*?\}[^{}]*?)*?[^{}]*?)\}" 61) 62LATEX_TESTOUT_RE = re.compile( 63 r"(?s)\\begin\{(?P<tag>testmessage|testprint|testresult)\}" 64 r"(?P<content>.*?)\\end\{(?P=tag)\}" 65) 66LATEX_TESTOUT_DELIM_RE = re.compile(r",") 67NUMBER_RE = re.compile(r"(\d*(?<!\.)\.\d+|\d+\.(?!\.)\d*|\d+)") 68LATEX_ARRAY_RE = re.compile( 69 r"(?s)\\begin\{testresult\}\\begin\{array\}\{l\}(.*?)" 70 r"\\end\{array\}\\end\{testresult\}" 71) 72LATEX_INLINE_END_RE = re.compile(r"(?s)(?P<all>\\lstinline'[^']*?'\}?[.,;:])") 73LATEX_CONSOLE_RE = re.compile(r"\\console\{(.*?)\}") 74 75ALLOWED_TAGS = ( 76 "dl", 77 "dd", 78 "dt", 79 "em", 80 "url", 81 "ul", 82 "i", 83 "ol", 84 "li", 85 "con", 86 "console", 87 "img", 88 "imgpng", 89 "ref", 90 "subsection", 91) 92ALLOWED_TAGS_RE = dict( 93 (allowed, re.compile("<(%s.*?)>" % allowed)) for allowed in ALLOWED_TAGS 94) 95 96SPECIAL_COMMANDS = { 97 "LaTeX": (r"<em>LaTeX</em>", r"\LaTeX{}"), 98 "Mathematica": ( 99 r"<em>Mathematica</em>®", 100 r"\emph{Mathematica}\textregistered{}", 101 ), 102 "Mathics": (r"<em>Mathics</em>", r"\emph{Mathics}"), 103 "Sage": (r"<em>Sage</em>", r"\emph{Sage}"), 104 "Wolfram": (r"<em>Wolfram</em>", r"\emph{Wolfram}"), 105 "skip": (r"<br /><br />", r"\bigskip"), 106} 107 108try: 109 with open(settings.DOC_XML_DATA, "rb") as xml_data_file: 110 xml_data = pickle.load(xml_data_file) 111except IOError: 112 xml_data = {} 113 114def get_submodule_names(object): 115 modpkgs = [] 116 if hasattr(object, '__path__'): 117 for importer, modname, ispkg in pkgutil.iter_modules(object.__path__): 118 modpkgs.append(modname) 119 modpkgs.sort() 120 return modpkgs 121 122 123 124def filter_comments(doc): 125 return "\n".join( 126 line for line in doc.splitlines() if not line.lstrip().startswith("##") 127 ) 128 129 130def strip_system_prefix(name): 131 if name.startswith("System`"): 132 stripped_name = name[len("System`") :] 133 # don't return Private`sym for System`Private`sym 134 if "`" not in stripped_name: 135 return stripped_name 136 return name 137 138 139def get_latex_escape_char(text): 140 for escape_char in ("'", "~", "@"): 141 if escape_char not in text: 142 return escape_char 143 raise ValueError 144 145 146def _replace_all(text, pairs): 147 for (i, j) in pairs: 148 text = text.replace(i, j) 149 return text 150 151 152def escape_latex_output(text): 153 " Escape Mathics output " 154 155 text = _replace_all( 156 text, 157 [ 158 ("\\", "\\\\"), 159 ("{", "\\{"), 160 ("}", "\\}"), 161 ("~", "\\~"), 162 ("&", "\\&"), 163 ("%", "\\%"), 164 ("$", r"\$"), 165 ("_", "\\_"), 166 ], 167 ) 168 return text 169 170 171def escape_latex_code(text): 172 " Escape verbatim Mathics input " 173 174 text = escape_latex_output(text) 175 escape_char = get_latex_escape_char(text) 176 return "\\lstinline%s%s%s" % (escape_char, text, escape_char) 177 178 179def escape_latex(text): 180 " Escape documentation text " 181 182 def repl_python(match): 183 return ( 184 r"""\begin{lstlisting}[style=python] 185%s 186\end{lstlisting}""" 187 % match.group(1).strip() 188 ) 189 190 text, post_substitutions = pre_sub(PYTHON_RE, text, repl_python) 191 192 text = _replace_all( 193 text, 194 [ 195 ("\\", "\\\\"), 196 ("{", "\\{"), 197 ("}", "\\}"), 198 ("~", "\\~{ }"), 199 ("&", "\\&"), 200 ("%", "\\%"), 201 ("#", "\\#"), 202 ], 203 ) 204 205 def repl(match): 206 text = match.group(1) 207 if text: 208 text = _replace_all(text, [("\\'", "'"), ("^", "\\^")]) 209 escape_char = get_latex_escape_char(text) 210 text = LATEX_RE.sub( 211 lambda m: "%s%s\\codevar{\\textit{%s}}%s\\lstinline%s" 212 % (escape_char, m.group(1), m.group(2), m.group(3), escape_char), 213 text, 214 ) 215 if text.startswith(" "): 216 text = r"\ " + text[1:] 217 if text.endswith(" "): 218 text = text[:-1] + r"\ " 219 return "\\code{\\lstinline%s%s%s}" % (escape_char, text, escape_char) 220 else: 221 # treat double '' literaly 222 return "''" 223 224 text = MATHICS_RE.sub(repl, text) 225 226 text = LATEX_RE.sub( 227 lambda m: "%s\\textit{%s}%s" % (m.group(1), m.group(2), m.group(3)), text 228 ) 229 230 text = text.replace("\\\\'", "'") 231 232 def repl_dl(match): 233 text = match.group(1) 234 text = DL_ITEM_RE.sub( 235 lambda m: "\\%(tag)s{%(content)s}\n" % m.groupdict(), text 236 ) 237 return "\\begin{definitions}%s\\end{definitions}" % text 238 239 text = DL_RE.sub(repl_dl, text) 240 241 def repl_list(match): 242 tag = match.group("tag") 243 content = match.group("content") 244 content = LIST_ITEM_RE.sub(lambda m: "\\item %s\n" % m.group(1), content) 245 env = "itemize" if tag == "ul" else "enumerate" 246 return "\\begin{%s}%s\\end{%s}" % (env, content, env) 247 248 text = LIST_RE.sub(repl_list, text) 249 250 text = _replace_all( 251 text, 252 [ 253 ("$", r"\$"), 254 ("\u22bc", "nand"), # \barwedge isn't working 255 ("\u22bd", "nor"), # \vebarr isn't working 256 ("\u03c0", r"$\pi$"), 257 ("\u2265", r"$\ge$"), 258 ("\u2264", r"$\le$"), 259 ("\u2260", r"$\ne$"), 260 ("\u00e7", r"\c{c}"), 261 ("\u00e9", r"\'e"), 262 ("\u00ea", r"\^e"), 263 ("\00f1", r"\~n"), 264 ("\u222b", r"\int"), 265 ("\uf74c", r"d"), 266 ], 267 ) 268 269 def repl_char(match): 270 char = match.group(1) 271 return { 272 "^": "$^\wedge$", 273 }[char] 274 275 text = LATEX_CHAR_RE.sub(repl_char, text) 276 277 def repl_img(match): 278 src = match.group("src") 279 title = match.group("title") 280 label = match.group("label") 281 return r"""\begin{figure*}[htp] 282\centering 283\includegraphics[width=\textwidth]{images/%(src)s} 284\caption{%(title)s} 285\label{%(label)s} 286\end{figure*}""" % { 287 "src": src, 288 "title": title, 289 "label": label, 290 } 291 292 text = IMG_RE.sub(repl_img, text) 293 294 def repl_imgpng(match): 295 src = match.group("src") 296 return r"\includegraphics[scale=1.0]{images/%(src)s}" % {"src": src} 297 298 text = IMG_PNG_RE.sub(repl_imgpng, text) 299 300 def repl_ref(match): 301 return r"figure \ref{%s}" % match.group("label") 302 303 text = REF_RE.sub(repl_ref, text) 304 305 def repl_quotation(match): 306 return r"``%s''" % match.group(1) 307 308 def repl_hypertext(match): 309 tag = match.group("tag") 310 content = match.group("content") 311 if tag == "em": 312 return r"\emph{%s}" % content 313 elif tag == "url": 314 return "\\url{%s}" % content 315 316 text = QUOTATIONS_RE.sub(repl_quotation, text) 317 text = HYPERTEXT_RE.sub(repl_hypertext, text) 318 319 def repl_console(match): 320 tag = match.group("tag") 321 content = match.group("content") 322 content = content.strip() 323 content = content.replace(r"\$", "$") 324 if tag == "con": 325 return "\\console{%s}" % content 326 else: 327 return "\\begin{lstlisting}\n%s\n\\end{lstlisting}" % content 328 329 text = CONSOLE_RE.sub(repl_console, text) 330 331 def repl_italic(match): 332 content = match.group("content") 333 return "\\emph{%s}" % content 334 335 text = ITALIC_RE.sub(repl_italic, text) 336 337 '''def repl_asy(match): 338 """ 339 Ensure \begin{asy} and \end{asy} are on their own line, 340 but there shall be no extra empty lines 341 """ 342 #tag = match.group(1) 343 #return '\n%s\n' % tag 344 #print "replace" 345 return '\\end{asy}\n\\begin{asy}' 346 text = LATEX_BETWEEN_ASY_RE.sub(repl_asy, text)''' 347 348 def repl_subsection(match): 349 return "\n\\subsection*{%s}\n" % match.group(1) 350 351 text = SUBSECTION_RE.sub(repl_subsection, text) 352 text = SUBSECTION_END_RE.sub("", text) 353 354 for key, (xml, tex) in SPECIAL_COMMANDS.items(): 355 # "\" has been escaped already => 2 \ 356 text = text.replace("\\\\" + key, tex) 357 358 text = post_sub(text, post_substitutions) 359 360 return text 361 362 363def post_process_latex(result): 364 """ 365 Some post-processing hacks of generated LaTeX code to handle linebreaks 366 """ 367 368 WORD_SPLIT_RE = re.compile(r"(\s+|\\newline\s*)") 369 370 def wrap_word(word): 371 if word.strip() == r"\newline": 372 return word 373 return r"\text{%s}" % word 374 375 def repl_text(match): 376 text = match.group(1) 377 if not text: 378 return r"\text{}" 379 words = WORD_SPLIT_RE.split(text) 380 assert len(words) >= 1 381 if len(words) > 1: 382 text = "" 383 index = 0 384 while index < len(words) - 1: 385 text += "%s%s\\allowbreak{}" % ( 386 wrap_word(words[index]), 387 wrap_word(words[index + 1]), 388 ) 389 index += 2 390 text += wrap_word(words[-1]) 391 else: 392 text = r"\text{%s}" % words[0] 393 if not text: 394 return r"\text{}" 395 text = text.replace("><", r">}\allowbreak\text{<") 396 return text 397 398 def repl_out_delim(match): 399 return ",\\allowbreak{}" 400 401 def repl_number(match): 402 guard = r"\allowbreak{}" 403 inter_groups_pre = r"\,\discretionary{\~{}}{\~{}}{}" 404 inter_groups_post = r"\discretionary{\~{}}{\~{}}{}" 405 number = match.group(1) 406 parts = number.split(".") 407 if len(number) <= 3: 408 return number 409 assert 1 <= len(parts) <= 2 410 pre_dec = parts[0] 411 groups = [] 412 while pre_dec: 413 groups.append(pre_dec[-3:]) 414 pre_dec = pre_dec[:-3] 415 pre_dec = inter_groups_pre.join(reversed(groups)) 416 if len(parts) == 2: 417 post_dec = parts[1] 418 groups = [] 419 while post_dec: 420 groups.append(post_dec[:3]) 421 post_dec = post_dec[3:] 422 post_dec = inter_groups_post.join(groups) 423 result = pre_dec + "." + post_dec 424 else: 425 result = pre_dec 426 return guard + result + guard 427 428 def repl_array(match): 429 content = match.group(1) 430 lines = content.split("\\\\") 431 content = "".join( 432 r"\begin{dmath*}%s\end{dmath*}" % line for line in lines if line.strip() 433 ) 434 return r"\begin{testresultlist}%s\end{testresultlist}" % content 435 436 def repl_out(match): 437 tag = match.group("tag") 438 content = match.group("content") 439 content = LATEX_TESTOUT_DELIM_RE.sub(repl_out_delim, content) 440 content = NUMBER_RE.sub(repl_number, content) 441 content = content.replace(r"\left[", r"\left[\allowbreak{}") 442 return "\\begin{%s}%s\\end{%s}" % (tag, content, tag) 443 444 def repl_inline_end(match): 445 " Prevent linebreaks between inline code and sentence delimeters " 446 447 code = match.group("all") 448 if code[-2] == "}": 449 code = code[:-2] + code[-1] + code[-2] 450 return r"\mbox{%s}" % code 451 452 def repl_console(match): 453 code = match.group(1) 454 code = code.replace("/", r"/\allowbreak{}") 455 return r"\console{%s}" % code 456 457 def repl_nonasy(match): 458 result = match.group(1) 459 result = LATEX_TEXT_RE.sub(repl_text, result) 460 result = LATEX_TESTOUT_RE.sub(repl_out, result) 461 result = LATEX_ARRAY_RE.sub(repl_array, result) 462 result = LATEX_INLINE_END_RE.sub(repl_inline_end, result) 463 result = LATEX_CONSOLE_RE.sub(repl_console, result) 464 return result 465 466 return OUTSIDE_ASY_RE.sub(repl_nonasy, result) 467 468 469POST_SUBSTITUTION_TAG = "_POST_SUBSTITUTION%d_" 470 471 472def pre_sub(re, text, repl_func): 473 post_substitutions = [] 474 475 def repl_pre(match): 476 repl = repl_func(match) 477 index = len(post_substitutions) 478 post_substitutions.append(repl) 479 return POST_SUBSTITUTION_TAG % index 480 481 text = re.sub(repl_pre, text) 482 483 return text, post_substitutions 484 485 486def post_sub(text, post_substitutions): 487 for index, sub in enumerate(post_substitutions): 488 text = text.replace(POST_SUBSTITUTION_TAG % index, sub) 489 return text 490 491 492# FIXME: can we replace this with Python 3's html.escape ? 493def escape_html(text, verbatim_mode=False, counters=None, single_line=False): 494 def repl_python(match): 495 return ( 496 r"""<pre><![CDATA[ 497%s 498]]></pre>""" 499 % match.group(1).strip() 500 ) 501 502 text, post_substitutions = pre_sub(PYTHON_RE, text, repl_python) 503 504 text = text.replace("&", "&").replace("<", "<").replace(">", ">") 505 506 if not verbatim_mode: 507 508 def repl_quotation(match): 509 return r"“%s”" % match.group(1) 510 511 text = QUOTATIONS_RE.sub(repl_quotation, text) 512 513 if counters is None: 514 counters = {} 515 516 text = text.replace('"', """) 517 if not verbatim_mode: 518 519 def repl_latex(match): 520 return "%s<var>%s</var>%s" % ( 521 match.group(1), 522 match.group(2), 523 match.group(3), 524 ) 525 526 text = LATEX_RE.sub(repl_latex, text) 527 528 def repl_mathics(match): 529 text = match.group(1) 530 text = text.replace("\\'", "'") 531 text = text.replace(" ", " ") 532 if text: 533 return "<code>%s</code>" % text 534 else: 535 return "'" 536 537 def repl_allowed(match): 538 content = _replace_all( 539 match.group(1), [("“", '"'), ("”", '"'), (""", '"')] 540 ) 541 return "<%s>" % content 542 543 text = MATHICS_RE.sub(repl_mathics, text) 544 for allowed in ALLOWED_TAGS: 545 text = ALLOWED_TAGS_RE[allowed].sub(repl_allowed, text) 546 text = text.replace("</%s>" % allowed, "</%s>" % allowed) 547 548 def repl_dl(match): 549 text = match.group(1) 550 text = DL_ITEM_RE.sub( 551 lambda m: "<%(tag)s>%(content)s</%(tag)s>\n" % m.groupdict(), text 552 ) 553 return "<dl>%s</dl>" % text 554 555 text = DL_RE.sub(repl_dl, text) 556 557 def repl_list(match): 558 tag = match.group("tag") 559 content = match.group("content") 560 content = LIST_ITEM_RE.sub(lambda m: "<li>%s</li>" % m.group(1), content) 561 return "<%s>%s</%s>" % (tag, content, tag) 562 563 text = LIST_RE.sub(repl_list, text) 564 565 def repl_hypertext(match): 566 tag = match.group("tag") 567 content = match.group("content") 568 if tag == "em": 569 return r"<em>%s</em>" % content 570 elif tag == "url": 571 return r'<a href="%s">%s</a>' % (content, content) 572 573 text = HYPERTEXT_RE.sub(repl_hypertext, text) 574 575 def repl_console(match): 576 tag = match.group("tag") 577 content = match.group("content") 578 tag = "div" if tag == "console" else "span" 579 content = content.strip() 580 pre = post = "" 581 582 # gets replaced for <br /> later by DocText.html() 583 content = content.replace("\n", "<br>") 584 585 return r'<%s class="console">%s%s%s</%s>' % (tag, pre, content, post, tag) 586 587 text = CONSOLE_RE.sub(repl_console, text) 588 589 def repl_img(match): 590 src = match.group("src") 591 title = match.group("title") 592 return ( 593 r'<a href="/media/doc/%(src)s.pdf">' 594 r'<img src="/media/doc/%(src)s.png" title="%(title)s" />' 595 r"</a>" 596 ) % {"src": src, "title": title} 597 598 text = IMG_RE.sub(repl_img, text) 599 600 def repl_imgpng(match): 601 src = match.group("src") 602 title = match.group("title") 603 return (r'<img src="/media/doc/%(src)s" title="%(title)s" />') % { 604 "src": src, 605 "title": title, 606 } 607 608 text = IMG_PNG_RE.sub(repl_imgpng, text) 609 610 def repl_ref(match): 611 # TODO: this is not an optimal solution - maybe we need figure 612 # numbers in the XML doc as well? 613 return r"the following figure" 614 615 text = REF_RE.sub(repl_ref, text) 616 617 def repl_subsection(match): 618 return '\n<h2 label="%s">%s</h2>\n' % (match.group(1), match.group(1)) 619 620 text = SUBSECTION_RE.sub(repl_subsection, text) 621 text = SUBSECTION_END_RE.sub("", text) 622 623 text = text.replace("\\'", "'") 624 else: 625 text = text.replace(" ", " ") 626 text = "<code>%s</code>" % text 627 text = text.replace("'", "'") 628 text = text.replace("---", "—") 629 for key, (xml, tex) in SPECIAL_COMMANDS.items(): 630 text = text.replace("\\" + key, xml) 631 632 if not single_line: 633 # FIXME: linebreaks() is not defined 634 # text = linebreaks(text) 635 text = text.replace("<br />", "\n").replace("<br>", "<br />") 636 637 text = post_sub(text, post_substitutions) 638 639 text = text.replace("<p><pre>", "<pre>").replace("</pre></p>", "</pre>") 640 641 return text 642 643 644class Tests(object): 645 def __init__(self, part, chapter, section, tests): 646 self.part, self.chapter = part, chapter 647 self.section, self.tests = section, tests 648 649 650class DocElement(object): 651 def href(self, ajax=False): 652 if ajax: 653 return "javascript:loadDoc('%s')" % self.get_url() 654 else: 655 return "/doc%s" % self.get_url() 656 657 def get_prev(self): 658 return self.get_prev_next()[0] 659 660 def get_next(self): 661 return self.get_prev_next()[1] 662 663 def get_collection(self): 664 return [] 665 666 def get_prev_next(self): 667 collection = self.get_collection() 668 index = collection.index(self) 669 prev = collection[index - 1] if index > 0 else None 670 next = collection[index + 1] if index < len(collection) - 1 else None 671 return prev, next 672 673 def get_title_html(self): 674 return escape_html(self.title, single_line=True) 675 676 677class Documentation(DocElement): 678 def __str__(self): 679 return "\n\n\n".join(str(part) for part in self.parts) 680 681 def get_tests(self): 682 for part in self.parts: 683 for chapter in part.chapters: 684 tests = chapter.doc.get_tests() 685 if tests: 686 yield Tests(part.title, chapter.title, "", tests) 687 for section in chapter.sections: 688 if section.installed: 689 tests = section.doc.get_tests() 690 if tests: 691 yield Tests(part.title, chapter.title, section.title, tests) 692 693 def get_part(self, part_slug): 694 return self.parts_by_slug.get(part_slug) 695 696 def get_chapter(self, part_slug, chapter_slug): 697 part = self.parts_by_slug.get(part_slug) 698 if part: 699 return part.chapters_by_slug.get(chapter_slug) 700 return None 701 """for part in self.parts: 702 if part.slug == part_slug: 703 for chapter in self: 704 pass""" 705 706 def get_section(self, part_slug, chapter_slug, section_slug): 707 part = self.parts_by_slug.get(part_slug) 708 if part: 709 chapter = part.chapters_by_slug.get(chapter_slug) 710 if chapter: 711 return chapter.sections_by_slug.get(section_slug) 712 return None 713 714 def latex(self, output): 715 parts = [] 716 appendix = False 717 for part in self.parts: 718 text = part.latex(output) 719 if part.is_appendix and not appendix: 720 appendix = True 721 text = "\n\\appendix\n" + text 722 parts.append(text) 723 result = "\n\n".join(parts) 724 result = post_process_latex(result) 725 return result 726 727 def get_url(self): 728 return "/" 729 730 def search(self, query): 731 query = query.strip() 732 query_parts = [q.strip().lower() for q in query.split()] 733 734 def matches(text): 735 text = text.lower() 736 return all(q in text for q in query_parts) 737 738 result = [] 739 for part in self.parts: 740 if matches(part.title): 741 result.append((False, part)) 742 for chapter in part.chapters: 743 if matches(chapter.title): 744 result.append((False, chapter)) 745 for section in chapter.sections: 746 if matches(section.title): 747 result.append((section.title == query, section)) 748 elif query == section.operator: 749 result.append((True, section)) 750 return result 751 752 753class MathicsMainDocumentation(Documentation): 754 def __init__(self): 755 self.title = "Overview" 756 self.parts = [] 757 self.parts_by_slug = {} 758 self.doc_dir = settings.DOC_DIR 759 self.xml_data_file = settings.DOC_XML_DATA 760 self.tex_data_file = settings.DOC_TEX_DATA 761 self.latex_file = settings.DOC_LATEX_FILE 762 self.pymathics_doc_loaded = False 763 files = listdir(self.doc_dir) 764 files.sort() 765 appendix = [] 766 767 for file in files: 768 part_title = file[2:] 769 if part_title.endswith(".mdoc"): 770 part_title = part_title[: -len(".mdoc")] 771 part = DocPart(self, part_title) 772 text = open(self.doc_dir + file, "rb").read().decode("utf8") 773 text = filter_comments(text) 774 chapters = CHAPTER_RE.findall(text) 775 for title, text in chapters: 776 chapter = DocChapter(part, title) 777 text += '<section title=""></section>' 778 sections = SECTION_RE.findall(text) 779 for pre_text, title, text in sections: 780 if not chapter.doc: 781 chapter.doc = Doc(pre_text) 782 if title: 783 section = DocSection(chapter, title, text) 784 chapter.sections.append(section) 785 part.chapters.append(chapter) 786 if file[0].isdigit(): 787 self.parts.append(part) 788 else: 789 part.is_appendix = True 790 appendix.append(part) 791 792 for title, modules, builtins_by_module, start in [ 793 ( 794 "Reference of Built-in Symbols", 795 builtin.modules, 796 builtin.builtins_by_module, 797 True, 798 ) 799 ]: # nopep8 800 # ("Reference of optional symbols", optional.modules, 801 # optional.optional_builtins_by_module, False)]: 802 803 builtin_part = DocPart(self, title, is_reference=start) 804 for module in modules: 805 title, text = get_module_doc(module) 806 chapter = DocChapter(builtin_part, title, Doc(text)) 807 builtins = builtins_by_module[module.__name__] 808 809 if module.__file__.endswith("__init__.py"): 810 section_names = get_submodule_names(module) 811 else: 812 section_names = builtins 813 814 for instance in section_names: 815 installed = True 816 for package in getattr(instance, "requires", []): 817 try: 818 importlib.import_module(package) 819 except ImportError: 820 installed = False 821 break 822 if isinstance(instance, str): 823 section = DocSection( 824 chapter, 825 instance, 826 "", 827 None, 828 installed=installed, 829 ) 830 else: 831 section = DocSection( 832 chapter, 833 strip_system_prefix(instance.get_name()), 834 instance.__doc__ or "", 835 operator=instance.get_operator(), 836 installed=installed, 837 ) 838 chapter.sections.append(section) 839 builtin_part.chapters.append(chapter) 840 self.parts.append(builtin_part) 841 842 for part in appendix: 843 self.parts.append(part) 844 845 # set keys of tests 846 for tests in self.get_tests(): 847 for test in tests.tests: 848 test.key = (tests.part, tests.chapter, tests.section, test.index) 849 850 def load_pymathics_doc(self): 851 if self.pymathics_doc_loaded: 852 return 853 from mathics.settings import default_pymathics_modules 854 855 pymathicspart = None 856 # Look the "Pymathics Modules" part, and if it does not exist, create it. 857 for part in self.parts: 858 if part.title == "Pymathics Modules": 859 pymathicspart = part 860 if pymathicspart is None: 861 pymathicspart = DocPart(self, "Pymathics Modules", is_reference=True) 862 self.parts.append(pymathicspart) 863 864 # For each module, create the documentation object and load the chapters in the pymathics part. 865 for pymmodule in default_pymathics_modules: 866 pymathicsdoc = PyMathicsDocumentation(pymmodule) 867 for part in pymathicsdoc.parts: 868 for ch in part.chapters: 869 ch.title = f"{pymmodule} {part.title} {ch.title}" 870 ch.part = pymathicspart 871 pymathicspart.chapters_by_slug[ch.slug] = ch 872 pymathicspart.chapters.append(ch) 873 874 self.pymathics_doc_loaded = True 875 876 877class PyMathicsDocumentation(Documentation): 878 def __init__(self, module=None): 879 self.title = "Overview" 880 self.parts = [] 881 self.parts_by_slug = {} 882 self.doc_dir = None 883 self.xml_data_file = None 884 self.tex_data_file = None 885 self.latex_file = None 886 self.symbols = {} 887 if module is None: 888 return 889 890 import importlib 891 892 # Load the module and verifies it is a pymathics module 893 try: 894 self.pymathicsmodule = importlib.import_module(module) 895 except ImportError: 896 print("Module does not exist") 897 mainfolder = "" 898 self.pymathicsmodule = None 899 self.parts = [] 900 return 901 902 try: 903 mainfolder = self.pymathicsmodule.__path__[0] 904 if "name" in self.pymathicsmodule.pymathics_version_data: 905 self.name = self.version = self.pymathicsmodule.pymathics_version_data[ 906 "name" 907 ] 908 else: 909 self.name = (self.pymathicsmodule.__package__)[10:] 910 self.version = self.pymathicsmodule.pymathics_version_data["version"] 911 self.author = self.pymathicsmodule.pymathics_version_data["author"] 912 except (AttributeError, KeyError, IndexError): 913 print(module + " is not a pymathics module.") 914 mainfolder = "" 915 self.pymathicsmodule = None 916 self.parts = [] 917 return 918 919 # Paths 920 self.doc_dir = self.pymathicsmodule.__path__[0] + "/doc/" 921 self.xml_data_file = self.doc_dir + "xml/data" 922 self.tex_data_file = self.doc_dir + "tex/data" 923 self.latex_file = self.doc_dir + "tex/documentation.tex" 924 925 # Load the dictionary of mathics symbols defined in the module 926 self.symbols = {} 927 from mathics.builtin import is_builtin, Builtin 928 929 print("loading symbols") 930 for name in dir(self.pymathicsmodule): 931 var = getattr(self.pymathicsmodule, name) 932 if ( 933 hasattr(var, "__module__") 934 and var.__module__ != "mathics.builtin.base" 935 and is_builtin(var) 936 and not name.startswith("_") 937 and var.__module__[: len(self.pymathicsmodule.__name__)] 938 == self.pymathicsmodule.__name__ 939 ): # nopep8 940 instance = var(expression=False) 941 if isinstance(instance, Builtin): 942 self.symbols[instance.get_name()] = instance 943 # Defines de default first part, in case we are building an independent documentation module. 944 self.title = "Overview" 945 self.parts = [] 946 self.parts_by_slug = {} 947 try: 948 files = listdir(self.doc_dir) 949 files.sort() 950 except FileNotFoundError: 951 self.doc_dir = "" 952 self.xml_data_file = "" 953 self.tex_data_file = "" 954 self.latex_file = "" 955 files = [] 956 appendix = [] 957 for file in files: 958 part_title = file[2:] 959 if part_title.endswith(".mdoc"): 960 part_title = part_title[: -len(".mdoc")] 961 part = DocPart(self, part_title) 962 text = open(self.doc_dir + file, "rb").read().decode("utf8") 963 text = filter_comments(text) 964 chapters = CHAPTER_RE.findall(text) 965 for title, text in chapters: 966 chapter = DocChapter(part, title) 967 text += '<section title=""></section>' 968 sections = SECTION_RE.findall(text) 969 for pre_text, title, text in sections: 970 if not chapter.doc: 971 chapter.doc = Doc(pre_text) 972 if title: 973 section = DocSection(chapter, title, text) 974 chapter.sections.append(section) 975 part.chapters.append(chapter) 976 if file[0].isdigit(): 977 self.parts.append(part) 978 else: 979 part.is_appendix = True 980 appendix.append(part) 981 982 # Builds the automatic documentation 983 builtin_part = DocPart(self, "Pymathics Modules", is_reference=True) 984 title, text = get_module_doc(self.pymathicsmodule) 985 chapter = DocChapter(builtin_part, title, Doc(text)) 986 for name in self.symbols: 987 instance = self.symbols[name] 988 installed = True 989 for package in getattr(instance, "requires", []): 990 try: 991 importlib.import_module(package) 992 except ImportError: 993 installed = False 994 break 995 section = DocSection( 996 chapter, 997 strip_system_prefix(name), 998 instance.__doc__ or "", 999 operator=instance.get_operator(), 1000 installed=installed, 1001 ) 1002 chapter.sections.append(section) 1003 builtin_part.chapters.append(chapter) 1004 self.parts.append(builtin_part) 1005 # Adds possible appendices 1006 for part in appendix: 1007 self.parts.append(part) 1008 1009 # set keys of tests 1010 for tests in self.get_tests(): 1011 for test in tests.tests: 1012 test.key = (tests.part, tests.chapter, tests.section, test.index) 1013 1014 1015class DocPart(DocElement): 1016 def __init__(self, doc, title, is_reference=False): 1017 self.doc = doc 1018 self.title = title 1019 self.slug = slugify(title) 1020 self.chapters = [] 1021 self.chapters_by_slug = {} 1022 self.is_reference = is_reference 1023 self.is_appendix = False 1024 doc.parts_by_slug[self.slug] = self 1025 1026 def __str__(self): 1027 return "%s\n\n%s" % ( 1028 self.title, 1029 "\n".join(str(chapter) for chapter in self.chapters), 1030 ) 1031 1032 def latex(self, output): 1033 result = "\n\n\\part{%s}\n\n" % escape_latex(self.title) + ( 1034 "\n\n".join(chapter.latex(output) for chapter in self.chapters) 1035 ) 1036 if self.is_reference: 1037 result = "\n\n\\referencestart" + result 1038 return result 1039 1040 def get_url(self): 1041 return f"/{self.slug}/" 1042 1043 def get_collection(self): 1044 return self.doc.parts 1045 1046 1047class DocChapter(DocElement): 1048 def __init__(self, part, title, doc=None): 1049 self.part = part 1050 self.title = title 1051 self.slug = slugify(title) 1052 self.doc = doc 1053 self.sections = [] 1054 self.sections_by_slug = {} 1055 part.chapters_by_slug[self.slug] = self 1056 1057 def __str__(self): 1058 sections = "\n".join(str(section) for section in self.sections) 1059 return f"= {self.title} =\n\n{sections}" 1060 1061 def latex(self, output): 1062 intro = self.doc.latex(output).strip() 1063 if intro: 1064 short = "short" if len(intro) < 300 else "" 1065 intro = "\\begin{chapterintro%s}\n%s\n\n\\end{chapterintro%s}" % ( 1066 short, 1067 intro, 1068 short, 1069 ) 1070 return "".join( 1071 [ 1072 "\n\n\\chapter{%(title)s}\n\\chapterstart\n\n%(intro)s" 1073 % {"title": escape_latex(self.title), "intro": intro}, 1074 "\\chaptersections\n", 1075 "\n\n".join(section.latex(output) for section in self.sections), 1076 "\n\\chapterend\n", 1077 ] 1078 ) 1079 1080 def get_url(self): 1081 return f"/{self.part.slug}/{self.slug}/" 1082 1083 def get_collection(self): 1084 return self.part.chapters 1085 1086 1087class DocSection(DocElement): 1088 def __init__(self, chapter, title, text, operator=None, installed=True): 1089 self.chapter = chapter 1090 self.title = title 1091 self.slug = slugify(title) 1092 if text.count("<dl>") != text.count("</dl>"): 1093 raise ValueError( 1094 "Missing opening or closing <dl> tag in " 1095 "{} documentation".format(title) 1096 ) 1097 self.doc = Doc(text) 1098 self.operator = operator 1099 self.installed = installed 1100 chapter.sections_by_slug[self.slug] = self 1101 1102 def __str__(self): 1103 return f"== {self.title} ==\n{self.doc}" 1104 1105 def latex(self, output): 1106 title = escape_latex(self.title) 1107 if self.operator: 1108 title += " (\\code{%s})" % escape_latex_code(self.operator) 1109 index = ( 1110 "\index{%s}" % escape_latex(self.title) 1111 if self.chapter.part.is_reference 1112 else "" 1113 ) 1114 return ( 1115 "\n\n\\section*{%(title)s}%(index)s\n" 1116 "\\sectionstart\n\n%(content)s\\sectionend" 1117 "\\addcontentsline{toc}{section}{%(title)s}" 1118 ) % {"title": title, "index": index, "content": self.doc.latex(output)} 1119 1120 def get_url(self): 1121 return f"/{self.chapter.part.slug}/{self.chapter.slug}/{self.slug}/" 1122 1123 def get_collection(self): 1124 return self.chapter.sections 1125 1126 def html_data(self): 1127 indices = set() 1128 for test in self.doc.items: 1129 indices.update(test.test_indices()) 1130 result = {} 1131 for index in indices: 1132 result[index] = xml_data.get( 1133 (self.chapter.part.title, self.chapter.title, self.title, index) 1134 ) 1135 return result 1136 1137 1138class Doc(object): 1139 def __init__(self, doc): 1140 self.items = [] 1141 # remove commented lines 1142 doc = filter_comments(doc) 1143 # pre-substitute Python code because it might contain tests 1144 doc, post_substitutions = pre_sub( 1145 PYTHON_RE, doc, lambda m: "<python>%s</python>" % m.group(1) 1146 ) 1147 # HACK: Artificially construct a last testcase to get the "intertext" 1148 # after the last (real) testcase. Ignore the test, of course. 1149 doc += "\n>> test\n = test" 1150 testcases = TESTCASE_RE.findall(doc) 1151 tests = None 1152 for index in range(len(testcases)): 1153 testcase = list(testcases[index]) 1154 text = testcase.pop(0).strip() 1155 if text: 1156 if tests is not None: 1157 self.items.append(tests) 1158 tests = None 1159 text = post_sub(text, post_substitutions) 1160 self.items.append(DocText(text)) 1161 tests = None 1162 if index < len(testcases) - 1: 1163 test = DocTest(index, testcase) 1164 if tests is None: 1165 tests = DocTests() 1166 tests.tests.append(test) 1167 if tests is not None: 1168 self.items.append(tests) 1169 tests = None 1170 1171 def __str__(self): 1172 return "\n".join(str(item) for item in self.items) 1173 1174 def text(self, detail_level): 1175 # used for introspection 1176 # TODO parse XML and pretty print 1177 # HACK 1178 item = str(self.items[0]) 1179 item = "\n".join(line.strip() for line in item.split("\n")) 1180 item = item.replace("<dl>", "") 1181 item = item.replace("</dl>", "") 1182 item = item.replace("<dt>", " ") 1183 item = item.replace("</dt>", "") 1184 item = item.replace("<dd>", " ") 1185 item = item.replace("</dd>", "") 1186 item = "\n".join(line for line in item.split("\n") if not line.isspace()) 1187 return item 1188 1189 def get_tests(self): 1190 tests = [] 1191 for item in self.items: 1192 tests.extend(item.get_tests()) 1193 return tests 1194 1195 def latex(self, output): 1196 return "\n".join( 1197 item.latex(output) for item in self.items if not item.is_private() 1198 ) 1199 1200 def html(self): 1201 counters = {} 1202 return html_escape( 1203 "\n".join( 1204 item.html(counters) for item in self.items if not item.is_private() 1205 ) 1206 ) 1207 1208 1209class DocText(object): 1210 def __init__(self, text): 1211 self.text = text 1212 1213 def get_tests(self): 1214 return [] 1215 1216 def is_private(self): 1217 return False 1218 1219 def __str__(self): 1220 return self.text 1221 1222 def latex(self, output): 1223 return escape_latex(self.text) 1224 1225 def html(self, counters=None): 1226 result = escape_html(self.text, counters=counters) 1227 return result 1228 1229 def test_indices(self): 1230 return [] 1231 1232 1233class DocTests(object): 1234 def __init__(self): 1235 self.tests = [] 1236 1237 def get_tests(self): 1238 return self.tests 1239 1240 def is_private(self): 1241 return all(test.private for test in self.tests) 1242 1243 def __str__(self): 1244 return "\n".join(str(test) for test in self.tests) 1245 1246 def latex(self, output): 1247 if len(self.tests) == 0: 1248 return "\n" 1249 1250 testLatexStrings = [ 1251 test.latex(output) for test in self.tests if not test.private 1252 ] 1253 testLatexStrings = [t for t in testLatexStrings if len(t) > 1] 1254 if len(testLatexStrings) == 0: 1255 return "\n" 1256 1257 return "\\begin{tests}%%\n%s%%\n\\end{tests}" % ("%\n".join(testLatexStrings)) 1258 1259 def html(self, counters=None): 1260 if len(self.tests) == 0: 1261 return "\n" 1262 return '<ul class="tests">%s</ul>' % ( 1263 "\n".join( 1264 "<li>%s</li>" % test.html() for test in self.tests if not test.private 1265 ) 1266 ) 1267 1268 def test_indices(self): 1269 return [test.index for test in self.tests] 1270 1271 1272# This string is used so we can indicate a trailing blank at the end of a line by 1273# adding this string to the end of the line which gets stripped off. 1274# Some editors and formatters like to strip off trailing blanks at the ends of lines. 1275END_LINE_SENTINAL = "#<--#" 1276 1277 1278class DocTest(object): 1279 """ 1280 DocTest formatting rules: 1281 1282 * `>>` Marks test case; it will also appear as part of 1283 the documentation. 1284 * `#>` Marks test private or one that does not appear as part of 1285 the documentation. 1286 * `X>` Shows the example in the docs, but disables testing the example. 1287 * `S>` Shows the example in the docs, but disables testing if environment 1288 variable SANDBOX is set. 1289 * `=` Compares the result text. 1290 * `:` Compares an (error) message. 1291 `|` Prints output. 1292 """ 1293 1294 def __init__(self, index, testcase): 1295 def strip_sentinal(line): 1296 """Remove END_LINE_SENTINAL from the end of a line if it appears. 1297 1298 Some editors like to strip blanks at the end of a line. 1299 Since the line ends in END_LINE_SENTINAL which isn't blank, 1300 any blanks that appear before will be preserved. 1301 1302 Some tests require some lines to be blank or entry because 1303 Mathics output can be that way 1304 """ 1305 if line.endswith(END_LINE_SENTINAL): 1306 line = line[: -len(END_LINE_SENTINAL)] 1307 1308 # Also remove any remaining trailing blanks since that 1309 # seems *also* what we want to do. 1310 return line.strip() 1311 1312 self.index = index 1313 self.result = None 1314 self.outs = [] 1315 1316 # Private test cases are executed, but NOT shown as part of the docs 1317 self.private = testcase[0] == "#" 1318 1319 # Ignored test cases are NOT executed, but shown as part of the docs 1320 # Sandboxed test cases are NOT executed if environtment SANDBOX is set 1321 if testcase[0] == "X" or (testcase[0] == "S" and getenv("SANDBOX", False)): 1322 self.ignore = True 1323 # substitute '>' again so we get the correct formatting 1324 testcase[0] = ">" 1325 else: 1326 self.ignore = False 1327 1328 self.test = strip_sentinal(testcase[1]) 1329 1330 self.key = None 1331 outs = testcase[2].splitlines() 1332 for line in outs: 1333 line = strip_sentinal(line) 1334 if line: 1335 if line.startswith("."): 1336 text = line[1:] 1337 if text.startswith(" "): 1338 text = text[1:] 1339 text = "\n" + text 1340 if self.result is not None: 1341 self.result += text 1342 elif self.outs: 1343 self.outs[-1].text += text 1344 continue 1345 1346 match = TESTCASE_OUT_RE.match(line) 1347 symbol, text = match.group(1), match.group(2) 1348 text = text.strip() 1349 if symbol == "=": 1350 self.result = text 1351 elif symbol == ":": 1352 out = Message("", "", text) 1353 self.outs.append(out) 1354 elif symbol == "|": 1355 out = Print(text) 1356 self.outs.append(out) 1357 1358 def __str__(self): 1359 return self.test 1360 1361 def latex(self, output): 1362 text = "" 1363 text += "\\begin{testcase}\n" 1364 text += "\\test{%s}\n" % escape_latex_code(self.test) 1365 if self.key is None: 1366 return "" 1367 results = output[self.key]["results"] 1368 for result in results: 1369 for out in result["out"]: 1370 kind = "message" if out["message"] else "print" 1371 text += "\\begin{test%s}%s\\end{test%s}" % ( 1372 kind, 1373 escape_latex_output(out["text"]), 1374 kind, 1375 ) 1376 if result["result"]: # is not None and result['result'].strip(): 1377 text += "\\begin{testresult}%s\\end{testresult}" % result["result"] 1378 text += "\\end{testcase}" 1379 return text 1380 1381 def html(self): 1382 result = '<div class="test"><span class="move"></span>' 1383 result += '<ul class="test" id="test_%d">' % self.index 1384 result += '<li class="test">%s</li>' % escape_html(self.test, True) 1385 result += "</ul>" 1386 result += "</div>" 1387 return result 1388