1""" 2SuperFences. 3 4pymdownx.superfences 5Nested Fenced Code Blocks 6 7This is a modification of the original Fenced Code Extension. 8Algorithm has been rewritten to allow for fenced blocks in blockquotes, 9lists, etc. And also , allow for special UML fences like 'flow' for flowcharts 10and `sequence` for sequence diagrams. 11 12Modified: 2014 - 2017 Isaac Muse <isaacmuse@gmail.com> 13--- 14 15Fenced Code Extension for Python Markdown 16========================================= 17 18This extension adds Fenced Code Blocks to Python-Markdown. 19 20See <https://pythonhosted.org/Markdown/extensions/fenced_code_blocks.html> 21for documentation. 22 23Original code Copyright 2007-2008 [Waylan Limberg](http://achinghead.com/). 24 25 26All changes Copyright 2008-2014 The Python Markdown Project 27 28License: [BSD](http://www.opensource.org/licenses/bsd-license.php) 29""" 30 31from markdown.extensions import Extension 32from markdown.preprocessors import Preprocessor 33from markdown.blockprocessors import CodeBlockProcessor 34from markdown.extensions.attr_list import get_attrs 35from markdown import util as md_util 36import functools 37import re 38 39SOH = '\u0001' # start 40EOT = '\u0004' # end 41 42PREFIX_CHARS = ('>', ' ', '\t') 43 44RE_NESTED_FENCE_START = re.compile( 45 r'''(?x) 46 (?P<fence>~{3,}|`{3,})[ \t]* # Fence opening 47 (?:(\{(?P<attrs>[^\}\n]*)\})?| # Optional attributes or 48 (?:\.?(?P<lang>[\w#.+-]*))?[ \t]* # Language 49 (?P<options> 50 (?: 51 (?:\b[a-zA-Z][a-zA-Z0-9_]*(?:=(?P<quot>"|').*?(?P=quot))?[ \t]*) | # Options 52 )* 53 ) 54 )[ \t]*$ 55 ''' 56) 57 58RE_HL_LINES = re.compile(r'^(?P<hl_lines>\d+(?:-\d+)?(?:[ \t]+\d+(?:-\d+)?)*)$') 59RE_LINENUMS = re.compile(r'(?P<linestart>[\d]+)(?:[ \t]+(?P<linestep>[\d]+))?(?:[ \t]+(?P<linespecial>[\d]+))?') 60RE_OPTIONS = re.compile( 61 r'''(?x) 62 (?: 63 (?P<key>[a-zA-Z][a-zA-Z0-9_]*)(?:=(?P<quot>"|')(?P<value>.*?)(?P=quot))? 64 ) 65 ''' 66) 67 68NESTED_FENCE_END = r'%s[ \t]*$' 69 70FENCED_BLOCK_RE = re.compile( 71 r'^([\> ]*)%s(%s)%s$' % ( 72 md_util.HTML_PLACEHOLDER[0], 73 md_util.HTML_PLACEHOLDER[1:-1] % r'([0-9]+)', 74 md_util.HTML_PLACEHOLDER[-1] 75 ) 76) 77 78 79def _escape(txt): 80 """Basic html escaping.""" 81 82 txt = txt.replace('&', '&') 83 txt = txt.replace('<', '<') 84 txt = txt.replace('>', '>') 85 return txt 86 87 88class CodeStash(object): 89 """ 90 Stash code for later retrieval. 91 92 Store original fenced code here in case we were 93 too greedy and need to restore in an indented code 94 block. 95 """ 96 97 def __init__(self): 98 """Initialize.""" 99 100 self.stash = {} 101 102 def __len__(self): # pragma: no cover 103 """Length of stash.""" 104 105 return len(self.stash) 106 107 def get(self, key, default=None): 108 """Get the code from the key.""" 109 110 code = self.stash.get(key, default) 111 return code 112 113 def remove(self, key): 114 """Remove the stashed code.""" 115 116 del self.stash[key] 117 118 def store(self, key, code, indent_level): 119 """Store the code in the stash.""" 120 121 self.stash[key] = (code, indent_level) 122 123 def clear_stash(self): 124 """Clear the stash.""" 125 126 self.stash = {} 127 128 129def fence_code_format(source, language, class_name, options, md, **kwargs): 130 """Format source as code blocks.""" 131 132 classes = kwargs['classes'] 133 id_value = kwargs['id_value'] 134 attrs = kwargs['attrs'] 135 136 if class_name: 137 classes.insert(0, class_name) 138 139 id_value = ' id="{}"'.format(id_value) if id_value else '' 140 classes = ' class="{}"'.format(' '.join(classes)) if classes else '' 141 attrs = ' ' + ' '.join('{k}="{v}"'.format(k=k, v=v) for k, v in attrs.items()) if attrs else '' 142 143 return '<pre%s%s%s><code>%s</code></pre>' % (id_value, classes, attrs, _escape(source)) 144 145 146def fence_div_format(source, language, class_name, options, md, **kwargs): 147 """Format source as div.""" 148 149 classes = kwargs['classes'] 150 id_value = kwargs['id_value'] 151 attrs = kwargs['attrs'] 152 153 if class_name: 154 classes.insert(0, class_name) 155 156 id_value = ' id="{}"'.format(id_value) if id_value else '' 157 classes = ' class="{}"'.format(' '.join(classes)) if classes else '' 158 attrs = ' ' + ' '.join('{k}="{v}"'.format(k=k, v=v) for k, v in attrs.items()) if attrs else '' 159 160 return '<div%s%s%s>%s</div>' % (id_value, classes, attrs, _escape(source)) 161 162 163def highlight_validator(language, inputs, options, attrs, md): 164 """Highlight validator.""" 165 166 use_pygments = md.preprocessors['fenced_code_block'].use_pygments 167 168 for k, v in inputs.items(): 169 matched = False 170 if use_pygments: 171 if k.startswith('data-'): 172 attrs[k] = v 173 continue 174 for opt, validator in (('hl_lines', RE_HL_LINES), ('linenums', RE_LINENUMS), ('title', None)): 175 if k == opt: 176 if v is not True and (validator is None or validator.match(v) is not None): 177 options[k] = v 178 matched = True 179 break 180 if not matched: 181 attrs[k] = v 182 183 return True 184 185 186def default_validator(language, inputs, options, attrs, md): 187 """Default validator.""" 188 189 for k, v in inputs.items(): 190 attrs[k] = v 191 return True 192 193 194def _validator(language, inputs, options, attrs, md, validator=None): 195 """Validator wrapper.""" 196 197 md.preprocessors['fenced_code_block'].get_hl_settings() 198 return validator(language, inputs, options, attrs, md) 199 200 201def _formatter(src='', language='', options=None, md=None, class_name="", _fmt=None, **kwargs): 202 """Formatter wrapper.""" 203 204 return _fmt(src, language, class_name, options, md, **kwargs) 205 206 207def _test(language, test_language=None): 208 """Test language.""" 209 210 return test_language is None or test_language == "*" or language == test_language 211 212 213class SuperFencesCodeExtension(Extension): 214 """SuperFences code block extension.""" 215 216 def __init__(self, *args, **kwargs): 217 """Initialize.""" 218 219 self.superfences = [] 220 self.config = { 221 'disable_indented_code_blocks': [False, "Disable indented code blocks - Default: False"], 222 'custom_fences': [[], 'Specify custom fences. Default: See documentation.'], 223 'css_class': [ 224 '', 225 "Set class name for wrapper element. The default of CodeHilite or Highlight will be used" 226 "if nothing is set. - " 227 "Default: ''" 228 ], 229 'preserve_tabs': [False, "Preserve tabs in fences - Default: False"] 230 } 231 super().__init__(*args, **kwargs) 232 233 def extend_super_fences(self, name, formatter, validator): 234 """Extend SuperFences with the given name, language, and formatter.""" 235 236 obj = { 237 "name": name, 238 "test": functools.partial(_test, test_language=name), 239 "formatter": formatter, 240 "validator": validator 241 } 242 243 if name == '*': 244 self.superfences[0] = obj 245 else: 246 self.superfences.append(obj) 247 248 def extendMarkdown(self, md): 249 """Add fenced block preprocessor to the Markdown instance.""" 250 251 # Not super yet, so let's make it super 252 md.registerExtension(self) 253 config = self.getConfigs() 254 255 # Default fenced blocks 256 self.superfences.insert( 257 0, 258 { 259 "name": "superfences", 260 "test": _test, 261 "formatter": None, 262 "validator": functools.partial(_validator, validator=highlight_validator) 263 } 264 ) 265 266 # Custom Fences 267 custom_fences = config.get('custom_fences', []) 268 for custom in custom_fences: 269 name = custom.get('name') 270 class_name = custom.get('class') 271 fence_format = custom.get('format', fence_code_format) 272 validator = custom.get('validator', default_validator) 273 if name is not None and class_name is not None: 274 self.extend_super_fences( 275 name, 276 functools.partial(_formatter, class_name=class_name, _fmt=fence_format), 277 functools.partial(_validator, validator=validator) 278 ) 279 280 self.md = md 281 self.patch_fenced_rule() 282 self.stash = CodeStash() 283 284 def patch_fenced_rule(self): 285 """ 286 Patch Python Markdown with our own fenced block extension. 287 288 We don't attempt to protect against a user loading the `fenced_code` extension with this. 289 Most likely they will have issues, but they shouldn't have loaded them together in the first place :). 290 """ 291 292 config = self.getConfigs() 293 294 fenced = SuperFencesBlockPreprocessor(self.md) 295 fenced.config = config 296 fenced.extension = self 297 if self.superfences[0]['name'] == "superfences": 298 self.superfences[0]["formatter"] = fenced.highlight 299 self.md.preprocessors.register(fenced, "fenced_code_block", 25) 300 301 indented_code = SuperFencesCodeBlockProcessor(self.md.parser) 302 indented_code.config = config 303 indented_code.extension = self 304 self.md.parser.blockprocessors.register(indented_code, "code", 80) 305 306 if config["preserve_tabs"]: 307 # Need to squeeze in right after critic. 308 raw_fenced = SuperFencesRawBlockPreprocessor(self.md) 309 raw_fenced.config = config 310 raw_fenced.extension = self 311 self.md.preprocessors.register(raw_fenced, "fenced_raw_block", 31.05) 312 self.md.registerExtensions(["pymdownx._bypassnorm"], {}) 313 314 # Add the highlight extension, but do so in a disabled state so we can just retrieve default configurations 315 self.md.registerExtensions(["pymdownx.highlight"], {"pymdownx.highlight": {"_enabled": False}}) 316 317 def reset(self): 318 """Clear the stash.""" 319 320 self.stash.clear_stash() 321 322 323class SuperFencesBlockPreprocessor(Preprocessor): 324 """ 325 Preprocessor to find fenced code blocks. 326 327 Because this is done as a preprocessor, it might be too greedy. 328 We will stash the blocks code and restore if we mistakenly processed 329 text from an indented code block. 330 """ 331 332 CODE_WRAP = '<pre%s><code%s>%s</code></pre>' 333 334 def __init__(self, md): 335 """Initialize.""" 336 337 super().__init__(md) 338 self.tab_len = self.md.tab_length 339 self.checked_hl_settings = False 340 self.codehilite_conf = {} 341 342 def normalize_ws(self, text): 343 """Normalize whitespace.""" 344 345 return text.expandtabs(self.tab_len) 346 347 def rebuild_block(self, lines): 348 """Dedent the fenced block lines.""" 349 350 return '\n'.join([line[self.ws_virtual_len:] for line in lines]) 351 352 def get_hl_settings(self): 353 """Check for Highlight extension to get its configurations.""" 354 355 if not self.checked_hl_settings: 356 self.checked_hl_settings = True 357 358 config = None 359 self.highlighter = None 360 for ext in self.md.registeredExtensions: 361 self.highlight_ext = ext 362 try: 363 config = getattr(ext, "get_pymdownx_highlight_settings")() 364 self.highlighter = getattr(ext, "get_pymdownx_highlighter")() 365 break 366 except AttributeError: 367 pass 368 369 self.attr_list = 'attr_list' in self.md.treeprocessors 370 371 css_class = self.config['css_class'] 372 self.css_class = css_class if css_class else config['css_class'] 373 374 self.extend_pygments_lang = config.get('extend_pygments_lang', None) 375 self.guess_lang = config['guess_lang'] 376 self.pygments_style = config['pygments_style'] 377 self.use_pygments = config['use_pygments'] 378 self.noclasses = config['noclasses'] 379 self.linenums = config['linenums'] 380 self.linenums_style = config.get('linenums_style', 'table') 381 self.linenums_class = config.get('linenums_class', 'linenums') 382 self.linenums_special = config.get('linenums_special', -1) 383 self.language_prefix = config.get('language_prefix', 'language-') 384 self.code_attr_on_pre = config.get('code_attr_on_pre', False) 385 self.auto_title = config.get('auto_title', False) 386 self.auto_title_map = config.get('auto_title_map', {}) 387 self.line_spans = config.get('line_spans', '') 388 self.line_anchors = config.get('line_anchors', '') 389 self.anchor_linenums = config.get('anchor_linenums', False) 390 391 def clear(self): 392 """Reset the class variables.""" 393 394 self.ws = None 395 self.ws_len = 0 396 self.ws_virtual_len = 0 397 self.fence = None 398 self.lang = None 399 self.quote_level = 0 400 self.code = [] 401 self.empty_lines = 0 402 self.fence_end = None 403 self.options = {} 404 self.classes = [] 405 self.id = '' 406 self.attrs = {} 407 self.formatter = None 408 409 def eval_fence(self, ws, content, start, end): 410 """Evaluate a normal fence.""" 411 412 if (ws + content).strip() == '': 413 # Empty line is okay 414 self.empty_lines += 1 415 self.code.append(ws + content) 416 elif len(ws) != self.ws_virtual_len and content != '': 417 # Not indented enough 418 self.clear() 419 elif self.fence_end.match(content) is not None and not content.startswith((' ', '\t')): 420 # End of fence 421 try: 422 self.process_nested_block(ws, content, start, end) 423 except Exception: 424 self.clear() 425 else: 426 # Content line 427 self.empty_lines = 0 428 self.code.append(ws + content) 429 430 def eval_quoted(self, ws, content, quote_level, start, end): 431 """Evaluate fence inside a blockquote.""" 432 433 if quote_level > self.quote_level: 434 # Quote level exceeds the starting quote level 435 self.clear() 436 elif quote_level <= self.quote_level: 437 if content == '': 438 # Empty line is okay 439 self.code.append(ws + content) 440 self.empty_lines += 1 441 elif len(ws) < self.ws_len: 442 # Not indented enough 443 self.clear() 444 elif self.empty_lines and quote_level < self.quote_level: 445 # Quote levels don't match and we are signified 446 # the end of the block with an empty line 447 self.clear() 448 elif self.fence_end.match(content) is not None: 449 # End of fence 450 self.process_nested_block(ws, content, start, end) 451 else: 452 # Content line 453 self.empty_lines = 0 454 self.code.append(ws + content) 455 456 def process_nested_block(self, ws, content, start, end): 457 """Process the contents of the nested block.""" 458 459 self.last = ws + self.normalize_ws(content) 460 code = None 461 if self.formatter is not None: 462 self.line_count = end - start - 2 463 464 code = self.formatter( 465 src=self.rebuild_block(self.code), 466 language=self.lang, 467 md=self.md, 468 options=self.options, 469 classes=self.classes, 470 id_value=self.id, 471 attrs=self.attrs if self.attr_list else {} 472 ) 473 474 if code is not None: 475 self._store(self.normalize_ws('\n'.join(self.code)) + '\n', code, start, end) 476 self.clear() 477 478 def normalize_hl_line(self, number): 479 """ 480 Normalize highlight line number. 481 482 Clamp outrages numbers. Numbers out of range will be only one increment out range. 483 This prevents people from create massive buffers of line numbers that exceed real 484 number of code lines. 485 """ 486 487 number = int(number) 488 if number < 1: 489 number = 0 490 elif number > self.line_count: 491 number = self.line_count + 1 492 return number 493 494 def parse_hl_lines(self, hl_lines): 495 """Parse the lines to highlight.""" 496 497 lines = [] 498 if hl_lines: 499 for entry in hl_lines.split(): 500 line_range = [self.normalize_hl_line(e) for e in entry.split('-')] 501 if len(line_range) > 1: 502 if line_range[0] <= line_range[1]: 503 lines.extend(list(range(line_range[0], line_range[1] + 1))) 504 elif 1 <= line_range[0] <= self.line_count: 505 lines.extend(line_range) 506 return lines 507 508 def parse_line_start(self, linestart): 509 """Parse line start.""" 510 511 return int(linestart) if linestart else -1 512 513 def parse_line_step(self, linestep): 514 """Parse line start.""" 515 516 step = int(linestep) if linestep else -1 517 518 return step if step > 1 else -1 519 520 def parse_line_special(self, linespecial): 521 """Parse line start.""" 522 523 return int(linespecial) if linespecial else -1 524 525 def parse_fence_line(self, line): 526 """Parse fence line.""" 527 528 ws_len = 0 529 ws_virtual_len = 0 530 ws = [] 531 index = 0 532 for c in line: 533 if ws_virtual_len >= self.ws_virtual_len: 534 break 535 if c not in PREFIX_CHARS: 536 break 537 ws_len += 1 538 if c == '\t': 539 tab_size = self.tab_len - (index % self.tab_len) 540 ws_virtual_len += tab_size 541 ws.append(' ' * tab_size) 542 else: 543 tab_size = 1 544 ws_virtual_len += 1 545 ws.append(c) 546 index += tab_size 547 548 return ''.join(ws), line[ws_len:] 549 550 def parse_whitespace(self, line): 551 """Parse the whitespace (blockquote syntax is counted as well).""" 552 553 self.ws_len = 0 554 self.ws_virtual_len = 0 555 ws = [] 556 for c in line: 557 if c not in PREFIX_CHARS: 558 break 559 self.ws_len += 1 560 ws.append(c) 561 562 ws = self.normalize_ws(''.join(ws)) 563 self.ws_virtual_len = len(ws) 564 565 return ws 566 567 def parse_options(self, m): 568 """Get options.""" 569 570 okay = False 571 572 if m.group('lang'): 573 self.lang = m.group('lang') 574 575 string = m.group('options') 576 577 self.options = {} 578 self.attrs = {} 579 self.formatter = None 580 values = {} 581 if string: 582 for m in RE_OPTIONS.finditer(string): 583 key = m.group('key') 584 value = m.group('value') 585 if value is None: 586 value = key 587 values[key] = value 588 589 # Run per language validator 590 for entry in reversed(self.extension.superfences): 591 if entry["test"](self.lang): 592 options = {} 593 attrs = {} 594 validator = entry.get("validator", functools.partial(_validator, validator=default_validator)) 595 try: 596 okay = validator(self.lang, values, options, attrs, self.md) 597 except Exception: 598 pass 599 if attrs: 600 okay = False 601 if okay: 602 self.formatter = entry.get("formatter") 603 self.options = options 604 break 605 606 return okay 607 608 def handle_attrs(self, m): 609 """Handle attribute list.""" 610 611 okay = False 612 attributes = get_attrs(m.group('attrs').replace('\t', ' ' * self.tab_len)) 613 614 self.options = {} 615 self.attrs = {} 616 self.formatter = None 617 values = {} 618 for k, v in attributes: 619 if k == 'id': 620 self.id = v 621 elif k == '.': 622 self.classes.append(v) 623 else: 624 values[k] = v 625 626 self.lang = self.classes.pop(0) if self.classes else '' 627 628 # Run per language validator 629 for entry in reversed(self.extension.superfences): 630 if entry["test"](self.lang): 631 options = {} 632 attrs = {} 633 validator = entry.get("validator", functools.partial(_validator, validator=default_validator)) 634 try: 635 okay = validator(self.lang, values, options, attrs, self.md) 636 except Exception: 637 pass 638 if okay: 639 self.formatter = entry.get("formatter") 640 self.options = options 641 if self.attr_list: 642 self.attrs = attrs 643 break 644 645 return okay 646 647 def search_nested(self, lines): 648 """Search for nested fenced blocks.""" 649 650 count = 0 651 for line in lines: 652 # Strip carriage returns if the lines end with them. 653 # This is necessary since we are handling preserved tabs 654 # Before whitespace normalization. 655 line = line.rstrip('\r') 656 if self.fence is None: 657 ws = self.parse_whitespace(line) 658 659 # Found the start of a fenced block. 660 m = RE_NESTED_FENCE_START.match(line, self.ws_len) 661 if m is not None: 662 663 # Parse options 664 if m.group('attrs'): 665 okay = self.handle_attrs(m) 666 else: 667 okay = self.parse_options(m) 668 669 if okay: 670 # Valid fence options, handle fence 671 start = count 672 self.first = ws + self.normalize_ws(m.group(0)) 673 self.ws = ws 674 self.quote_level = self.ws.count(">") 675 self.empty_lines = 0 676 self.fence = m.group('fence') 677 self.fence_end = re.compile(NESTED_FENCE_END % self.fence) 678 else: 679 # Option parsing failed, abandon fence 680 self.clear() 681 else: 682 # Evaluate lines 683 # - Determine if it is the ending line or content line 684 # - If is a content line, make sure it is all indented 685 # with the opening and closing lines (lines with just 686 # whitespace will be stripped so those don't matter). 687 # - When content lines are inside blockquotes, make sure 688 # the nested block quote levels make sense according to 689 # blockquote rules. 690 ws, content = self.parse_fence_line(line) 691 692 end = count + 1 693 quote_level = ws.count(">") 694 695 if self.quote_level: 696 # Handle blockquotes 697 self.eval_quoted(ws, content, quote_level, start, end) 698 elif quote_level == 0: 699 # Handle all other cases 700 self.eval_fence(ws, content, start, end) 701 else: 702 # Looks like we got a blockquote line 703 # when not in a blockquote. 704 self.clear() 705 706 count += 1 707 708 return self.reassemble(lines) 709 710 def reassemble(self, lines): 711 """Reassemble text.""" 712 713 # Now that we are done iterating the lines, 714 # let's replace the original content with the 715 # fenced blocks. 716 while len(self.stack): 717 fenced, start, end = self.stack.pop() 718 lines = lines[:start] + [fenced] + lines[end:] 719 return lines 720 721 def highlight(self, src="", language="", options=None, md=None, **kwargs): 722 """ 723 Syntax highlight the code block. 724 725 If configuration is not empty, then the CodeHilite extension 726 is enabled, so we call into it to highlight the code. 727 """ 728 729 classes = kwargs['classes'] 730 id_value = kwargs['id_value'] 731 attrs = kwargs['attrs'] 732 733 if classes is None: # pragma: no cover 734 classes = [] 735 736 # Default format options 737 linestep = None 738 linestart = None 739 linespecial = None 740 hl_lines = None 741 title = None 742 743 if self.use_pygments: 744 if 'hl_lines' in options: 745 m = RE_HL_LINES.match(options['hl_lines']) 746 hl_lines = m.group('hl_lines') 747 del options['hl_lines'] 748 if 'linenums' in options: 749 m = RE_LINENUMS.match(options['linenums']) 750 linestart = m.group('linestart') 751 linestep = m.group('linestep') 752 linespecial = m.group('linespecial') 753 del options['linenums'] 754 if 'title' in options: 755 title = options['title'] 756 del options['title'] 757 758 linestep = self.parse_line_step(linestep) 759 linestart = self.parse_line_start(linestart) 760 linespecial = self.parse_line_special(linespecial) 761 hl_lines = self.parse_hl_lines(hl_lines) 762 763 self.highlight_ext.pygments_code_block += 1 764 765 el = self.highlighter( 766 guess_lang=self.guess_lang, 767 pygments_style=self.pygments_style, 768 use_pygments=self.use_pygments, 769 noclasses=self.noclasses, 770 linenums=self.linenums, 771 linenums_style=self.linenums_style, 772 linenums_special=self.linenums_special, 773 linenums_class=self.linenums_class, 774 extend_pygments_lang=self.extend_pygments_lang, 775 language_prefix=self.language_prefix, 776 code_attr_on_pre=self.code_attr_on_pre, 777 auto_title=self.auto_title, 778 auto_title_map=self.auto_title_map, 779 line_spans=self.line_spans, 780 line_anchors=self.line_anchors, 781 anchor_linenums=self.anchor_linenums 782 ).highlight( 783 src, 784 language, 785 self.css_class, 786 hl_lines=hl_lines, 787 linestart=linestart, 788 linestep=linestep, 789 linespecial=linespecial, 790 classes=classes, 791 id_value=id_value, 792 attrs=attrs, 793 title=title, 794 code_block_count=self.highlight_ext.pygments_code_block 795 ) 796 797 return el 798 799 def _store(self, source, code, start, end): 800 """ 801 Store the fenced blocks in the stack to be replaced when done iterating. 802 803 Store the original text in case we need to restore if we are too greedy. 804 """ 805 # Save the fenced blocks to add once we are done iterating the lines 806 placeholder = self.md.htmlStash.store(code) 807 self.stack.append(('%s%s' % (self.ws, placeholder), start, end)) 808 if not self.disabled_indented: 809 # If an indented block consumes this placeholder, 810 # we can restore the original source 811 self.extension.stash.store( 812 placeholder[1:-1], 813 "%s\n%s%s" % (self.first, self.normalize_ws(source), self.last), 814 self.ws_virtual_len 815 ) 816 817 def reindent(self, text, pos, level): 818 """Reindent the code to where it is supposed to be.""" 819 820 indented = [] 821 for line in text.split('\n'): 822 index = pos - level 823 indented.append(line[index:]) 824 return indented 825 826 def restore_raw_text(self, lines): 827 """Revert a prematurely converted fenced block.""" 828 829 new_lines = [] 830 for line in lines: 831 m = FENCED_BLOCK_RE.match(line) 832 if m: 833 key = m.group(2) 834 indent_level = len(m.group(1)) 835 original = None 836 original, pos = self.extension.stash.get(key, (None, None)) 837 if original is not None: 838 code = self.reindent(original, pos, indent_level) 839 new_lines.extend(code) 840 self.extension.stash.remove(key) 841 if original is None: # pragma: no cover 842 # Too much work to test this. This is just a fall back in case 843 # we find a placeholder, and we went to revert it and it wasn't in our stash. 844 # Most likely this would be caused by someone else. We just want to put it 845 # back in the block if we can't revert it. Maybe we can do a more directed 846 # unit test in the future. 847 new_lines.append(line) 848 else: 849 new_lines.append(line) 850 return new_lines 851 852 def run(self, lines): 853 """Search for fenced blocks.""" 854 855 self.get_hl_settings() 856 self.clear() 857 self.stack = [] 858 self.disabled_indented = self.config.get("disable_indented_code_blocks", False) 859 self.preserve_tabs = self.config.get("preserve_tabs", False) 860 861 if self.preserve_tabs: 862 lines = self.restore_raw_text(lines) 863 return self.search_nested(lines) 864 865 866class SuperFencesRawBlockPreprocessor(SuperFencesBlockPreprocessor): 867 """Special class for preserving tabs before normalizing whitespace.""" 868 869 def process_nested_block(self, ws, content, start, end): 870 """Process the contents of the nested block.""" 871 872 self.last = ws + self.normalize_ws(content) 873 code = '\n'.join(self.code) 874 self._store(code + '\n', code, start, end) 875 self.clear() 876 877 def _store(self, source, code, start, end): 878 """ 879 Store the fenced blocks in the stack to be replaced when done iterating. 880 881 Store the original text in case we need to restore if we are too greedy. 882 """ 883 # Just get a placeholder, we won't ever actually retrieve this source 884 placeholder = self.md.htmlStash.store('') 885 self.stack.append(('%s%s' % (self.ws, placeholder), start, end)) 886 # Here is the source we'll actually retrieve. 887 self.extension.stash.store( 888 placeholder[1:-1], 889 "%s\n%s%s" % (self.first, source, self.last), 890 self.ws_virtual_len 891 ) 892 893 def reassemble(self, lines): 894 """Reassemble text.""" 895 896 # Now that we are done iterating the lines, 897 # let's replace the original content with the 898 # fenced blocks. 899 while len(self.stack): 900 fenced, start, end = self.stack.pop() 901 lines = lines[:start] + [fenced.replace(md_util.STX, SOH, 1)[:-1] + EOT] + lines[end:] 902 return lines 903 904 def run(self, lines): 905 """Search for fenced blocks.""" 906 907 self.get_hl_settings() 908 self.clear() 909 self.stack = [] 910 self.disabled_indented = self.config.get("disable_indented_code_blocks", False) 911 return self.search_nested(lines) 912 913 914class SuperFencesCodeBlockProcessor(CodeBlockProcessor): 915 """Process indented code blocks to see if we accidentally processed its content as a fenced block.""" 916 917 def test(self, parent, block): 918 """Test method that is one day to be deprecated.""" 919 920 return True 921 922 def reindent(self, text, pos, level): 923 """Reindent the code to where it is supposed to be.""" 924 925 indented = [] 926 for line in text.split('\n'): 927 index = pos - level 928 indented.append(line[index:]) 929 return '\n'.join(indented) 930 931 def revert_greedy_fences(self, block): 932 """Revert a prematurely converted fenced block.""" 933 934 new_block = [] 935 for line in block.split('\n'): 936 m = FENCED_BLOCK_RE.match(line) 937 if m: 938 key = m.group(2) 939 indent_level = len(m.group(1)) 940 original = None 941 original, pos = self.extension.stash.get(key) 942 if original is not None: 943 code = self.reindent(original, pos, indent_level) 944 new_block.append(code) 945 self.extension.stash.remove(key) 946 if original is None: # pragma: no cover 947 # Too much work to test this. This is just a fall back in case 948 # we find a placeholder, and we went to revert it and it wasn't in our stash. 949 # Most likely this would be caused by someone else. We just want to put it 950 # back in the block if we can't revert it. Maybe we can do a more directed 951 # unit test in the future. 952 new_block.append(line) 953 else: 954 new_block.append(line) 955 return '\n'.join(new_block) 956 957 def run(self, parent, blocks): 958 """Look for and parse code block.""" 959 960 handled = False 961 962 if not self.config.get("disable_indented_code_blocks", False): 963 handled = CodeBlockProcessor.test(self, parent, blocks[0]) 964 if handled: 965 if self.config.get("nested", True): 966 blocks[0] = self.revert_greedy_fences(blocks[0]) 967 handled = CodeBlockProcessor.run(self, parent, blocks) is not False 968 return handled 969 970 971def makeExtension(*args, **kwargs): 972 """Return extension.""" 973 974 return SuperFencesCodeExtension(*args, **kwargs) 975