1# Copyright David Abrahams 2004.
2# Copyright Daniel Wallin 2006.
3# Distributed under the Boost
4# Software License, Version 1.0. (See accompanying
5# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
6
7import os
8import tempfile
9import litre
10import re
11import sys
12import traceback
13
14# Thanks to Jean Brouwers for this snippet
15def _caller(up=0):
16    '''Get file name, line number, function name and
17       source text of the caller's caller as 4-tuple:
18       (file, line, func, text).
19
20       The optional argument 'up' allows retrieval of
21       a caller further back up into the call stack.
22
23       Note, the source text may be None and function
24       name may be '?' in the returned result.  In
25       Python 2.3+ the file name may be an absolute
26       path.
27    '''
28    try:  # just get a few frames'
29        f = traceback.extract_stack(limit=up+2)
30        if f:
31           return f[0]
32    except:
33        pass
34    # running with psyco?
35    return ('', 0, '', None)
36
37class Example:
38    closed = False
39    in_emph = None
40
41    def __init__(self, node, section, line_offset, line_hash = '#'):
42        # A list of text fragments comprising the Example.  Start with a #line
43        # directive
44        self.section = section
45        self.line_hash = line_hash
46        self.node = node
47        self.body = []
48        self.line_offset = line_offset
49        self._number_of_prefixes = 0
50
51        self.emphasized = [] # indices of text strings that have been
52                             # emphasized.  These are generally expected to be
53                             # invalid C++ and will need special treatment
54
55    def begin_emphasis(self):
56        self.in_emph = len(self.body)
57
58    def end_emphasis(self):
59        self.emphasized.append( (self.in_emph, len(self.body)) )
60
61    def append(self, s):
62        self.append_raw(self._make_line(s))
63
64    def prepend(self, s):
65        self.prepend_raw(self._make_line(s))
66
67    def append_raw(self, s):
68        self.body.append(s)
69
70    def prepend_raw(self, s):
71        self.body.insert(0,s)
72        self.emphasized = [ (x[0]+1,x[1]+1) for x in self.emphasized ]
73        self._number_of_prefixes += 1
74
75    def replace(self, s1, s2):
76        self.body = [x.replace(s1,s2) for x in self.body]
77
78    def sub(self, pattern, repl, count = 1, flags = re.MULTILINE):
79        pat = re.compile(pattern, flags)
80        for i,txt in enumerate(self.body):
81            if count > 0:
82                x, subs = pat.subn(repl, txt, count)
83                self.body[i] = x
84                count -= subs
85
86    def wrap(self, s1, s2):
87        self.append_raw(self._make_line(s2))
88        self.prepend_raw(self._make_line(s1, offset = -s1.count('\n')))
89
90    def replace_emphasis(self, s, index = 0):
91        """replace the index'th emphasized text with s"""
92        e = self.emphasized[index]
93        self.body[e[0]:e[1]] = [s]
94        del self.emphasized[index]
95
96    elipsis = re.compile('^([ \t]*)([.][.][.][ \t]*)$', re.MULTILINE)
97
98    def __str__(self):
99        # Comment out any remaining emphasized sections
100        b = [self.elipsis.sub(r'\1// \2', s) for s in self.body]
101        emph = self.emphasized
102        emph.reverse()
103        for e in emph:
104            b.insert(e[1], ' */')
105            b.insert(e[0], '/* ')
106        emph.reverse()
107
108        # Add initial #line
109        b.insert(
110            self._number_of_prefixes,
111            self._line_directive(self.node.line, self.node.source)
112            )
113
114        # Add trailing newline to avoid warnings
115        b.append('\n')
116        return ''.join(b)
117
118    def __repr__(self):
119        return "Example: " + repr(str(self))
120
121    def raw(self):
122        return ''.join(self.body)
123
124    def _make_line(self, s, offset = 0):
125        c = _caller(2)[1::-1]
126        offset -= s.count('\n')
127        return '\n%s%s\n' % (self._line_directive(offset = offset, *c), s.strip('\n'))
128
129    def _line_directive(self, line, source, offset = None):
130        if self.line_hash is None:
131            return '\n'
132
133        if offset is None:
134            offset = self.line_offset
135
136        if line is None or line <= -offset:
137            line = 1
138        else:
139            line += offset
140
141        if source is None:
142            return '%sline %d\n' % (self.line_hash, line)
143        else:
144            return '%sline %d "%s"\n' % (self.line_hash, line, source)
145
146
147def syscmd(
148      cmd
149    , expect_error = False
150    , input = None
151    , max_output_lines = None
152    ):
153
154    # On windows close() returns the exit code, on *nix it doesn't so
155    # we need to use popen2.Popen4 instead.
156    if sys.platform == 'win32':
157        stdin, stdout_stderr = os.popen4(cmd)
158        if input: stdin.write(input)
159        stdin.close()
160
161        out = stdout_stderr.read()
162        status = stdout_stderr.close()
163    else:
164        import popen2
165        process = popen2.Popen4(cmd)
166        if input: process.tochild.write(input)
167        out = process.fromchild.read()
168        status = process.wait()
169
170    if max_output_lines is not None:
171        out = '\n'.join(out.split('\n')[:max_output_lines])
172
173    if expect_error:
174        status = not status
175
176    if status:
177        print
178        print '========== offending command ==========='
179        print cmd
180        print '------------ stdout/stderr -------------'
181        print expect_error and 'Error expected, but none seen' or out
182    elif expect_error > 1:
183        print
184        print '------ Output of Expected Error --------'
185        print out
186        print '----------------------------------------'
187
188    sys.stdout.flush()
189
190    return (status,out)
191
192
193def expand_vars(path):
194    if os.name == 'nt':
195        re_env = re.compile(r'%\w+%')
196        return re_env.sub(
197              lambda m: os.environ.get( m.group(0)[1:-1] )
198            , path
199            )
200    else:
201        return os.path.expandvars(path)
202
203def remove_directory_and_contents(path):
204    for root, dirs, files in os.walk(path, topdown=False):
205        for name in files:
206            os.remove(os.path.join(root, name))
207        for name in dirs:
208            os.rmdir(os.path.join(root, name))
209    os.rmdir(path)
210
211class BuildResult:
212    def __init__(self, path):
213        self.path = path
214
215    def __repr__(self):
216        return self.path
217
218    def __del__(self):
219        remove_directory_and_contents(self.path)
220
221class CPlusPlusTranslator(litre.LitreTranslator):
222
223    _exposed_attrs = ['compile', 'test', 'ignore', 'match_stdout', 'stack', 'config'
224                      , 'example', 'prefix', 'preprocessors', 'litre_directory',
225                      'litre_translator', 'includes', 'build', 'jam_prefix',
226                      'run_python']
227
228    last_run_output = ''
229
230    """Attributes that will be made available to litre code"""
231
232    def __init__(self, document, config):
233        litre.LitreTranslator.__init__(self, document, config)
234        self.in_literal = False
235        self.in_table = True
236        self.preprocessors = []
237        self.stack = []
238        self.example = None
239        self.prefix = []
240        self.includes = config.includes
241        self.litre_directory = os.path.split(__file__)[0]
242        self.config = config
243        self.litre_translator = self
244        self.line_offset = 0
245        self.last_source = None
246        self.jam_prefix = []
247
248        self.globals = { 'test_literals_in_tables' : False }
249        for m in self._exposed_attrs:
250            self.globals[m] = getattr(self, m)
251
252        self.examples = {}
253        self.current_section = None
254
255    #
256    # Stuff for use by docutils writer framework
257    #
258    def visit_emphasis(self, node):
259        if self.in_literal:
260            self.example.begin_emphasis()
261
262    def depart_emphasis(self, node):
263        if self.in_literal:
264            self.example.end_emphasis()
265
266    def visit_section(self, node):
267        self.current_section = node['ids'][0]
268
269    def visit_literal_block(self, node):
270        if node.source is None:
271            node.source = self.last_source
272        self.last_source = node.source
273
274        # create a new example
275        self.example = Example(node, self.current_section, line_offset = self.line_offset, line_hash = self.config.line_hash)
276
277        self.stack.append(self.example)
278
279        self.in_literal = True
280
281    def depart_literal_block(self, node):
282        self.in_literal = False
283
284    def visit_literal(self, node):
285        if self.in_table and self.globals['test_literals_in_tables']:
286            self.visit_literal_block(node)
287        else:
288            litre.LitreTranslator.visit_literal(self,node)
289
290    def depart_literal(self, node):
291        if self.in_table and self.globals['test_literals_in_tables']:
292            self.depart_literal_block(node)
293        else:
294            litre.LitreTranslator.depart_literal(self,node)
295
296    def visit_table(self,node):
297        self.in_table = True
298        litre.LitreTranslator.visit_table(self,node)
299
300    def depart_table(self,node):
301        self.in_table = False
302        litre.LitreTranslator.depart_table(self,node)
303
304    def visit_Text(self, node):
305        if self.in_literal:
306            self.example.append_raw(node.astext())
307
308    def depart_document(self, node):
309        self.write_examples()
310
311    #
312    # Private stuff
313    #
314
315    def handled(self, n = 1):
316        r = self.stack[-n:]
317        del self.stack[-n:]
318        return r
319
320    def _execute(self, code):
321        """Override of litre._execute; sets up variable context before
322        evaluating code
323        """
324        self.globals['example'] = self.example
325        eval(code, self.globals)
326
327    #
328    # Stuff for use by embedded python code
329    #
330
331    def match_stdout(self, expected = None):
332
333        if expected is None:
334            expected = self.example.raw()
335            self.handled()
336
337        if not re.search(expected, self.last_run_output, re.MULTILINE):
338        #if self.last_run_output.strip('\n') != expected.strip('\n'):
339            print 'output failed to match example'
340            print '-------- Actual Output -------------'
341            print repr(self.last_run_output)
342            print '-------- Expected Output -----------'
343            print repr(expected)
344            print '------------------------------------'
345            sys.stdout.flush()
346
347    def ignore(self, n = 1):
348        if n == 'all':
349            n = len(self.stack)
350        return self.handled(n)
351
352    def wrap(self, n, s1, s2):
353        self.stack[-1].append(s2)
354        self.stack[-n].prepend(s1)
355
356
357    def compile(
358          self
359        , howmany = 1
360        , pop = -1
361        , expect_error = False
362        , extension = '.o'
363        , options = ['-c']
364        , built_handler = lambda built_file: None
365        , source_file = None
366        , source_suffix = '.cpp'
367          # C-style comments by default; handles C++ and YACC
368        , make_comment = lambda text: '/*\n%s\n*/' % text
369        , built_file = None
370        , command = None
371        ):
372        """
373        Compile examples on the stack, whose topmost item is the last example
374        seen but not yet handled so far.
375
376        :howmany: How many of the topmost examples on the stack to compile.
377           You can pass a number, or 'all' to indicate that all examples should
378           be compiled.
379
380        :pop: How many of the topmost examples to discard.  By default, all of
381           the examples that are compiled are discarded.
382
383        :expect_error: Whether a compilation error is to be expected.  Any value
384           > 1 will cause the expected diagnostic's text to be dumped for
385           diagnostic purposes.  It's common to expect an error but see a
386           completely unrelated one because of bugs in the example (you can get
387           this behavior for all examples by setting show_expected_error_output
388           in your config).
389
390        :extension: The extension of the file to build (set to .exe for
391           run)
392
393        :options: Compiler flags
394
395        :built_file: A path to use for the built file.  By default, a temp
396            filename is conjured up
397
398        :built_handler: A function that's called with the name of the built file
399           upon success.
400
401        :source_file: The full name of the source file to write
402
403        :source_suffix: If source_file is None, the suffix to use for the source file
404
405        :make_comment: A function that transforms text into an appropriate comment.
406
407        :command: A function that is passed (includes, opts, target, source), where
408           opts is a string representing compiler options, target is the name of
409           the file to build, and source is the name of the file into which the
410           example code is written.  By default, the function formats
411           litre.config.compiler with its argument tuple.
412        """
413
414        # Grab one example by default
415        if howmany == 'all':
416            howmany = len(self.stack)
417
418        source = '\n'.join(
419            self.prefix
420            + [str(x) for x in self.stack[-howmany:]]
421            )
422
423        source = reduce(lambda s, f: f(s), self.preprocessors, source)
424
425        if pop:
426            if pop < 0:
427                pop = howmany
428            del self.stack[-pop:]
429
430        if len(self.stack):
431            self.example = self.stack[-1]
432
433        cpp = self._source_file_path(source_file, source_suffix)
434
435        if built_file is None:
436            built_file = self._output_file_path(source_file, extension)
437
438        opts = ' '.join(options)
439
440        includes = ' '.join(['-I%s' % d for d in self.includes])
441        if not command:
442            command = self.config.compiler
443
444        if type(command) == str:
445            command = lambda i, o, t, s, c = command: c % (i, o, t, s)
446
447        cmd = command(includes, opts, expand_vars(built_file), expand_vars(cpp))
448
449        if expect_error and self.config.show_expected_error_output:
450            expect_error += 1
451
452
453        comment_cmd = command(includes, opts, built_file, os.path.basename(cpp))
454        comment = make_comment(config.comment_text(comment_cmd, expect_error))
455
456        self._write_source(cpp, '\n'.join([comment, source]))
457
458        #print 'wrote in', cpp
459        #print 'trying command', cmd
460
461        status, output = syscmd(cmd, expect_error)
462
463        if status or expect_error > 1:
464            print
465            if expect_error and expect_error < 2:
466                print 'Compilation failure expected, but none seen'
467            print '------------ begin offending source ------------'
468            print open(cpp).read()
469            print '------------ end offending source ------------'
470
471            if self.config.save_cpp:
472                print 'saved in', repr(cpp)
473            else:
474                self._remove_source(cpp)
475
476            sys.stdout.flush()
477        else:
478            print '.',
479            sys.stdout.flush()
480            built_handler(built_file)
481
482            self._remove_source(cpp)
483
484        try:
485            self._unlink(built_file)
486        except:
487            if not expect_error:
488                print 'failed to unlink', built_file
489
490        return status
491
492    def test(
493          self
494        , rule = 'run'
495        , howmany = 1
496        , pop = -1
497        , expect_error = False
498        , requirements = ''
499        , input = ''
500        ):
501
502        # Grab one example by default
503        if howmany == 'all':
504            howmany = len(self.stack)
505
506        source = '\n'.join(
507            self.prefix
508            + [str(x) for x in self.stack[-howmany:]]
509            )
510
511        source = reduce(lambda s, f: f(s), self.preprocessors, source)
512
513        id = self.example.section
514        if not id:
515            id = 'top-level'
516
517        if not self.examples.has_key(self.example.section):
518            self.examples[id] = [(rule, source)]
519        else:
520            self.examples[id].append((rule, source))
521
522        if pop:
523            if pop < 0:
524                pop = howmany
525            del self.stack[-pop:]
526
527        if len(self.stack):
528            self.example = self.stack[-1]
529
530    def write_examples(self):
531        jam = open(os.path.join(self.config.dump_dir, 'Jamfile.v2'), 'w')
532
533        jam.write('''
534import testing ;
535
536''')
537
538        for id,examples in self.examples.items():
539            for i in range(len(examples)):
540                cpp = '%s%d.cpp' % (id, i)
541
542                jam.write('%s %s ;\n' % (examples[i][0], cpp))
543
544                outfile = os.path.join(self.config.dump_dir, cpp)
545                print cpp,
546                try:
547                    if open(outfile, 'r').read() == examples[i][1]:
548                        print ' .. skip'
549                        continue
550                except:
551                    pass
552
553                open(outfile, 'w').write(examples[i][1])
554                print ' .. written'
555
556        jam.close()
557
558    def build(
559          self
560        , howmany = 1
561        , pop = -1
562        , source_file = 'example.cpp'
563        , expect_error = False
564        , target_rule = 'obj'
565        , requirements = ''
566        , input = ''
567        , output = 'example_output'
568        ):
569
570        # Grab one example by default
571        if howmany == 'all':
572            howmany = len(self.stack)
573
574        source = '\n'.join(
575            self.prefix
576            + [str(x) for x in self.stack[-howmany:]]
577            )
578
579        source = reduce(lambda s, f: f(s), self.preprocessors, source)
580
581        if pop:
582            if pop < 0:
583                pop = howmany
584            del self.stack[-pop:]
585
586        if len(self.stack):
587            self.example = self.stack[-1]
588
589        dir = tempfile.mkdtemp()
590        cpp = os.path.join(dir, source_file)
591        self._write_source(cpp, source)
592        self._write_jamfile(
593            dir
594          , target_rule = target_rule
595          , requirements = requirements
596          , input = input
597          , output = output
598          )
599
600        cmd = 'bjam'
601        if self.config.bjam_options:
602            cmd += ' %s' % self.config.bjam_options
603
604        os.chdir(dir)
605        status, output = syscmd(cmd, expect_error)
606
607        if status or expect_error > 1:
608            print
609            if expect_error and expect_error < 2:
610                print 'Compilation failure expected, but none seen'
611            print '------------ begin offending source ------------'
612            print open(cpp).read()
613            print '------------ begin offending Jamfile -----------'
614            print open(os.path.join(dir, 'Jamroot')).read()
615            print '------------ end offending Jamfile -------------'
616
617            sys.stdout.flush()
618        else:
619            print '.',
620            sys.stdout.flush()
621
622        if status: return None
623        else: return BuildResult(dir)
624
625    def _write_jamfile(self, path, target_rule, requirements, input, output):
626        jamfile = open(os.path.join(path, 'Jamroot'), 'w')
627        contents = r"""
628import modules ;
629
630BOOST_ROOT = [ modules.peek : BOOST_ROOT ] ;
631use-project /boost : $(BOOST_ROOT) ;
632
633%s
634
635%s %s
636  : example.cpp %s
637  : <include>.
638    %s
639    %s
640  ;
641        """ % (
642            '\n'.join(self.jam_prefix)
643          , target_rule
644          , output
645          , input
646          , ' '.join(['<include>%s' % d for d in self.includes])
647          , requirements
648          )
649
650        jamfile.write(contents)
651
652    def run_python(
653          self
654        , howmany = 1
655        , pop = -1
656        , module_path = []
657        , expect_error = False
658        ):
659        # Grab one example by default
660        if howmany == 'all':
661            howmany = len(self.stack)
662
663        if module_path == None: module_path = []
664
665        if isinstance(module_path, BuildResult) or type(module_path) == str:
666            module_path = [module_path]
667
668        module_path = map(lambda p: str(p), module_path)
669
670        source = '\n'.join(
671            self.prefix
672            + [str(x) for x in self.stack[-howmany:]]
673            )
674
675        if pop:
676            if pop < 0:
677                pop = howmany
678            del self.stack[-pop:]
679
680        if len(self.stack):
681            self.example = self.stack[-1]
682
683        r = re.compile(r'^(>>>|\.\.\.) (.*)$', re.MULTILINE)
684        source = r.sub(r'\2', source)
685        py = self._source_file_path(source_file = None, source_suffix = 'py')
686        open(py, 'w').write(source)
687
688        old_path = os.getenv('PYTHONPATH')
689        if old_path == None:
690            pythonpath = ':'.join(module_path)
691            old_path = ''
692        else:
693            pythonpath = old_path + ':%s' % ':'.join(module_path)
694
695        os.putenv('PYTHONPATH', pythonpath)
696        status, output = syscmd('python %s' % py)
697
698        if status or expect_error > 1:
699            print
700            if expect_error and expect_error < 2:
701                print 'Compilation failure expected, but none seen'
702            print '------------ begin offending source ------------'
703            print open(py).read()
704            print '------------ end offending Jamfile -------------'
705
706            sys.stdout.flush()
707        else:
708            print '.',
709            sys.stdout.flush()
710
711        self.last_run_output = output
712        os.putenv('PYTHONPATH', old_path)
713        self._unlink(py)
714
715    def _write_source(self, filename, contents):
716        open(filename,'w').write(contents)
717
718    def _remove_source(self, source_path):
719        os.unlink(source_path)
720
721    def _source_file_path(self, source_file, source_suffix):
722        if source_file is None:
723            cpp = tempfile.mktemp(suffix=source_suffix)
724        else:
725            cpp = os.path.join(tempfile.gettempdir(), source_file)
726        return cpp
727
728    def _output_file_path(self, source_file, extension):
729        return tempfile.mktemp(suffix=extension)
730
731    def _unlink(self, file):
732        file = expand_vars(file)
733        if os.path.exists(file):
734            os.unlink(file)
735
736    def _launch(self, exe, stdin = None):
737        status, output = syscmd(exe, input = stdin)
738        self.last_run_output = output
739
740    def run_(self, howmany = 1, stdin = None, **kw):
741        new_kw = { 'options':[], 'extension':'.exe' }
742        new_kw.update(kw)
743
744        self.compile(
745            howmany
746          , built_handler = lambda exe: self._launch(exe, stdin = stdin)
747          , **new_kw
748        )
749
750    def astext(self):
751        return ""
752        return '\n\n ---------------- Unhandled Fragment ------------ \n\n'.join(
753            [''] # generates a leading announcement
754            + [ unicode(s) for s in self.stack]
755            )
756
757class DumpTranslator(CPlusPlusTranslator):
758    example_index = 1
759
760    def _source_file_path(self, source_file, source_suffix):
761        if source_file is None:
762            source_file = 'example%s%s' % (self.example_index, source_suffix)
763            self.example_index += 1
764
765        cpp = os.path.join(config.dump_dir, source_file)
766        return cpp
767
768    def _output_file_path(self, source_file, extension):
769        chapter = os.path.basename(config.dump_dir)
770        return '%%TEMP%%\metaprogram-%s-example%s%s' \
771            % ( chapter, self.example_index - 1, extension)
772
773    def _remove_source(self, source_path):
774        pass
775
776
777class WorkaroundTranslator(DumpTranslator):
778    """Translator used to test/dump workaround examples for vc6 and vc7.  Just
779    like a DumpTranslator except that we leave existing files alone.
780
781    Warning: not sensitive to changes in .rst source!!  If you change the actual
782    examples in source files you will have to move the example files out of the
783    way and regenerate them, then re-incorporate the workarounds.
784    """
785    def _write_source(self, filename, contents):
786        if not os.path.exists(filename):
787            DumpTranslator._write_source(self, filename, contents)
788
789class Config:
790    save_cpp = False
791    line_hash = '#'
792    show_expected_error_output = False
793    max_output_lines = None
794
795class Writer(litre.Writer):
796    translator = CPlusPlusTranslator
797
798    def __init__(
799        self
800      , config
801    ):
802        litre.Writer.__init__(self)
803        self._config = Config()
804        defaults = Config.__dict__
805
806        # update config elements
807        self._config.__dict__.update(config.__dict__)
808#             dict([i for i in config.__dict__.items()
809#                   if i[0] in config.__all__]))
810
811