1"""
2Test harness for the venv module.
3
4Copyright (C) 2011-2012 Vinay Sajip.
5Licensed to the PSF under a contributor agreement.
6"""
7
8import ensurepip
9import os
10import os.path
11import re
12import shutil
13import struct
14import subprocess
15import sys
16import tempfile
17from test.support import (captured_stdout, captured_stderr, requires_zlib,
18                          skip_if_broken_multiprocessing_synchronize, verbose)
19from test.support.os_helper import (can_symlink, EnvironmentVarGuard, rmtree)
20import unittest
21import venv
22from unittest.mock import patch
23
24try:
25    import ctypes
26except ImportError:
27    ctypes = None
28
29# Platforms that set sys._base_executable can create venvs from within
30# another venv, so no need to skip tests that require venv.create().
31requireVenvCreate = unittest.skipUnless(
32    sys.prefix == sys.base_prefix
33    or sys._base_executable != sys.executable,
34    'cannot run venv.create from within a venv on this platform')
35
36def check_output(cmd, encoding=None):
37    p = subprocess.Popen(cmd,
38        stdout=subprocess.PIPE,
39        stderr=subprocess.PIPE,
40        encoding=encoding)
41    out, err = p.communicate()
42    if p.returncode:
43        if verbose and err:
44            print(err.decode('utf-8', 'backslashreplace'))
45        raise subprocess.CalledProcessError(
46            p.returncode, cmd, out, err)
47    return out, err
48
49class BaseTest(unittest.TestCase):
50    """Base class for venv tests."""
51    maxDiff = 80 * 50
52
53    def setUp(self):
54        self.env_dir = os.path.realpath(tempfile.mkdtemp())
55        if os.name == 'nt':
56            self.bindir = 'Scripts'
57            self.lib = ('Lib',)
58            self.include = 'Include'
59        else:
60            self.bindir = 'bin'
61            self.lib = ('lib', 'python%d.%d' % sys.version_info[:2])
62            self.include = 'include'
63        executable = sys._base_executable
64        self.exe = os.path.split(executable)[-1]
65        if (sys.platform == 'win32'
66            and os.path.lexists(executable)
67            and not os.path.exists(executable)):
68            self.cannot_link_exe = True
69        else:
70            self.cannot_link_exe = False
71
72    def tearDown(self):
73        rmtree(self.env_dir)
74
75    def run_with_capture(self, func, *args, **kwargs):
76        with captured_stdout() as output:
77            with captured_stderr() as error:
78                func(*args, **kwargs)
79        return output.getvalue(), error.getvalue()
80
81    def get_env_file(self, *args):
82        return os.path.join(self.env_dir, *args)
83
84    def get_text_file_contents(self, *args, encoding='utf-8'):
85        with open(self.get_env_file(*args), 'r', encoding=encoding) as f:
86            result = f.read()
87        return result
88
89class BasicTest(BaseTest):
90    """Test venv module functionality."""
91
92    def isdir(self, *args):
93        fn = self.get_env_file(*args)
94        self.assertTrue(os.path.isdir(fn))
95
96    def test_defaults(self):
97        """
98        Test the create function with default arguments.
99        """
100        rmtree(self.env_dir)
101        self.run_with_capture(venv.create, self.env_dir)
102        self.isdir(self.bindir)
103        self.isdir(self.include)
104        self.isdir(*self.lib)
105        # Issue 21197
106        p = self.get_env_file('lib64')
107        conditions = ((struct.calcsize('P') == 8) and (os.name == 'posix') and
108                      (sys.platform != 'darwin'))
109        if conditions:
110            self.assertTrue(os.path.islink(p))
111        else:
112            self.assertFalse(os.path.exists(p))
113        data = self.get_text_file_contents('pyvenv.cfg')
114        executable = sys._base_executable
115        path = os.path.dirname(executable)
116        self.assertIn('home = %s' % path, data)
117        fn = self.get_env_file(self.bindir, self.exe)
118        if not os.path.exists(fn):  # diagnostics for Windows buildbot failures
119            bd = self.get_env_file(self.bindir)
120            print('Contents of %r:' % bd)
121            print('    %r' % os.listdir(bd))
122        self.assertTrue(os.path.exists(fn), 'File %r should exist.' % fn)
123
124    def test_prompt(self):
125        env_name = os.path.split(self.env_dir)[1]
126
127        rmtree(self.env_dir)
128        builder = venv.EnvBuilder()
129        self.run_with_capture(builder.create, self.env_dir)
130        context = builder.ensure_directories(self.env_dir)
131        data = self.get_text_file_contents('pyvenv.cfg')
132        self.assertEqual(context.prompt, '(%s) ' % env_name)
133        self.assertNotIn("prompt = ", data)
134
135        rmtree(self.env_dir)
136        builder = venv.EnvBuilder(prompt='My prompt')
137        self.run_with_capture(builder.create, self.env_dir)
138        context = builder.ensure_directories(self.env_dir)
139        data = self.get_text_file_contents('pyvenv.cfg')
140        self.assertEqual(context.prompt, '(My prompt) ')
141        self.assertIn("prompt = 'My prompt'\n", data)
142
143        rmtree(self.env_dir)
144        builder = venv.EnvBuilder(prompt='.')
145        cwd = os.path.basename(os.getcwd())
146        self.run_with_capture(builder.create, self.env_dir)
147        context = builder.ensure_directories(self.env_dir)
148        data = self.get_text_file_contents('pyvenv.cfg')
149        self.assertEqual(context.prompt, '(%s) ' % cwd)
150        self.assertIn("prompt = '%s'\n" % cwd, data)
151
152    def test_upgrade_dependencies(self):
153        builder = venv.EnvBuilder()
154        bin_path = 'Scripts' if sys.platform == 'win32' else 'bin'
155        python_exe = os.path.split(sys.executable)[1]
156        with tempfile.TemporaryDirectory() as fake_env_dir:
157            expect_exe = os.path.normcase(
158                os.path.join(fake_env_dir, bin_path, python_exe)
159            )
160            if sys.platform == 'win32':
161                expect_exe = os.path.normcase(os.path.realpath(expect_exe))
162
163            def pip_cmd_checker(cmd):
164                cmd[0] = os.path.normcase(cmd[0])
165                self.assertEqual(
166                    cmd,
167                    [
168                        expect_exe,
169                        '-m',
170                        'pip',
171                        'install',
172                        '--upgrade',
173                        'pip',
174                        'setuptools'
175                    ]
176                )
177
178            fake_context = builder.ensure_directories(fake_env_dir)
179            with patch('venv.subprocess.check_call', pip_cmd_checker):
180                builder.upgrade_dependencies(fake_context)
181
182    @requireVenvCreate
183    def test_prefixes(self):
184        """
185        Test that the prefix values are as expected.
186        """
187        # check a venv's prefixes
188        rmtree(self.env_dir)
189        self.run_with_capture(venv.create, self.env_dir)
190        envpy = os.path.join(self.env_dir, self.bindir, self.exe)
191        cmd = [envpy, '-c', None]
192        for prefix, expected in (
193            ('prefix', self.env_dir),
194            ('exec_prefix', self.env_dir),
195            ('base_prefix', sys.base_prefix),
196            ('base_exec_prefix', sys.base_exec_prefix)):
197            cmd[2] = 'import sys; print(sys.%s)' % prefix
198            out, err = check_output(cmd)
199            self.assertEqual(out.strip(), expected.encode(), prefix)
200
201    if sys.platform == 'win32':
202        ENV_SUBDIRS = (
203            ('Scripts',),
204            ('Include',),
205            ('Lib',),
206            ('Lib', 'site-packages'),
207        )
208    else:
209        ENV_SUBDIRS = (
210            ('bin',),
211            ('include',),
212            ('lib',),
213            ('lib', 'python%d.%d' % sys.version_info[:2]),
214            ('lib', 'python%d.%d' % sys.version_info[:2], 'site-packages'),
215        )
216
217    def create_contents(self, paths, filename):
218        """
219        Create some files in the environment which are unrelated
220        to the virtual environment.
221        """
222        for subdirs in paths:
223            d = os.path.join(self.env_dir, *subdirs)
224            os.mkdir(d)
225            fn = os.path.join(d, filename)
226            with open(fn, 'wb') as f:
227                f.write(b'Still here?')
228
229    def test_overwrite_existing(self):
230        """
231        Test creating environment in an existing directory.
232        """
233        self.create_contents(self.ENV_SUBDIRS, 'foo')
234        venv.create(self.env_dir)
235        for subdirs in self.ENV_SUBDIRS:
236            fn = os.path.join(self.env_dir, *(subdirs + ('foo',)))
237            self.assertTrue(os.path.exists(fn))
238            with open(fn, 'rb') as f:
239                self.assertEqual(f.read(), b'Still here?')
240
241        builder = venv.EnvBuilder(clear=True)
242        builder.create(self.env_dir)
243        for subdirs in self.ENV_SUBDIRS:
244            fn = os.path.join(self.env_dir, *(subdirs + ('foo',)))
245            self.assertFalse(os.path.exists(fn))
246
247    def clear_directory(self, path):
248        for fn in os.listdir(path):
249            fn = os.path.join(path, fn)
250            if os.path.islink(fn) or os.path.isfile(fn):
251                os.remove(fn)
252            elif os.path.isdir(fn):
253                rmtree(fn)
254
255    def test_unoverwritable_fails(self):
256        #create a file clashing with directories in the env dir
257        for paths in self.ENV_SUBDIRS[:3]:
258            fn = os.path.join(self.env_dir, *paths)
259            with open(fn, 'wb') as f:
260                f.write(b'')
261            self.assertRaises((ValueError, OSError), venv.create, self.env_dir)
262            self.clear_directory(self.env_dir)
263
264    def test_upgrade(self):
265        """
266        Test upgrading an existing environment directory.
267        """
268        # See Issue #21643: the loop needs to run twice to ensure
269        # that everything works on the upgrade (the first run just creates
270        # the venv).
271        for upgrade in (False, True):
272            builder = venv.EnvBuilder(upgrade=upgrade)
273            self.run_with_capture(builder.create, self.env_dir)
274            self.isdir(self.bindir)
275            self.isdir(self.include)
276            self.isdir(*self.lib)
277            fn = self.get_env_file(self.bindir, self.exe)
278            if not os.path.exists(fn):
279                # diagnostics for Windows buildbot failures
280                bd = self.get_env_file(self.bindir)
281                print('Contents of %r:' % bd)
282                print('    %r' % os.listdir(bd))
283            self.assertTrue(os.path.exists(fn), 'File %r should exist.' % fn)
284
285    def test_isolation(self):
286        """
287        Test isolation from system site-packages
288        """
289        for ssp, s in ((True, 'true'), (False, 'false')):
290            builder = venv.EnvBuilder(clear=True, system_site_packages=ssp)
291            builder.create(self.env_dir)
292            data = self.get_text_file_contents('pyvenv.cfg')
293            self.assertIn('include-system-site-packages = %s\n' % s, data)
294
295    @unittest.skipUnless(can_symlink(), 'Needs symlinks')
296    def test_symlinking(self):
297        """
298        Test symlinking works as expected
299        """
300        for usl in (False, True):
301            builder = venv.EnvBuilder(clear=True, symlinks=usl)
302            builder.create(self.env_dir)
303            fn = self.get_env_file(self.bindir, self.exe)
304            # Don't test when False, because e.g. 'python' is always
305            # symlinked to 'python3.3' in the env, even when symlinking in
306            # general isn't wanted.
307            if usl:
308                if self.cannot_link_exe:
309                    # Symlinking is skipped when our executable is already a
310                    # special app symlink
311                    self.assertFalse(os.path.islink(fn))
312                else:
313                    self.assertTrue(os.path.islink(fn))
314
315    # If a venv is created from a source build and that venv is used to
316    # run the test, the pyvenv.cfg in the venv created in the test will
317    # point to the venv being used to run the test, and we lose the link
318    # to the source build - so Python can't initialise properly.
319    @requireVenvCreate
320    def test_executable(self):
321        """
322        Test that the sys.executable value is as expected.
323        """
324        rmtree(self.env_dir)
325        self.run_with_capture(venv.create, self.env_dir)
326        envpy = os.path.join(os.path.realpath(self.env_dir),
327                             self.bindir, self.exe)
328        out, err = check_output([envpy, '-c',
329            'import sys; print(sys.executable)'])
330        self.assertEqual(out.strip(), envpy.encode())
331
332    @unittest.skipUnless(can_symlink(), 'Needs symlinks')
333    def test_executable_symlinks(self):
334        """
335        Test that the sys.executable value is as expected.
336        """
337        rmtree(self.env_dir)
338        builder = venv.EnvBuilder(clear=True, symlinks=True)
339        builder.create(self.env_dir)
340        envpy = os.path.join(os.path.realpath(self.env_dir),
341                             self.bindir, self.exe)
342        out, err = check_output([envpy, '-c',
343            'import sys; print(sys.executable)'])
344        self.assertEqual(out.strip(), envpy.encode())
345
346    @unittest.skipUnless(os.name == 'nt', 'only relevant on Windows')
347    def test_unicode_in_batch_file(self):
348        """
349        Test handling of Unicode paths
350        """
351        rmtree(self.env_dir)
352        env_dir = os.path.join(os.path.realpath(self.env_dir), 'ϼўТλФЙ')
353        builder = venv.EnvBuilder(clear=True)
354        builder.create(env_dir)
355        activate = os.path.join(env_dir, self.bindir, 'activate.bat')
356        envpy = os.path.join(env_dir, self.bindir, self.exe)
357        out, err = check_output(
358            [activate, '&', self.exe, '-c', 'print(0)'],
359            encoding='oem',
360        )
361        self.assertEqual(out.strip(), '0')
362
363    @requireVenvCreate
364    def test_multiprocessing(self):
365        """
366        Test that the multiprocessing is able to spawn.
367        """
368        # bpo-36342: Instantiation of a Pool object imports the
369        # multiprocessing.synchronize module. Skip the test if this module
370        # cannot be imported.
371        skip_if_broken_multiprocessing_synchronize()
372
373        rmtree(self.env_dir)
374        self.run_with_capture(venv.create, self.env_dir)
375        envpy = os.path.join(os.path.realpath(self.env_dir),
376                             self.bindir, self.exe)
377        out, err = check_output([envpy, '-c',
378            'from multiprocessing import Pool; '
379            'pool = Pool(1); '
380            'print(pool.apply_async("Python".lower).get(3)); '
381            'pool.terminate()'])
382        self.assertEqual(out.strip(), "python".encode())
383
384    @unittest.skipIf(os.name == 'nt', 'not relevant on Windows')
385    def test_deactivate_with_strict_bash_opts(self):
386        bash = shutil.which("bash")
387        if bash is None:
388            self.skipTest("bash required for this test")
389        rmtree(self.env_dir)
390        builder = venv.EnvBuilder(clear=True)
391        builder.create(self.env_dir)
392        activate = os.path.join(self.env_dir, self.bindir, "activate")
393        test_script = os.path.join(self.env_dir, "test_strict.sh")
394        with open(test_script, "w") as f:
395            f.write("set -euo pipefail\n"
396                    f"source {activate}\n"
397                    "deactivate\n")
398        out, err = check_output([bash, test_script])
399        self.assertEqual(out, "".encode())
400        self.assertEqual(err, "".encode())
401
402
403    @unittest.skipUnless(sys.platform == 'darwin', 'only relevant on macOS')
404    def test_macos_env(self):
405        rmtree(self.env_dir)
406        builder = venv.EnvBuilder()
407        builder.create(self.env_dir)
408
409        envpy = os.path.join(os.path.realpath(self.env_dir),
410                             self.bindir, self.exe)
411        out, err = check_output([envpy, '-c',
412            'import os; print("__PYVENV_LAUNCHER__" in os.environ)'])
413        self.assertEqual(out.strip(), 'False'.encode())
414
415@requireVenvCreate
416class EnsurePipTest(BaseTest):
417    """Test venv module installation of pip."""
418    def assert_pip_not_installed(self):
419        envpy = os.path.join(os.path.realpath(self.env_dir),
420                             self.bindir, self.exe)
421        out, err = check_output([envpy, '-c',
422            'try:\n import pip\nexcept ImportError:\n print("OK")'])
423        # We force everything to text, so unittest gives the detailed diff
424        # if we get unexpected results
425        err = err.decode("latin-1") # Force to text, prevent decoding errors
426        self.assertEqual(err, "")
427        out = out.decode("latin-1") # Force to text, prevent decoding errors
428        self.assertEqual(out.strip(), "OK")
429
430
431    def test_no_pip_by_default(self):
432        rmtree(self.env_dir)
433        self.run_with_capture(venv.create, self.env_dir)
434        self.assert_pip_not_installed()
435
436    def test_explicit_no_pip(self):
437        rmtree(self.env_dir)
438        self.run_with_capture(venv.create, self.env_dir, with_pip=False)
439        self.assert_pip_not_installed()
440
441    def test_devnull(self):
442        # Fix for issue #20053 uses os.devnull to force a config file to
443        # appear empty. However http://bugs.python.org/issue20541 means
444        # that doesn't currently work properly on Windows. Once that is
445        # fixed, the "win_location" part of test_with_pip should be restored
446        with open(os.devnull, "rb") as f:
447            self.assertEqual(f.read(), b"")
448
449        self.assertTrue(os.path.exists(os.devnull))
450
451    def do_test_with_pip(self, system_site_packages):
452        rmtree(self.env_dir)
453        with EnvironmentVarGuard() as envvars:
454            # pip's cross-version compatibility may trigger deprecation
455            # warnings in current versions of Python. Ensure related
456            # environment settings don't cause venv to fail.
457            envvars["PYTHONWARNINGS"] = "ignore"
458            # ensurepip is different enough from a normal pip invocation
459            # that we want to ensure it ignores the normal pip environment
460            # variable settings. We set PIP_NO_INSTALL here specifically
461            # to check that ensurepip (and hence venv) ignores it.
462            # See http://bugs.python.org/issue19734
463            envvars["PIP_NO_INSTALL"] = "1"
464            # Also check that we ignore the pip configuration file
465            # See http://bugs.python.org/issue20053
466            with tempfile.TemporaryDirectory() as home_dir:
467                envvars["HOME"] = home_dir
468                bad_config = "[global]\nno-install=1"
469                # Write to both config file names on all platforms to reduce
470                # cross-platform variation in test code behaviour
471                win_location = ("pip", "pip.ini")
472                posix_location = (".pip", "pip.conf")
473                # Skips win_location due to http://bugs.python.org/issue20541
474                for dirname, fname in (posix_location,):
475                    dirpath = os.path.join(home_dir, dirname)
476                    os.mkdir(dirpath)
477                    fpath = os.path.join(dirpath, fname)
478                    with open(fpath, 'w') as f:
479                        f.write(bad_config)
480
481                # Actually run the create command with all that unhelpful
482                # config in place to ensure we ignore it
483                try:
484                    self.run_with_capture(venv.create, self.env_dir,
485                                          system_site_packages=system_site_packages,
486                                          with_pip=True)
487                except subprocess.CalledProcessError as exc:
488                    # The output this produces can be a little hard to read,
489                    # but at least it has all the details
490                    details = exc.output.decode(errors="replace")
491                    msg = "{}\n\n**Subprocess Output**\n{}"
492                    self.fail(msg.format(exc, details))
493        # Ensure pip is available in the virtual environment
494        envpy = os.path.join(os.path.realpath(self.env_dir), self.bindir, self.exe)
495        # Ignore DeprecationWarning since pip code is not part of Python
496        out, err = check_output([envpy, '-W', 'ignore::DeprecationWarning',
497               '-W', 'ignore::ImportWarning', '-I',
498               '-m', 'pip', '--version'])
499        # We force everything to text, so unittest gives the detailed diff
500        # if we get unexpected results
501        err = err.decode("latin-1") # Force to text, prevent decoding errors
502        self.assertEqual(err, "")
503        out = out.decode("latin-1") # Force to text, prevent decoding errors
504        expected_version = "pip {}".format(ensurepip.version())
505        self.assertEqual(out[:len(expected_version)], expected_version)
506        env_dir = os.fsencode(self.env_dir).decode("latin-1")
507        self.assertIn(env_dir, out)
508
509        # http://bugs.python.org/issue19728
510        # Check the private uninstall command provided for the Windows
511        # installers works (at least in a virtual environment)
512        with EnvironmentVarGuard() as envvars:
513            # It seems ensurepip._uninstall calls subprocesses which do not
514            # inherit the interpreter settings.
515            envvars["PYTHONWARNINGS"] = "ignore"
516            out, err = check_output([envpy,
517                '-W', 'ignore::DeprecationWarning',
518                '-W', 'ignore::ImportWarning', '-I',
519                '-m', 'ensurepip._uninstall'])
520        # We force everything to text, so unittest gives the detailed diff
521        # if we get unexpected results
522        err = err.decode("latin-1") # Force to text, prevent decoding errors
523        # Ignore the warning:
524        #   "The directory '$HOME/.cache/pip/http' or its parent directory
525        #    is not owned by the current user and the cache has been disabled.
526        #    Please check the permissions and owner of that directory. If
527        #    executing pip with sudo, you may want sudo's -H flag."
528        # where $HOME is replaced by the HOME environment variable.
529        err = re.sub("^(WARNING: )?The directory .* or its parent directory "
530                     "is not owned or is not writable by the current user.*$", "",
531                     err, flags=re.MULTILINE)
532        self.assertEqual(err.rstrip(), "")
533        # Being fairly specific regarding the expected behaviour for the
534        # initial bundling phase in Python 3.4. If the output changes in
535        # future pip versions, this test can likely be relaxed further.
536        out = out.decode("latin-1") # Force to text, prevent decoding errors
537        self.assertIn("Successfully uninstalled pip", out)
538        self.assertIn("Successfully uninstalled setuptools", out)
539        # Check pip is now gone from the virtual environment. This only
540        # applies in the system_site_packages=False case, because in the
541        # other case, pip may still be available in the system site-packages
542        if not system_site_packages:
543            self.assert_pip_not_installed()
544
545    # Issue #26610: pip/pep425tags.py requires ctypes
546    @unittest.skipUnless(ctypes, 'pip requires ctypes')
547    @requires_zlib()
548    def test_with_pip(self):
549        self.do_test_with_pip(False)
550        self.do_test_with_pip(True)
551
552if __name__ == "__main__":
553    unittest.main()
554