1"""
2Test harness for the venv module.
3
4Copyright (C) 2011-2012 Vinay Sajip.
5Licensed to the PSF under a contributor agreement.
6"""
7
8import ensurepip
9import os
10import os.path
11import re
12import shutil
13import struct
14import subprocess
15import sys
16import tempfile
17from test.support import (captured_stdout, captured_stderr, requires_zlib,
18                          can_symlink, EnvironmentVarGuard, rmtree,
19                          import_module,
20                          skip_if_broken_multiprocessing_synchronize)
21import threading
22import unittest
23import venv
24
25try:
26    import ctypes
27except ImportError:
28    ctypes = None
29
30# Platforms that set sys._base_executable can create venvs from within
31# another venv, so no need to skip tests that require venv.create().
32requireVenvCreate = unittest.skipUnless(
33    sys.prefix == sys.base_prefix
34    or sys._base_executable != sys.executable,
35    'cannot run venv.create from within a venv on this platform')
36
37def check_output(cmd, encoding=None):
38    p = subprocess.Popen(cmd,
39        stdout=subprocess.PIPE,
40        stderr=subprocess.PIPE,
41        encoding=encoding)
42    out, err = p.communicate()
43    if p.returncode:
44        raise subprocess.CalledProcessError(
45            p.returncode, cmd, out, err)
46    return out, err
47
48class BaseTest(unittest.TestCase):
49    """Base class for venv tests."""
50    maxDiff = 80 * 50
51
52    def setUp(self):
53        self.env_dir = os.path.realpath(tempfile.mkdtemp())
54        if os.name == 'nt':
55            self.bindir = 'Scripts'
56            self.lib = ('Lib',)
57            self.include = 'Include'
58        else:
59            self.bindir = 'bin'
60            self.lib = ('lib', 'python%d.%d' % sys.version_info[:2])
61            self.include = 'include'
62        executable = sys._base_executable
63        self.exe = os.path.split(executable)[-1]
64        if (sys.platform == 'win32'
65            and os.path.lexists(executable)
66            and not os.path.exists(executable)):
67            self.cannot_link_exe = True
68        else:
69            self.cannot_link_exe = False
70
71    def tearDown(self):
72        rmtree(self.env_dir)
73
74    def run_with_capture(self, func, *args, **kwargs):
75        with captured_stdout() as output:
76            with captured_stderr() as error:
77                func(*args, **kwargs)
78        return output.getvalue(), error.getvalue()
79
80    def get_env_file(self, *args):
81        return os.path.join(self.env_dir, *args)
82
83    def get_text_file_contents(self, *args):
84        with open(self.get_env_file(*args), 'r') as f:
85            result = f.read()
86        return result
87
88class BasicTest(BaseTest):
89    """Test venv module functionality."""
90
91    def isdir(self, *args):
92        fn = self.get_env_file(*args)
93        self.assertTrue(os.path.isdir(fn))
94
95    def test_defaults(self):
96        """
97        Test the create function with default arguments.
98        """
99        rmtree(self.env_dir)
100        self.run_with_capture(venv.create, self.env_dir)
101        self.isdir(self.bindir)
102        self.isdir(self.include)
103        self.isdir(*self.lib)
104        # Issue 21197
105        p = self.get_env_file('lib64')
106        conditions = ((struct.calcsize('P') == 8) and (os.name == 'posix') and
107                      (sys.platform != 'darwin'))
108        if conditions:
109            self.assertTrue(os.path.islink(p))
110        else:
111            self.assertFalse(os.path.exists(p))
112        data = self.get_text_file_contents('pyvenv.cfg')
113        executable = sys._base_executable
114        path = os.path.dirname(executable)
115        self.assertIn('home = %s' % path, data)
116        fn = self.get_env_file(self.bindir, self.exe)
117        if not os.path.exists(fn):  # diagnostics for Windows buildbot failures
118            bd = self.get_env_file(self.bindir)
119            print('Contents of %r:' % bd)
120            print('    %r' % os.listdir(bd))
121        self.assertTrue(os.path.exists(fn), 'File %r should exist.' % fn)
122
123    def test_prompt(self):
124        env_name = os.path.split(self.env_dir)[1]
125
126        rmtree(self.env_dir)
127        builder = venv.EnvBuilder()
128        self.run_with_capture(builder.create, self.env_dir)
129        context = builder.ensure_directories(self.env_dir)
130        data = self.get_text_file_contents('pyvenv.cfg')
131        self.assertEqual(context.prompt, '(%s) ' % env_name)
132        self.assertNotIn("prompt = ", data)
133
134        rmtree(self.env_dir)
135        builder = venv.EnvBuilder(prompt='My prompt')
136        self.run_with_capture(builder.create, self.env_dir)
137        context = builder.ensure_directories(self.env_dir)
138        data = self.get_text_file_contents('pyvenv.cfg')
139        self.assertEqual(context.prompt, '(My prompt) ')
140        self.assertIn("prompt = 'My prompt'\n", data)
141
142    @requireVenvCreate
143    def test_prefixes(self):
144        """
145        Test that the prefix values are as expected.
146        """
147        # check a venv's prefixes
148        rmtree(self.env_dir)
149        self.run_with_capture(venv.create, self.env_dir)
150        envpy = os.path.join(self.env_dir, self.bindir, self.exe)
151        cmd = [envpy, '-c', None]
152        for prefix, expected in (
153            ('prefix', self.env_dir),
154            ('exec_prefix', self.env_dir),
155            ('base_prefix', sys.base_prefix),
156            ('base_exec_prefix', sys.base_exec_prefix)):
157            cmd[2] = 'import sys; print(sys.%s)' % prefix
158            out, err = check_output(cmd)
159            self.assertEqual(out.strip(), expected.encode())
160
161    if sys.platform == 'win32':
162        ENV_SUBDIRS = (
163            ('Scripts',),
164            ('Include',),
165            ('Lib',),
166            ('Lib', 'site-packages'),
167        )
168    else:
169        ENV_SUBDIRS = (
170            ('bin',),
171            ('include',),
172            ('lib',),
173            ('lib', 'python%d.%d' % sys.version_info[:2]),
174            ('lib', 'python%d.%d' % sys.version_info[:2], 'site-packages'),
175        )
176
177    def create_contents(self, paths, filename):
178        """
179        Create some files in the environment which are unrelated
180        to the virtual environment.
181        """
182        for subdirs in paths:
183            d = os.path.join(self.env_dir, *subdirs)
184            os.mkdir(d)
185            fn = os.path.join(d, filename)
186            with open(fn, 'wb') as f:
187                f.write(b'Still here?')
188
189    def test_overwrite_existing(self):
190        """
191        Test creating environment in an existing directory.
192        """
193        self.create_contents(self.ENV_SUBDIRS, 'foo')
194        venv.create(self.env_dir)
195        for subdirs in self.ENV_SUBDIRS:
196            fn = os.path.join(self.env_dir, *(subdirs + ('foo',)))
197            self.assertTrue(os.path.exists(fn))
198            with open(fn, 'rb') as f:
199                self.assertEqual(f.read(), b'Still here?')
200
201        builder = venv.EnvBuilder(clear=True)
202        builder.create(self.env_dir)
203        for subdirs in self.ENV_SUBDIRS:
204            fn = os.path.join(self.env_dir, *(subdirs + ('foo',)))
205            self.assertFalse(os.path.exists(fn))
206
207    def clear_directory(self, path):
208        for fn in os.listdir(path):
209            fn = os.path.join(path, fn)
210            if os.path.islink(fn) or os.path.isfile(fn):
211                os.remove(fn)
212            elif os.path.isdir(fn):
213                rmtree(fn)
214
215    def test_unoverwritable_fails(self):
216        #create a file clashing with directories in the env dir
217        for paths in self.ENV_SUBDIRS[:3]:
218            fn = os.path.join(self.env_dir, *paths)
219            with open(fn, 'wb') as f:
220                f.write(b'')
221            self.assertRaises((ValueError, OSError), venv.create, self.env_dir)
222            self.clear_directory(self.env_dir)
223
224    def test_upgrade(self):
225        """
226        Test upgrading an existing environment directory.
227        """
228        # See Issue #21643: the loop needs to run twice to ensure
229        # that everything works on the upgrade (the first run just creates
230        # the venv).
231        for upgrade in (False, True):
232            builder = venv.EnvBuilder(upgrade=upgrade)
233            self.run_with_capture(builder.create, self.env_dir)
234            self.isdir(self.bindir)
235            self.isdir(self.include)
236            self.isdir(*self.lib)
237            fn = self.get_env_file(self.bindir, self.exe)
238            if not os.path.exists(fn):
239                # diagnostics for Windows buildbot failures
240                bd = self.get_env_file(self.bindir)
241                print('Contents of %r:' % bd)
242                print('    %r' % os.listdir(bd))
243            self.assertTrue(os.path.exists(fn), 'File %r should exist.' % fn)
244
245    def test_isolation(self):
246        """
247        Test isolation from system site-packages
248        """
249        for ssp, s in ((True, 'true'), (False, 'false')):
250            builder = venv.EnvBuilder(clear=True, system_site_packages=ssp)
251            builder.create(self.env_dir)
252            data = self.get_text_file_contents('pyvenv.cfg')
253            self.assertIn('include-system-site-packages = %s\n' % s, data)
254
255    @unittest.skipUnless(can_symlink(), 'Needs symlinks')
256    def test_symlinking(self):
257        """
258        Test symlinking works as expected
259        """
260        for usl in (False, True):
261            builder = venv.EnvBuilder(clear=True, symlinks=usl)
262            builder.create(self.env_dir)
263            fn = self.get_env_file(self.bindir, self.exe)
264            # Don't test when False, because e.g. 'python' is always
265            # symlinked to 'python3.3' in the env, even when symlinking in
266            # general isn't wanted.
267            if usl:
268                if self.cannot_link_exe:
269                    # Symlinking is skipped when our executable is already a
270                    # special app symlink
271                    self.assertFalse(os.path.islink(fn))
272                else:
273                    self.assertTrue(os.path.islink(fn))
274
275    # If a venv is created from a source build and that venv is used to
276    # run the test, the pyvenv.cfg in the venv created in the test will
277    # point to the venv being used to run the test, and we lose the link
278    # to the source build - so Python can't initialise properly.
279    @requireVenvCreate
280    def test_executable(self):
281        """
282        Test that the sys.executable value is as expected.
283        """
284        rmtree(self.env_dir)
285        self.run_with_capture(venv.create, self.env_dir)
286        envpy = os.path.join(os.path.realpath(self.env_dir),
287                             self.bindir, self.exe)
288        out, err = check_output([envpy, '-c',
289            'import sys; print(sys.executable)'])
290        self.assertEqual(out.strip(), envpy.encode())
291
292    @unittest.skipUnless(can_symlink(), 'Needs symlinks')
293    def test_executable_symlinks(self):
294        """
295        Test that the sys.executable value is as expected.
296        """
297        rmtree(self.env_dir)
298        builder = venv.EnvBuilder(clear=True, symlinks=True)
299        builder.create(self.env_dir)
300        envpy = os.path.join(os.path.realpath(self.env_dir),
301                             self.bindir, self.exe)
302        out, err = check_output([envpy, '-c',
303            'import sys; print(sys.executable)'])
304        self.assertEqual(out.strip(), envpy.encode())
305
306    @unittest.skipUnless(os.name == 'nt', 'only relevant on Windows')
307    def test_unicode_in_batch_file(self):
308        """
309        Test handling of Unicode paths
310        """
311        rmtree(self.env_dir)
312        env_dir = os.path.join(os.path.realpath(self.env_dir), 'ϼўТλФЙ')
313        builder = venv.EnvBuilder(clear=True)
314        builder.create(env_dir)
315        activate = os.path.join(env_dir, self.bindir, 'activate.bat')
316        envpy = os.path.join(env_dir, self.bindir, self.exe)
317        out, err = check_output(
318            [activate, '&', self.exe, '-c', 'print(0)'],
319            encoding='oem',
320        )
321        self.assertEqual(out.strip(), '0')
322
323    @requireVenvCreate
324    def test_multiprocessing(self):
325        """
326        Test that the multiprocessing is able to spawn.
327        """
328        # bpo-36342: Instantiation of a Pool object imports the
329        # multiprocessing.synchronize module. Skip the test if this module
330        # cannot be imported.
331        skip_if_broken_multiprocessing_synchronize()
332
333        rmtree(self.env_dir)
334        self.run_with_capture(venv.create, self.env_dir)
335        envpy = os.path.join(os.path.realpath(self.env_dir),
336                             self.bindir, self.exe)
337        out, err = check_output([envpy, '-c',
338            'from multiprocessing import Pool; '
339            'pool = Pool(1); '
340            'print(pool.apply_async("Python".lower).get(3)); '
341            'pool.terminate()'])
342        self.assertEqual(out.strip(), "python".encode())
343
344    @unittest.skipIf(os.name == 'nt', 'not relevant on Windows')
345    def test_deactivate_with_strict_bash_opts(self):
346        bash = shutil.which("bash")
347        if bash is None:
348            self.skipTest("bash required for this test")
349        rmtree(self.env_dir)
350        builder = venv.EnvBuilder(clear=True)
351        builder.create(self.env_dir)
352        activate = os.path.join(self.env_dir, self.bindir, "activate")
353        test_script = os.path.join(self.env_dir, "test_strict.sh")
354        with open(test_script, "w") as f:
355            f.write("set -euo pipefail\n"
356                    f"source {activate}\n"
357                    "deactivate\n")
358        out, err = check_output([bash, test_script])
359        self.assertEqual(out, "".encode())
360        self.assertEqual(err, "".encode())
361
362
363    @unittest.skipUnless(sys.platform == 'darwin', 'only relevant on macOS')
364    def test_macos_env(self):
365        rmtree(self.env_dir)
366        builder = venv.EnvBuilder()
367        builder.create(self.env_dir)
368
369        envpy = os.path.join(os.path.realpath(self.env_dir),
370                             self.bindir, self.exe)
371        out, err = check_output([envpy, '-c',
372            'import os; print("__PYVENV_LAUNCHER__" in os.environ)'])
373        self.assertEqual(out.strip(), 'False'.encode())
374
375@requireVenvCreate
376class EnsurePipTest(BaseTest):
377    """Test venv module installation of pip."""
378    def assert_pip_not_installed(self):
379        envpy = os.path.join(os.path.realpath(self.env_dir),
380                             self.bindir, self.exe)
381        out, err = check_output([envpy, '-c',
382            'try:\n import pip\nexcept ImportError:\n print("OK")'])
383        # We force everything to text, so unittest gives the detailed diff
384        # if we get unexpected results
385        err = err.decode("latin-1") # Force to text, prevent decoding errors
386        self.assertEqual(err, "")
387        out = out.decode("latin-1") # Force to text, prevent decoding errors
388        self.assertEqual(out.strip(), "OK")
389
390
391    def test_no_pip_by_default(self):
392        rmtree(self.env_dir)
393        self.run_with_capture(venv.create, self.env_dir)
394        self.assert_pip_not_installed()
395
396    def test_explicit_no_pip(self):
397        rmtree(self.env_dir)
398        self.run_with_capture(venv.create, self.env_dir, with_pip=False)
399        self.assert_pip_not_installed()
400
401    def test_devnull(self):
402        # Fix for issue #20053 uses os.devnull to force a config file to
403        # appear empty. However http://bugs.python.org/issue20541 means
404        # that doesn't currently work properly on Windows. Once that is
405        # fixed, the "win_location" part of test_with_pip should be restored
406        with open(os.devnull, "rb") as f:
407            self.assertEqual(f.read(), b"")
408
409        self.assertTrue(os.path.exists(os.devnull))
410
411    def do_test_with_pip(self, system_site_packages):
412        rmtree(self.env_dir)
413        with EnvironmentVarGuard() as envvars:
414            # pip's cross-version compatibility may trigger deprecation
415            # warnings in current versions of Python. Ensure related
416            # environment settings don't cause venv to fail.
417            envvars["PYTHONWARNINGS"] = "e"
418            # ensurepip is different enough from a normal pip invocation
419            # that we want to ensure it ignores the normal pip environment
420            # variable settings. We set PIP_NO_INSTALL here specifically
421            # to check that ensurepip (and hence venv) ignores it.
422            # See http://bugs.python.org/issue19734
423            envvars["PIP_NO_INSTALL"] = "1"
424            # Also check that we ignore the pip configuration file
425            # See http://bugs.python.org/issue20053
426            with tempfile.TemporaryDirectory() as home_dir:
427                envvars["HOME"] = home_dir
428                bad_config = "[global]\nno-install=1"
429                # Write to both config file names on all platforms to reduce
430                # cross-platform variation in test code behaviour
431                win_location = ("pip", "pip.ini")
432                posix_location = (".pip", "pip.conf")
433                # Skips win_location due to http://bugs.python.org/issue20541
434                for dirname, fname in (posix_location,):
435                    dirpath = os.path.join(home_dir, dirname)
436                    os.mkdir(dirpath)
437                    fpath = os.path.join(dirpath, fname)
438                    with open(fpath, 'w') as f:
439                        f.write(bad_config)
440
441                # Actually run the create command with all that unhelpful
442                # config in place to ensure we ignore it
443                try:
444                    self.run_with_capture(venv.create, self.env_dir,
445                                          system_site_packages=system_site_packages,
446                                          with_pip=True)
447                except subprocess.CalledProcessError as exc:
448                    # The output this produces can be a little hard to read,
449                    # but at least it has all the details
450                    details = exc.output.decode(errors="replace")
451                    msg = "{}\n\n**Subprocess Output**\n{}"
452                    self.fail(msg.format(exc, details))
453        # Ensure pip is available in the virtual environment
454        envpy = os.path.join(os.path.realpath(self.env_dir), self.bindir, self.exe)
455        # Ignore DeprecationWarning since pip code is not part of Python
456        out, err = check_output([envpy, '-W', 'ignore::DeprecationWarning', '-I',
457               '-m', 'pip', '--version'])
458        # We force everything to text, so unittest gives the detailed diff
459        # if we get unexpected results
460        err = err.decode("latin-1") # Force to text, prevent decoding errors
461        self.assertEqual(err, "")
462        out = out.decode("latin-1") # Force to text, prevent decoding errors
463        expected_version = "pip {}".format(ensurepip.version())
464        self.assertEqual(out[:len(expected_version)], expected_version)
465        env_dir = os.fsencode(self.env_dir).decode("latin-1")
466        self.assertIn(env_dir, out)
467
468        # http://bugs.python.org/issue19728
469        # Check the private uninstall command provided for the Windows
470        # installers works (at least in a virtual environment)
471        with EnvironmentVarGuard() as envvars:
472            out, err = check_output([envpy,
473                '-W', 'ignore::DeprecationWarning', '-I',
474                '-m', 'ensurepip._uninstall'])
475        # We force everything to text, so unittest gives the detailed diff
476        # if we get unexpected results
477        err = err.decode("latin-1") # Force to text, prevent decoding errors
478        # Ignore the warning:
479        #   "The directory '$HOME/.cache/pip/http' or its parent directory
480        #    is not owned by the current user and the cache has been disabled.
481        #    Please check the permissions and owner of that directory. If
482        #    executing pip with sudo, you may want sudo's -H flag."
483        # where $HOME is replaced by the HOME environment variable.
484        err = re.sub("^(WARNING: )?The directory .* or its parent directory "
485                     "is not owned or is not writable by the current user.*$", "",
486                     err, flags=re.MULTILINE)
487        self.assertEqual(err.rstrip(), "")
488        # Being fairly specific regarding the expected behaviour for the
489        # initial bundling phase in Python 3.4. If the output changes in
490        # future pip versions, this test can likely be relaxed further.
491        out = out.decode("latin-1") # Force to text, prevent decoding errors
492        self.assertIn("Successfully uninstalled pip", out)
493        self.assertIn("Successfully uninstalled setuptools", out)
494        # Check pip is now gone from the virtual environment. This only
495        # applies in the system_site_packages=False case, because in the
496        # other case, pip may still be available in the system site-packages
497        if not system_site_packages:
498            self.assert_pip_not_installed()
499
500    # Issue #26610: pip/pep425tags.py requires ctypes
501    @unittest.skipUnless(ctypes, 'pip requires ctypes')
502    @requires_zlib
503    def test_with_pip(self):
504        self.do_test_with_pip(False)
505        self.do_test_with_pip(True)
506
507if __name__ == "__main__":
508    unittest.main()
509