1"""
2tests.unit.states.pip_test
3~~~~~~~~~~~~~~~~~~~~~~~~~~
4"""
5
6import logging
7import os
8import subprocess
9import sys
10
11import pytest
12import salt.states.pip_state as pip_state
13import salt.utils.path
14import salt.version
15from salt.modules.virtualenv_mod import KNOWN_BINARY_NAMES
16from tests.support.helpers import VirtualEnv, dedent
17from tests.support.mixins import LoaderModuleMockMixin, SaltReturnAssertsMixin
18from tests.support.mock import MagicMock, patch
19from tests.support.runtests import RUNTIME_VARS
20from tests.support.unit import TestCase, skipIf
21
22try:
23    import pip
24
25    HAS_PIP = True
26except ImportError:
27    HAS_PIP = False
28
29
30log = logging.getLogger(__name__)
31
32
33@skipIf(not HAS_PIP, "The 'pip' library is not importable(installed system-wide)")
34class PipStateTest(TestCase, SaltReturnAssertsMixin, LoaderModuleMockMixin):
35    def setup_loader_modules(self):
36        return {
37            pip_state: {
38                "__env__": "base",
39                "__opts__": {"test": False},
40                "__salt__": {"cmd.which_bin": lambda _: "pip"},
41            }
42        }
43
44    def test_install_requirements_parsing(self):
45        log.debug("Real pip version is %s", pip.__version__)
46        mock = MagicMock(return_value={"retcode": 0, "stdout": ""})
47        pip_list = MagicMock(return_value={"pep8": "1.3.3"})
48        pip_version = pip.__version__
49        mock_pip_version = MagicMock(return_value=pip_version)
50        with patch.dict(pip_state.__salt__, {"pip.version": mock_pip_version}):
51            with patch.dict(
52                pip_state.__salt__, {"cmd.run_all": mock, "pip.list": pip_list}
53            ):
54                with patch.dict(pip_state.__opts__, {"test": True}):
55                    log.debug(
56                        "pip_state._from_line globals: %s",
57                        [name for name in pip_state._from_line.__globals__],
58                    )
59                    ret = pip_state.installed("pep8=1.3.2")
60                    self.assertSaltFalseReturn({"test": ret})
61                    self.assertInSaltComment(
62                        "Invalid version specification in package pep8=1.3.2. "
63                        "'=' is not supported, use '==' instead.",
64                        {"test": ret},
65                    )
66
67            mock = MagicMock(return_value={"retcode": 0, "stdout": ""})
68            pip_list = MagicMock(return_value={"pep8": "1.3.3"})
69            pip_install = MagicMock(return_value={"retcode": 0})
70            with patch.dict(
71                pip_state.__salt__,
72                {"cmd.run_all": mock, "pip.list": pip_list, "pip.install": pip_install},
73            ):
74                with patch.dict(pip_state.__opts__, {"test": True}):
75                    ret = pip_state.installed("pep8>=1.3.2")
76                    self.assertSaltTrueReturn({"test": ret})
77                    self.assertInSaltComment(
78                        "Python package pep8>=1.3.2 was already installed",
79                        {"test": ret},
80                    )
81
82            mock = MagicMock(return_value={"retcode": 0, "stdout": ""})
83            pip_list = MagicMock(return_value={"pep8": "1.3.3"})
84            with patch.dict(
85                pip_state.__salt__, {"cmd.run_all": mock, "pip.list": pip_list}
86            ):
87                with patch.dict(pip_state.__opts__, {"test": True}):
88                    ret = pip_state.installed("pep8<1.3.2")
89                    self.assertSaltNoneReturn({"test": ret})
90                    self.assertInSaltComment(
91                        "Python package pep8<1.3.2 is set to be installed",
92                        {"test": ret},
93                    )
94
95            mock = MagicMock(return_value={"retcode": 0, "stdout": ""})
96            pip_list = MagicMock(return_value={"pep8": "1.3.2"})
97            pip_install = MagicMock(return_value={"retcode": 0})
98            with patch.dict(
99                pip_state.__salt__,
100                {"cmd.run_all": mock, "pip.list": pip_list, "pip.install": pip_install},
101            ):
102                with patch.dict(pip_state.__opts__, {"test": True}):
103                    ret = pip_state.installed("pep8>1.3.1,<1.3.3")
104                    self.assertSaltTrueReturn({"test": ret})
105                    self.assertInSaltComment(
106                        "Python package pep8>1.3.1,<1.3.3 was already installed",
107                        {"test": ret},
108                    )
109
110            mock = MagicMock(return_value={"retcode": 0, "stdout": ""})
111            pip_list = MagicMock(return_value={"pep8": "1.3.1"})
112            pip_install = MagicMock(return_value={"retcode": 0})
113            with patch.dict(
114                pip_state.__salt__,
115                {"cmd.run_all": mock, "pip.list": pip_list, "pip.install": pip_install},
116            ):
117                with patch.dict(pip_state.__opts__, {"test": True}):
118                    ret = pip_state.installed("pep8>1.3.1,<1.3.3")
119                    self.assertSaltNoneReturn({"test": ret})
120                    self.assertInSaltComment(
121                        "Python package pep8>1.3.1,<1.3.3 is set to be installed",
122                        {"test": ret},
123                    )
124
125            mock = MagicMock(return_value={"retcode": 0, "stdout": ""})
126            pip_list = MagicMock(return_value={"pep8": "1.3.1"})
127            with patch.dict(
128                pip_state.__salt__, {"cmd.run_all": mock, "pip.list": pip_list}
129            ):
130                with patch.dict(pip_state.__opts__, {"test": True}):
131                    ret = pip_state.installed(
132                        "git+https://github.com/saltstack/salt-testing.git#egg=SaltTesting>=0.5.1"
133                    )
134                    self.assertSaltNoneReturn({"test": ret})
135                    self.assertInSaltComment(
136                        "Python package git+https://github.com/saltstack/"
137                        "salt-testing.git#egg=SaltTesting>=0.5.1 is set to be "
138                        "installed",
139                        {"test": ret},
140                    )
141
142            mock = MagicMock(return_value={"retcode": 0, "stdout": ""})
143            pip_list = MagicMock(return_value={"pep8": "1.3.1"})
144            with patch.dict(
145                pip_state.__salt__, {"cmd.run_all": mock, "pip.list": pip_list}
146            ):
147                with patch.dict(pip_state.__opts__, {"test": True}):
148                    ret = pip_state.installed(
149                        "git+https://github.com/saltstack/salt-testing.git#egg=SaltTesting"
150                    )
151                    self.assertSaltNoneReturn({"test": ret})
152                    self.assertInSaltComment(
153                        "Python package git+https://github.com/saltstack/"
154                        "salt-testing.git#egg=SaltTesting is set to be "
155                        "installed",
156                        {"test": ret},
157                    )
158
159            mock = MagicMock(return_value={"retcode": 0, "stdout": ""})
160            pip_list = MagicMock(return_value={"pep8": "1.3.1"})
161            with patch.dict(
162                pip_state.__salt__, {"cmd.run_all": mock, "pip.list": pip_list}
163            ):
164                with patch.dict(pip_state.__opts__, {"test": True}):
165                    ret = pip_state.installed(
166                        "https://pypi.python.org/packages/source/S/SaltTesting/"
167                        "SaltTesting-0.5.0.tar.gz"
168                        "#md5=e6760af92b7165f8be53b5763e40bc24"
169                    )
170                    self.assertSaltNoneReturn({"test": ret})
171                    self.assertInSaltComment(
172                        "Python package https://pypi.python.org/packages/source/"
173                        "S/SaltTesting/SaltTesting-0.5.0.tar.gz"
174                        "#md5=e6760af92b7165f8be53b5763e40bc24 is set to be "
175                        "installed",
176                        {"test": ret},
177                    )
178
179            mock = MagicMock(return_value={"retcode": 0, "stdout": ""})
180            pip_list = MagicMock(return_value={"SaltTesting": "0.5.0"})
181            pip_install = MagicMock(
182                return_value={
183                    "retcode": 0,
184                    "stderr": "",
185                    "stdout": (
186                        "Downloading/unpacking https://pypi.python.org/packages"
187                        "/source/S/SaltTesting/SaltTesting-0.5.0.tar.gz\n  "
188                        "Downloading SaltTesting-0.5.0.tar.gz\n  Running "
189                        "setup.py egg_info for package from "
190                        "https://pypi.python.org/packages/source/S/SaltTesting/"
191                        "SaltTesting-0.5.0.tar.gz\n    \nCleaning up..."
192                    ),
193                }
194            )
195            with patch.dict(
196                pip_state.__salt__,
197                {"cmd.run_all": mock, "pip.list": pip_list, "pip.install": pip_install},
198            ):
199                ret = pip_state.installed(
200                    "https://pypi.python.org/packages/source/S/SaltTesting/"
201                    "SaltTesting-0.5.0.tar.gz"
202                    "#md5=e6760af92b7165f8be53b5763e40bc24"
203                )
204                self.assertSaltTrueReturn({"test": ret})
205                self.assertInSaltComment(
206                    "All packages were successfully installed", {"test": ret}
207                )
208                self.assertInSaltReturn(
209                    "Installed",
210                    {"test": ret},
211                    (
212                        "changes",
213                        "https://pypi.python.org/packages/source/S/"
214                        "SaltTesting/SaltTesting-0.5.0.tar.gz"
215                        "#md5=e6760af92b7165f8be53b5763e40bc24==???",
216                    ),
217                )
218
219            mock = MagicMock(return_value={"retcode": 0, "stdout": ""})
220            pip_list = MagicMock(return_value={"SaltTesting": "0.5.0"})
221            pip_install = MagicMock(
222                return_value={"retcode": 0, "stderr": "", "stdout": "Cloned!"}
223            )
224            with patch.dict(
225                pip_state.__salt__,
226                {"cmd.run_all": mock, "pip.list": pip_list, "pip.install": pip_install},
227            ):
228                with patch.dict(pip_state.__opts__, {"test": False}):
229                    ret = pip_state.installed(
230                        "git+https://github.com/saltstack/salt-testing.git#egg=SaltTesting"
231                    )
232                    self.assertSaltTrueReturn({"test": ret})
233                    self.assertInSaltComment(
234                        "packages are already installed", {"test": ret}
235                    )
236
237            mock = MagicMock(return_value={"retcode": 0, "stdout": ""})
238            pip_list = MagicMock(return_value={"pep8": "1.3.1"})
239            pip_install = MagicMock(return_value={"retcode": 0})
240            with patch.dict(
241                pip_state.__salt__,
242                {"cmd.run_all": mock, "pip.list": pip_list, "pip.install": pip_install},
243            ):
244                with patch.dict(pip_state.__opts__, {"test": False}):
245                    ret = pip_state.installed(
246                        "arbitrary ID that should be ignored due to requirements"
247                        " specified",
248                        requirements="/tmp/non-existing-requirements.txt",
249                    )
250                    self.assertSaltTrueReturn({"test": ret})
251
252            # Test VCS installations using git+git://
253            mock = MagicMock(return_value={"retcode": 0, "stdout": ""})
254            pip_list = MagicMock(return_value={"SaltTesting": "0.5.0"})
255            pip_install = MagicMock(
256                return_value={"retcode": 0, "stderr": "", "stdout": "Cloned!"}
257            )
258            with patch.dict(
259                pip_state.__salt__,
260                {"cmd.run_all": mock, "pip.list": pip_list, "pip.install": pip_install},
261            ):
262                with patch.dict(pip_state.__opts__, {"test": False}):
263                    ret = pip_state.installed(
264                        "git+git://github.com/saltstack/salt-testing.git#egg=SaltTesting"
265                    )
266                    self.assertSaltTrueReturn({"test": ret})
267                    self.assertInSaltComment(
268                        "packages are already installed", {"test": ret}
269                    )
270
271    def test_install_requirements_custom_pypi(self):
272        """
273        test requirement parsing for both when a custom
274        pypi index-url is set and when it is not and
275        the requirement is already installed.
276        """
277
278        # create requirements file
279        req_filename = os.path.join(
280            RUNTIME_VARS.TMP_STATE_TREE, "custom-pypi-requirements.txt"
281        )
282        with salt.utils.files.fopen(req_filename, "wb") as reqf:
283            reqf.write(b"pep8\n")
284
285        site_pkgs = "/tmp/pip-env/lib/python3.7/site-packages"
286        check_stdout = [
287            "Looking in indexes: https://custom-pypi-url.org,"
288            "https://pypi.org/simple/\nRequirement already satisfied: pep8 in {1}"
289            "(from -r /tmp/files/prod/{0} (line 1)) (1.7.1)".format(
290                req_filename, site_pkgs
291            ),
292            "Requirement already satisfied: pep8 in {1}"
293            "(from -r /tmp/files/prod/{0} (line1)) (1.7.1)".format(
294                req_filename, site_pkgs
295            ),
296        ]
297        pip_version = pip.__version__
298        mock_pip_version = MagicMock(return_value=pip_version)
299
300        for stdout in check_stdout:
301            pip_install = MagicMock(return_value={"retcode": 0, "stdout": stdout})
302            with patch.dict(pip_state.__salt__, {"pip.version": mock_pip_version}):
303                with patch.dict(pip_state.__salt__, {"pip.install": pip_install}):
304                    ret = pip_state.installed(name="", requirements=req_filename)
305                    self.assertSaltTrueReturn({"test": ret})
306                    assert "Requirements were already installed." == ret["comment"]
307
308    def test_install_requirements_custom_pypi_changes(self):
309        """
310        test requirement parsing for both when a custom
311        pypi index-url is set and when it is not and
312        the requirement is not installed.
313        """
314
315        # create requirements file
316        req_filename = os.path.join(
317            RUNTIME_VARS.TMP_STATE_TREE, "custom-pypi-requirements.txt"
318        )
319        with salt.utils.files.fopen(req_filename, "wb") as reqf:
320            reqf.write(b"pep8\n")
321
322        site_pkgs = "/tmp/pip-env/lib/python3.7/site-packages"
323        check_stdout = [
324            "Looking in indexes:"
325            " https://custom-pypi-url.org,https://pypi.org/simple/\nCollecting pep8\n "
326            " Using"
327            " cachedhttps://custom-pypi-url.org//packages/42/3f/669429cef5acb4/pep8-1.7.1-py2.py3-none-any.whl"
328            " (41 kB)\nInstalling collected packages: pep8\nSuccessfully installed"
329            " pep8-1.7.1",
330            "Collecting pep8\n  Using"
331            " cachedhttps://custom-pypi-url.org//packages/42/3f/669429cef5acb4/pep8-1.7.1-py2.py3-none-any.whl"
332            " (41 kB)\nInstalling collected packages: pep8\nSuccessfully installed"
333            " pep8-1.7.1",
334        ]
335
336        pip_version = pip.__version__
337        mock_pip_version = MagicMock(return_value=pip_version)
338
339        for stdout in check_stdout:
340            pip_install = MagicMock(return_value={"retcode": 0, "stdout": stdout})
341            with patch.dict(pip_state.__salt__, {"pip.version": mock_pip_version}):
342                with patch.dict(pip_state.__salt__, {"pip.install": pip_install}):
343                    ret = pip_state.installed(name="", requirements=req_filename)
344                    self.assertSaltTrueReturn({"test": ret})
345                    assert (
346                        "Successfully processed requirements file {}.".format(
347                            req_filename
348                        )
349                        == ret["comment"]
350                    )
351
352    def test_install_in_editable_mode(self):
353        """
354        Check that `name` parameter containing bad characters is not parsed by
355        pip when package is being installed in editable mode.
356        For more information, see issue #21890.
357        """
358        mock = MagicMock(return_value={"retcode": 0, "stdout": ""})
359        pip_list = MagicMock(return_value={})
360        pip_install = MagicMock(
361            return_value={"retcode": 0, "stderr": "", "stdout": "Cloned!"}
362        )
363        pip_version = MagicMock(return_value="10.0.1")
364        with patch.dict(
365            pip_state.__salt__,
366            {
367                "cmd.run_all": mock,
368                "pip.list": pip_list,
369                "pip.install": pip_install,
370                "pip.version": pip_version,
371            },
372        ):
373            ret = pip_state.installed(
374                "state@name", cwd="/path/to/project", editable=["."]
375            )
376            self.assertSaltTrueReturn({"test": ret})
377            self.assertInSaltComment("successfully installed", {"test": ret})
378
379
380class PipStateUtilsTest(TestCase):
381    def test_has_internal_exceptions_mod_function(self):
382        assert pip_state.pip_has_internal_exceptions_mod("10.0")
383        assert pip_state.pip_has_internal_exceptions_mod("18.1")
384        assert not pip_state.pip_has_internal_exceptions_mod("9.99")
385
386    def test_has_exceptions_mod_function(self):
387        assert pip_state.pip_has_exceptions_mod("1.0")
388        assert not pip_state.pip_has_exceptions_mod("0.1")
389        assert not pip_state.pip_has_exceptions_mod("10.0")
390
391    def test_pip_purge_method_with_pip(self):
392        mock_modules = sys.modules.copy()
393        mock_modules.pop("pip", None)
394        mock_modules["pip"] = object()
395        with patch("sys.modules", mock_modules):
396            pip_state.purge_pip()
397        assert "pip" not in mock_modules
398
399    def test_pip_purge_method_without_pip(self):
400        mock_modules = sys.modules.copy()
401        mock_modules.pop("pip", None)
402        with patch("sys.modules", mock_modules):
403            pip_state.purge_pip()
404
405
406@skipIf(
407    salt.utils.path.which_bin(KNOWN_BINARY_NAMES) is None, "virtualenv not installed"
408)
409@pytest.mark.requires_network
410class PipStateInstallationErrorTest(TestCase):
411    @pytest.mark.slow_test
412    def test_importable_installation_error(self):
413        extra_requirements = []
414        for name, version in salt.version.dependency_information():
415            if name in ["PyYAML"]:
416                extra_requirements.append("{}=={}".format(name, version))
417        failures = {}
418        pip_version_requirements = [
419            # Latest pip 8
420            "<9.0",
421            # Latest pip 9
422            "<10.0",
423            # Latest pip 18
424            "<19.0",
425            # Latest pip 19
426            "<20.0",
427            # Latest pip 20
428            "<21.0",
429            # Latest pip
430            None,
431        ]
432        code = dedent(
433            """\
434        import sys
435        import traceback
436        try:
437            import salt.states.pip_state
438            salt.states.pip_state.InstallationError
439        except ImportError as exc:
440            traceback.print_exc(file=sys.stdout)
441            sys.stdout.flush()
442            sys.exit(1)
443        except AttributeError as exc:
444            traceback.print_exc(file=sys.stdout)
445            sys.stdout.flush()
446            sys.exit(2)
447        except Exception as exc:
448            traceback.print_exc(exc, file=sys.stdout)
449            sys.stdout.flush()
450            sys.exit(3)
451        sys.exit(0)
452        """
453        )
454        for requirement in list(pip_version_requirements):
455            try:
456                with VirtualEnv() as venv:
457                    venv.install(*extra_requirements)
458                    if requirement:
459                        venv.install("pip{}".format(requirement))
460                    try:
461                        subprocess.check_output([venv.venv_python, "-c", code])
462                    except subprocess.CalledProcessError as exc:
463                        if exc.returncode == 1:
464                            failures[requirement] = "Failed to import pip:\n{}".format(
465                                exc.output
466                            )
467                        elif exc.returncode == 2:
468                            failures[
469                                requirement
470                            ] = "Failed to import InstallationError from pip:\n{}".format(
471                                exc.output
472                            )
473                        else:
474                            failures[requirement] = exc.output
475            except Exception as exc:  # pylint: disable=broad-except
476                failures[requirement] = str(exc)
477        if failures:
478            errors = ""
479            for requirement, exception in failures.items():
480                errors += "pip{}: {}\n\n".format(requirement or "", exception)
481            self.fail(
482                "Failed to get InstallationError exception under at least one pip"
483                " version:\n{}".format(errors)
484            )
485