1#!/usr/local/bin/python3.8
2# -*- coding: utf-8 -*-
3# (c) 2018, Ansible Project
4#
5# This file is part of Ansible
6#
7# Ansible is free software: you can redistribute it and/or modify
8# it under the terms of the GNU General Public License as published by
9# the Free Software Foundation, either version 3 of the License, or
10# (at your option) any later version.
11#
12# Ansible is distributed in the hope that it will be useful,
13# but WITHOUT ANY WARRANTY; without even the implied warranty of
14# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15# GNU General Public License for more details.
16#
17# You should have received a copy of the GNU General Public License
18# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
19"""
20This test checks whether the libraries we're bundling are out of date and need to be synced with
21a newer upstream release.
22"""
23
24
25from __future__ import (absolute_import, division, print_function)
26__metaclass__ = type
27
28import fnmatch
29import json
30import re
31import sys
32from distutils.version import LooseVersion
33
34import packaging.specifiers
35
36from ansible.module_utils.urls import open_url
37
38
39BUNDLED_RE = re.compile(b'\\b_BUNDLED_METADATA\\b')
40
41
42def get_bundled_libs(paths):
43    """
44    Return the set of known bundled libraries
45
46    :arg paths: The paths which the test has been instructed to check
47    :returns: The list of all files which we know to contain bundled libraries.  If a bundled
48        library consists of multiple files, this should be the file which has metadata included.
49    """
50    bundled_libs = set()
51    for filename in fnmatch.filter(paths, 'lib/ansible/compat/*/__init__.py'):
52        bundled_libs.add(filename)
53
54    bundled_libs.add('lib/ansible/module_utils/compat/selectors.py')
55    bundled_libs.add('lib/ansible/module_utils/distro/__init__.py')
56    bundled_libs.add('lib/ansible/module_utils/six/__init__.py')
57    # backports.ssl_match_hostname should be moved to its own file in the future
58    bundled_libs.add('lib/ansible/module_utils/urls.py')
59
60    return bundled_libs
61
62
63def get_files_with_bundled_metadata(paths):
64    """
65    Search for any files which have bundled metadata inside of them
66
67    :arg paths: Iterable of filenames to search for metadata inside of
68    :returns: A set of pathnames which contained metadata
69    """
70
71    with_metadata = set()
72    for path in paths:
73        with open(path, 'rb') as f:
74            body = f.read()
75
76        if BUNDLED_RE.search(body):
77            with_metadata.add(path)
78
79    return with_metadata
80
81
82def get_bundled_metadata(filename):
83    """
84    Retrieve the metadata about a bundled library from a python file
85
86    :arg filename: The filename to look inside for the metadata
87    :raises ValueError: If we're unable to extract metadata from the file
88    :returns: The metadata from the python file
89    """
90    with open(filename, 'r') as module:
91        for line in module:
92            if line.strip().startswith('# NOT_BUNDLED'):
93                return None
94
95            if line.strip().startswith('# CANT_UPDATE'):
96                print(
97                    '{0} marked as CANT_UPDATE, so skipping. Manual '
98                    'check for CVEs required.'.format(filename))
99                return None
100
101            if line.strip().startswith('_BUNDLED_METADATA'):
102                data = line[line.index('{'):].strip()
103                break
104        else:
105            raise ValueError('Unable to check bundled library for update.  Please add'
106                             ' _BUNDLED_METADATA dictionary to the library file with'
107                             ' information on pypi name and bundled version.')
108        metadata = json.loads(data)
109    return metadata
110
111
112def get_latest_applicable_version(pypi_data, constraints=None):
113    """Get the latest pypi version of the package that we allow
114
115    :arg pypi_data: Pypi information about the data as returned by
116        ``https://pypi.org/pypi/{pkg_name}/json``
117    :kwarg constraints: version constraints on what we're allowed to use as specified by
118        the bundled metadata
119    :returns: The most recent version on pypi that are allowed by ``constraints``
120    """
121    latest_version = "0"
122    if constraints:
123        version_specification = packaging.specifiers.SpecifierSet(constraints)
124        for version in pypi_data['releases']:
125            if version in version_specification:
126                if LooseVersion(version) > LooseVersion(latest_version):
127                    latest_version = version
128    else:
129        latest_version = pypi_data['info']['version']
130
131    return latest_version
132
133
134def main():
135    """Entrypoint to the script"""
136
137    paths = sys.argv[1:] or sys.stdin.read().splitlines()
138
139    bundled_libs = get_bundled_libs(paths)
140    files_with_bundled_metadata = get_files_with_bundled_metadata(paths)
141
142    for filename in files_with_bundled_metadata.difference(bundled_libs):
143        print('{0}: ERROR: File contains _BUNDLED_METADATA but needs to be added to'
144              ' test/sanity/code-smell/update-bundled.py'.format(filename))
145
146    for filename in bundled_libs:
147        try:
148            metadata = get_bundled_metadata(filename)
149        except ValueError as e:
150            print('{0}: ERROR: {1}'.format(filename, e))
151            continue
152        except (IOError, OSError) as e:
153            if e.errno == 2:
154                print('{0}: ERROR: {1}.  Perhaps the bundled library has been removed'
155                      ' or moved and the bundled library test needs to be modified as'
156                      ' well?'.format(filename, e))
157
158        if metadata is None:
159            continue
160
161        pypi_fh = open_url('https://pypi.org/pypi/{0}/json'.format(metadata['pypi_name']))
162        pypi_data = json.loads(pypi_fh.read().decode('utf-8'))
163
164        constraints = metadata.get('version_constraints', None)
165        latest_version = get_latest_applicable_version(pypi_data, constraints)
166
167        if LooseVersion(metadata['version']) < LooseVersion(latest_version):
168            print('{0}: UPDATE {1} from {2} to {3} {4}'.format(
169                filename,
170                metadata['pypi_name'],
171                metadata['version'],
172                latest_version,
173                'https://pypi.org/pypi/{0}/json'.format(metadata['pypi_name'])))
174
175
176if __name__ == '__main__':
177    main()
178