1#!/usr/local/bin/python3.8 2# -*- coding: utf-8 -*- 3# (c) 2018, Ansible Project 4# 5# This file is part of Ansible 6# 7# Ansible is free software: you can redistribute it and/or modify 8# it under the terms of the GNU General Public License as published by 9# the Free Software Foundation, either version 3 of the License, or 10# (at your option) any later version. 11# 12# Ansible is distributed in the hope that it will be useful, 13# but WITHOUT ANY WARRANTY; without even the implied warranty of 14# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15# GNU General Public License for more details. 16# 17# You should have received a copy of the GNU General Public License 18# along with Ansible. If not, see <http://www.gnu.org/licenses/>. 19""" 20This test checks whether the libraries we're bundling are out of date and need to be synced with 21a newer upstream release. 22""" 23 24 25from __future__ import (absolute_import, division, print_function) 26__metaclass__ = type 27 28import fnmatch 29import json 30import re 31import sys 32from distutils.version import LooseVersion 33 34import packaging.specifiers 35 36from ansible.module_utils.urls import open_url 37 38 39BUNDLED_RE = re.compile(b'\\b_BUNDLED_METADATA\\b') 40 41 42def get_bundled_libs(paths): 43 """ 44 Return the set of known bundled libraries 45 46 :arg paths: The paths which the test has been instructed to check 47 :returns: The list of all files which we know to contain bundled libraries. If a bundled 48 library consists of multiple files, this should be the file which has metadata included. 49 """ 50 bundled_libs = set() 51 for filename in fnmatch.filter(paths, 'lib/ansible/compat/*/__init__.py'): 52 bundled_libs.add(filename) 53 54 bundled_libs.add('lib/ansible/module_utils/compat/selectors.py') 55 bundled_libs.add('lib/ansible/module_utils/distro/__init__.py') 56 bundled_libs.add('lib/ansible/module_utils/six/__init__.py') 57 # backports.ssl_match_hostname should be moved to its own file in the future 58 bundled_libs.add('lib/ansible/module_utils/urls.py') 59 60 return bundled_libs 61 62 63def get_files_with_bundled_metadata(paths): 64 """ 65 Search for any files which have bundled metadata inside of them 66 67 :arg paths: Iterable of filenames to search for metadata inside of 68 :returns: A set of pathnames which contained metadata 69 """ 70 71 with_metadata = set() 72 for path in paths: 73 with open(path, 'rb') as f: 74 body = f.read() 75 76 if BUNDLED_RE.search(body): 77 with_metadata.add(path) 78 79 return with_metadata 80 81 82def get_bundled_metadata(filename): 83 """ 84 Retrieve the metadata about a bundled library from a python file 85 86 :arg filename: The filename to look inside for the metadata 87 :raises ValueError: If we're unable to extract metadata from the file 88 :returns: The metadata from the python file 89 """ 90 with open(filename, 'r') as module: 91 for line in module: 92 if line.strip().startswith('# NOT_BUNDLED'): 93 return None 94 95 if line.strip().startswith('# CANT_UPDATE'): 96 print( 97 '{0} marked as CANT_UPDATE, so skipping. Manual ' 98 'check for CVEs required.'.format(filename)) 99 return None 100 101 if line.strip().startswith('_BUNDLED_METADATA'): 102 data = line[line.index('{'):].strip() 103 break 104 else: 105 raise ValueError('Unable to check bundled library for update. Please add' 106 ' _BUNDLED_METADATA dictionary to the library file with' 107 ' information on pypi name and bundled version.') 108 metadata = json.loads(data) 109 return metadata 110 111 112def get_latest_applicable_version(pypi_data, constraints=None): 113 """Get the latest pypi version of the package that we allow 114 115 :arg pypi_data: Pypi information about the data as returned by 116 ``https://pypi.org/pypi/{pkg_name}/json`` 117 :kwarg constraints: version constraints on what we're allowed to use as specified by 118 the bundled metadata 119 :returns: The most recent version on pypi that are allowed by ``constraints`` 120 """ 121 latest_version = "0" 122 if constraints: 123 version_specification = packaging.specifiers.SpecifierSet(constraints) 124 for version in pypi_data['releases']: 125 if version in version_specification: 126 if LooseVersion(version) > LooseVersion(latest_version): 127 latest_version = version 128 else: 129 latest_version = pypi_data['info']['version'] 130 131 return latest_version 132 133 134def main(): 135 """Entrypoint to the script""" 136 137 paths = sys.argv[1:] or sys.stdin.read().splitlines() 138 139 bundled_libs = get_bundled_libs(paths) 140 files_with_bundled_metadata = get_files_with_bundled_metadata(paths) 141 142 for filename in files_with_bundled_metadata.difference(bundled_libs): 143 print('{0}: ERROR: File contains _BUNDLED_METADATA but needs to be added to' 144 ' test/sanity/code-smell/update-bundled.py'.format(filename)) 145 146 for filename in bundled_libs: 147 try: 148 metadata = get_bundled_metadata(filename) 149 except ValueError as e: 150 print('{0}: ERROR: {1}'.format(filename, e)) 151 continue 152 except (IOError, OSError) as e: 153 if e.errno == 2: 154 print('{0}: ERROR: {1}. Perhaps the bundled library has been removed' 155 ' or moved and the bundled library test needs to be modified as' 156 ' well?'.format(filename, e)) 157 158 if metadata is None: 159 continue 160 161 pypi_fh = open_url('https://pypi.org/pypi/{0}/json'.format(metadata['pypi_name'])) 162 pypi_data = json.loads(pypi_fh.read().decode('utf-8')) 163 164 constraints = metadata.get('version_constraints', None) 165 latest_version = get_latest_applicable_version(pypi_data, constraints) 166 167 if LooseVersion(metadata['version']) < LooseVersion(latest_version): 168 print('{0}: UPDATE {1} from {2} to {3} {4}'.format( 169 filename, 170 metadata['pypi_name'], 171 metadata['version'], 172 latest_version, 173 'https://pypi.org/pypi/{0}/json'.format(metadata['pypi_name']))) 174 175 176if __name__ == '__main__': 177 main() 178