1"""Support code for working with Shippable.""" 2from __future__ import (absolute_import, division, print_function) 3__metaclass__ = type 4 5import os 6import re 7import time 8 9from .. import types as t 10 11from ..config import ( 12 CommonConfig, 13 TestConfig, 14) 15 16from ..git import ( 17 Git, 18) 19 20from ..http import ( 21 HttpClient, 22 urlencode, 23) 24 25from ..util import ( 26 ApplicationError, 27 display, 28 MissingEnvironmentVariable, 29 SubprocessError, 30) 31 32from . import ( 33 AuthContext, 34 ChangeDetectionNotSupported, 35 CIProvider, 36 OpenSSLAuthHelper, 37) 38 39 40CODE = 'shippable' 41 42 43class Shippable(CIProvider): 44 """CI provider implementation for Shippable.""" 45 def __init__(self): 46 self.auth = ShippableAuthHelper() 47 48 @staticmethod 49 def is_supported(): # type: () -> bool 50 """Return True if this provider is supported in the current running environment.""" 51 return os.environ.get('SHIPPABLE') == 'true' 52 53 @property 54 def code(self): # type: () -> str 55 """Return a unique code representing this provider.""" 56 return CODE 57 58 @property 59 def name(self): # type: () -> str 60 """Return descriptive name for this provider.""" 61 return 'Shippable' 62 63 def generate_resource_prefix(self): # type: () -> str 64 """Return a resource prefix specific to this CI provider.""" 65 try: 66 prefix = 'shippable-%s-%s' % ( 67 os.environ['SHIPPABLE_BUILD_NUMBER'], 68 os.environ['SHIPPABLE_JOB_NUMBER'], 69 ) 70 except KeyError as ex: 71 raise MissingEnvironmentVariable(name=ex.args[0]) 72 73 return prefix 74 75 def get_base_branch(self): # type: () -> str 76 """Return the base branch or an empty string.""" 77 base_branch = os.environ.get('BASE_BRANCH') 78 79 if base_branch: 80 base_branch = 'origin/%s' % base_branch 81 82 return base_branch or '' 83 84 def detect_changes(self, args): # type: (TestConfig) -> t.Optional[t.List[str]] 85 """Initialize change detection.""" 86 result = ShippableChanges(args) 87 88 if result.is_pr: 89 job_type = 'pull request' 90 elif result.is_tag: 91 job_type = 'tag' 92 else: 93 job_type = 'merge commit' 94 95 display.info('Processing %s for branch %s commit %s' % (job_type, result.branch, result.commit)) 96 97 if not args.metadata.changes: 98 args.metadata.populate_changes(result.diff) 99 100 if result.paths is None: 101 # There are several likely causes of this: 102 # - First run on a new branch. 103 # - Too many pull requests passed since the last merge run passed. 104 display.warning('No successful commit found. All tests will be executed.') 105 106 return result.paths 107 108 def supports_core_ci_auth(self, context): # type: (AuthContext) -> bool 109 """Return True if Ansible Core CI is supported.""" 110 return True 111 112 def prepare_core_ci_auth(self, context): # type: (AuthContext) -> t.Dict[str, t.Any] 113 """Return authentication details for Ansible Core CI.""" 114 try: 115 request = dict( 116 run_id=os.environ['SHIPPABLE_BUILD_ID'], 117 job_number=int(os.environ['SHIPPABLE_JOB_NUMBER']), 118 ) 119 except KeyError as ex: 120 raise MissingEnvironmentVariable(name=ex.args[0]) 121 122 self.auth.sign_request(request) 123 124 auth = dict( 125 shippable=request, 126 ) 127 128 return auth 129 130 def get_git_details(self, args): # type: (CommonConfig) -> t.Optional[t.Dict[str, t.Any]] 131 """Return details about git in the current environment.""" 132 commit = os.environ.get('COMMIT') 133 base_commit = os.environ.get('BASE_COMMIT') 134 135 details = dict( 136 base_commit=base_commit, 137 commit=commit, 138 merged_commit=self._get_merged_commit(args, commit), 139 ) 140 141 return details 142 143 # noinspection PyUnusedLocal 144 def _get_merged_commit(self, args, commit): # type: (CommonConfig, str) -> t.Optional[str] # pylint: disable=unused-argument 145 """Find the merged commit that should be present.""" 146 if not commit: 147 return None 148 149 git = Git() 150 151 try: 152 show_commit = git.run_git(['show', '--no-patch', '--no-abbrev', commit]) 153 except SubprocessError as ex: 154 # This should only fail for pull requests where the commit does not exist. 155 # Merge runs would fail much earlier when attempting to checkout the commit. 156 raise ApplicationError('Commit %s was not found:\n\n%s\n\n' 157 'GitHub may not have fully replicated the commit across their infrastructure.\n' 158 'It is also possible the commit was removed by a force push between job creation and execution.\n' 159 'Find the latest run for the pull request and restart failed jobs as needed.' 160 % (commit, ex.stderr.strip())) 161 162 head_commit = git.run_git(['show', '--no-patch', '--no-abbrev', 'HEAD']) 163 164 if show_commit == head_commit: 165 # Commit is HEAD, so this is not a pull request or the base branch for the pull request is up-to-date. 166 return None 167 168 match_merge = re.search(r'^Merge: (?P<parents>[0-9a-f]{40} [0-9a-f]{40})$', head_commit, flags=re.MULTILINE) 169 170 if not match_merge: 171 # The most likely scenarios resulting in a failure here are: 172 # A new run should or does supersede this job, but it wasn't cancelled in time. 173 # A job was superseded and then later restarted. 174 raise ApplicationError('HEAD is not commit %s or a merge commit:\n\n%s\n\n' 175 'This job has likely been superseded by another run due to additional commits being pushed.\n' 176 'Find the latest run for the pull request and restart failed jobs as needed.' 177 % (commit, head_commit.strip())) 178 179 parents = set(match_merge.group('parents').split(' ')) 180 181 if len(parents) != 2: 182 raise ApplicationError('HEAD is a %d-way octopus merge.' % len(parents)) 183 184 if commit not in parents: 185 raise ApplicationError('Commit %s is not a parent of HEAD.' % commit) 186 187 parents.remove(commit) 188 189 last_commit = parents.pop() 190 191 return last_commit 192 193 194class ShippableAuthHelper(OpenSSLAuthHelper): 195 """ 196 Authentication helper for Shippable. 197 Based on OpenSSL since cryptography is not provided by the default Shippable environment. 198 """ 199 def publish_public_key(self, public_key_pem): # type: (str) -> None 200 """Publish the given public key.""" 201 # display the public key as a single line to avoid mangling such as when prefixing each line with a timestamp 202 display.info(public_key_pem.replace('\n', ' ')) 203 # allow time for logs to become available to reduce repeated API calls 204 time.sleep(3) 205 206 207class ShippableChanges: 208 """Change information for Shippable build.""" 209 def __init__(self, args): # type: (TestConfig) -> None 210 self.args = args 211 self.git = Git() 212 213 try: 214 self.branch = os.environ['BRANCH'] 215 self.is_pr = os.environ['IS_PULL_REQUEST'] == 'true' 216 self.is_tag = os.environ['IS_GIT_TAG'] == 'true' 217 self.commit = os.environ['COMMIT'] 218 self.project_id = os.environ['PROJECT_ID'] 219 self.commit_range = os.environ['SHIPPABLE_COMMIT_RANGE'] 220 except KeyError as ex: 221 raise MissingEnvironmentVariable(name=ex.args[0]) 222 223 if self.is_tag: 224 raise ChangeDetectionNotSupported('Change detection is not supported for tags.') 225 226 if self.is_pr: 227 self.paths = sorted(self.git.get_diff_names([self.commit_range])) 228 self.diff = self.git.get_diff([self.commit_range]) 229 else: 230 commits = self.get_successful_merge_run_commits(self.project_id, self.branch) 231 last_successful_commit = self.get_last_successful_commit(commits) 232 233 if last_successful_commit: 234 self.paths = sorted(self.git.get_diff_names([last_successful_commit, self.commit])) 235 self.diff = self.git.get_diff([last_successful_commit, self.commit]) 236 else: 237 # first run for branch 238 self.paths = None # act as though change detection not enabled, do not filter targets 239 self.diff = [] 240 241 def get_successful_merge_run_commits(self, project_id, branch): # type: (str, str) -> t.Set[str] 242 """Return a set of recent successsful merge commits from Shippable for the given project and branch.""" 243 parameters = dict( 244 isPullRequest='false', 245 projectIds=project_id, 246 branch=branch, 247 ) 248 249 url = 'https://api.shippable.com/runs?%s' % urlencode(parameters) 250 251 http = HttpClient(self.args, always=True) 252 response = http.get(url) 253 result = response.json() 254 255 if 'id' in result and result['id'] == 4004: 256 # most likely due to a private project, which returns an HTTP 200 response with JSON 257 display.warning('Unable to find project. Cannot determine changes. All tests will be executed.') 258 return set() 259 260 commits = set(run['commitSha'] for run in result if run['statusCode'] == 30) 261 262 return commits 263 264 def get_last_successful_commit(self, successful_commits): # type: (t.Set[str]) -> t.Optional[str] 265 """Return the last successful commit from git history that is found in the given commit list, or None.""" 266 commit_history = self.git.get_rev_list(max_count=100) 267 ordered_successful_commits = [commit for commit in commit_history if commit in successful_commits] 268 last_successful_commit = ordered_successful_commits[0] if ordered_successful_commits else None 269 return last_successful_commit 270