1# This file is part of Buildbot. Buildbot is free software: you can 2# redistribute it and/or modify it under the terms of the GNU General Public 3# License as published by the Free Software Foundation, version 2. 4# 5# This program is distributed in the hope that it will be useful, but WITHOUT 6# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 7# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 8# details. 9# 10# You should have received a copy of the GNU General Public License along with 11# this program; if not, write to the Free Software Foundation, Inc., 51 12# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 13# 14# Copyright Buildbot Team Members 15 16from twisted.internet import defer 17 18from buildbot.data import base 19from buildbot.data import types 20from buildbot.db.buildrequests import AlreadyClaimedError 21from buildbot.db.buildrequests import NotClaimedError 22from buildbot.process import results 23from buildbot.process.results import RETRY 24 25 26class Db2DataMixin: 27 28 def _generate_filtered_properties(self, props, filters): 29 """ 30 This method returns Build's properties according to property filters. 31 32 :param props: Properties as a dict (from db) 33 :param filters: Desired properties keys as a list (from API URI) 34 """ 35 # by default no properties are returned 36 if props and filters: 37 return (props 38 if '*' in filters 39 else dict(((k, v) for k, v in props.items() if k in filters))) 40 return None 41 42 @defer.inlineCallbacks 43 def addPropertiesToBuildRequest(self, buildrequest, filters): 44 if not filters: 45 return None 46 props = yield self.master.db.buildsets.getBuildsetProperties(buildrequest['buildsetid']) 47 filtered_properties = self._generate_filtered_properties(props, filters) 48 if filtered_properties: 49 buildrequest['properties'] = filtered_properties 50 return None 51 52 def db2data(self, dbdict): 53 data = { 54 'buildrequestid': dbdict['buildrequestid'], 55 'buildsetid': dbdict['buildsetid'], 56 'builderid': dbdict['builderid'], 57 'priority': dbdict['priority'], 58 'claimed': dbdict['claimed'], 59 'claimed_at': dbdict['claimed_at'], 60 'claimed_by_masterid': dbdict['claimed_by_masterid'], 61 'complete': dbdict['complete'], 62 'results': dbdict['results'], 63 'submitted_at': dbdict['submitted_at'], 64 'complete_at': dbdict['complete_at'], 65 'waited_for': dbdict['waited_for'], 66 'properties': dbdict.get('properties'), 67 } 68 return defer.succeed(data) 69 fieldMapping = { 70 'buildrequestid': 'buildrequests.id', 71 'buildsetid': 'buildrequests.buildsetid', 72 'builderid': 'buildrequests.builderid', 73 'priority': 'buildrequests.priority', 74 'complete': 'buildrequests.complete', 75 'results': 'buildrequests.results', 76 'submitted_at': 'buildrequests.submitted_at', 77 'complete_at': 'buildrequests.complete_at', 78 'waited_for': 'buildrequests.waited_for', 79 # br claim 80 'claimed_at': 'buildrequest_claims.claimed_at', 81 'claimed_by_masterid': 'buildrequest_claims.masterid', 82 } 83 84 85class BuildRequestEndpoint(Db2DataMixin, base.Endpoint): 86 87 isCollection = False 88 pathPatterns = """ 89 /buildrequests/n:buildrequestid 90 """ 91 92 @defer.inlineCallbacks 93 def get(self, resultSpec, kwargs): 94 buildrequest = yield self.master.db.buildrequests.getBuildRequest(kwargs['buildrequestid']) 95 96 if buildrequest: 97 filters = resultSpec.popProperties() if hasattr(resultSpec, 'popProperties') else [] 98 yield self.addPropertiesToBuildRequest(buildrequest, filters) 99 return (yield self.db2data(buildrequest)) 100 return None 101 102 @defer.inlineCallbacks 103 def control(self, action, args, kwargs): 104 if action != "cancel": 105 raise ValueError("action: {} is not supported".format(action)) 106 brid = kwargs['buildrequestid'] 107 # first, try to claim the request; if this fails, then it's too late to 108 # cancel the build anyway 109 try: 110 b = yield self.master.db.buildrequests.claimBuildRequests(brids=[brid]) 111 except AlreadyClaimedError: 112 # XXX race condition 113 # - After a buildrequest was claimed, and 114 # - Before creating a build, 115 # the claiming master still 116 # needs to do some processing, (send a message to the message queue, 117 # call maybeStartBuild on the related builder). 118 # In that case we won't have the related builds here. We don't have 119 # an alternative to letting them run without stopping them for now. 120 builds = yield self.master.data.get(("buildrequests", brid, "builds")) 121 122 # Don't call the data API here, as the buildrequests might have been 123 # taken by another master. We just send the stop message and forget 124 # about those. 125 mqArgs = {'reason': args.get('reason', 'no reason')} 126 for b in builds: 127 self.master.mq.produce(("control", "builds", str(b['buildid']), "stop"), 128 mqArgs) 129 return None 130 131 # then complete it with 'CANCELLED'; this is the closest we can get to 132 # cancelling a request without running into trouble with dangling 133 # references. 134 yield self.master.data.updates.completeBuildRequests([brid], 135 results.CANCELLED) 136 return None 137 138 139class BuildRequestsEndpoint(Db2DataMixin, base.Endpoint): 140 141 isCollection = True 142 pathPatterns = """ 143 /buildrequests 144 /builders/n:builderid/buildrequests 145 """ 146 rootLinkName = 'buildrequests' 147 148 @defer.inlineCallbacks 149 def get(self, resultSpec, kwargs): 150 builderid = kwargs.get("builderid", None) 151 complete = resultSpec.popBooleanFilter('complete') 152 claimed_by_masterid = resultSpec.popBooleanFilter( 153 'claimed_by_masterid') 154 if claimed_by_masterid: 155 # claimed_by_masterid takes precedence over 'claimed' filter 156 # (no need to check consistency with 'claimed' filter even if 157 # 'claimed'=False with 'claimed_by_masterid' set, doesn't make sense) 158 claimed = claimed_by_masterid 159 else: 160 claimed = resultSpec.popBooleanFilter('claimed') 161 162 bsid = resultSpec.popOneFilter('buildsetid', 'eq') 163 resultSpec.fieldMapping = self.fieldMapping 164 buildrequests = yield self.master.db.buildrequests.getBuildRequests( 165 builderid=builderid, 166 complete=complete, 167 claimed=claimed, 168 bsid=bsid, 169 resultSpec=resultSpec) 170 results = [] 171 filters = resultSpec.popProperties() if hasattr(resultSpec, 'popProperties') else [] 172 for br in buildrequests: 173 yield self.addPropertiesToBuildRequest(br, filters) 174 results.append((yield self.db2data(br))) 175 return results 176 177 178class BuildRequest(base.ResourceType): 179 180 name = "buildrequest" 181 plural = "buildrequests" 182 endpoints = [BuildRequestEndpoint, BuildRequestsEndpoint] 183 keyField = 'buildrequestid' 184 eventPathPatterns = """ 185 /buildsets/:buildsetid/builders/:builderid/buildrequests/:buildrequestid 186 /buildrequests/:buildrequestid 187 /builders/:builderid/buildrequests/:buildrequestid 188 """ 189 190 subresources = ["Build"] 191 192 class EntityType(types.Entity): 193 buildrequestid = types.Integer() 194 buildsetid = types.Integer() 195 builderid = types.Integer() 196 priority = types.Integer() 197 claimed = types.Boolean() 198 claimed_at = types.NoneOk(types.DateTime()) 199 claimed_by_masterid = types.NoneOk(types.Integer()) 200 complete = types.Boolean() 201 results = types.NoneOk(types.Integer()) 202 submitted_at = types.DateTime() 203 complete_at = types.NoneOk(types.DateTime()) 204 waited_for = types.Boolean() 205 properties = types.NoneOk(types.SourcedProperties()) 206 entityType = EntityType(name, 'Buildrequest') 207 208 @defer.inlineCallbacks 209 def generateEvent(self, brids, event): 210 for brid in brids: 211 # get the build and munge the result for the notification 212 br = yield self.master.data.get(('buildrequests', str(brid))) 213 self.produceEvent(br, event) 214 215 @defer.inlineCallbacks 216 def callDbBuildRequests(self, brids, db_callable, event, **kw): 217 if not brids: 218 # empty buildrequest list. No need to call db API 219 return True 220 try: 221 yield db_callable(brids, **kw) 222 except AlreadyClaimedError: 223 # the db layer returned an AlreadyClaimedError exception, usually 224 # because one of the buildrequests has already been claimed by 225 # another master 226 return False 227 yield self.generateEvent(brids, event) 228 return True 229 230 @base.updateMethod 231 def claimBuildRequests(self, brids, claimed_at=None): 232 return self.callDbBuildRequests(brids, 233 self.master.db.buildrequests.claimBuildRequests, 234 event="claimed", 235 claimed_at=claimed_at) 236 237 @base.updateMethod 238 @defer.inlineCallbacks 239 def unclaimBuildRequests(self, brids): 240 if brids: 241 yield self.master.db.buildrequests.unclaimBuildRequests(brids) 242 yield self.generateEvent(brids, "unclaimed") 243 244 @base.updateMethod 245 @defer.inlineCallbacks 246 def completeBuildRequests(self, brids, results, complete_at=None): 247 assert results != RETRY, "a buildrequest cannot be completed with a retry status!" 248 if not brids: 249 # empty buildrequest list. No need to call db API 250 return True 251 try: 252 yield self.master.db.buildrequests.completeBuildRequests( 253 brids, 254 results, 255 complete_at=complete_at) 256 except NotClaimedError: 257 # the db layer returned a NotClaimedError exception, usually 258 # because one of the buildrequests has been claimed by another 259 # master 260 return False 261 yield self.generateEvent(brids, "complete") 262 263 # check for completed buildsets -- one call for each build request with 264 # a unique bsid 265 seen_bsids = set() 266 for brid in brids: 267 brdict = yield self.master.db.buildrequests.getBuildRequest(brid) 268 269 if brdict: 270 bsid = brdict['buildsetid'] 271 if bsid in seen_bsids: 272 continue 273 seen_bsids.add(bsid) 274 yield self.master.data.updates.maybeBuildsetComplete(bsid) 275 276 return True 277 278 @base.updateMethod 279 @defer.inlineCallbacks 280 def rebuildBuildrequest(self, buildrequest): 281 282 # goal is to make a copy of the original buildset 283 buildset = yield self.master.data.get(('buildsets', buildrequest['buildsetid'])) 284 properties = yield self.master.data.get(('buildsets', buildrequest['buildsetid'], 285 'properties')) 286 ssids = [ss['ssid'] for ss in buildset['sourcestamps']] 287 res = yield self.master.data.updates.addBuildset( 288 waited_for=False, scheduler='rebuild', sourcestamps=ssids, reason='rebuild', 289 properties=properties, 290 builderids=[buildrequest['builderid']], 291 external_idstring=buildset['external_idstring'], 292 parent_buildid=buildset['parent_buildid'], 293 parent_relationship=buildset['parent_relationship']) 294 return res 295