Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(310)

Side by Side Diff: third_party/buildbot_7_12/buildbot/process/base.py

Issue 12207158: Bye bye buildbot 0.7.12. (Closed) Base URL: svn://svn.chromium.org/chrome/trunk/tools/build
Patch Set: Created 7 years, 10 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
OLDNEW
(Empty)
1 # -*- test-case-name: buildbot.test.test_step -*-
2
3 import types
4
5 from zope.interface import implements
6 from twisted.python import log
7 from twisted.python.failure import Failure
8 from twisted.internet import reactor, defer, error
9
10 from buildbot import interfaces, locks
11 from buildbot.status.builder import SUCCESS, WARNINGS, FAILURE, EXCEPTION
12 from buildbot.status.builder import Results, BuildRequestStatus
13 from buildbot.status.progress import BuildProgress
14 from buildbot.process.properties import Properties
15
16 class BuildRequest:
17 """I represent a request to a specific Builder to run a single build.
18
19 I have a SourceStamp which specifies what sources I will build. This may
20 specify a specific revision of the source tree (so source.branch,
21 source.revision, and source.patch are used). The .patch attribute is
22 either None or a tuple of (patchlevel, diff), consisting of a number to
23 use in 'patch -pN', and a unified-format context diff.
24
25 Alternatively, the SourceStamp may specify a set of Changes to be built,
26 contained in source.changes. In this case, I may be mergeable with other
27 BuildRequests on the same branch.
28
29 I may be part of a BuildSet, in which case I will report status results
30 to it.
31
32 I am paired with a BuildRequestStatus object, to which I feed status
33 information.
34
35 @type source: a L{buildbot.sourcestamp.SourceStamp} instance.
36 @ivar source: the source code that this BuildRequest use
37
38 @type reason: string
39 @ivar reason: the reason this Build is being requested. Schedulers
40 provide this, but for forced builds the user requesting the
41 build will provide a string.
42
43 @type properties: Properties object
44 @ivar properties: properties that should be applied to this build
45 'owner' property is used by Build objects to collect
46 the list returned by getInterestedUsers
47
48 @ivar status: the IBuildStatus object which tracks our status
49
50 @ivar submittedAt: a timestamp (seconds since epoch) when this request
51 was submitted to the Builder. This is used by the CVS
52 step to compute a checkout timestamp, as well as the
53 master to prioritize build requests from oldest to
54 newest.
55 """
56
57 source = None
58 builder = None
59 startCount = 0 # how many times we have tried to start this build
60 submittedAt = None
61
62 implements(interfaces.IBuildRequestControl)
63
64 def __init__(self, reason, source, builderName, properties=None):
65 assert interfaces.ISourceStamp(source, None)
66 self.reason = reason
67 self.source = source
68
69 self.properties = Properties()
70 if properties:
71 self.properties.updateFromProperties(properties)
72
73 self.start_watchers = []
74 self.finish_watchers = []
75 self.status = BuildRequestStatus(source, builderName)
76
77 def canBeMergedWith(self, other):
78 return self.source.canBeMergedWith(other.source)
79
80 def mergeWith(self, others):
81 return self.source.mergeWith([o.source for o in others])
82
83 def mergeReasons(self, others):
84 """Return a reason for the merged build request."""
85 reasons = []
86 for req in [self] + others:
87 if req.reason and req.reason not in reasons:
88 reasons.append(req.reason)
89 return ", ".join(reasons)
90
91 def waitUntilFinished(self):
92 """Get a Deferred that will fire (with a
93 L{buildbot.interfaces.IBuildStatus} instance when the build
94 finishes."""
95 d = defer.Deferred()
96 self.finish_watchers.append(d)
97 return d
98
99 # these are called by the Builder
100
101 def requestSubmitted(self, builder):
102 # the request has been placed on the queue
103 self.builder = builder
104
105 def buildStarted(self, build, buildstatus):
106 """This is called by the Builder when a Build has been started in the
107 hopes of satifying this BuildRequest. It may be called multiple
108 times, since interrupted builds and lost buildslaves may force
109 multiple Builds to be run until the fate of the BuildRequest is known
110 for certain."""
111 for o in self.start_watchers[:]:
112 # these observers get the IBuildControl
113 o(build)
114 # while these get the IBuildStatus
115 self.status.buildStarted(buildstatus)
116
117 def finished(self, buildstatus):
118 """This is called by the Builder when the BuildRequest has been
119 retired. This happens when its Build has either succeeded (yay!) or
120 failed (boo!). TODO: If it is halted due to an exception (oops!), or
121 some other retryable error, C{finished} will not be called yet."""
122
123 for w in self.finish_watchers:
124 w.callback(buildstatus)
125 self.finish_watchers = []
126
127 # IBuildRequestControl
128
129 def subscribe(self, observer):
130 self.start_watchers.append(observer)
131 def unsubscribe(self, observer):
132 self.start_watchers.remove(observer)
133
134 def cancel(self):
135 """Cancel this request. This can only be successful if the Build has
136 not yet been started.
137
138 @return: a boolean indicating if the cancel was successful."""
139 if self.builder:
140 return self.builder.cancelBuildRequest(self)
141 return False
142
143 def setSubmitTime(self, t):
144 self.submittedAt = t
145 self.status.setSubmitTime(t)
146
147 def getSubmitTime(self):
148 return self.submittedAt
149
150
151 class Build:
152 """I represent a single build by a single slave. Specialized Builders can
153 use subclasses of Build to hold status information unique to those build
154 processes.
155
156 I control B{how} the build proceeds. The actual build is broken up into a
157 series of steps, saved in the .buildSteps[] array as a list of
158 L{buildbot.process.step.BuildStep} objects. Each step is a single remote
159 command, possibly a shell command.
160
161 During the build, I put status information into my C{BuildStatus}
162 gatherer.
163
164 After the build, I go away.
165
166 I can be used by a factory by setting buildClass on
167 L{buildbot.process.factory.BuildFactory}
168
169 @ivar requests: the list of L{BuildRequest}s that triggered me
170 @ivar build_status: the L{buildbot.status.builder.BuildStatus} that
171 collects our status
172 """
173
174 implements(interfaces.IBuildControl)
175
176 workdir = "build"
177 build_status = None
178 reason = "changes"
179 finished = False
180 results = None
181
182 def __init__(self, requests):
183 self.requests = requests
184 for req in self.requests:
185 req.startCount += 1
186 self.locks = []
187 # build a source stamp
188 self.source = requests[0].mergeWith(requests[1:])
189 self.reason = requests[0].mergeReasons(requests[1:])
190
191 self.progress = None
192 self.currentStep = None
193 self.slaveEnvironment = {}
194
195 self.terminate = False
196
197 def setBuilder(self, builder):
198 """
199 Set the given builder as our builder.
200
201 @type builder: L{buildbot.process.builder.Builder}
202 """
203 self.builder = builder
204
205 def setLocks(self, locks):
206 self.locks = locks
207
208 def setSlaveEnvironment(self, env):
209 self.slaveEnvironment = env
210
211 def getSourceStamp(self):
212 return self.source
213
214 def setProperty(self, propname, value, source):
215 """Set a property on this build. This may only be called after the
216 build has started, so that it has a BuildStatus object where the
217 properties can live."""
218 self.build_status.setProperty(propname, value, source)
219
220 def getProperties(self):
221 return self.build_status.getProperties()
222
223 def getProperty(self, propname):
224 return self.build_status.getProperty(propname)
225
226 def allChanges(self):
227 return self.source.changes
228
229 def allFiles(self):
230 # return a list of all source files that were changed
231 files = []
232 havedirs = 0
233 for c in self.allChanges():
234 for f in c.files:
235 files.append(f)
236 if c.isdir:
237 havedirs = 1
238 return files
239
240 def __repr__(self):
241 return "<Build %s>" % (self.builder.name,)
242
243 def blamelist(self):
244 blamelist = []
245 for c in self.allChanges():
246 if c.who not in blamelist:
247 blamelist.append(c.who)
248 blamelist.sort()
249 return blamelist
250
251 def changesText(self):
252 changetext = ""
253 for c in self.allChanges():
254 changetext += "-" * 60 + "\n\n" + c.asText() + "\n"
255 # consider sorting these by number
256 return changetext
257
258 def setStepFactories(self, step_factories):
259 """Set a list of 'step factories', which are tuples of (class,
260 kwargs), where 'class' is generally a subclass of step.BuildStep .
261 These are used to create the Steps themselves when the Build starts
262 (as opposed to when it is first created). By creating the steps
263 later, their __init__ method will have access to things like
264 build.allFiles() ."""
265 self.stepFactories = list(step_factories)
266
267
268
269 useProgress = True
270
271 def getSlaveCommandVersion(self, command, oldversion=None):
272 return self.slavebuilder.getSlaveCommandVersion(command, oldversion)
273 def getSlaveName(self):
274 return self.slavebuilder.slave.slavename
275
276 def setupProperties(self):
277 props = self.getProperties()
278
279 # start with global properties from the configuration
280 buildmaster = self.builder.botmaster.parent
281 props.updateFromProperties(buildmaster.properties)
282
283 # get any properties from requests (this is the path through
284 # which schedulers will send us properties)
285 for rq in self.requests:
286 props.updateFromProperties(rq.properties)
287
288 # and finally, from the SourceStamp, which has properties via Change
289 for change in self.source.changes:
290 props.updateFromProperties(change.properties)
291
292 # now set some properties of our own, corresponding to the
293 # build itself
294 props.setProperty("buildername", self.builder.name, "Build")
295 props.setProperty("buildnumber", self.build_status.number, "Build")
296 props.setProperty("branch", self.source.branch, "Build")
297 props.setProperty("revision", self.source.revision, "Build")
298
299 def setupSlaveBuilder(self, slavebuilder):
300 self.slavebuilder = slavebuilder
301
302 # navigate our way back to the L{buildbot.buildslave.BuildSlave}
303 # object that came from the config, and get its properties
304 buildslave_properties = slavebuilder.slave.properties
305 self.getProperties().updateFromProperties(buildslave_properties)
306
307 self.slavename = slavebuilder.slave.slavename
308 self.build_status.setSlavename(self.slavename)
309
310 def startBuild(self, build_status, expectations, slavebuilder):
311 """This method sets up the build, then starts it by invoking the
312 first Step. It returns a Deferred which will fire when the build
313 finishes. This Deferred is guaranteed to never errback."""
314
315 # we are taking responsibility for watching the connection to the
316 # remote. This responsibility was held by the Builder until our
317 # startBuild was called, and will not return to them until we fire
318 # the Deferred returned by this method.
319
320 log.msg("%s.startBuild" % self)
321 self.build_status = build_status
322 # now that we have a build_status, we can set properties
323 self.setupProperties()
324 self.setupSlaveBuilder(slavebuilder)
325 slavebuilder.slave.updateSlaveStatus(buildStarted=build_status)
326
327 # convert all locks into their real forms
328 lock_list = []
329 for access in self.locks:
330 if not isinstance(access, locks.LockAccess):
331 # Buildbot 0.7.7 compability: user did not specify access
332 access = access.defaultAccess()
333 lock = self.builder.botmaster.getLockByID(access.lockid)
334 lock_list.append((lock, access))
335 self.locks = lock_list
336 # then narrow SlaveLocks down to the right slave
337 self.locks = [(l.getLock(self.slavebuilder), la)
338 for l, la in self.locks]
339 self.remote = slavebuilder.remote
340 self.remote.notifyOnDisconnect(self.lostRemote)
341 d = self.deferred = defer.Deferred()
342 def _release_slave(res, slave, bs):
343 self.slavebuilder.buildFinished()
344 slave.updateSlaveStatus(buildFinished=bs)
345 return res
346 d.addCallback(_release_slave, self.slavebuilder.slave, build_status)
347
348 try:
349 self.setupBuild(expectations) # create .steps
350 except:
351 # the build hasn't started yet, so log the exception as a point
352 # event instead of flunking the build. TODO: associate this
353 # failure with the build instead. this involves doing
354 # self.build_status.buildStarted() from within the exception
355 # handler
356 log.msg("Build.setupBuild failed")
357 log.err(Failure())
358 self.builder.builder_status.addPointEvent(["setupBuild",
359 "exception"])
360 self.finished = True
361 self.results = FAILURE
362 self.deferred = None
363 d.callback(self)
364 return d
365
366 self.acquireLocks().addCallback(self._startBuild_2)
367 return d
368
369 def acquireLocks(self, res=None):
370 log.msg("acquireLocks(step %s, locks %s)" % (self, self.locks))
371 if not self.locks:
372 return defer.succeed(None)
373 for lock, access in self.locks:
374 if not lock.isAvailable(access):
375 log.msg("Build %s waiting for lock %s" % (self, lock))
376 d = lock.waitUntilMaybeAvailable(self, access)
377 d.addCallback(self.acquireLocks)
378 return d
379 # all locks are available, claim them all
380 for lock, access in self.locks:
381 lock.claim(self, access)
382 return defer.succeed(None)
383
384 def _startBuild_2(self, res):
385 self.build_status.buildStarted(self)
386 self.startNextStep()
387
388 def setupBuild(self, expectations):
389 # create the actual BuildSteps. If there are any name collisions, we
390 # add a count to the loser until it is unique.
391 self.steps = []
392 self.stepStatuses = {}
393 stepnames = {}
394 sps = []
395
396 for factory, args in self.stepFactories:
397 args = args.copy()
398 try:
399 step = factory(**args)
400 except:
401 log.msg("error while creating step, factory=%s, args=%s"
402 % (factory, args))
403 raise
404 step.setBuild(self)
405 step.setBuildSlave(self.slavebuilder.slave)
406 step.setDefaultWorkdir(self.workdir)
407 name = step.name
408 if stepnames.has_key(name):
409 count = stepnames[name]
410 count += 1
411 stepnames[name] = count
412 name = step.name + "_%d" % count
413 else:
414 stepnames[name] = 0
415 step.name = name
416 self.steps.append(step)
417
418 # tell the BuildStatus about the step. This will create a
419 # BuildStepStatus and bind it to the Step.
420 step_status = self.build_status.addStepWithName(name)
421 step.setStepStatus(step_status)
422
423 sp = None
424 if self.useProgress:
425 # XXX: maybe bail if step.progressMetrics is empty? or skip
426 # progress for that one step (i.e. "it is fast"), or have a
427 # separate "variable" flag that makes us bail on progress
428 # tracking
429 sp = step.setupProgress()
430 if sp:
431 sps.append(sp)
432
433 # Create a buildbot.status.progress.BuildProgress object. This is
434 # called once at startup to figure out how to build the long-term
435 # Expectations object, and again at the start of each build to get a
436 # fresh BuildProgress object to track progress for that individual
437 # build. TODO: revisit at-startup call
438
439 if self.useProgress:
440 self.progress = BuildProgress(sps)
441 if self.progress and expectations:
442 self.progress.setExpectationsFrom(expectations)
443
444 # we are now ready to set up our BuildStatus.
445 self.build_status.setSourceStamp(self.source)
446 self.build_status.setRequests([req.status for req in self.requests])
447 self.build_status.setReason(self.reason)
448 self.build_status.setBlamelist(self.blamelist())
449 self.build_status.setProgress(self.progress)
450
451 # gather owners from build requests
452 owners = [r.properties['owner'] for r in self.requests
453 if r.properties.has_key('owner')]
454 if owners: self.setProperty('owners', owners, self.reason)
455
456 self.results = [] # list of FAILURE, SUCCESS, WARNINGS, SKIPPED
457 self.result = SUCCESS # overall result, may downgrade after each step
458 self.text = [] # list of text string lists (text2)
459
460 def getNextStep(self):
461 """This method is called to obtain the next BuildStep for this build.
462 When it returns None (or raises a StopIteration exception), the build
463 is complete."""
464 if not self.steps:
465 return None
466 if self.terminate:
467 while True:
468 s = self.steps.pop(0)
469 if s.alwaysRun:
470 return s
471 if not self.steps:
472 return None
473 else:
474 return self.steps.pop(0)
475
476 def startNextStep(self):
477 try:
478 s = self.getNextStep()
479 except StopIteration:
480 s = None
481 if not s:
482 return self.allStepsDone()
483 self.currentStep = s
484 d = defer.maybeDeferred(s.startStep, self.remote)
485 d.addCallback(self._stepDone, s)
486 d.addErrback(self.buildException)
487
488 def _stepDone(self, results, step):
489 self.currentStep = None
490 if self.finished:
491 return # build was interrupted, don't keep building
492 terminate = self.stepDone(results, step) # interpret/merge results
493 if terminate:
494 self.terminate = True
495 return self.startNextStep()
496
497 def stepDone(self, result, step):
498 """This method is called when the BuildStep completes. It is passed a
499 status object from the BuildStep and is responsible for merging the
500 Step's results into those of the overall Build."""
501
502 terminate = False
503 text = None
504 if type(result) == types.TupleType:
505 result, text = result
506 assert type(result) == type(SUCCESS)
507 log.msg(" step '%s' complete: %s" % (step.name, Results[result]))
508 self.results.append(result)
509 if text:
510 self.text.extend(text)
511 if not self.remote:
512 terminate = True
513 if result == FAILURE:
514 if step.warnOnFailure:
515 if self.result != FAILURE:
516 self.result = WARNINGS
517 if step.flunkOnFailure:
518 self.result = FAILURE
519 if step.haltOnFailure:
520 terminate = True
521 elif result == WARNINGS:
522 if step.warnOnWarnings:
523 if self.result != FAILURE:
524 self.result = WARNINGS
525 if step.flunkOnWarnings:
526 self.result = FAILURE
527 elif result == EXCEPTION:
528 self.result = EXCEPTION
529 terminate = True
530 return terminate
531
532 def lostRemote(self, remote=None):
533 # the slave went away. There are several possible reasons for this,
534 # and they aren't necessarily fatal. For now, kill the build, but
535 # TODO: see if we can resume the build when it reconnects.
536 log.msg("%s.lostRemote" % self)
537 self.remote = None
538 if self.currentStep:
539 # this should cause the step to finish.
540 log.msg(" stopping currentStep", self.currentStep)
541 self.currentStep.interrupt(Failure(error.ConnectionLost()))
542
543 def stopBuild(self, reason="<no reason given>"):
544 # the idea here is to let the user cancel a build because, e.g.,
545 # they realized they committed a bug and they don't want to waste
546 # the time building something that they know will fail. Another
547 # reason might be to abandon a stuck build. We want to mark the
548 # build as failed quickly rather than waiting for the slave's
549 # timeout to kill it on its own.
550
551 log.msg(" %s: stopping build: %s" % (self, reason))
552 if self.finished:
553 return
554 # TODO: include 'reason' in this point event
555 self.builder.builder_status.addPointEvent(['interrupt'])
556 self.currentStep.interrupt(reason)
557 if 0:
558 # TODO: maybe let its deferred do buildFinished
559 if self.currentStep and self.currentStep.progress:
560 # XXX: really .fail or something
561 self.currentStep.progress.finish()
562 text = ["stopped", reason]
563 self.buildFinished(text, FAILURE)
564
565 def allStepsDone(self):
566 if self.result == FAILURE:
567 text = ["failed"]
568 elif self.result == WARNINGS:
569 text = ["warnings"]
570 elif self.result == EXCEPTION:
571 text = ["exception"]
572 else:
573 text = ["build", "successful"]
574 text.extend(self.text)
575 return self.buildFinished(text, self.result)
576
577 def buildException(self, why):
578 log.msg("%s.buildException" % self)
579 log.err(why)
580 self.buildFinished(["build", "exception"], FAILURE)
581
582 def buildFinished(self, text, results):
583 """This method must be called when the last Step has completed. It
584 marks the Build as complete and returns the Builder to the 'idle'
585 state.
586
587 It takes two arguments which describe the overall build status:
588 text, results. 'results' is one of SUCCESS, WARNINGS, or FAILURE.
589
590 If 'results' is SUCCESS or WARNINGS, we will permit any dependant
591 builds to start. If it is 'FAILURE', those builds will be
592 abandoned."""
593
594 self.finished = True
595 if self.remote:
596 self.remote.dontNotifyOnDisconnect(self.lostRemote)
597 self.results = results
598
599 log.msg(" %s: build finished" % self)
600 self.build_status.setText(text)
601 self.build_status.setResults(results)
602 self.build_status.buildFinished()
603 if self.progress and results == SUCCESS:
604 # XXX: also test a 'timing consistent' flag?
605 log.msg(" setting expectations for next time")
606 self.builder.setExpectations(self.progress)
607 reactor.callLater(0, self.releaseLocks)
608 self.deferred.callback(self)
609 self.deferred = None
610
611 def releaseLocks(self):
612 log.msg("releaseLocks(%s): %s" % (self, self.locks))
613 for lock, access in self.locks:
614 lock.release(self, access)
615
616 # IBuildControl
617
618 def getStatus(self):
619 return self.build_status
620
621 # stopBuild is defined earlier
622
OLDNEW
« no previous file with comments | « third_party/buildbot_7_12/buildbot/process/__init__.py ('k') | third_party/buildbot_7_12/buildbot/process/builder.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698