Differentiate between tests that needs to be run or need to wait for completion.

verification/try_job_on_rietveld.py

 # coding=utf8
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 """Sends patches to the Try server and reads back results.
 
 - RietveldTryJobs contains RietveldTryJob, one per try job on a builder.
 - TryRunnerRietveld uses Rietveld to signal and poll job results.
 """
 
@@ skipping 12 matching lines @@
   A job that occured more than 4 days ago or more than 200 commits behind
   is 'expired'.
   """
   if timestamp < (time.time() - 4*24*60*60):
     return True
   if checkout.revisions(revision, None) >= 200:
     return True
   return False
 
 
+class RietveldTryJobPending(model.PersistentMixIn):
+  """Represents a pending try job for a pending commit that we care about.
+  """
+  persistent = [
+    'builder', 'revision', 'requested_steps',
+    'clobber', 'tries',
+  ]
+  def __init__(self, builder, revision, requested_steps, clobber, tries):
+    super(RietveldTryJobPending, self).__init__()
+    self.builder = builder
+    self.revision = revision
+    self.requested_steps = requested_steps
+    self.clobber = clobber
+    # Number of retries for this configuration. Initial try is 1.
+    self.tries = tries
+
+
 class RietveldTryJob(model.PersistentMixIn):
   """Represents a try job for a pending commit that we care about.
 
   This data can be regenerated by parsing all the try job names but it is a bit
   hard on the try server.
-
-  TODO(maruel): Should use __getstate__(), __setstate__() and __reduce__().
   """
   persistent = [
     'builder', 'build', 'revision', 'requested_steps', 'started',
     'steps_passed', 'steps_failed', 'clobber', 'completed', 'tries',
   ]
 
   def __init__(
       self, builder, build, revision, requested_steps, started, passed, failed,
       clobber, completed, tries):
     super(RietveldTryJob, self).__init__()
     self.builder = builder
     self.build = build
     self.revision = revision
     self.requested_steps = requested_steps
+    # The timestamp when the build started.
     self.started = started
     self.steps_passed = passed
     self.steps_failed = failed
     self.clobber = clobber
     self.completed = completed
     # Number of retries for this configuration. Initial try is 1.
     self.tries = tries
 
   @property
   def result(self):
@@ skipping 31 matching lines @@
     # An dict of RietveldTryJob objects per key.
     self.try_jobs = {}
     # The try job keys we ignore because they can't be used to give a good
     # signal: either they are too old (old revision) or they were not triggerd
     # by Rietveld, so we don't know if the diff is 100% good.
     self.irrelevant = []
     # When NOTRY=true is specified.
     self.skipped = False
     self.builders_and_tests = {}
     # Jobs that have been sent but are not found yet. Likely a builder is fully
-    # utilized or the try server hasn't polled Rietveld yet.
+    # utilized or the try server hasn't polled Rietveld yet. list of
+    # RietveldTryJobPending() instances.
     self.pendings = []
 
   def get_state(self):
-    if self.skipped or (not self.remaining() and not self.pendings):
+    if self.skipped or not self.tests_waiting_for_result():
       return base.SUCCEEDED
     if (self.pendings or
         not all(t.completed for t in self.try_jobs.itervalues())):
       return base.PROCESSING
     logging.debug('Not pending, all %d jobs completed' % len(self.try_jobs))
     return base.FAILED
 
-  def remaining(self):
-    """Returns what remains to be tested.
+  def tests_need_to_be_run(self):
+    """Returns which tests need to be run.
 
-    This excludes tests from any pending build so they are not retried
-    unnecessarily.
+    These are the tests that are not pending on any try job, either running or
+    in the pending list.
     """
     # What needs to be run.
     all_tests = dict(
         (builder, set(tests))
         for builder, tests in self.builders_and_tests.iteritems())
 
-    def clean(job):
-      if job.builder in all_tests:
-        all_tests[job.builder] -= set(job.steps_passed)
+    # Removes what was run and almost to be run, e.g. the build is running.
+    for try_job in self.try_jobs.itervalues():
+      if try_job.builder in all_tests:
+        all_tests[try_job.builder] -= set(try_job.steps_passed)
         # If it was requested but still not run, do not add it either. Only do
         # that if the job hasn't completed yet to catch issues like: a test was
         # requested on a builder but the test was not run because it is not in
         # the BuildFactory for the Builder.
-        if not job.completed:
-          to_be_run = set(job.requested_steps) - set(job.steps_failed)
-          all_tests[job.builder] -= to_be_run
+        if not try_job.completed:
+          to_be_run = set(try_job.requested_steps) - set(try_job.steps_failed)
+          all_tests[try_job.builder] -= to_be_run
+
+    # Removes what is queued to be run but hasn't started yet.
+    for try_job in self.pendings:
+      if try_job.builder in all_tests:
+        all_tests[try_job.builder] -= set(try_job.requested_steps)
+
+    return dict(
+        (builder, list(tests)) for builder, tests in all_tests.iteritems()
+        if tests)
+
+  def tests_waiting_for_result(self):
+    """Returns the tests that we are waiting for results on pending or running
+    builds.
+    """
+    all_tests = dict(
+        (builder, set(tests))
+        for builder, tests in self.builders_and_tests.iteritems())
 
     # Removes what was run.
     for try_job in self.try_jobs.itervalues():
-      clean(try_job)
-    for try_job in self.pendings:
-      clean(try_job)
+      if try_job.builder in all_tests:
+        all_tests[try_job.builder] -= set(try_job.steps_passed)
 
     return dict(
         (builder, list(tests)) for builder, tests in all_tests.iteritems()
         if tests)
 
   def update_jobs_from_rietveld(self, data, status, checkout):
     """Retrieves the jobs statuses from rietveld and updates its state.
 
     Arguments:
     - data: Patchset properties as returned from Rietveld.
@@ skipping 54 matching lines @@
           if is_job_expired(build.revision, build.start_time, checkout):
             logging.debug('Ignoring %s/%d; expired', builder, buildnumber)
             self.irrelevant.append(key)
             continue
       else:
         requested_steps = job.requested_steps
         tries = job.tries
 
       passed = [s.name for s in build.steps if s.simplified_result]
       failed = [s.name for s in build.steps if s.simplified_result is False]
+      # The steps in neither passed or failed were skipped.
       logging.info(
           'Found Job success: %s/%d: %s',
           builder, buildnumber, ','.join(passed))
       new_job = RietveldTryJob(
           builder,
           buildnumber,
           build.revision,
           requested_steps,
           build.start_time,
           passed,
@@ skipping 18 matching lines @@
   Analysis goes as following:
   - compile step itself is not flaky. compile.py already takes care of most
     flakiness and clobber build is done by default. If compile step fails, try
     again with clobber=True
   - test steps are flaky and can be retried as necessary.
 
   1. For each existing try jobs from rietveld.
     1. Fetch result from try server.
     2. If try job was generated from rietveld;
       1. If not is_job_expired();
-        1. Strips from jobs to run for each succeeded steps
-        2. If no step remaining, mark job as succeeded.
+        1. Skip any scheduled test that succeeded on this builder.
+  2. For each builder with tests scheduled;
+    1. If no step waiting to be triggered, skip this builder completely.
   2. For each non succeeded job;
     1. Send try jobs to rietveld.
 
   Note: It needs rietveld, hence it uses VerifierCheckout, but it doesn't need a
   checkout.
   """
   name = 'try job rietveld'
 
   # Only updates a job status once every 60 seconds.
   update_latency = 60
@@ skipping 55 matching lines @@
       # Send any necessary job. Noop if not needed.
       self._send_jobs(pending, jobs)
 
   def _send_jobs(self, pending, jobs):
     """Prepares the RietveldTryJobs instance |jobs| to send try jobs to the try
     server.
     """
     if jobs.error_message:
       # Too late.
       return
-    remaining = jobs.remaining()
+    remaining = jobs.tests_need_to_be_run()
     if not remaining:
       return
     # Send them in order to simplify testing.
     for builder in sorted(remaining):
       tests = remaining[builder]
       # Find if there was a previous try.
       previous_jobs = [
         job for job in jobs.try_jobs.itervalues()
         if job.builder == builder
       ]
@@ skipping 16 matching lines @@
 
       # Do one request per builder so each can have a different 'clobber'
       # setting.
       logging.debug(
           'Sending job %s for %s: %s', pending.issue, builder, ','.join(tests))
       # TODO(maruel): Use keys.
       self.context.rietveld.trigger_try_jobs(
           pending.issue, pending.patchset, 'CQ', clobber, None,
           {builder: tests})
       jobs.pendings.append(
-          RietveldTryJob(
-            builder, None, None, tests, time.time(), [], [],
-            clobber, False, tries + 1))
+          RietveldTryJobPending(builder, None, tests, clobber, tries + 1))
       # Update the status on the AppEngine status to signal a new try job was
       # sent.
       info = {
         'builder': builder,
         'clobber': clobber,
         'job_name': 'CQ',
         'revision': None, #revision,
       }
       self.send_status(pending, info)
 
@@ skipping 15 matching lines @@
       'build': job.build,
       'builder': job.builder,
       'duration': build.duration,
       'job_name': 'CQ',
       'result': job.result,
       'revision': job.revision,
       'url': self._build_status_url(job),
     }
     self.send_status(pending, info)
 
-    remaining = jobs.remaining()
+    remaining = jobs.tests_need_to_be_run()
     retry = [
       step for step in job.steps_failed
       if step in remaining.get(job.builder, [])
     ]
     if job.tries > 2:
       jobs.error_message = 'Retried try job too often for step(s) %s' % (
           ', '.join(retry))
       logging.info(jobs.error_message)
     elif 'update' in job.steps_failed:
       jobs.error_message = (
@@ skipping 16 matching lines @@
     return match and match.group(1).lower() == 'true'
 
   def _update_jobs_from_rietveld(self, pending, jobs):
     """Grabs data from Rietveld and pass it to
     RietveldTryJobs.update_jobs_from_rietveld().
     """
     data = self.context.rietveld.get_patchset_properties(
         pending.issue, pending.patchset)
     return jobs.update_jobs_from_rietveld(
         data, self.status, self.context.checkout)