Add retries to the InfraTests bot's "update go pkgs" step

infra/bots/recipes/swarm_infra.py

Index: infra/bots/recipes/swarm_infra.py
diff --git a/infra/bots/recipes/swarm_infra.py b/infra/bots/recipes/swarm_infra.py
index 3c32faa262fe6f5a5f8247ed791e7a93adaa25a3..bfc004b1385fd38b83352e8ad6dbd8223c65c41c 100644
--- a/infra/bots/recipes/swarm_infra.py
+++ b/infra/bots/recipes/swarm_infra.py
@@ -15,16 +15,32 @@ DEPS = [
 ]
 
 
+UPDATE_GO_ATTEMPTS = 5
+
+
 def RunSteps(api):
   api.vars.setup()
   api.core.checkout_steps()
 
+  # Attempt to update go dependencies. This fails flakily sometimes, so perform
+  # multiple attempts.
   gopath = api.vars.checkout_root.join('gopath')
   env = {'GOPATH': gopath}
-  api.step('update_go_pkgs',
-           cmd=['go', 'get', '-u', 'go.skia.org/infra/...'],
-           env=env)
+  name = 'update go pkgs'
+  for attempt in xrange(UPDATE_GO_ATTEMPTS):
+    step_name = name
+    if attempt > 0:
+      step_name += ' (attempt %d)' % (attempt + 1)
+    try:
+      api.step(step_name,
+               cmd=['go', 'get', '-u', 'go.skia.org/infra/...'],
+               env=env)
+      break
+    except api.step.StepFailure:
+      if attempt == UPDATE_GO_ATTEMPTS - 1:
+        raise
 
+  # Run the infra tests.
   infra_tests = api.vars.skia_dir.join(
       'infra', 'bots', 'infra_tests.py')
   api.step('infra_tests',
@@ -44,3 +60,31 @@ def GenTests(api):
                      path_config='kitchen',
                      swarm_out_dir='[SWARM_OUT_DIR]')
   )
+
+  yield (
+    api.test('failed_one_update') +
+      api.properties(buildername='Housekeeper-PerCommit-InfraTests',
+                     mastername='client.skia.fyi',
+                     slavename='dummy-slave',
+                     buildnumber=5,
+                     revision='abc123',
+                     path_config='kitchen',
+                     swarm_out_dir='[SWARM_OUT_DIR]') +
+    api.step_data('update go pkgs', retcode=1)
+  )
+
+  yield (
+    api.test('failed_all_updates') +
+      api.properties(buildername='Housekeeper-PerCommit-InfraTests',
+                     mastername='client.skia.fyi',
+                     slavename='dummy-slave',
+                     buildnumber=5,
+                     revision='abc123',
+                     path_config='kitchen',
+                     swarm_out_dir='[SWARM_OUT_DIR]') +
+    api.step_data('update go pkgs', retcode=1) +
+    api.step_data('update go pkgs (attempt 2)', retcode=1) +
+    api.step_data('update go pkgs (attempt 3)', retcode=1) +
+    api.step_data('update go pkgs (attempt 4)', retcode=1) +
+    api.step_data('update go pkgs (attempt 5)', retcode=1)
+  )