Add a module to fetch builds from different types of builders.

tools/auto_bisect/fetch_build_test.py

Index: tools/auto_bisect/fetch_build_test.py
diff --git a/tools/auto_bisect/fetch_build_test.py b/tools/auto_bisect/fetch_build_test.py
new file mode 100644
index 0000000000000000000000000000000000000000..4dc511c62d4569122d5b69d18be8d582bbab10d1
--- /dev/null
+++ b/tools/auto_bisect/fetch_build_test.py
@@ -0,0 +1,198 @@
+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Unit tests for the fetch_builds module."""
+
+import errno
+import os
+import sys
+import unittest
+
+import fetch_build
+import mock_wrapper as mock
+
+
+# The tests below test private functions (W0212).
+# Some methods don't reference self because they use the mock module (R0201).
+# pylint: disable=R0201,W0212
+class FetchBuildTest(unittest.TestCase):
+

[ghost stip (do not use), 2014/09/17 00:12:08]

remove blank line

[qyearsley, 2014/09/18 22:58:59]

Google style guide says: "One blank line ... between the class line and the
first method."
https://google-styleguide.googlecode.com/svn/trunk/pyguide.html?showone=Blank...

PEP 8 doesn't say anything explicitly about it, but they use a blank line after
the class line in one example.
http://legacy.python.org/dev/peps/pep-0008/#blank-lines

+  def setUp(self):
+    # Don't print to console when running tests.
+    sys.stdout = open(os.devnull, 'w')

[ghost stip (do not use), 2014/09/17 00:12:08]

I don't think this is needed -- we want those debugging statements!

[qyearsley, 2014/09/18 22:58:59]

Done.

+
+  @mock.patch('fetch_build.os.path.exists')

[ghost stip (do not use), 2014/09/17 00:12:08]

I'm not sure if it makes it easier but I used patcher.start() and patcher.stop()
in setUp in
https://chromium.googlesource.com/chromium/tools/build/+/master/scripts/slave....
The decorator is probably fine though.

[qyearsley, 2014/09/18 22:58:59]

Using patcher.start() and patcher.stop() looks like it will be a little bit
nicer, because it means that I wouldn't need to repeat the decorator over every
method name; I plan to try this and upload another patch to this CL.

+  @mock.patch('fetch_build.cloud_storage')
+  def test_FetchFromCloudStorage_FileFound(
+      self, mock_cloud_storage, mock_os_path_exists):
+    mock_cloud_storage.Exists.return_value = True
+    mock_os_path_exists.return_value = True
+    local_path = fetch_build.FetchFromCloudStorage(
+        'my_bucket', 'remote/foo.zip', 'local')
+    self.assertEqual('local/foo.zip', local_path)
+    mock_cloud_storage.Get.assert_called_with(
+        'my_bucket', 'remote/foo.zip', 'local/foo.zip')
+
+  @mock.patch('fetch_build.cloud_storage')
+  def test_FetchFromCloudStorage_FileNotFound(
+      self, mock_cloud_storage):
+    mock_cloud_storage.Exists.return_value = False
+    local_path = fetch_build.FetchFromCloudStorage(
+        'my_bucket', 'remote/foo.zip', 'local')
+    self.assertIsNone(local_path)
+    self.assertFalse(mock_cloud_storage.Get.called)
+
+
+class BuildArchiveTest(unittest.TestCase):
+

[ghost stip (do not use), 2014/09/17 00:12:08]

remove blank line

[qyearsley, 2014/09/18 22:58:59]

Same as comment above.

+  @mock.patch('fetch_build.bisect_utils')
+  def test_CreatePerfBuildArchive(self, mock_bisect_utils):
+    self._SetPlatformLinux(mock_bisect_utils)
+    instance = fetch_build.BuildArchive.Create(fetch_build.PERF_BUILDER)
+    self._SetPlatformLinux(mock_bisect_utils)
+    self.assertEqual('chrome-perf', instance.BucketName())
+    self.assertTrue(isinstance(instance, fetch_build.PerfBuildArchive))
+
+  @mock.patch('fetch_build.bisect_utils')
+  def test_CreateFullBuildArchive(self, mock_bisect_utils):
+    self._SetPlatformLinux(mock_bisect_utils)
+    instance = fetch_build.BuildArchive.Create(fetch_build.FULL_BUILDER)
+    self.assertEqual('chromium-linux-archive', instance.BucketName())
+    self.assertTrue(isinstance(instance, fetch_build.FullBuildArchive))
+
+  @mock.patch('fetch_build.bisect_utils')
+  def test_BuildArchive_NonExistentType(self, mock_bisect_utils):
+    self._SetPlatformLinux(mock_bisect_utils)
+    self.assertRaises(NotImplementedError,
+                      fetch_build.BuildArchive.Create, 'other')
+
+  @mock.patch('fetch_build.bisect_utils')
+  def test_FullBuildArchive_Linux(self, mock_bisect_utils):
+    self._SetPlatformLinux(mock_bisect_utils)
+    archive = fetch_build.FullBuildArchive()
+    self.assertEqual('chromium-linux-archive', archive.BucketName())
+    self.assertEqual(
+        'chromium.linux/Linux Builder/full-build-linux_1234567890abcdef.zip',
+        archive.FilePath('1234567890abcdef'))
+
+  @mock.patch('fetch_build.bisect_utils')
+  def test_FullBuildArchive_Android(self, mock_bisect_utils):
+    self._SetPlatformLinux(mock_bisect_utils)
+    archive = fetch_build.FullBuildArchive(target_platform='android')
+    self.assertEqual('chromium-android', archive.BucketName())
+    self.assertEqual('android_main_rel/full-build-linux_1234567890abcdef.zip',
+        archive.FilePath('1234567890abcdef'))
+
+  @mock.patch('fetch_build.bisect_utils')
+  def test_PerfBuildArchive_Linux(self, mock_bisect_utils):
+    self._SetPlatformLinux(mock_bisect_utils)
+    archive = fetch_build.PerfBuildArchive()
+    self.assertEqual('chrome-perf', archive.BucketName())
+    self.assertEqual(
+        'Linux Builder/full-build-linux_1234567890abcdef.zip',
+        archive.FilePath('1234567890abcdef'))
+
+  @mock.patch('fetch_build.bisect_utils')
+  def test_PerfBuildArchive_Android(self, mock_bisect_utils):
+    self._SetPlatformLinux(mock_bisect_utils)
+    archive = fetch_build.PerfBuildArchive(target_platform='android')
+    self.assertEqual('chrome-perf', archive.BucketName())
+    self.assertEqual(
+        'android_perf_rel/full-build-linux_123456.zip',
+        archive.FilePath('123456'))
+
+  @mock.patch('fetch_build.bisect_utils')
+  def test_PerfBuildArchive_64BitWindows(self, mock_bisect_utils):
+    mock_bisect_utils.IsLinuxHost.return_value = False
+    mock_bisect_utils.IsMacHost.return_value = False
+    mock_bisect_utils.IsWindowsHost.return_value = True
+    mock_bisect_utils.Is64BitWindows.return_value = True
+    archive = fetch_build.PerfBuildArchive(target_arch='x64')
+    self.assertEqual('chrome-perf', archive.BucketName())
+    self.assertEqual(
+        'Win x64 Builder/full-build-win32_123456.zip',
+        archive.FilePath('123456'))
+
+  @mock.patch('fetch_build.bisect_utils')
+  def test_PerfBuildArchive_WithDepsPatchSha(self, mock_bisect_utils):
+    self._SetPlatformLinux(mock_bisect_utils)
+    archive = fetch_build.PerfBuildArchive()
+    self.assertEqual(
+        'Linux Builder/full-build-linux_123456'
+        '_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.zip',
+        archive.FilePath(123456, 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'))
+
+  @staticmethod
+  def _SetPlatformLinux(mock_bisect_utils):
+    mock_bisect_utils.IsLinuxHost.return_value = True
+    mock_bisect_utils.IsMacHost.return_value = False
+    mock_bisect_utils.IsWindowsHost.return_value = False
+    mock_bisect_utils.Is64BitWindows.return_value = False
+
+
+class UnzipTest(unittest.TestCase):
+
+  @mock.patch('fetch_build.bisect_utils')
+  @mock.patch('fetch_build._MakeDirectory')
+  @mock.patch('fetch_build._UnzipUsingCommand')
+  def test_Unzip_Linux(
+      self, mock_UnzipUsingCommand, mock_MakeDirectory, mock_bisect_utils):
+    mock_bisect_utils.IsLinuxHost.return_value = True
+    mock_bisect_utils.IsMacHost.return_value = False
+    mock_bisect_utils.IsWindowsHost.return_value = False
+    fetch_build.Unzip('x.zip', 'out_dir', verbose=False)
+    mock_MakeDirectory.assert_called_with('out_dir')
+    mock_UnzipUsingCommand.assert_called_with(
+        ['unzip', '-o'], 'x.zip', 'out_dir')
+
+  @mock.patch('fetch_build.os')
+  @mock.patch('fetch_build.bisect_utils')
+  @mock.patch('fetch_build._MakeDirectory')
+  @mock.patch('fetch_build._UnzipUsingZipFile')
+  def test_Unzip_Mac_LargeFile(
+      self, mock_UnzipUsingZipFile, mock_MakeDirectory, mock_bisect_utils,
+      mock_os):
+    # The zipfile module is used to unzip on mac when the file is > 4GB.
+    mock_bisect_utils.IsLinuxHost.return_value = False
+    mock_bisect_utils.IsMacHost.return_value = True
+    mock_bisect_utils.IsWindowsHost.return_value = False
+    mock_os.path.getsize.return_value = 2 ** 33  # 8GB
+    fetch_build.Unzip('x.zip', 'out_dir', verbose=False)
+    mock_MakeDirectory.assert_called_with('out_dir')
+    mock_UnzipUsingZipFile.assert_called_with('x.zip', 'out_dir', False)
+
+  @mock.patch('fetch_build.os')
+  @mock.patch('fetch_build.bisect_utils.RunProcess')
+  def test_UnzipUsingCommand(self, mock_RunProcess, mock_os):
+    # The _UnzipUsingCommand function should move to the output
+    # directory and run the command with the file's absolute path.
+    mock_os.path.abspath.return_value = '/foo/some/path/x.zip'
+    mock_os.getcwd.return_value = 'curr_dir'
+    mock_RunProcess.return_value = 0
+    fetch_build._UnzipUsingCommand(['unzip'], 'x.zip', 'out_dir')
+    mock_os.chdir.assert_has_calls(
+        [mock.call('out_dir'), mock.call('curr_dir')])
+    mock_RunProcess.assert_called_with(['unzip', '/foo/some/path/x.zip'])
+
+  @mock.patch('fetch_build.os')
+  def test_MakeDirectory(self, mock_os):
+    # _MakeDirectory uses os.makedirs.
+    fetch_build._MakeDirectory('some/path')
+    mock_os.makedirs.assert_called_with('some/path')
+
+  @mock.patch('fetch_build.os')
+  def test_MakeDirectory_RaisesError(self, mock_os):
+    mock_os.makedirs.side_effect = OSError()
+    self.assertRaises(OSError, fetch_build._MakeDirectory, 'some/path')
+
+  @mock.patch('fetch_build.os')
+  def test_MakeDirectory_SucceedsIfDirectoryExists(self, mock_os):
+    already_exists = OSError()
+    already_exists.errno = errno.EEXIST
+    mock_os.makedirs.side_effect = already_exists
+
+
+if __name__ == '__main__':
+  unittest.main()
+