OLD | NEW |
1 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | 1 # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 from datetime import datetime | 4 from datetime import datetime |
5 import glob | 5 import glob |
6 import optparse | 6 import optparse |
7 import os | 7 import os |
8 import re | 8 import re |
| 9 import tempfile |
9 | 10 |
10 from telemetry import test | 11 from telemetry import test |
11 from telemetry.core import bitmap | 12 from telemetry.core import bitmap |
| 13 from telemetry.page import cloud_storage |
12 from telemetry.page import page_test | 14 from telemetry.page import page_test |
13 | 15 |
14 test_data_dir = os.path.abspath(os.path.join( | 16 test_data_dir = os.path.abspath(os.path.join( |
15 os.path.dirname(__file__), '..', '..', 'data', 'gpu')) | 17 os.path.dirname(__file__), '..', '..', 'data', 'gpu')) |
16 | 18 |
17 default_generated_data_dir = os.path.join(test_data_dir, 'generated') | 19 default_generated_data_dir = os.path.join(test_data_dir, 'generated') |
18 default_reference_image_dir = os.path.join(test_data_dir, 'gpu_reference') | 20 default_reference_image_dir = os.path.join(test_data_dir, 'gpu_reference') |
19 | 21 |
| 22 error_image_cloud_storage_bucket = 'chromium-browser-gpu-tests' |
| 23 |
20 test_harness_script = r""" | 24 test_harness_script = r""" |
21 var domAutomationController = {}; | 25 var domAutomationController = {}; |
22 | 26 |
23 domAutomationController._succeeded = false; | 27 domAutomationController._succeeded = false; |
24 domAutomationController._finished = false; | 28 domAutomationController._finished = false; |
25 | 29 |
26 domAutomationController.setAutomationId = function(id) {} | 30 domAutomationController.setAutomationId = function(id) {} |
27 | 31 |
28 domAutomationController.send = function(msg) { | 32 domAutomationController.send = function(msg) { |
29 domAutomationController._finished = true; | 33 domAutomationController._finished = true; |
(...skipping 10 matching lines...) Expand all Loading... |
40 | 44 |
41 class PixelTestFailure(Exception): | 45 class PixelTestFailure(Exception): |
42 pass | 46 pass |
43 | 47 |
44 def _DidTestSucceed(tab): | 48 def _DidTestSucceed(tab): |
45 return tab.EvaluateJavaScript('domAutomationController._succeeded') | 49 return tab.EvaluateJavaScript('domAutomationController._succeeded') |
46 | 50 |
47 class PixelValidator(page_test.PageTest): | 51 class PixelValidator(page_test.PageTest): |
48 def __init__(self): | 52 def __init__(self): |
49 super(PixelValidator, self).__init__('ValidatePage') | 53 super(PixelValidator, self).__init__('ValidatePage') |
| 54 # Parameters for cloud storage reference images. |
| 55 self.vendor_id = None |
| 56 self.device_id = None |
| 57 self.vendor_string = None |
| 58 self.device_string = None |
50 | 59 |
51 def CustomizeBrowserOptions(self, options): | 60 def CustomizeBrowserOptions(self, options): |
52 options.AppendExtraBrowserArgs('--enable-gpu-benchmarking') | 61 options.AppendExtraBrowserArgs('--enable-gpu-benchmarking') |
53 | 62 |
54 def ValidatePage(self, page, tab, results): | 63 def ValidatePage(self, page, tab, results): |
55 if not _DidTestSucceed(tab): | 64 if not _DidTestSucceed(tab): |
56 raise page_test.Failure('Page indicated a failure') | 65 raise page_test.Failure('Page indicated a failure') |
57 | 66 |
58 if not tab.screenshot_supported: | 67 if not tab.screenshot_supported: |
59 raise page_test.Failure('Browser does not support screenshot capture') | 68 raise page_test.Failure('Browser does not support screenshot capture') |
60 | 69 |
61 screenshot = tab.Screenshot(5) | 70 screenshot = tab.Screenshot(5) |
62 | 71 |
63 if not screenshot: | 72 if not screenshot: |
64 raise page_test.Failure('Could not capture screenshot') | 73 raise page_test.Failure('Could not capture screenshot') |
65 | 74 |
66 if hasattr(page, 'test_rect'): | 75 if hasattr(page, 'test_rect'): |
67 screenshot = screenshot.Crop( | 76 screenshot = screenshot.Crop( |
68 page.test_rect[0], page.test_rect[1], | 77 page.test_rect[0], page.test_rect[1], |
69 page.test_rect[2], page.test_rect[3]) | 78 page.test_rect[2], page.test_rect[3]) |
70 | 79 |
71 image_name = PixelValidator.UrlToImageName(page.display_name) | 80 image_name = PixelValidator.UrlToImageName(page.display_name) |
72 | 81 |
73 ref_png = PixelValidator.GetReferenceImage(self.options.reference_dir, | 82 if self.options.upload_refimg_to_cloud_storage: |
74 image_name, page.revision, screenshot) | 83 if self._ConditionallyUploadToCloudStorage(image_name, page, tab, |
| 84 screenshot): |
| 85 # This is the new reference image; there's nothing to compare against. |
| 86 ref_png = screenshot |
| 87 else: |
| 88 # There was a preexisting reference image, so we might as well |
| 89 # compare against it. |
| 90 ref_png = self._DownloadFromCloudStorage(image_name, page, tab) |
| 91 elif self.options.download_refimg_from_cloud_storage: |
| 92 # This bot doesn't have the ability to properly generate a |
| 93 # reference image, so download it from cloud storage. |
| 94 ref_png = self._DownloadFromCloudStorage(image_name, page, tab) |
| 95 else: |
| 96 # Legacy path using on-disk results. |
| 97 ref_png = PixelValidator.GetReferenceImage(self.options.reference_dir, |
| 98 image_name, page.revision, screenshot) |
75 | 99 |
76 # Test new snapshot against existing reference image | 100 # Test new snapshot against existing reference image |
77 if not ref_png.IsEqual(screenshot, tolerance=2): | 101 if not ref_png.IsEqual(screenshot, tolerance=2): |
78 PixelValidator.WriteErrorImages(self.options.generated_dir, image_name, | 102 if self.options.test_machine_name: |
79 self.options.build_revision, screenshot, ref_png) | 103 self._UploadErrorImagesToCloudStorage(image_name, ref_png, screenshot) |
| 104 else: |
| 105 PixelValidator.WriteErrorImages(self.options.generated_dir, image_name, |
| 106 self.options.build_revision, screenshot, ref_png) |
80 raise page_test.Failure('Reference image did not match captured screen') | 107 raise page_test.Failure('Reference image did not match captured screen') |
81 | 108 |
82 @staticmethod | 109 @staticmethod |
83 def UrlToImageName(url): | 110 def UrlToImageName(url): |
84 image_name = re.sub(r'^(http|https|file)://(/*)', '', url) | 111 image_name = re.sub(r'^(http|https|file)://(/*)', '', url) |
85 image_name = re.sub(r'\.\./', '', image_name) | 112 image_name = re.sub(r'\.\./', '', image_name) |
86 image_name = re.sub(r'(\.|/|-)', '_', image_name) | 113 image_name = re.sub(r'(\.|/|-)', '_', image_name) |
87 return image_name | 114 return image_name |
88 | 115 |
89 @staticmethod | 116 @staticmethod |
(...skipping 50 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
140 os.path.join(img_dir, 'DIFF_' + full_image_name), diff_png) | 167 os.path.join(img_dir, 'DIFF_' + full_image_name), diff_png) |
141 | 168 |
142 @staticmethod | 169 @staticmethod |
143 def WriteImage(image_path, png_image): | 170 def WriteImage(image_path, png_image): |
144 output_dir = os.path.dirname(image_path) | 171 output_dir = os.path.dirname(image_path) |
145 if not os.path.exists(output_dir): | 172 if not os.path.exists(output_dir): |
146 os.makedirs(output_dir) | 173 os.makedirs(output_dir) |
147 | 174 |
148 png_image.WritePngFile(image_path) | 175 png_image.WritePngFile(image_path) |
149 | 176 |
| 177 def _ComputeGpuInfo(self, tab): |
| 178 if ((self.vendor_id and self.device_id) or |
| 179 (self.vendor_string and self.device_string)): |
| 180 return |
| 181 browser = tab.browser |
| 182 if not browser.supports_system_info: |
| 183 raise Exception('System info must be supported by the browser') |
| 184 system_info = browser.GetSystemInfo() |
| 185 if not system_info.gpu: |
| 186 raise Exception('GPU information was absent') |
| 187 device = system_info.gpu.devices[0] |
| 188 if device.vendor_id and device.device_id: |
| 189 self.vendor_id = device.vendor_id |
| 190 self.device_id = device.device_id |
| 191 elif device.vendor_string and device.device_string: |
| 192 self.vendor_string = device.vendor_string |
| 193 self.device_string = device.device_string |
| 194 else: |
| 195 raise Exception('GPU device information was incomplete') |
| 196 |
| 197 def _FormatGpuInfo(self, tab): |
| 198 self._ComputeGpuInfo(tab) |
| 199 if self.vendor_id: |
| 200 return '%s_%04x_%04x' % ( |
| 201 self.options.os_type, self.vendor_id, self.device_id) |
| 202 else: |
| 203 return '%s_%s_%s' % ( |
| 204 self.options.os_type, self.vendor_string, self.device_string) |
| 205 |
| 206 def _FormatReferenceImageName(self, img_name, page, tab): |
| 207 return '%s_v%s_%s.png' % ( |
| 208 img_name, |
| 209 page.revision, |
| 210 self._FormatGpuInfo(tab)) |
| 211 |
| 212 def _WriteToTempPng(self, screenshot): |
| 213 f = tempfile.NamedTemporaryFile() |
| 214 screenshot.WritePngToFile(f) |
| 215 f.flush() |
| 216 return f |
| 217 |
| 218 def _UploadBitmapToCloudStorage(self, bucket, name, bitmap, public=False): |
| 219 f = self._WriteToTempPng(bitmap) |
| 220 cloud_storage.Insert(bucket, name, f.name, publicly_readable=public) |
| 221 f.close() |
| 222 |
| 223 def _ConditionallyUploadToCloudStorage(self, img_name, page, tab, screenshot): |
| 224 """Uploads the screenshot to cloud storage as the reference image |
| 225 for this test, unless it already exists. Returns True if the |
| 226 upload was actually performed.""" |
| 227 cloud_name = self._FormatReferenceImageName(img_name, page, tab) |
| 228 if not cloud_storage.Exists(self.options.refimg_cloud_storage_bucket, |
| 229 cloud_name): |
| 230 self._UploadBitmapToCloudStorage(self.options.refimg_cloud_storage_bucket, |
| 231 cloud_name, |
| 232 screenshot) |
| 233 return True |
| 234 return False |
| 235 |
| 236 def _DownloadFromCloudStorage(self, img_name, page, tab): |
| 237 """Downloads the reference image for the given test from cloud |
| 238 storage, returning it as a Telemetry Bitmap object.""" |
| 239 # TODO(kbr): there's a race condition between the deletion of the |
| 240 # temporary file and gsutil's overwriting it. |
| 241 f = tempfile.NamedTemporaryFile() |
| 242 filename = f.name |
| 243 f.close() |
| 244 cloud_storage.Get(self.options.refimg_cloud_storage_bucket, |
| 245 self._FormatReferenceImageName(img_name, page, tab), |
| 246 filename) |
| 247 return bitmap.Bitmap.FromPngFile(filename) |
| 248 |
| 249 def _UploadErrorImagesToCloudStorage(self, image_name, ref_img, screenshot): |
| 250 """For a failing run, uploads the reference image, failing image, |
| 251 and diff image to cloud storage. This subsumes the functionality |
| 252 of the archive_gpu_pixel_test_results.py script.""" |
| 253 machine_name = re.sub('\W+', '_', self.options.test_machine_name) |
| 254 upload_dir = '%s_%s_telemetry' % (self.options.build_revision, machine_name) |
| 255 base_bucket = '%s/runs/%s' % (error_image_cloud_storage_bucket, upload_dir) |
| 256 image_name_with_revision = '%s_%s.png' % ( |
| 257 image_name, self.options.build_revision) |
| 258 self._UploadBitmapToCloudStorage( |
| 259 base_bucket + '/ref', image_name_with_revision, ref_img, public=True) |
| 260 self._UploadBitmapToCloudStorage( |
| 261 base_bucket + '/gen', image_name_with_revision, screenshot, |
| 262 public=True) |
| 263 diff_img = screenshot.Diff(ref_img) |
| 264 self._UploadBitmapToCloudStorage( |
| 265 base_bucket + '/diff', image_name_with_revision, diff_img, |
| 266 public=True) |
| 267 print ('See http://%s.commondatastorage.googleapis.com/' |
| 268 'view_test_results.html?%s for this run\'s test results') % ( |
| 269 error_image_cloud_storage_bucket, upload_dir) |
| 270 |
150 class Pixel(test.Test): | 271 class Pixel(test.Test): |
151 test = PixelValidator | 272 test = PixelValidator |
152 page_set = 'page_sets/pixel_tests.json' | 273 page_set = 'page_sets/pixel_tests.json' |
153 | 274 |
154 @staticmethod | 275 @staticmethod |
155 def AddTestCommandLineOptions(parser): | 276 def AddTestCommandLineOptions(parser): |
156 group = optparse.OptionGroup(parser, 'Pixel test options') | 277 group = optparse.OptionGroup(parser, 'Pixel test options') |
157 group.add_option('--generated-dir', | 278 group.add_option('--generated-dir', |
158 help='Overrides the default location for generated test images that do ' | 279 help='Overrides the default location for generated test images that do ' |
159 'not match reference images', | 280 'not match reference images', |
160 default=default_generated_data_dir) | 281 default=default_generated_data_dir) |
161 group.add_option('--reference-dir', | 282 group.add_option('--reference-dir', |
162 help='Overrides the default location for reference images', | 283 help='Overrides the default location for reference images', |
163 default=default_reference_image_dir) | 284 default=default_reference_image_dir) |
164 group.add_option('--build-revision', | 285 group.add_option('--build-revision', |
165 help='Chrome revision being tested.', | 286 help='Chrome revision being tested.', |
166 default="unknownrev") | 287 default="unknownrev") |
| 288 group.add_option('--upload-refimg-to-cloud-storage', |
| 289 dest='upload_refimg_to_cloud_storage', |
| 290 action='store_true', default=False, |
| 291 help='Upload resulting images to cloud storage as reference images') |
| 292 group.add_option('--download-refimg-from-cloud-storage', |
| 293 dest='download_refimg_from_cloud_storage', |
| 294 action='store_true', default=False, |
| 295 help='Download reference images from cloud storage') |
| 296 group.add_option('--refimg-cloud-storage-bucket', |
| 297 help='Name of the cloud storage bucket to use for reference images; ' |
| 298 'required with --upload-refimg-to-cloud-storage and ' |
| 299 '--download-refimg-from-cloud-storage. Example: ' |
| 300 '"chromium-gpu-archive/reference-images"', |
| 301 default='') |
| 302 group.add_option('--os-type', |
| 303 help='Type of operating system on which the pixel test is being run, ' |
| 304 'used only to distinguish different operating systems with the same ' |
| 305 'graphics card. Any value is acceptable, but canonical values are ' |
| 306 '"win", "mac", and "linux", and probably, eventually, "chromeos" ' |
| 307 'and "android").', |
| 308 default='') |
| 309 group.add_option('--test-machine-name', |
| 310 help='Name of the test machine. Specifying this argument causes this ' |
| 311 'script to upload failure images and diffs to cloud storage directly, ' |
| 312 'instead of relying on the archive_gpu_pixel_test_results.py script.', |
| 313 default='') |
167 parser.add_option_group(group) | 314 parser.add_option_group(group) |
168 | 315 |
169 def CreatePageSet(self, options): | 316 def CreatePageSet(self, options): |
170 page_set = super(Pixel, self).CreatePageSet(options) | 317 page_set = super(Pixel, self).CreatePageSet(options) |
171 for page in page_set.pages: | 318 for page in page_set.pages: |
172 page.script_to_evaluate_on_commit = test_harness_script | 319 page.script_to_evaluate_on_commit = test_harness_script |
173 return page_set | 320 return page_set |
OLD | NEW |