| OLD | NEW |
|---|
| (Empty) |
| 1 # Copyright (c) 2010 Google Inc. All rights reserved. | |
| 2 # | |
| 3 # Redistribution and use in source and binary forms, with or without | |
| 4 # modification, are permitted provided that the following conditions are | |
| 5 # met: | |
| 6 # | |
| 7 # * Redistributions of source code must retain the above copyright | |
| 8 # notice, this list of conditions and the following disclaimer. | |
| 9 # * Redistributions in binary form must reproduce the above | |
| 10 # copyright notice, this list of conditions and the following disclaimer | |
| 11 # in the documentation and/or other materials provided with the | |
| 12 # distribution. | |
| 13 # * Neither the name of Google Inc. nor the names of its | |
| 14 # contributors may be used to endorse or promote products derived from | |
| 15 # this software without specific prior written permission. | |
| 16 # | |
| 17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | |
| 18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | |
| 19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | |
| 20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | |
| 21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | |
| 22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | |
| 23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | |
| 24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | |
| 25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | |
| 26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | |
| 27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
| 28 | |
| 29 import codecs | |
| 30 import logging | |
| 31 import os.path | |
| 32 | |
| 33 from webkitpy.common.net.layouttestresults import path_for_layout_test, LayoutTe
stResults | |
| 34 from webkitpy.common.config import urls | |
| 35 from webkitpy.tool.bot.botinfo import BotInfo | |
| 36 from webkitpy.tool.grammar import plural, pluralize, join_with_separators | |
| 37 | |
| 38 _log = logging.getLogger(__name__) | |
| 39 | |
| 40 | |
| 41 class FlakyTestReporter(object): | |
| 42 def __init__(self, tool, bot_name): | |
| 43 self._tool = tool | |
| 44 self._bot_name = bot_name | |
| 45 # FIXME: Use the real port object | |
| 46 self._bot_info = BotInfo(tool, tool.deprecated_port().name()) | |
| 47 | |
| 48 def _author_emails_for_test(self, flaky_test): | |
| 49 test_path = path_for_layout_test(flaky_test) | |
| 50 commit_infos = self._tool.checkout().recent_commit_infos_for_files([test
_path]) | |
| 51 # This ignores authors which are not committers because we don't have th
eir bugzilla_email. | |
| 52 return set([commit_info.author().bugzilla_email() for commit_info in com
mit_infos if commit_info.author()]) | |
| 53 | |
| 54 def _bugzilla_email(self): | |
| 55 # FIXME: This is kinda a funny way to get the bugzilla email, | |
| 56 # we could also just create a Credentials object directly | |
| 57 # but some of the Credentials logic is in bugzilla.py too... | |
| 58 self._tool.bugs.authenticate() | |
| 59 return self._tool.bugs.username | |
| 60 | |
| 61 # FIXME: This should move into common.config | |
| 62 _bot_emails = set([ | |
| 63 "commit-queue@webkit.org", # commit-queue | |
| 64 "eseidel@chromium.org", # old commit-queue | |
| 65 "webkit.review.bot@gmail.com", # style-queue, sheriff-bot, CrLx/Gtk EWS | |
| 66 "buildbot@hotmail.com", # Win EWS | |
| 67 # Mac EWS currently uses eric@webkit.org, but that's not normally a bot | |
| 68 ]) | |
| 69 | |
| 70 def _lookup_bug_for_flaky_test(self, flaky_test): | |
| 71 bugs = self._tool.bugs.queries.fetch_bugs_matching_search(search_string=
flaky_test) | |
| 72 if not bugs: | |
| 73 return None | |
| 74 # Match any bugs which are from known bots or the email this bot is usin
g. | |
| 75 allowed_emails = self._bot_emails | set([self._bugzilla_email]) | |
| 76 bugs = filter(lambda bug: bug.reporter_email() in allowed_emails, bugs) | |
| 77 if not bugs: | |
| 78 return None | |
| 79 if len(bugs) > 1: | |
| 80 # FIXME: There are probably heuristics we could use for finding | |
| 81 # the right bug instead of the first, like open vs. closed. | |
| 82 _log.warn("Found %s %s matching '%s' filed by a bot, using the first
." % (pluralize('bug', len(bugs)), [bug.id() for bug in bugs], flaky_test)) | |
| 83 return bugs[0] | |
| 84 | |
| 85 def _view_source_url_for_test(self, test_path): | |
| 86 return urls.view_source_url("LayoutTests/%s" % test_path) | |
| 87 | |
| 88 def _create_bug_for_flaky_test(self, flaky_test, author_emails, latest_flake
_message): | |
| 89 format_values = { | |
| 90 'test': flaky_test, | |
| 91 'authors': join_with_separators(sorted(author_emails)), | |
| 92 'flake_message': latest_flake_message, | |
| 93 'test_url': self._view_source_url_for_test(flaky_test), | |
| 94 'bot_name': self._bot_name, | |
| 95 } | |
| 96 title = "Flaky Test: %(test)s" % format_values | |
| 97 description = """This is an automatically generated bug from the %(bot_n
ame)s. | |
| 98 %(test)s has been flaky on the %(bot_name)s. | |
| 99 | |
| 100 %(test)s was authored by %(authors)s. | |
| 101 %(test_url)s | |
| 102 | |
| 103 %(flake_message)s | |
| 104 | |
| 105 The bots will update this with information from each new failure. | |
| 106 | |
| 107 If you believe this bug to be fixed or invalid, feel free to close. The bots wi
ll re-open if the flake re-occurs. | |
| 108 | |
| 109 If you would like to track this test fix with another bug, please close this bug
as a duplicate. The bots will follow the duplicate chain when making future co
mments. | |
| 110 """ % format_values | |
| 111 | |
| 112 master_flake_bug = 50856 # MASTER: Flaky tests found by the commit-queu
e | |
| 113 return self._tool.bugs.create_bug(title, description, | |
| 114 component="Tools / Tests", | |
| 115 cc=",".join(author_emails), | |
| 116 blocked="50856") | |
| 117 | |
| 118 # This is over-engineered, but it makes for pretty bug messages. | |
| 119 def _optional_author_string(self, author_emails): | |
| 120 if not author_emails: | |
| 121 return "" | |
| 122 heading_string = plural('author') if len(author_emails) > 1 else 'author
' | |
| 123 authors_string = join_with_separators(sorted(author_emails)) | |
| 124 return " (%s: %s)" % (heading_string, authors_string) | |
| 125 | |
| 126 def _latest_flake_message(self, flaky_result, patch): | |
| 127 failure_messages = [failure.message() for failure in flaky_result.failur
es] | |
| 128 flake_message = "The %s just saw %s flake (%s) while processing attachme
nt %s on bug %s." % (self._bot_name, flaky_result.test_name, ", ".join(failure_m
essages), patch.id(), patch.bug_id()) | |
| 129 return "%s\n%s" % (flake_message, self._bot_info.summary_text()) | |
| 130 | |
| 131 def _results_diff_path_for_test(self, test_path): | |
| 132 # FIXME: This is a big hack. We should get this path from results.json | |
| 133 # except that old-run-webkit-tests doesn't produce a results.json | |
| 134 # so we just guess at the file path. | |
| 135 (test_path_root, _) = os.path.splitext(test_path) | |
| 136 return "%s-diffs.txt" % test_path_root | |
| 137 | |
| 138 def _follow_duplicate_chain(self, bug): | |
| 139 while bug.is_closed() and bug.duplicate_of(): | |
| 140 bug = self._tool.bugs.fetch_bug(bug.duplicate_of()) | |
| 141 return bug | |
| 142 | |
| 143 # Maybe this logic should move into Bugzilla? a reopen=True arg to post_comm
ent? | |
| 144 def _update_bug_for_flaky_test(self, bug, latest_flake_message): | |
| 145 if bug.is_closed(): | |
| 146 self._tool.bugs.reopen_bug(bug.id(), latest_flake_message) | |
| 147 else: | |
| 148 self._tool.bugs.post_comment_to_bug(bug.id(), latest_flake_message) | |
| 149 | |
| 150 # This method is needed because our archive paths include a leading tmp/layo
ut-test-results | |
| 151 def _find_in_archive(self, path, archive): | |
| 152 for archived_path in archive.namelist(): | |
| 153 # Archives are currently created with full paths. | |
| 154 if archived_path.endswith(path): | |
| 155 return archived_path | |
| 156 return None | |
| 157 | |
| 158 def _attach_failure_diff(self, flake_bug_id, flaky_test, results_archive_zip
): | |
| 159 results_diff_path = self._results_diff_path_for_test(flaky_test) | |
| 160 # Check to make sure that the path makes sense. | |
| 161 # Since we're not actually getting this path from the results.html | |
| 162 # there is a chance it's wrong. | |
| 163 bot_id = self._tool.status_server.bot_id or "bot" | |
| 164 archive_path = self._find_in_archive(results_diff_path, results_archive_
zip) | |
| 165 if archive_path: | |
| 166 results_diff = results_archive_zip.read(archive_path) | |
| 167 description = "Failure diff from %s" % bot_id | |
| 168 self._tool.bugs.add_attachment_to_bug(flake_bug_id, results_diff, de
scription, filename="failure.diff") | |
| 169 else: | |
| 170 _log.warn("%s does not exist in results archive, uploading entire ar
chive." % results_diff_path) | |
| 171 description = "Archive of layout-test-results from %s" % bot_id | |
| 172 # results_archive is a ZipFile object, grab the File object (.fp) to
pass to Mechanize for uploading. | |
| 173 results_archive_file = results_archive_zip.fp | |
| 174 # Rewind the file object to start (since Mechanize won't do that aut
omatically) | |
| 175 # See https://bugs.webkit.org/show_bug.cgi?id=54593 | |
| 176 results_archive_file.seek(0) | |
| 177 self._tool.bugs.add_attachment_to_bug(flake_bug_id, results_archive_
file, description, filename="layout-test-results.zip") | |
| 178 | |
| 179 def report_flaky_tests(self, patch, flaky_test_results, results_archive): | |
| 180 message = "The %s encountered the following flaky tests while processing
attachment %s:\n\n" % (self._bot_name, patch.id()) | |
| 181 for flaky_result in flaky_test_results: | |
| 182 flaky_test = flaky_result.test_name | |
| 183 bug = self._lookup_bug_for_flaky_test(flaky_test) | |
| 184 latest_flake_message = self._latest_flake_message(flaky_result, patc
h) | |
| 185 author_emails = self._author_emails_for_test(flaky_test) | |
| 186 if not bug: | |
| 187 _log.info("Bug does not already exist for %s, creating." % flaky
_test) | |
| 188 flake_bug_id = self._create_bug_for_flaky_test(flaky_test, autho
r_emails, latest_flake_message) | |
| 189 else: | |
| 190 bug = self._follow_duplicate_chain(bug) | |
| 191 # FIXME: Ideally we'd only make one comment per flake, not two.
But that's not possible | |
| 192 # in all cases (e.g. when reopening), so for now file attachment
and comment are separate. | |
| 193 self._update_bug_for_flaky_test(bug, latest_flake_message) | |
| 194 flake_bug_id = bug.id() | |
| 195 | |
| 196 self._attach_failure_diff(flake_bug_id, flaky_test, results_archive) | |
| 197 message += "%s bug %s%s\n" % (flaky_test, flake_bug_id, self._option
al_author_string(author_emails)) | |
| 198 | |
| 199 message += "The %s is continuing to process your patch." % self._bot_nam
e | |
| 200 self._tool.bugs.post_comment_to_bug(patch.bug_id(), message) | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 15416008:
Remove a bunch of dead code from webkitpy (Closed)
Base URL: svn://svn.chromium.org/blink/trunk